// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time}; use super::object_lock::ObjectLockApi; use super::versioning::VersioningApi; use super::{quota::BucketQuota, target::BucketTargets}; use crate::bucket::utils::deserialize; use crate::config::com::{read_config, save_config}; use crate::disk::BUCKET_META_PREFIX; use crate::error::{Error, Result}; use crate::runtime::sources as runtime_sources; use crate::store::ECStore; use byteorder::{BigEndian, ByteOrder, LittleEndian}; use rustfs_policy::policy::BucketPolicy; use s3s::dto::{ AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration, RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration, }; use serde::Serializer; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::io::{Read, Write}; use std::sync::Arc; use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset}; use tracing::error; fn read_msgp_str(rd: &mut R) -> Result { let len = rmp::decode::read_str_len(rd)? as usize; let mut buf = vec![0u8; len]; rd.read_exact(&mut buf)?; Ok(String::from_utf8(buf)?) } fn read_msgp_bool(rd: &mut R) -> Result { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; match marker { rmp::Marker::True => Ok(true), rmp::Marker::False => Ok(false), rmp::Marker::FixPos(v) => Ok(v != 0), rmp::Marker::U8 => Ok(read_u8(rd)? != 0), rmp::Marker::U16 => Ok(read_u16_raw(rd)? != 0), rmp::Marker::U32 => Ok(read_u32_raw(rd)? != 0), rmp::Marker::U64 => Ok(read_u64_raw(rd)? != 0), rmp::Marker::I8 => Ok(read_i8_raw(rd)? != 0), rmp::Marker::I16 => Ok(read_i16_raw(rd)? != 0), rmp::Marker::I32 => Ok(read_i32_raw(rd)? != 0), rmp::Marker::I64 => Ok(read_i64_raw(rd)? != 0), rmp::Marker::FixNeg(v) => Ok(v != 0), _ => Err(Error::other(format!("expected bool or int-like bool, got marker: {marker:?}"))), } } fn read_msgp_time_value(rd: &mut R) -> Result { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; match marker { rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH), rmp::Marker::Ext8 => read_msgp_ext8_time(rd), rmp::Marker::FixArray(len) => read_msgp_legacy_compact_time(rd, u32::from(len)), rmp::Marker::Array16 => { let len = read_u16_raw(rd)?; read_msgp_legacy_compact_time(rd, u32::from(len)) } rmp::Marker::Array32 => { let len = read_u32_raw(rd)?; read_msgp_legacy_compact_time(rd, len) } rmp::Marker::Bin8 => { let len = usize::from(read_u8(rd)?); read_msgp_time_value_from_embedded_bin(rd, len) } rmp::Marker::Bin16 => { let len = usize::from(read_u16_raw(rd)?); read_msgp_time_value_from_embedded_bin(rd, len) } rmp::Marker::Bin32 => { let len = read_u32_raw(rd)? as usize; read_msgp_time_value_from_embedded_bin(rd, len) } _ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))), } } fn read_u8(rd: &mut R) -> Result { let mut buf = [0u8; 1]; rd.read_exact(&mut buf)?; Ok(buf[0]) } fn read_u16_raw(rd: &mut R) -> Result { let mut buf = [0u8; 2]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_u16(&buf)) } fn read_u32_raw(rd: &mut R) -> Result { let mut buf = [0u8; 4]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_u32(&buf)) } fn read_u64_raw(rd: &mut R) -> Result { let mut buf = [0u8; 8]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_u64(&buf)) } fn read_i8_raw(rd: &mut R) -> Result { Ok(read_u8(rd)? as i8) } fn read_i16_raw(rd: &mut R) -> Result { let mut buf = [0u8; 2]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_i16(&buf)) } fn read_i32_raw(rd: &mut R) -> Result { let mut buf = [0u8; 4]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_i32(&buf)) } fn read_i64_raw(rd: &mut R) -> Result { let mut buf = [0u8; 8]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_i64(&buf)) } fn read_msgp_time_value_from_embedded_bin(rd: &mut R, len: usize) -> Result { let mut buf = vec![0u8; len]; rd.read_exact(&mut buf)?; let mut cur = std::io::Cursor::new(buf); read_msgp_time_value(&mut cur) } fn read_msgp_legacy_compact_time(rd: &mut R, len: u32) -> Result { if len != 9 { return Err(Error::other(format!("invalid legacy compact time len: {len}"))); } let year: i32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let ordinal: u16 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let hour: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let minute: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let second: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let nanosecond: u32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let offset_hour: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let offset_minute: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let offset_second: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let date = Date::from_ordinal_date(year, ordinal).map_err(|e| Error::other(format!("invalid legacy compact time date: {e}")))?; let time = CivilTime::from_hms_nano(hour, minute, second, nanosecond) .map_err(|e| Error::other(format!("invalid legacy compact time time: {e}")))?; let offset = UtcOffset::from_hms(offset_hour, offset_minute, offset_second) .map_err(|e| Error::other(format!("invalid legacy compact time offset: {e}")))?; Ok(PrimitiveDateTime::new(date, time) .assume_offset(offset) .to_offset(UtcOffset::UTC)) } fn read_msgp_bin(rd: &mut R) -> Result> { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; match marker { rmp::Marker::Null => Ok(Vec::new()), rmp::Marker::Bin8 => { let len = usize::from(read_u8(rd)?); read_exact_bytes(rd, len) } rmp::Marker::Bin16 => { let len = usize::from(read_u16_raw(rd)?); read_exact_bytes(rd, len) } rmp::Marker::Bin32 => { let len = read_u32_raw(rd)? as usize; read_exact_bytes(rd, len) } rmp::Marker::FixArray(len) => read_msgp_legacy_byte_array(rd, u32::from(len)), rmp::Marker::Array16 => { let len = read_u16_raw(rd)?; read_msgp_legacy_byte_array(rd, u32::from(len)) } rmp::Marker::Array32 => { let len = read_u32_raw(rd)?; read_msgp_legacy_byte_array(rd, len) } _ => Err(Error::other(format!("expected bin or byte array, got marker: {marker:?}"))), } } fn read_exact_bytes(rd: &mut R, len: usize) -> Result> { let mut buf = vec![0u8; len]; rd.read_exact(&mut buf)?; Ok(buf) } fn read_msgp_legacy_byte_array(rd: &mut R, len: u32) -> Result> { let mut buf = Vec::with_capacity(len as usize); for _ in 0..len { let value: i64 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let byte = u8::try_from(value).map_err(|_| Error::other(format!("byte value out of range: {value}")))?; buf.push(byte); } Ok(buf) } fn write_bin_field(wr: &mut W, key: &str, val: &[u8]) -> Result<()> { rmp::encode::write_str(wr, key)?; rmp::encode::write_bin(wr, val)?; Ok(()) } pub const BUCKET_METADATA_FILE: &str = ".metadata.bin"; pub const BUCKET_METADATA_FORMAT: u16 = 1; pub const BUCKET_METADATA_VERSION: u16 = 1; pub const BUCKET_POLICY_CONFIG: &str = "policy.json"; pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml"; pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml"; pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml"; pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml"; pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json"; pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml"; pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml"; pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml"; pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json"; pub const BUCKET_CORS_CONFIG: &str = "cors.xml"; pub const BUCKET_LOGGING_CONFIG: &str = "logging.xml"; pub const BUCKET_WEBSITE_CONFIG: &str = "website.xml"; pub const BUCKET_ACCELERATE_CONFIG: &str = "accelerate.xml"; pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml"; pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml"; pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json"; pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json"; pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json"; pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table"; pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog"; pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets"; pub fn table_catalog_path_hash(value: &str) -> String { let digest = Sha256::digest(value.as_bytes()); let mut output = String::with_capacity(digest.len() * 2); const HEX: &[u8; 16] = b"0123456789abcdef"; for byte in digest { output.push(char::from(HEX[usize::from(byte >> 4)])); output.push(char::from(HEX[usize::from(byte & 0x0f)])); } output } pub fn table_bucket_catalog_metadata_prefix(bucket: &str) -> String { format!( "{}/{}/{}", BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, table_catalog_path_hash(bucket) ) } #[derive(Debug, Clone)] pub struct BucketMetadata { pub name: String, pub created: OffsetDateTime, pub lock_enabled: bool, // While marked as unused, it may need to be retained pub policy_config_json: Vec, pub notification_config_xml: Vec, pub lifecycle_config_xml: Vec, pub object_lock_config_xml: Vec, pub versioning_config_xml: Vec, pub encryption_config_xml: Vec, pub tagging_config_xml: Vec, pub quota_config_json: Vec, pub replication_config_xml: Vec, pub bucket_targets_config_json: Vec, pub bucket_targets_config_meta_json: Vec, pub cors_config_xml: Vec, pub logging_config_xml: Vec, pub website_config_xml: Vec, pub accelerate_config_xml: Vec, pub request_payment_config_xml: Vec, pub public_access_block_config_xml: Vec, pub bucket_acl_config_json: Vec, pub table_bucket_config_json: Vec, pub durability_config_json: Vec, pub policy_config_updated_at: OffsetDateTime, pub object_lock_config_updated_at: OffsetDateTime, pub encryption_config_updated_at: OffsetDateTime, pub tagging_config_updated_at: OffsetDateTime, pub quota_config_updated_at: OffsetDateTime, pub replication_config_updated_at: OffsetDateTime, pub versioning_config_updated_at: OffsetDateTime, pub lifecycle_config_updated_at: OffsetDateTime, pub notification_config_updated_at: OffsetDateTime, pub bucket_targets_config_updated_at: OffsetDateTime, pub bucket_targets_config_meta_updated_at: OffsetDateTime, pub cors_config_updated_at: OffsetDateTime, pub logging_config_updated_at: OffsetDateTime, pub website_config_updated_at: OffsetDateTime, pub accelerate_config_updated_at: OffsetDateTime, pub request_payment_config_updated_at: OffsetDateTime, pub public_access_block_config_updated_at: OffsetDateTime, pub bucket_acl_config_updated_at: OffsetDateTime, pub table_bucket_config_updated_at: OffsetDateTime, pub durability_config_updated_at: OffsetDateTime, pub new_field_updated_at: OffsetDateTime, pub policy_config: Option, pub notification_config: Option, pub lifecycle_config: Option, pub object_lock_config: Option, pub versioning_config: Option, pub sse_config: Option, pub tagging_config: Option, pub quota_config: Option, pub replication_config: Option, pub bucket_target_config: Option, pub bucket_target_config_meta: Option>, pub cors_config: Option, pub logging_config: Option, pub website_config: Option, pub accelerate_config: Option, pub request_payment_config: Option, pub public_access_block_config: Option, pub bucket_acl_config: Option, } impl Default for BucketMetadata { fn default() -> Self { Self { name: Default::default(), created: OffsetDateTime::UNIX_EPOCH, lock_enabled: Default::default(), policy_config_json: Default::default(), notification_config_xml: Default::default(), lifecycle_config_xml: Default::default(), object_lock_config_xml: Default::default(), versioning_config_xml: Default::default(), encryption_config_xml: Default::default(), tagging_config_xml: Default::default(), quota_config_json: Default::default(), replication_config_xml: Default::default(), bucket_targets_config_json: Default::default(), bucket_targets_config_meta_json: Default::default(), cors_config_xml: Default::default(), logging_config_xml: Default::default(), website_config_xml: Default::default(), accelerate_config_xml: Default::default(), request_payment_config_xml: Default::default(), public_access_block_config_xml: Default::default(), bucket_acl_config_json: Default::default(), table_bucket_config_json: Default::default(), durability_config_json: Default::default(), policy_config_updated_at: OffsetDateTime::UNIX_EPOCH, object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH, encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH, tagging_config_updated_at: OffsetDateTime::UNIX_EPOCH, quota_config_updated_at: OffsetDateTime::UNIX_EPOCH, replication_config_updated_at: OffsetDateTime::UNIX_EPOCH, versioning_config_updated_at: OffsetDateTime::UNIX_EPOCH, lifecycle_config_updated_at: OffsetDateTime::UNIX_EPOCH, notification_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH, cors_config_updated_at: OffsetDateTime::UNIX_EPOCH, logging_config_updated_at: OffsetDateTime::UNIX_EPOCH, website_config_updated_at: OffsetDateTime::UNIX_EPOCH, accelerate_config_updated_at: OffsetDateTime::UNIX_EPOCH, request_payment_config_updated_at: OffsetDateTime::UNIX_EPOCH, public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH, table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH, durability_config_updated_at: OffsetDateTime::UNIX_EPOCH, new_field_updated_at: OffsetDateTime::UNIX_EPOCH, policy_config: Default::default(), notification_config: Default::default(), lifecycle_config: Default::default(), object_lock_config: Default::default(), versioning_config: Default::default(), sse_config: Default::default(), tagging_config: Default::default(), quota_config: Default::default(), replication_config: Default::default(), bucket_target_config: Default::default(), bucket_target_config_meta: Default::default(), cors_config: Default::default(), logging_config: Default::default(), website_config: Default::default(), accelerate_config: Default::default(), request_payment_config: Default::default(), public_access_block_config: Default::default(), bucket_acl_config: Default::default(), } } } impl BucketMetadata { pub fn new(name: &str) -> Self { BucketMetadata { name: name.to_string(), ..Default::default() } } pub fn save_file_path(&self) -> String { format!("{}/{}/{}", BUCKET_META_PREFIX, self.name.as_str(), BUCKET_METADATA_FILE) } pub fn versioning(&self) -> bool { self.lock_enabled || (self.object_lock_config.as_ref().is_some_and(|v| v.enabled()) || self.versioning_config.as_ref().is_some_and(|v| v.enabled())) } pub fn object_locking(&self) -> bool { self.lock_enabled || self.object_lock_config.as_ref().is_some_and(|v| v.enabled()) } pub fn table_bucket_enabled(&self) -> bool { !self.table_bucket_config_json.is_empty() } /// Parsed per-bucket durability override, if a valid one is stored. /// /// Absent/empty/unparsable payloads all mean "no override" (the bucket /// follows the global durability mode); a parse failure is logged so a /// corrupted entry cannot silently change fsync behavior. pub fn durability_config(&self) -> Option { if self.durability_config_json.is_empty() { return None; } match serde_json::from_slice(&self.durability_config_json) { Ok(cfg) => Some(cfg), Err(e) => { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "durability", error = %e, "Failed to parse bucket metadata config" ); None } } } /// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { let mut fields = rmp::decode::read_map_len(rd)?; *self = Self::default(); while fields > 0 { fields -= 1; let key_len = rmp::decode::read_str_len(rd)?; let mut key_buf = vec![0u8; key_len as usize]; rd.read_exact(&mut key_buf)?; let key = String::from_utf8(key_buf)?; match key.as_str() { "Name" => self.name = read_msgp_str(rd)?, "Created" => self.created = read_msgp_time_value(rd)?, "LockEnabled" => self.lock_enabled = read_msgp_bool(rd)?, "PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?, "NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?, "LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?, "ObjectLockConfigXML" | "ObjectLockConfigXml" => self.object_lock_config_xml = read_msgp_bin(rd)?, "VersioningConfigXML" | "VersioningConfigXml" => self.versioning_config_xml = read_msgp_bin(rd)?, "EncryptionConfigXML" | "EncryptionConfigXml" => self.encryption_config_xml = read_msgp_bin(rd)?, "TaggingConfigXML" | "TaggingConfigXml" => self.tagging_config_xml = read_msgp_bin(rd)?, "QuotaConfigJSON" | "QuotaConfigJson" => self.quota_config_json = read_msgp_bin(rd)?, "ReplicationConfigXML" | "ReplicationConfigXml" => self.replication_config_xml = read_msgp_bin(rd)?, "BucketTargetsConfigJSON" | "BucketTargetsConfigJson" => self.bucket_targets_config_json = read_msgp_bin(rd)?, "BucketTargetsConfigMetaJSON" | "BucketTargetsConfigMetaJson" => { self.bucket_targets_config_meta_json = read_msgp_bin(rd)? } "PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?, "ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?, "EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?, "TaggingConfigUpdatedAt" => self.tagging_config_updated_at = read_msgp_time_value(rd)?, "QuotaConfigUpdatedAt" => self.quota_config_updated_at = read_msgp_time_value(rd)?, "ReplicationConfigUpdatedAt" => self.replication_config_updated_at = read_msgp_time_value(rd)?, "VersioningConfigUpdatedAt" => self.versioning_config_updated_at = read_msgp_time_value(rd)?, "LifecycleConfigUpdatedAt" => self.lifecycle_config_updated_at = read_msgp_time_value(rd)?, "NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?, "BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?, "BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?, "CorsConfigXML" | "CorsConfigXml" => self.cors_config_xml = read_msgp_bin(rd)?, "LoggingConfigXML" | "LoggingConfigXml" => self.logging_config_xml = read_msgp_bin(rd)?, "WebsiteConfigXML" | "WebsiteConfigXml" => self.website_config_xml = read_msgp_bin(rd)?, "AccelerateConfigXML" | "AccelerateConfigXml" => self.accelerate_config_xml = read_msgp_bin(rd)?, "RequestPaymentConfigXML" | "RequestPaymentConfigXml" => self.request_payment_config_xml = read_msgp_bin(rd)?, "PublicAccessBlockConfigXML" | "PublicAccessBlockConfigXml" => { self.public_access_block_config_xml = read_msgp_bin(rd)? } "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?, "TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?, "DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?, "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?, "LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?, "WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?, "AccelerateConfigUpdatedAt" => self.accelerate_config_updated_at = read_msgp_time_value(rd)?, "RequestPaymentConfigUpdatedAt" => self.request_payment_config_updated_at = read_msgp_time_value(rd)?, "PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?, "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?, "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?, "DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?, other => { tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field"); skip_msgp_value(rd)?; } } } Ok(()) } /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn encode_to(&self, wr: &mut W) -> Result<()> { // Map size: MinIO fields (25) + RustFS extensions (18) let map_len: u32 = 43; rmp::encode::write_map_len(wr, map_len)?; // MinIO field order (same as Go struct) rmp::encode::write_str(wr, "Name")?; rmp::encode::write_str(wr, &self.name)?; rmp::encode::write_str(wr, "Created")?; write_msgp_time(wr, self.created)?; rmp::encode::write_str(wr, "LockEnabled")?; rmp::encode::write_bool(wr, self.lock_enabled)?; write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?; write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?; write_bin_field(wr, "LifecycleConfigXML", &self.lifecycle_config_xml)?; write_bin_field(wr, "ObjectLockConfigXML", &self.object_lock_config_xml)?; write_bin_field(wr, "VersioningConfigXML", &self.versioning_config_xml)?; write_bin_field(wr, "EncryptionConfigXML", &self.encryption_config_xml)?; write_bin_field(wr, "TaggingConfigXML", &self.tagging_config_xml)?; write_bin_field(wr, "QuotaConfigJSON", &self.quota_config_json)?; write_bin_field(wr, "ReplicationConfigXML", &self.replication_config_xml)?; write_bin_field(wr, "BucketTargetsConfigJSON", &self.bucket_targets_config_json)?; write_bin_field(wr, "BucketTargetsConfigMetaJSON", &self.bucket_targets_config_meta_json)?; rmp::encode::write_str(wr, "PolicyConfigUpdatedAt")?; write_msgp_time(wr, self.policy_config_updated_at)?; rmp::encode::write_str(wr, "ObjectLockConfigUpdatedAt")?; write_msgp_time(wr, self.object_lock_config_updated_at)?; rmp::encode::write_str(wr, "EncryptionConfigUpdatedAt")?; write_msgp_time(wr, self.encryption_config_updated_at)?; rmp::encode::write_str(wr, "TaggingConfigUpdatedAt")?; write_msgp_time(wr, self.tagging_config_updated_at)?; rmp::encode::write_str(wr, "QuotaConfigUpdatedAt")?; write_msgp_time(wr, self.quota_config_updated_at)?; rmp::encode::write_str(wr, "ReplicationConfigUpdatedAt")?; write_msgp_time(wr, self.replication_config_updated_at)?; rmp::encode::write_str(wr, "VersioningConfigUpdatedAt")?; write_msgp_time(wr, self.versioning_config_updated_at)?; rmp::encode::write_str(wr, "LifecycleConfigUpdatedAt")?; write_msgp_time(wr, self.lifecycle_config_updated_at)?; rmp::encode::write_str(wr, "NotificationConfigUpdatedAt")?; write_msgp_time(wr, self.notification_config_updated_at)?; rmp::encode::write_str(wr, "BucketTargetsConfigUpdatedAt")?; write_msgp_time(wr, self.bucket_targets_config_updated_at)?; rmp::encode::write_str(wr, "BucketTargetsConfigMetaUpdatedAt")?; write_msgp_time(wr, self.bucket_targets_config_meta_updated_at)?; // RustFS extensions write_bin_field(wr, "CorsConfigXML", &self.cors_config_xml)?; write_bin_field(wr, "LoggingConfigXML", &self.logging_config_xml)?; write_bin_field(wr, "WebsiteConfigXML", &self.website_config_xml)?; write_bin_field(wr, "AccelerateConfigXML", &self.accelerate_config_xml)?; write_bin_field(wr, "RequestPaymentConfigXML", &self.request_payment_config_xml)?; write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?; write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?; write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?; write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?; rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?; write_msgp_time(wr, self.cors_config_updated_at)?; rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?; write_msgp_time(wr, self.logging_config_updated_at)?; rmp::encode::write_str(wr, "WebsiteConfigUpdatedAt")?; write_msgp_time(wr, self.website_config_updated_at)?; rmp::encode::write_str(wr, "AccelerateConfigUpdatedAt")?; write_msgp_time(wr, self.accelerate_config_updated_at)?; rmp::encode::write_str(wr, "RequestPaymentConfigUpdatedAt")?; write_msgp_time(wr, self.request_payment_config_updated_at)?; rmp::encode::write_str(wr, "PublicAccessBlockConfigUpdatedAt")?; write_msgp_time(wr, self.public_access_block_config_updated_at)?; rmp::encode::write_str(wr, "BucketAclConfigUpdatedAt")?; write_msgp_time(wr, self.bucket_acl_config_updated_at)?; rmp::encode::write_str(wr, "TableBucketConfigUpdatedAt")?; write_msgp_time(wr, self.table_bucket_config_updated_at)?; rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?; write_msgp_time(wr, self.durability_config_updated_at)?; Ok(()) } pub fn marshal_msg(&self) -> Result> { let mut buf = Vec::new(); self.encode_to(&mut buf)?; Ok(buf) } pub fn unmarshal(buf: &[u8]) -> Result { let mut bm = Self::default(); let mut cur = std::io::Cursor::new(buf); bm.decode_from(&mut cur)?; Ok(bm) } pub fn check_header(buf: &[u8]) -> Result<()> { if buf.len() <= 4 { return Err(Error::other("read_bucket_metadata: data invalid")); } let format = LittleEndian::read_u16(&buf[0..2]); let version = LittleEndian::read_u16(&buf[2..4]); match format { BUCKET_METADATA_FORMAT => {} _ => return Err(Error::other("read_bucket_metadata: format invalid")), } match version { BUCKET_METADATA_VERSION => {} _ => return Err(Error::other("read_bucket_metadata: version invalid")), } Ok(()) } pub(crate) fn default_timestamps(&mut self) { if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.policy_config_updated_at = self.created } if self.encryption_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.encryption_config_updated_at = self.created } if self.tagging_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.tagging_config_updated_at = self.created } if self.object_lock_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.object_lock_config_updated_at = self.created } if self.quota_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.quota_config_updated_at = self.created } if self.replication_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.replication_config_updated_at = self.created } if self.versioning_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.versioning_config_updated_at = self.created } if self.lifecycle_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.lifecycle_config_updated_at = self.created } if self.notification_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.notification_config_updated_at = self.created } if self.bucket_targets_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.bucket_targets_config_updated_at = self.created } if self.bucket_targets_config_meta_updated_at == OffsetDateTime::UNIX_EPOCH { self.bucket_targets_config_meta_updated_at = self.created } if self.public_access_block_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.public_access_block_config_updated_at = self.created } if self.logging_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.logging_config_updated_at = self.created } if self.website_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.website_config_updated_at = self.created } if self.accelerate_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.accelerate_config_updated_at = self.created } if self.request_payment_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.request_payment_config_updated_at = self.created } if self.bucket_acl_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.bucket_acl_config_updated_at = self.created } if self.table_bucket_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.table_bucket_config_updated_at = self.created } if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.durability_config_updated_at = self.created } } pub fn update_config(&mut self, config_file: &str, data: Vec) -> Result { let updated = OffsetDateTime::now_utc(); match config_file { BUCKET_POLICY_CONFIG => { self.policy_config_json = data; self.policy_config_updated_at = updated; } BUCKET_NOTIFICATION_CONFIG => { self.notification_config_xml = data; self.notification_config_updated_at = updated; } BUCKET_LIFECYCLE_CONFIG => { self.lifecycle_config_xml = data; self.lifecycle_config = None; self.lifecycle_config_updated_at = updated; } BUCKET_SSECONFIG => { self.encryption_config_xml = data; self.encryption_config_updated_at = updated; } BUCKET_TAGGING_CONFIG => { self.tagging_config_xml = data; // Drop the parsed form (like lifecycle above) so clearing the // payload can't leave stale parsed tags to be cached. self.tagging_config = None; self.tagging_config_updated_at = updated; } BUCKET_QUOTA_CONFIG_FILE => { self.quota_config_json = data; self.quota_config_updated_at = updated; } OBJECT_LOCK_CONFIG => { self.object_lock_config_xml = data; self.object_lock_config_updated_at = updated; } BUCKET_VERSIONING_CONFIG => { self.versioning_config_xml = data; self.versioning_config_updated_at = updated; } BUCKET_REPLICATION_CONFIG => { self.replication_config_xml = data; self.replication_config_updated_at = updated; } BUCKET_TARGETS_FILE => { // let x = data.clone(); // let str = std::str::from_utf8(&x).expect("Invalid UTF-8"); // println!("update config:{}", str); self.bucket_targets_config_json = data; self.bucket_targets_config_updated_at = updated; } BUCKET_CORS_CONFIG => { self.cors_config_xml = data; self.cors_config_updated_at = updated; } BUCKET_LOGGING_CONFIG => { self.logging_config_xml = data; self.logging_config_updated_at = updated; } BUCKET_WEBSITE_CONFIG => { self.website_config_xml = data; self.website_config_updated_at = updated; } BUCKET_ACCELERATE_CONFIG => { self.accelerate_config_xml = data; self.accelerate_config_updated_at = updated; } BUCKET_REQUEST_PAYMENT_CONFIG => { self.request_payment_config_xml = data; self.request_payment_config_updated_at = updated; } BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => { self.public_access_block_config_xml = data; self.public_access_block_config_updated_at = updated; } BUCKET_ACL_CONFIG => { self.bucket_acl_config_json = data; self.bucket_acl_config_updated_at = updated; } BUCKET_TABLE_CONFIG => { self.table_bucket_config_json = data; self.table_bucket_config_updated_at = updated; } BUCKET_DURABILITY_CONFIG => { self.durability_config_json = data; self.durability_config_updated_at = updated; } _ => return Err(Error::other(format!("config file not found : {config_file}"))), } Ok(updated) } pub fn set_created(&mut self, created: Option) { self.created = created.unwrap_or_else(OffsetDateTime::now_utc) } pub async fn save(&mut self) -> Result<()> { let Some(store) = runtime_sources::object_store_handle() else { return Err(Error::other("errServerNotInitialized")); }; self.save_with_store(store).await } /// Persist this metadata through an explicit store (backlog#1052 S7): the /// owning instance's metadata system passes its own store so a second /// server's bucket metadata lands in that server's `.rustfs.sys`, not the /// ambient (first) one. [`BucketMetadata::save`] keeps the ambient default. pub async fn save_with_store(&mut self, store: std::sync::Arc) -> Result<()> { self.parse_all_configs()?; let mut buf: Vec = vec![0; 4]; LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT); LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION); let data = self .marshal_msg() .map_err(|e| Error::other(format!("save bucket metadata failed: {e}")))?; buf.extend_from_slice(&data); save_config(store, self.save_file_path().as_str(), buf).await?; Ok(()) } fn parse_policy_config(&mut self) -> Result<()> { if !self.policy_config_json.is_empty() { self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?); } else { self.policy_config = None; } Ok(()) } fn parse_all_configs(&mut self) -> Result<()> { if let Err(e) = self.parse_policy_config() { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "policy", error = %e, "Failed to parse bucket metadata config" ); } if !self.notification_config_xml.is_empty() && let Err(e) = deserialize::(&self.notification_config_xml) .map(|c| self.notification_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "notification", error = %e, "Failed to parse bucket metadata config" ); } if !self.lifecycle_config_xml.is_empty() && let Err(e) = deserialize::(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "lifecycle", error = %e, "Failed to parse bucket metadata config" ); } if !self.object_lock_config_xml.is_empty() && let Err(e) = deserialize::(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "object_lock", error = %e, "Failed to parse bucket metadata config" ); } if !self.versioning_config_xml.is_empty() && let Err(e) = deserialize::(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "versioning", error = %e, "Failed to parse bucket metadata config" ); } if !self.encryption_config_xml.is_empty() && let Err(e) = deserialize::(&self.encryption_config_xml).map(|c| self.sse_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "encryption", error = %e, "Failed to parse bucket metadata config" ); } if !self.tagging_config_xml.is_empty() && let Err(e) = deserialize::(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "tagging", error = %e, "Failed to parse bucket metadata config" ); } if !self.quota_config_json.is_empty() && let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "quota", error = %e, "Failed to parse bucket metadata config" ); } if !self.replication_config_xml.is_empty() && let Err(e) = deserialize::(&self.replication_config_xml).map(|c| self.replication_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "replication", error = %e, "Failed to parse bucket metadata config" ); } if !self.bucket_targets_config_json.is_empty() { if let Err(e) = serde_json::from_slice::(&self.bucket_targets_config_json) .map(|t| self.bucket_target_config = Some(t)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "bucket_targets", error = %e, "Failed to parse bucket metadata config" ); self.bucket_target_config = Some(BucketTargets::default()); } } else { self.bucket_target_config = Some(BucketTargets::default()); } if !self.cors_config_xml.is_empty() && let Err(e) = deserialize::(&self.cors_config_xml).map(|c| self.cors_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "cors", error = %e, "Failed to parse bucket metadata config" ); } if !self.logging_config_xml.is_empty() && let Err(e) = deserialize::(&self.logging_config_xml).map(|c| self.logging_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "logging", error = %e, "Failed to parse bucket metadata config" ); } if !self.website_config_xml.is_empty() && let Err(e) = deserialize::(&self.website_config_xml).map(|c| self.website_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "website", error = %e, "Failed to parse bucket metadata config" ); } if !self.accelerate_config_xml.is_empty() && let Err(e) = deserialize::(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "accelerate", error = %e, "Failed to parse bucket metadata config" ); } if !self.request_payment_config_xml.is_empty() && let Err(e) = deserialize::(&self.request_payment_config_xml) .map(|c| self.request_payment_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "request_payment", error = %e, "Failed to parse bucket metadata config" ); } if !self.public_access_block_config_xml.is_empty() && let Err(e) = deserialize::(&self.public_access_block_config_xml) .map(|c| self.public_access_block_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "public_access_block", error = %e, "Failed to parse bucket metadata config" ); } if !self.bucket_acl_config_json.is_empty() && let Err(e) = String::from_utf8(self.bucket_acl_config_json.clone()).map(|acl| self.bucket_acl_config = Some(acl)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "bucket_acl", error = %e, "Failed to parse bucket metadata config" ); } Ok(()) } } pub async fn load_bucket_metadata(api: Arc, bucket: &str) -> Result { load_bucket_metadata_parse(api, bucket, true).await } pub async fn load_bucket_metadata_parse(api: Arc, bucket: &str, parse: bool) -> Result { Ok(load_bucket_metadata_parse_with_presence(api, bucket, parse).await?.0) } /// The returned `bool` reports whether the metadata was actually read from /// persisted storage; `false` means no metadata exists for this bucket on this /// store and the returned value is a fabricated in-memory default. pub(crate) async fn load_bucket_metadata_parse_with_presence( api: Arc, bucket: &str, parse: bool, ) -> Result<(BucketMetadata, bool)> { let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await { Ok(res) => (res, true), Err(err) => { if err != Error::ConfigNotFound { return Err(err); } (BucketMetadata::new(bucket), false) } }; bm.default_timestamps(); if parse { bm.parse_all_configs()?; } Ok((bm, persisted)) } async fn read_bucket_metadata(api: Arc, bucket: &str) -> Result { if bucket.is_empty() { error!("bucket name empty"); return Err(Error::other("invalid argument")); } let bm = BucketMetadata::new(bucket); let file_path = bm.save_file_path(); let data = read_config(api, &file_path).await?; BucketMetadata::check_header(&data)?; let bm = BucketMetadata::unmarshal(&data[4..])?; Ok(bm) } fn _write_time(t: &OffsetDateTime, s: S) -> std::result::Result where S: Serializer, { let mut buf = vec![0x0; 15]; let sec = t.unix_timestamp() - 62135596800; let nsec = t.nanosecond(); buf[0] = 0xc7; // mext8 buf[1] = 0x0c; // Length buf[2] = 0x05; // Time extension type BigEndian::write_u64(&mut buf[3..], sec as u64); BigEndian::write_u32(&mut buf[11..], nsec); s.serialize_bytes(&buf) } #[cfg(test)] mod test { use super::*; /// Decode a whitespace-tolerant hex fixture into bytes. fn decode_hex(s: &str) -> Vec { let s: String = s.chars().filter(|c| !c.is_whitespace()).collect(); (0..s.len()) .step_by(2) .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("valid hex fixture")) .collect() } /// backlog#580: prove RustFS parses a real MinIO-written bucket `.metadata.bin` /// blob without loss. The fixture is the raw `.metadata.bin` object body /// (4-byte `format|version` header + msgpack) carved from the /// `.minio.sys/buckets/interop/.metadata.bin` object written by MinIO /// `RELEASE.2025-07-23` — see `tests/fixtures/minio/README.md`. #[test] fn parses_real_minio_bucket_metadata_blob_without_loss() { let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")); // Same 4-byte format|version header (1|1) and msgpack layout as MinIO. BucketMetadata::check_header(&blob).expect("valid .metadata.bin header"); let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata"); // Raw config fields survive the msgpack decode (PascalCase MinIO field names). assert_eq!(bm.name, "interop"); assert!(!bm.policy_config_json.is_empty(), "policy JSON present"); assert!(!bm.lifecycle_config_xml.is_empty(), "lifecycle XML present"); assert!(!bm.object_lock_config_xml.is_empty(), "object-lock XML present"); assert!(!bm.versioning_config_xml.is_empty(), "versioning XML present"); assert!(!bm.tagging_config_xml.is_empty(), "tagging XML present"); assert!(!bm.quota_config_json.is_empty(), "quota JSON present"); // Typed parse of each stored config must succeed. `parse_all_configs` // logs+skips on error, so a None here means a real MinIO-compat parse gap. bm.parse_all_configs().expect("parse_all_configs"); assert!(bm.policy_config.is_some(), "policy parsed"); assert!(bm.versioning_config.is_some(), "versioning parsed"); assert!(bm.object_lock_config.is_some(), "object-lock parsed"); assert!(bm.tagging_config.is_some(), "tagging parsed"); assert!(bm.quota_config.is_some(), "quota parsed"); assert!( bm.lifecycle_config.is_some(), "lifecycle parsed (MinIO writes an extension element)" ); assert!(bm.notification_config.is_some(), "notification parsed"); assert!(bm.sse_config.is_some(), "encryption (SSE) parsed"); assert!(bm.replication_config.is_some(), "replication parsed"); // Object lock is expressed through the parsed config, not the legacy // `LockEnabled` flag (MinIO leaves that false for config-based locks). assert!(bm.object_locking(), "object lock active via parsed config"); } /// backlog#580: KNOWN GAP (weisd 2026-03-06 "inline_data 前缀不同"). RustFS's /// inline-data extraction does not yet recover the object body from a /// MinIO-written bucket-metadata object: `into_fileinfo(read_data=true).data` /// returns bytes that are not the `.metadata.bin` blob (no `format|version` /// header). Kept as an ignored, documented reproduction until the MinIO /// inline-data framing is handled on the read path. /// backlog#580: prove RustFS reads a MinIO-written **inlined** bucket-metadata /// object end-to-end. MinIO stores inline data as `[bitrot hash][object body]` /// (the "`inline_data` 前缀不同" that weisd flagged on 2026-03-06 is that /// bitrot prefix, not a format incompatibility). Running the raw inline shard /// through RustFS's `BitrotReader` with the default `HighwayHash256S` must /// verify the checksum and yield the exact `.metadata.bin` blob. #[tokio::test] async fn reads_minio_inline_bucket_metadata_via_bitrot() { use crate::erasure::coding::BitrotReader; use rustfs_utils::HashAlgorithm; let xlmeta = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata_full.xlmeta.hex")); let fm = rustfs_filemeta::FileMeta::load(&xlmeta).expect("parse MinIO xl.meta"); let fi = fm .into_fileinfo("interop", ".metadata.bin", "", true, false, false) .expect("into_fileinfo"); // The raw inline shard is `[HighwayHash256 (32B)][object body]`. let inline = fi.data.expect("inline shard present"); let algo = HashAlgorithm::HighwayHash256S; let body_len = inline.len() - algo.size(); let mut reader = BitrotReader::new(std::io::Cursor::new(inline.to_vec()), body_len, algo, false); let mut body = vec![0u8; body_len]; let read = reader.read(&mut body).await.expect("bitrot verify + read MinIO inline shard"); assert_eq!(read, body_len); // The verified body is exactly the `.metadata.bin` blob, and it parses. BucketMetadata::check_header(&body).expect("recovered body is a valid .metadata.bin"); let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob"); assert_eq!(bm.name, "interop"); bm.parse_all_configs().expect("parse recovered configs"); assert!(bm.lifecycle_config.is_some()); } #[tokio::test] async fn marshal_msg() { // write_time(OffsetDateTime::UNIX_EPOCH).unwrap(); let bm = BucketMetadata::new("dada"); let buf = bm.marshal_msg().unwrap(); let new = BucketMetadata::unmarshal(&buf).unwrap(); assert_eq!(bm.name, new.name); } #[test] fn object_locking_requires_lock_metadata_not_plain_versioning() { use s3s::dto::ObjectLockEnabled; let mut bm = BucketMetadata::new("test-bucket"); bm.versioning_config = Some(VersioningConfiguration { status: Some(s3s::dto::BucketVersioningStatus::from_static("Enabled")), ..Default::default() }); assert!(!bm.object_locking()); bm.object_lock_config = Some(ObjectLockConfiguration { object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), ..Default::default() }); assert!(bm.object_locking()); } #[test] fn parse_all_configs_parses_stored_configs_without_store_dependency() { let mut bm = BucketMetadata::new("test-bucket"); bm.policy_config_json = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(); bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.amazonaws.com","targetbucket":"target-bucket","arn":"arn:aws:s3:::target-bucket"}]}"# .to_vec(); bm.parse_all_configs().unwrap(); assert!(bm.policy_config.is_some()); let bucket_targets = bm.bucket_target_config.unwrap(); assert_eq!(bucket_targets.targets.len(), 1); assert_eq!(bucket_targets.targets[0].endpoint, "s3.amazonaws.com"); assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket"); } #[test] fn lifecycle_update_config_clears_parsed_config_on_delete() { let mut bm = BucketMetadata::new("test-bucket"); let lifecycle_xml = br#"rule1Enabled30"#; bm.update_config(BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.to_vec()) .expect("lifecycle config should update"); bm.parse_all_configs().expect("lifecycle config should parse"); assert!(bm.lifecycle_config.is_some()); bm.update_config(BUCKET_LIFECYCLE_CONFIG, Vec::new()) .expect("lifecycle config delete should update metadata"); assert!(bm.lifecycle_config_xml.is_empty()); assert!(bm.lifecycle_config.is_none()); } /// Companion to the lifecycle case above. `parse_all_configs` skips empty /// XML rather than clearing, so without the explicit reset a cleared /// tagging config would keep serving the previously parsed tags. #[test] fn tagging_update_config_clears_parsed_config_on_delete() { let mut bm = BucketMetadata::new("test-bucket"); let tagging_xml = br#"envprod"#; bm.update_config(BUCKET_TAGGING_CONFIG, tagging_xml.to_vec()) .expect("tagging config should update"); bm.parse_all_configs().expect("tagging config should parse"); assert!(bm.tagging_config.is_some()); bm.update_config(BUCKET_TAGGING_CONFIG, Vec::new()) .expect("tagging config delete should update metadata"); assert!(bm.tagging_config_xml.is_empty()); assert!(bm.tagging_config.is_none()); // A re-parse must not resurrect them either. bm.parse_all_configs().expect("cleared tagging should parse"); assert!(bm.tagging_config.is_none()); } #[tokio::test] async fn marshal_msg_complete_example() { // Create a complete BucketMetadata with various configurations let mut bm = BucketMetadata::new("test-bucket"); // Set creation time to current time bm.created = OffsetDateTime::now_utc(); bm.lock_enabled = true; // Add policy configuration let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}"#; bm.policy_config_json = policy_json.as_bytes().to_vec(); bm.policy_config_updated_at = OffsetDateTime::now_utc(); // Add lifecycle configuration let lifecycle_xml = r#"rule1Enabled30"#; bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec(); bm.lifecycle_config_updated_at = OffsetDateTime::now_utc(); // Add versioning configuration let versioning_xml = r#"Enabled"#; bm.versioning_config_xml = versioning_xml.as_bytes().to_vec(); bm.versioning_config_updated_at = OffsetDateTime::now_utc(); // Add encryption configuration let encryption_xml = r#"AES256"#; bm.encryption_config_xml = encryption_xml.as_bytes().to_vec(); bm.encryption_config_updated_at = OffsetDateTime::now_utc(); // Add tagging configuration let tagging_xml = r#"EnvironmentTestOwnerRustFS"#; bm.tagging_config_xml = tagging_xml.as_bytes().to_vec(); bm.tagging_config_updated_at = OffsetDateTime::now_utc(); // Add quota configuration let quota_json = r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; // 1GB quota bm.quota_config_json = quota_json.as_bytes().to_vec(); bm.quota_config_updated_at = OffsetDateTime::now_utc(); // Add object lock configuration let object_lock_xml = r#"EnabledGOVERNANCE7"#; bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec(); bm.object_lock_config_updated_at = OffsetDateTime::now_utc(); // Add notification configuration let notification_xml = r#"notification1s3:ObjectCreated:*test-log-group"#; bm.notification_config_xml = notification_xml.as_bytes().to_vec(); bm.notification_config_updated_at = OffsetDateTime::now_utc(); // Add replication configuration let replication_xml = r#"arn:aws:iam::123456789012:role/replication-rolerule1Enableddocuments/arn:aws:s3:::destination-bucket"#; bm.replication_config_xml = replication_xml.as_bytes().to_vec(); bm.replication_config_updated_at = OffsetDateTime::now_utc(); // Add bucket targets configuration let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#; bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec(); bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc(); // Add bucket targets meta configuration let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#; bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec(); bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc(); // Add public access block configuration let public_access_block_xml = r#"truetruetruefalse"#; bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec(); bm.public_access_block_config_updated_at = OffsetDateTime::now_utc(); let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#; bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec(); bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc(); let table_bucket_marker = r#"{"enabled":true}"#; bm.table_bucket_config_json = table_bucket_marker.as_bytes().to_vec(); bm.table_bucket_config_updated_at = OffsetDateTime::now_utc(); // Test serialization let buf = bm.marshal_msg().unwrap(); assert!(!buf.is_empty(), "Serialized buffer should not be empty"); // Test deserialization let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap(); // Verify all fields are correctly serialized and deserialized assert_eq!(bm.name, deserialized_bm.name); assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp()); assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled); // Verify configuration data assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json); assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml); assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml); assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml); assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml); assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json); assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml); assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json); assert_eq!(bm.table_bucket_config_json, deserialized_bm.table_bucket_config_json); assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml); assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml); assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml); assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json); assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json); // Verify timestamps (comparing unix timestamps to avoid precision issues) assert_eq!( bm.policy_config_updated_at.unix_timestamp(), deserialized_bm.policy_config_updated_at.unix_timestamp() ); assert_eq!( bm.lifecycle_config_updated_at.unix_timestamp(), deserialized_bm.lifecycle_config_updated_at.unix_timestamp() ); assert_eq!( bm.versioning_config_updated_at.unix_timestamp(), deserialized_bm.versioning_config_updated_at.unix_timestamp() ); assert_eq!( bm.encryption_config_updated_at.unix_timestamp(), deserialized_bm.encryption_config_updated_at.unix_timestamp() ); assert_eq!( bm.tagging_config_updated_at.unix_timestamp(), deserialized_bm.tagging_config_updated_at.unix_timestamp() ); assert_eq!( bm.quota_config_updated_at.unix_timestamp(), deserialized_bm.quota_config_updated_at.unix_timestamp() ); assert_eq!( bm.object_lock_config_updated_at.unix_timestamp(), deserialized_bm.object_lock_config_updated_at.unix_timestamp() ); assert_eq!( bm.notification_config_updated_at.unix_timestamp(), deserialized_bm.notification_config_updated_at.unix_timestamp() ); assert_eq!( bm.replication_config_updated_at.unix_timestamp(), deserialized_bm.replication_config_updated_at.unix_timestamp() ); assert_eq!( bm.bucket_targets_config_updated_at.unix_timestamp(), deserialized_bm.bucket_targets_config_updated_at.unix_timestamp() ); assert_eq!( bm.bucket_targets_config_meta_updated_at.unix_timestamp(), deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp() ); assert_eq!( bm.table_bucket_config_updated_at.unix_timestamp(), deserialized_bm.table_bucket_config_updated_at.unix_timestamp() ); assert!(deserialized_bm.table_bucket_enabled()); // Test that the serialized data contains expected content let buf_str = String::from_utf8_lossy(&buf); assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name"); // Verify the buffer size is reasonable (should be larger due to all the config data) assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations"); println!("✅ Complete BucketMetadata serialization test passed"); println!(" - Bucket name: {}", deserialized_bm.name); println!(" - Lock enabled: {}", deserialized_bm.lock_enabled); println!(" - Policy config size: {} bytes", deserialized_bm.policy_config_json.len()); println!(" - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len()); println!(" - Serialized buffer size: {} bytes", buf.len()); } #[test] fn table_bucket_marker_tracks_config_presence() { let mut bm = BucketMetadata::new("table-bucket"); assert!(!bm.table_bucket_enabled()); bm.update_config(BUCKET_TABLE_CONFIG, br#"{"enabled":true}"#.to_vec()) .unwrap(); assert!(bm.table_bucket_enabled()); assert!(!bm.table_bucket_config_json.is_empty()); bm.update_config(BUCKET_TABLE_CONFIG, Vec::new()).unwrap(); assert!(!bm.table_bucket_enabled()); } /// HP-5b (rustfs/backlog#938): the durability override is a RustFS /// extension entry and must survive an encode/decode round trip. #[test] fn durability_config_round_trips_and_tracks_updates() { let mut bm = BucketMetadata::new("durability-bucket"); assert!(bm.durability_config().is_none(), "fresh metadata carries no override"); bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec()) .unwrap(); assert_ne!(bm.durability_config_updated_at, OffsetDateTime::UNIX_EPOCH); let buf = bm.marshal_msg().unwrap(); let back = BucketMetadata::unmarshal(&buf).unwrap(); assert_eq!(back.durability_config_json, bm.durability_config_json); assert_eq!( back.durability_config_updated_at.unix_timestamp(), bm.durability_config_updated_at.unix_timestamp() ); assert_eq!(back.durability_config().and_then(|c| c.normalized_mode()).as_deref(), Some("relaxed")); // Clearing the entry removes the override. bm.update_config(BUCKET_DURABILITY_CONFIG, Vec::new()).unwrap(); assert!(bm.durability_config().is_none()); // Corrupted payloads must degrade to "no override", never to a tier. bm.durability_config_json = b"not-json".to_vec(); assert!(bm.durability_config().is_none()); } /// After policy deletion (policy_config_json cleared), parse_policy_config sets policy_config to None. #[test] fn test_parse_policy_config_clears_cache_when_json_empty() { let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}"#; let mut bm = BucketMetadata::new("b"); bm.policy_config_json = policy_json.as_bytes().to_vec(); bm.parse_policy_config().unwrap(); assert!(bm.policy_config.is_some(), "policy_config should be set when JSON non-empty"); bm.policy_config_json.clear(); bm.parse_policy_config().unwrap(); assert!( bm.policy_config.is_none(), "policy_config should be None after JSON cleared (e.g. policy deleted)" ); } }