gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/rustfs/s3-rustfs into feature/ilm

Browse Source 
# Conflicts:
#	Cargo.lock
#	Cargo.toml
#	crates/utils/Cargo.toml
#	crates/utils/src/net.rs
#	ecstore/Cargo.toml
#	ecstore/src/set_disk.rs
#	rustfs/src/storage/ecfs.rs
This commit is contained in:
likewu
2025-06-23 16:42:18 +08:00
parent 2c176dd864 dc93398768
commit 81f0c9763f
225 changed files with 14913 additions and 6941 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/error.rs
+17 -1
View File
@@ -111,7 +111,20 @@ impl Clone for Error {
impl From<std::io::Error> for Error {
fn from(e: std::io::Error) -> Self {
Error::Io(e)
match e.kind() {
std::io::ErrorKind::UnexpectedEof => Error::Unexpected,
_ => Error::Io(e),
}
}
}
impl From<Error> for std::io::Error {
fn from(e: Error) -> Self {
match e {
Error::Unexpected => std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "Unexpected EOF"),
Error::Io(e) => e,
_ => std::io::Error::other(e.to_string()),
}
}
}
@@ -414,6 +427,9 @@ mod tests {
let filemeta_error: Error = io_error.into();
match filemeta_error {
Error::Unexpected => {
assert_eq!(kind, ErrorKind::UnexpectedEof);
}
Error::Io(extracted_io_error) => {
assert_eq!(extracted_io_error.kind(), kind);
assert!(extracted_io_error.to_string().contains("test error"));
crates/filemeta/src/fileinfo.rs
+27 -15
View File
@@ -1,5 +1,7 @@
use crate::error::{Error, Result};
use crate::headers::RESERVED_METADATA_PREFIX_LOWER;
use crate::headers::RUSTFS_HEALING;
use bytes::Bytes;
use rmp_serde::Serializer;
use rustfs_utils::HashAlgorithm;
use serde::Deserialize;
@@ -8,9 +10,6 @@ use std::collections::HashMap;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::headers::RESERVED_METADATA_PREFIX;
use crate::headers::RUSTFS_HEALING;
pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
@@ -27,10 +26,10 @@ pub struct ObjectPartInfo {
pub etag: String,
pub number: usize,
pub size: usize,
pub actual_size: usize, // Original data size
pub actual_size: i64, // Original data size
pub mod_time: Option<OffsetDateTime>,
// Index holds the index of the part in the erasure coding
pub index: Option<Vec<u8>>,
pub index: Option<Bytes>,
// Checksums holds checksums of the part
pub checksums: Option<HashMap<String, String>>,
}
@@ -40,7 +39,7 @@ pub struct ObjectPartInfo {
pub struct ChecksumInfo {
pub part_number: usize,
pub algorithm: HashAlgorithm,
pub hash: Vec<u8>,
pub hash: Bytes,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)]
@@ -121,15 +120,21 @@ impl ErasureInfo {
}
/// Calculate the total erasure file size for a given original size.
// Returns the final erasure size from the original size
pub fn shard_file_size(&self, total_length: usize) -> usize {
pub fn shard_file_size(&self, total_length: i64) -> i64 {
if total_length == 0 {
return 0;
}
if total_length < 0 {
return total_length;
}
let total_length = total_length as usize;
let num_shards = total_length / self.block_size;
let last_block_size = total_length % self.block_size;
let last_shard_size = calc_shard_size(last_block_size, self.data_blocks);
num_shards * self.shard_size() + last_shard_size
(num_shards * self.shard_size() + last_shard_size) as i64
}
/// Check if this ErasureInfo equals another ErasureInfo
@@ -158,7 +163,7 @@ pub struct FileInfo {
pub expire_restored: bool,
pub data_dir: Option<Uuid>,
pub mod_time: Option<OffsetDateTime>,
pub size: usize,
pub size: i64,
// File mode bits
pub mode: Option<u32>,
// WrittenByVersion is the unix time stamp of the version that created this version of the object
@@ -170,13 +175,13 @@ pub struct FileInfo {
pub mark_deleted: bool,
// ReplicationState - Internal replication state to be passed back in ObjectInfo
// pub replication_state: Option<ReplicationState>, // TODO: implement ReplicationState
pub data: Option<Vec<u8>>,
pub data: Option<Bytes>,
pub num_versions: usize,
pub successor_mod_time: Option<OffsetDateTime>,
pub fresh: bool,
pub idx: usize,
// Combined checksum when object was uploaded
pub checksum: Option<Vec<u8>>,
pub checksum: Option<Bytes>,
pub versioned: bool,
}
@@ -261,7 +266,8 @@ impl FileInfo {
etag: String,
part_size: usize,
mod_time: Option<OffsetDateTime>,
actual_size: usize,
actual_size: i64,
index: Option<Bytes>,
) {
let part = ObjectPartInfo {
etag,
@@ -269,7 +275,7 @@ impl FileInfo {
size: part_size,
mod_time,
actual_size,
index: None,
index,
checksums: None,
};
@@ -341,6 +347,12 @@ impl FileInfo {
self.metadata
.insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
}
pub fn set_data_moved(&mut self) {
self.metadata
.insert(format!("{}data-moved", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
}
pub fn inline_data(&self) -> bool {
self.metadata
.contains_key(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).as_str())
@@ -350,7 +362,7 @@ impl FileInfo {
/// Check if the object is compressed
pub fn is_compressed(&self) -> bool {
self.metadata
.contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX))
.contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX_LOWER))
}
/// Check if the object is remote (transitioned to another tier)
@@ -464,7 +476,7 @@ impl FileInfoVersions {
}
/// Calculate the total size of all versions for this object
pub fn size(&self) -> usize {
pub fn size(&self) -> i64 {
self.versions.iter().map(|v| v.size).sum()
}
}
crates/filemeta/src/filemeta.rs
+45 -38
View File
@@ -6,10 +6,12 @@ use crate::headers::{
RESERVED_METADATA_PREFIX_LOWER, VERSION_PURGE_STATUS_KEY,
};
use byteorder::ByteOrder;
use bytes::Bytes;
use rmp::Marker;
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::convert::TryFrom;
use std::hash::Hasher;
use std::io::{Read, Write};
use std::{collections::HashMap, io::Cursor};
@@ -433,7 +435,7 @@ impl FileMeta {
if let Some(ref data) = fi.data {
let key = vid.unwrap_or_default().to_string();
self.data.replace(&key, data.clone())?;
self.data.replace(&key, data.to_vec())?;
}
let version = FileMetaVersion::from(fi);
@@ -629,7 +631,10 @@ impl FileMeta {
}
if read_data {
fi.data = self.data.find(fi.version_id.unwrap_or_default().to_string().as_str())?;
fi.data = self
.data
.find(fi.version_id.unwrap_or_default().to_string().as_str())?
.map(bytes::Bytes::from);
}
fi.num_versions = self.versions.len();
@@ -1462,9 +1467,9 @@ pub struct MetaObject {
pub part_numbers: Vec<usize>, // Part Numbers
pub part_etags: Vec<String>, // Part ETags
pub part_sizes: Vec<usize>, // Part Sizes
pub part_actual_sizes: Vec<usize>, // Part ActualSizes (compression)
pub part_indices: Vec<Vec<u8>>, // Part Indexes (compression)
pub size: usize, // Object version size
pub part_actual_sizes: Vec<i64>, // Part ActualSizes (compression)
pub part_indices: Vec<Bytes>, // Part Indexes (compression)
pub size: i64, // Object version size
pub mod_time: Option<OffsetDateTime>, // Object version modified time
pub meta_sys: HashMap<String, Vec<u8>>, // Object version internal metadata
pub meta_user: HashMap<String, String>, // Object version metadata set by user
@@ -1621,7 +1626,7 @@ impl MetaObject {
let mut buf = vec![0u8; blen as usize];
cur.read_exact(&mut buf)?;
indices.push(buf);
indices.push(Bytes::from(buf));
}
self.part_indices = indices;
@@ -1893,13 +1898,16 @@ impl MetaObject {
}
for (k, v) in &self.meta_sys {
if k == AMZ_STORAGE_CLASS && v == b"STANDARD" {
continue;
}
if k.starts_with(RESERVED_METADATA_PREFIX)
|| k.starts_with(RESERVED_METADATA_PREFIX_LOWER)
|| k == VERSION_PURGE_STATUS_KEY
{
continue;
metadata.insert(k.to_owned(), String::from_utf8(v.to_owned()).unwrap_or_default());
}
metadata.insert(k.to_owned(), String::from_utf8(v.to_owned()).unwrap_or_default());
}
// todo: ReplicationState,Delete
@@ -2616,7 +2624,6 @@ pub async fn read_xl_meta_no_data<R: AsyncRead + Unpin>(reader: &mut R, size: us
}
#[cfg(test)]
mod test {
use super::*;
use crate::test_data::*;
@@ -2736,7 +2743,7 @@ mod test {
// 验证基本属性
assert_eq!(fm.meta_ver, XL_META_VERSION);
assert_eq!(fm.versions.len(), 3, "应该有3个版本（1个对象，1个删除标记，1个Legacy）");
assert_eq!(fm.versions.len(), 3, "应该有 3 个版本（1 个对象，1 个删除标记，1 个 Legacy）");
// 验证版本类型
let mut object_count = 0;
@@ -2752,9 +2759,9 @@ mod test {
}
}
assert_eq!(object_count, 1, "应该有1个对象版本");
assert_eq!(delete_count, 1, "应该有1个删除标记");
assert_eq!(legacy_count, 1, "应该有1个Legacy版本");
assert_eq!(object_count, 1, "应该有 1 个对象版本");
assert_eq!(delete_count, 1, "应该有 1 个删除标记");
assert_eq!(legacy_count, 1, "应该有 1 个 Legacy 版本");
// 验证兼容性
assert!(fm.is_compatible_with_meta(), "应该与 xl 格式兼容");
@@ -2777,7 +2784,7 @@ mod test {
let fm = FileMeta::load(&data).expect("解析复杂数据失败");
// 验证版本数量
assert!(fm.versions.len() >= 10, "应该有至少10个版本");
assert!(fm.versions.len() >= 10, "应该有至少 10 个版本");
// 验证版本排序
assert!(fm.is_sorted_by_mod_time(), "版本应该按修改时间排序");
@@ -2798,7 +2805,7 @@ mod test {
let data = create_xlmeta_with_inline_data().expect("创建内联数据测试失败");
let fm = FileMeta::load(&data).expect("解析内联数据失败");
assert_eq!(fm.versions.len(), 1, "应该有1个版本");
assert_eq!(fm.versions.len(), 1, "应该有 1 个版本");
assert!(!fm.data.as_slice().is_empty(), "应该包含内联数据");
// 验证内联数据内容
@@ -2845,7 +2852,7 @@ mod test {
for version in &fm.versions {
let signature = version.header.get_signature();
assert_eq!(signature.len(), 4, "签名应该是4字节");
assert_eq!(signature.len(), 4, "签名应该是 4 字节");
// 验证相同版本的签名一致性
let signature2 = version.header.get_signature();
@@ -2888,7 +2895,7 @@ mod test {
// 验证版本内容一致性
for (v1, v2) in fm.versions.iter().zip(fm2.versions.iter()) {
assert_eq!(v1.header.version_type, v2.header.version_type, "版本类型应该一致");
assert_eq!(v1.header.version_id, v2.header.version_id, "版本ID应该一致");
assert_eq!(v1.header.version_id, v2.header.version_id, "版本 ID 应该一致");
}
}
@@ -2909,26 +2916,26 @@ mod test {
let _serialized = fm.marshal_msg().expect("序列化失败");
let serialization_time = start.elapsed();
println!("性能测试结果:");
println!(" 创建时间: {:?}", creation_time);
println!(" 解析时间: {:?}", parsing_time);
println!(" 序列化时间: {:?}", serialization_time);
println!("性能测试结果：");
println!(" 创建时间：{:?}", creation_time);
println!(" 解析时间：{:?}", parsing_time);
println!(" 序列化时间：{:?}", serialization_time);
// 基本性能断言（这些值可能需要根据实际性能调整）
assert!(parsing_time.as_millis() < 100, "解析时间应该小于100ms");
assert!(serialization_time.as_millis() < 100, "序列化时间应该小于100ms");
assert!(parsing_time.as_millis() < 100, "解析时间应该小于 100ms");
assert!(serialization_time.as_millis() < 100, "序列化时间应该小于 100ms");
}
#[test]
fn test_edge_cases() {
// 测试边界情况
// 1. 测试空版本ID
// 1. 测试空版本 ID
let mut fm = FileMeta::new();
let version = FileMetaVersion {
version_type: VersionType::Object,
object: Some(MetaObject {
version_id: None, // 空版本ID
version_id: None, // 空版本 ID
data_dir: None,
erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
erasure_m: 1,
@@ -2961,13 +2968,13 @@ mod test {
// 2. 测试极大的文件大小
let large_object = MetaObject {
size: usize::MAX,
size: i64::MAX,
part_sizes: vec![usize::MAX],
..Default::default()
};
// 应该能够处理大数值
assert_eq!(large_object.size, usize::MAX);
assert_eq!(large_object.size, i64::MAX);
}
#[tokio::test]
@@ -3024,9 +3031,9 @@ mod test {
let large_size = mem::size_of_val(&large_fm);
println!("Large FileMeta size: {} bytes", large_size);
// 验证内存使用是合理的（注意：size_of_val只计算栈上的大小，不包括堆分配）
// 对于包含Vec的结构体，size_of_val可能相同，因为Vec的容量在堆上
println!("版本数量: {}", large_fm.versions.len());
// 验证内存使用是合理的（注意：size_of_val 只计算栈上的大小，不包括堆分配）
// 对于包含 Vec 的结构体，size_of_val 可能相同，因为 Vec 的容量在堆上
println!("版本数量：{}", large_fm.versions.len());
assert!(!large_fm.versions.is_empty(), "应该有版本数据");
}
@@ -3097,8 +3104,8 @@ mod test {
};
// 验证参数的合理性
assert!(obj.erasure_m > 0, "数据块数量必须大于0");
assert!(obj.erasure_n > 0, "校验块数量必须大于0");
assert!(obj.erasure_m > 0, "数据块数量必须大于 0");
assert!(obj.erasure_n > 0, "校验块数量必须大于 0");
assert_eq!(obj.erasure_dist.len(), data_blocks + parity_blocks);
// 验证序列化和反序列化
@@ -3259,7 +3266,7 @@ mod test {
// 测试多个版本列表的合并
let merged = merge_file_meta_versions(1, false, 0, &[versions1.clone(), versions2.clone()]);
// 合并结果可能为空，这取决于版本的兼容性，这是正常的
println!("合并结果数量: {}", merged.len());
println!("合并结果数量：{}", merged.len());
}
#[test]
@@ -3269,12 +3276,12 @@ mod test {
for flag in flags {
let flag_value = flag as u8;
assert!(flag_value > 0, "标志位值应该大于0");
assert!(flag_value > 0, "标志位值应该大于 0");
// 测试标志位组合
let combined = Flags::FreeVersion as u8 | Flags::UsesDataDir as u8;
// 对于位运算，组合值可能不总是大于单个值，这是正常的
assert!(combined > 0, "组合标志位应该大于0");
assert!(combined > 0, "组合标志位应该大于 0");
}
}
@@ -3410,7 +3417,7 @@ mod test {
("tabs", "col1\tcol2\tcol3"),
("quotes", "\"quoted\" and 'single'"),
("backslashes", "path\\to\\file"),
("mixed", "Mixed: 中文, English, 123, !@#$%"),
("mixed", "Mixed: 中文，English, 123, !@#$%"),
];
for (key, value) in special_cases {
@@ -3432,7 +3439,7 @@ mod test {
("tabs", "col1\tcol2\tcol3"),
("quotes", "\"quoted\" and 'single'"),
("backslashes", "path\\to\\file"),
("mixed", "Mixed: 中文, English, 123, !@#$%"),
("mixed", "Mixed: 中文，English, 123, !@#$%"),
] {
assert_eq!(obj2.meta_user.get(key), Some(&expected_value.to_string()));
}
@@ -3529,7 +3536,7 @@ pub struct DetailedVersionStats {
pub free_versions: usize,
pub versions_with_data_dir: usize,
pub versions_with_inline_data: usize,
pub total_size: usize,
pub total_size: i64,
pub latest_mod_time: Option<OffsetDateTime>,
}
crates/filemeta/src/headers.rs
+2
View File
@@ -19,3 +19,5 @@ pub const X_RUSTFS_DATA_MOV: &str = "X-Rustfs-Internal-data-mov";
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
pub const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status";
pub const AMZ_DECODED_CONTENT_LENGTH: &str = "X-Amz-Decoded-Content-Length";
pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
crates/filemeta/src/test_data.rs
+4 -4
View File
@@ -91,7 +91,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
let mut fm = FileMeta::new();
// 创建10个版本的对象
for i in 0..10 {
for i in 0i64..10i64 {
let version_id = Uuid::new_v4();
let data_dir = if i % 3 == 0 { Some(Uuid::new_v4()) } else { None };
@@ -113,9 +113,9 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
part_numbers: vec![1],
part_etags: vec![format!("etag-{:08x}", i)],
part_sizes: vec![1024 * (i + 1) as usize],
part_actual_sizes: vec![1024 * (i + 1) as usize],
part_actual_sizes: vec![1024 * (i + 1)],
part_indices: Vec::new(),
size: 1024 * (i + 1) as usize,
size: 1024 * (i + 1),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200 + i * 60)?),
meta_sys: HashMap::new(),
meta_user: metadata,
@@ -221,7 +221,7 @@ pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
part_sizes: vec![inline_data.len()],
part_actual_sizes: Vec::new(),
part_indices: Vec::new(),
size: inline_data.len(),
size: inline_data.len() as i64,
mod_time: Some(OffsetDateTime::now_utc()),
meta_sys: HashMap::new(),
meta_user: HashMap::new(),