gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 21:46:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature/scanner performance optimization (#498)

Browse Source 
* Refactor: reimplement scanner

Signed-off-by: RustFS Developer <dandan@rustfs.com>

* comment lock

Signed-off-by: junxiang Mu <1948535941@qq.com>

* remove dirty file

Signed-off-by: junxiang Mu <1948535941@qq.com>

* Fix: fix rebase

* fix(scanner): Improve error handling and logging

Signed-off-by: junxiang Mu <1948535941@qq.com>

---------

Signed-off-by: RustFS Developer <dandan@rustfs.com>
Signed-off-by: junxiang Mu <1948535941@qq.com>
Co-authored-by: RustFS Developer <dandan@rustfs.com>
This commit is contained in:
guojidan
2025-09-08 18:35:45 +08:00
committed by GitHub
parent 44f3eb7244
commit 9d5ed1acac
47 changed files with 6710 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/filemeta.rs
+9 -9
View File
@@ -2710,7 +2710,7 @@ mod test {
ChecksumAlgo::HighwayHash => assert!(algo.valid()),
}
// 验证序列化和反序列化
// Verify serialization and deserialization
let data = obj.marshal_msg().unwrap();
let mut obj2 = MetaObject::default();
obj2.unmarshal_msg(&data).unwrap();
@@ -2741,7 +2741,7 @@ mod test {
assert!(obj.erasure_n > 0, "校验块数量必须大于 0");
assert_eq!(obj.erasure_dist.len(), data_blocks + parity_blocks);
// 验证序列化和反序列化
// Verify serialization and deserialization
let data = obj.marshal_msg().unwrap();
let mut obj2 = MetaObject::default();
obj2.unmarshal_msg(&data).unwrap();
@@ -3039,18 +3039,18 @@ mod test {
#[test]
fn test_special_characters_in_metadata() {
// 测试元数据中的特殊字符处理
// Test special character handling in metadata
let mut obj = MetaObject::default();
// 测试各种特殊字符
// Test various special characters
let special_cases = vec![
("empty", ""),
("unicode", "测试🚀🎉"),
("unicode", "test🚀🎉"),
("newlines", "line1\nline2\nline3"),
("tabs", "col1\tcol2\tcol3"),
("quotes", "\"quoted\" and 'single'"),
("backslashes", "path\\to\\file"),
("mixed", "Mixed: 中文，English, 123, !@#$%"),
("mixed", "Mixed: Chinese，English, 123, !@#$%"),
];
for (key, value) in special_cases {
@@ -3064,15 +3064,15 @@ mod test {
assert_eq!(obj.meta_user, obj2.meta_user);
// 验证每个特殊字符都被正确保存
// Verify each special character is correctly saved
for (key, expected_value) in [
("empty", ""),
("unicode", "测试🚀🎉"),
("unicode", "test🚀🎉"),
("newlines", "line1\nline2\nline3"),
("tabs", "col1\tcol2\tcol3"),
("quotes", "\"quoted\" and 'single'"),
("backslashes", "path\\to\\file"),
("mixed", "Mixed: 中文，English, 123, !@#$%"),
("mixed", "Mixed: Chinese，English, 123, !@#$%"),
] {
assert_eq!(obj2.meta_user.get(key), Some(&expected_value.to_string()));
}
crates/filemeta/src/test_data.rs
+46 -46
View File
@@ -18,11 +18,11 @@ use std::collections::HashMap;
use time::OffsetDateTime;
use uuid::Uuid;
/// 创建一个真实的 xl.meta 文件数据用于测试
/// Create real xl.meta file data for testing
pub fn create_real_xlmeta() -> Result<Vec<u8>> {
let mut fm = FileMeta::new();
// 创建一个真实的对象版本
// Create a real object version
let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef")?;
let data_dir = Uuid::parse_str("fedcba98-7654-3210-fedc-ba9876543210")?;
@@ -62,11 +62,11 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
fm.versions.push(shallow_version);
// 添加一个删除标记版本
// Add a delete marker version
let delete_version_id = Uuid::parse_str("11111111-2222-3333-4444-555555555555")?;
let delete_marker = MetaDeleteMarker {
version_id: Some(delete_version_id),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312260)?), // 1分钟后
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312260)?), // 1 minute later
meta_sys: None,
};
@@ -80,7 +80,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
fm.versions.push(delete_shallow_version);
// 添加一个 Legacy 版本用于测试
// Add a Legacy version for testing
let legacy_version_id = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")?;
let legacy_version = FileMetaVersion {
version_type: VersionType::Legacy,
@@ -91,20 +91,20 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
let mut legacy_shallow = FileMetaShallowVersion::try_from(legacy_version)?;
legacy_shallow.header.version_id = Some(legacy_version_id);
legacy_shallow.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1705312140)?); // 更早的时间
legacy_shallow.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1705312140)?); // earlier time
fm.versions.push(legacy_shallow);
// 按修改时间排序（最新的在前）
// Sort by modification time (newest first)
fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
fm.marshal_msg()
}
/// 创建一个包含多个版本的复杂 xl.meta 文件
/// Create a complex xl.meta file with multiple versions
pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
let mut fm = FileMeta::new();
// 创建10个版本的对象
// Create 10 object versions
for i in 0i64..10i64 {
let version_id = Uuid::new_v4();
let data_dir = if i % 3 == 0 { Some(Uuid::new_v4()) } else { None };
@@ -145,7 +145,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
fm.versions.push(shallow_version);
// 每隔3个版本添加一个删除标记
// Add a delete marker every 3 versions
if i % 3 == 2 {
let delete_version_id = Uuid::new_v4();
let delete_marker = MetaDeleteMarker {
@@ -166,56 +166,56 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
}
}
// 按修改时间排序（最新的在前）
// Sort by modification time (newest first)
fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
fm.marshal_msg()
}
/// 创建一个损坏的 xl.meta 文件用于错误处理测试
/// Create a corrupted xl.meta file for error handling tests
pub fn create_corrupted_xlmeta() -> Vec<u8> {
let mut data = vec![
// 正确的文件头
b'X', b'L', b'2', b' ', // 版本号
1, 0, 3, 0, // 版本号
0xc6, 0x00, 0x00, 0x00, 0x10, // 正确的 bin32 长度标记，但数据长度不匹配
// Correct file header
b'X', b'L', b'2', b' ', // version
1, 0, 3, 0, // version
0xc6, 0x00, 0x00, 0x00, 0x10, // correct bin32 length marker, but data length mismatch
];
// 添加不足的数据（少于声明的长度）
data.extend_from_slice(&[0x42; 8]); // 只有8字节，但声明了16字节
// Add insufficient data (less than declared length)
data.extend_from_slice(&[0x42; 8]); // only 8 bytes, but declared 16 bytes
data
}
/// 创建一个空的 xl.meta 文件
/// Create an empty xl.meta file
pub fn create_empty_xlmeta() -> Result<Vec<u8>> {
let fm = FileMeta::new();
fm.marshal_msg()
}
/// 验证解析结果的辅助函数
/// Helper function to verify parsing results
pub fn verify_parsed_metadata(fm: &FileMeta, expected_versions: usize) -> Result<()> {
assert_eq!(fm.versions.len(), expected_versions, "版本数量不匹配");
assert_eq!(fm.meta_ver, crate::filemeta::XL_META_VERSION, "元数据版本不匹配");
assert_eq!(fm.versions.len(), expected_versions, "Version count mismatch");
assert_eq!(fm.meta_ver, crate::filemeta::XL_META_VERSION, "Metadata version mismatch");
// 验证版本是否按修改时间排序
// Verify versions are sorted by modification time
for i in 1..fm.versions.len() {
let prev_time = fm.versions[i - 1].header.mod_time;
let curr_time = fm.versions[i].header.mod_time;
if let (Some(prev), Some(curr)) = (prev_time, curr_time) {
assert!(prev >= curr, "版本未按修改时间正确排序");
assert!(prev >= curr, "Versions not sorted correctly by modification time");
}
}
Ok(())
}
/// 创建一个包含内联数据的 xl.meta 文件
/// Create an xl.meta file with inline data
pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
let mut fm = FileMeta::new();
// 添加内联数据
// Add inline data
let inline_data = b"This is inline data for testing purposes";
let version_id = Uuid::new_v4();
fm.data.replace(&version_id.to_string(), inline_data.to_vec())?;
@@ -260,47 +260,47 @@ mod tests {
#[test]
fn test_create_real_xlmeta() {
let data = create_real_xlmeta().expect("创建测试数据失败");
assert!(!data.is_empty(), "生成的数据不应为空");
let data = create_real_xlmeta().expect("Failed to create test data");
assert!(!data.is_empty(), "Generated data should not be empty");
// 验证文件头
assert_eq!(&data[0..4], b"XL2 ", "文件头不正确");
// Verify file header
assert_eq!(&data[0..4], b"XL2 ", "Incorrect file header");
// 尝试解析
let fm = FileMeta::load(&data).expect("解析失败");
verify_parsed_metadata(&fm, 3).expect("验证失败");
// Try to parse
let fm = FileMeta::load(&data).expect("Failed to parse");
verify_parsed_metadata(&fm, 3).expect("Verification failed");
}
#[test]
fn test_create_complex_xlmeta() {
let data = create_complex_xlmeta().expect("创建复杂测试数据失败");
assert!(!data.is_empty(), "生成的数据不应为空");
let data = create_complex_xlmeta().expect("Failed to create complex test data");
assert!(!data.is_empty(), "Generated data should not be empty");
let fm = FileMeta::load(&data).expect("解析失败");
assert!(fm.versions.len() >= 10, "应该有至少10个版本");
let fm = FileMeta::load(&data).expect("Failed to parse");
assert!(fm.versions.len() >= 10, "Should have at least 10 versions");
}
#[test]
fn test_create_xlmeta_with_inline_data() {
let data = create_xlmeta_with_inline_data().expect("创建内联数据测试失败");
assert!(!data.is_empty(), "生成的数据不应为空");
let data = create_xlmeta_with_inline_data().expect("Failed to create inline data test");
assert!(!data.is_empty(), "Generated data should not be empty");
let fm = FileMeta::load(&data).expect("解析失败");
assert_eq!(fm.versions.len(), 1, "应该有1个版本");
assert!(!fm.data.as_slice().is_empty(), "应该包含内联数据");
let fm = FileMeta::load(&data).expect("Failed to parse");
assert_eq!(fm.versions.len(), 1, "Should have 1 version");
assert!(!fm.data.as_slice().is_empty(), "Should contain inline data");
}
#[test]
fn test_corrupted_xlmeta_handling() {
let data = create_corrupted_xlmeta();
let result = FileMeta::load(&data);
assert!(result.is_err(), "损坏的数据应该解析失败");
assert!(result.is_err(), "Corrupted data should fail to parse");
}
#[test]
fn test_empty_xlmeta() {
let data = create_empty_xlmeta().expect("创建空测试数据失败");
let fm = FileMeta::load(&data).expect("解析空数据失败");
assert_eq!(fm.versions.len(), 0, "空文件应该没有版本");
let data = create_empty_xlmeta().expect("Failed to create empty test data");
let fm = FileMeta::load(&data).expect("Failed to parse empty data");
assert_eq!(fm.versions.len(), 0, "Empty file should have no versions");
}
}