gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 21:46:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature/scanner performance optimization (#498)

Browse Source 
* Refactor: reimplement scanner

Signed-off-by: RustFS Developer <dandan@rustfs.com>

* comment lock

Signed-off-by: junxiang Mu <1948535941@qq.com>

* remove dirty file

Signed-off-by: junxiang Mu <1948535941@qq.com>

* Fix: fix rebase

* fix(scanner): Improve error handling and logging

Signed-off-by: junxiang Mu <1948535941@qq.com>

---------

Signed-off-by: RustFS Developer <dandan@rustfs.com>
Signed-off-by: junxiang Mu <1948535941@qq.com>
Co-authored-by: RustFS Developer <dandan@rustfs.com>
This commit is contained in:
guojidan
2025-09-08 18:35:45 +08:00
committed by GitHub
parent 44f3eb7244
commit 9d5ed1acac
47 changed files with 6710 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/rio/src/hash_reader.rs
+7 -7
View File
@@ -415,16 +415,16 @@ mod tests {
let reader = Cursor::new(data.clone());
let reader = BufReader::new(reader);
// 启用压缩测试
// Enable compression test
let is_compress = true;
let size = data.len() as i64;
let actual_size = data.len() as i64;
let reader = Box::new(WarpReader::new(reader));
// 创建 HashReader
// Create HashReader
let mut hr = HashReader::new(reader, size, actual_size, Some(expected.clone()), false).unwrap();
// 如果启用压缩，先压缩数据
// If compression is enabled, compress data first
let compressed_data = if is_compress {
let mut compressed_buf = Vec::new();
let compress_reader = CompressReader::new(hr, CompressionAlgorithm::Gzip);
@@ -435,7 +435,7 @@ mod tests {
compressed_buf
} else {
// 如果不压缩，直接读取原始数据
// If not compressing, read original data directly
let mut buf = Vec::new();
hr.read_to_end(&mut buf).await.unwrap();
buf
@@ -449,7 +449,7 @@ mod tests {
let is_encrypt = true;
if is_encrypt {
// 加密压缩后的数据
// Encrypt compressed data
let encrypt_reader = encrypt_reader::EncryptReader::new(WarpReader::new(Cursor::new(compressed_data)), key, nonce);
let mut encrypted_data = Vec::new();
let mut encrypt_reader = encrypt_reader;
@@ -457,14 +457,14 @@ mod tests {
println!("Encrypted size: {}", encrypted_data.len());
// 解密数据
// Decrypt data
let decrypt_reader = DecryptReader::new(WarpReader::new(Cursor::new(encrypted_data)), key, nonce);
let mut decrypt_reader = decrypt_reader;
let mut decrypted_data = Vec::new();
decrypt_reader.read_to_end(&mut decrypted_data).await.unwrap();
if is_compress {
// 如果使用了压缩，需要解压缩
// If compression was used, decompress is needed
let decompress_reader =
DecompressReader::new(WarpReader::new(Cursor::new(decrypted_data)), CompressionAlgorithm::Gzip);
let mut decompress_reader = decompress_reader;
crates/rio/src/http_reader.rs
+6 -6
View File
@@ -377,14 +377,14 @@ impl AsyncWrite for HttpWriter {
// let data = vec![42u8; 8];
// // Write
// // 添加 header X-Deny-Write = 1 模拟不可写入的情况
// // Add header X-Deny-Write = 1 to simulate non-writable situation
// let mut headers = HeaderMap::new();
// headers.insert("X-Deny-Write", "1".parse().unwrap());
// // 这里我们使用 PUT 方法
// // Here we use PUT method
// let writer_result = HttpWriter::new(url.clone(), Method::PUT, headers).await;
// match writer_result {
// Ok(mut writer) => {
// // 如果能创建成功，写入应该报错
// // If creation succeeds, write should fail
// let write_result = writer.write_all(&data).await;
// assert!(write_result.is_err(), "write_all should fail when server denies write");
// if let Err(e) = write_result {
@@ -396,7 +396,7 @@ impl AsyncWrite for HttpWriter {
// }
// }
// Err(e) => {
// // 直接构造失败也可以
// // Direct construction failure is also acceptable
// println!("HttpWriter::new error: {e}");
// assert!(
// e.to_string().contains("Empty PUT failed") || e.to_string().contains("Forbidden"),
@@ -411,11 +411,11 @@ impl AsyncWrite for HttpWriter {
// #[tokio::test]
// async fn test_http_writer_and_reader_ok() {
// // 使用本地 Go 测试服务器
// // Use local Go test server
// let url = "http://127.0.0.1:8081/testfile".to_string();
// let data = vec![99u8; 512 * 1024]; // 512KB of data
// // Write (不加 X-Deny-Write)
// // Write (without X-Deny-Write)
// let headers = HeaderMap::new();
// let mut writer = HttpWriter::new(url.clone(), Method::PUT, headers).await.unwrap();
// writer.write_all(&data).await.unwrap();