mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 03:46:37 +00:00
Feature/scanner performance optimization (#498)
* Refactor: reimplement scanner Signed-off-by: RustFS Developer <dandan@rustfs.com> * comment lock Signed-off-by: junxiang Mu <1948535941@qq.com> * remove dirty file Signed-off-by: junxiang Mu <1948535941@qq.com> * Fix: fix rebase * fix(scanner): Improve error handling and logging Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: RustFS Developer <dandan@rustfs.com> Signed-off-by: junxiang Mu <1948535941@qq.com> Co-authored-by: RustFS Developer <dandan@rustfs.com>
This commit is contained in:
@@ -41,14 +41,14 @@ impl<R> ParallelReader<R>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
// readers传入前应处理disk错误,确保每个reader达到可用数量的BitrotReader
|
||||
// Readers should handle disk errors before being passed in, ensuring each reader reaches the available number of BitrotReaders
|
||||
pub fn new(readers: Vec<Option<BitrotReader<R>>>, e: Erasure, offset: usize, total_length: usize) -> Self {
|
||||
let shard_size = e.shard_size();
|
||||
let shard_file_size = e.shard_file_size(total_length as i64) as usize;
|
||||
|
||||
let offset = (offset / e.block_size) * shard_size;
|
||||
|
||||
// 确保offset不超过shard_file_size
|
||||
// Ensure offset does not exceed shard_file_size
|
||||
|
||||
ParallelReader {
|
||||
readers,
|
||||
@@ -99,7 +99,7 @@ where
|
||||
}
|
||||
}) as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
|
||||
} else {
|
||||
// reader是None时返回FileNotFound错误
|
||||
// Return FileNotFound error when reader is None
|
||||
Box::pin(async move { (i, Err(Error::FileNotFound)) })
|
||||
as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
|
||||
};
|
||||
@@ -146,7 +146,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取数据块总长度
|
||||
/// Get the total length of data blocks
|
||||
fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
|
||||
let mut size = 0;
|
||||
for shard in shards.iter().take(data_blocks).flatten() {
|
||||
@@ -156,7 +156,7 @@ fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
|
||||
size
|
||||
}
|
||||
|
||||
/// 将编码块中的数据块写入目标,支持 offset 和 length
|
||||
/// Write data blocks from encoded blocks to target, supporting offset and length
|
||||
async fn write_data_blocks<W>(
|
||||
writer: &mut W,
|
||||
en_blocks: &[Option<Vec<u8>>],
|
||||
|
||||
Reference in New Issue
Block a user