gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 13:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(perf): Add configurable bitrot skip for reads (#2110)

Browse Source 
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
evan slack
2026-03-10 20:59:00 -06:00
committed by GitHub
parent f00d01ec2d
commit 4b480727d6
9 changed files with 47 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bitrot.rs
+11 -4
View File
@@ -39,6 +39,7 @@ pub async fn create_bitrot_reader(
length: usize,
shard_size: usize,
checksum_algo: HashAlgorithm,
skip_verify: bool,
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
// Calculate the total length to read, including the checksum overhead
let length = length.div_ceil(shard_size) * checksum_algo.size() + length;
@@ -47,13 +48,18 @@ pub async fn create_bitrot_reader(
// Use inline data
let mut rd = Cursor::new(data.to_vec());
rd.set_position(offset as u64);
let reader = BitrotReader::new(Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>, shard_size, checksum_algo);
let reader = BitrotReader::new(
Box::new(rd) as Box<dyn AsyncRead + Send + Sync + Unpin>,
shard_size,
checksum_algo,
skip_verify,
);
Ok(Some(reader))
} else if let Some(disk) = disk {
// Read from disk
match disk.read_file_stream(bucket, path, offset, length - offset).await {
Ok(rd) => {
let reader = BitrotReader::new(rd, shard_size, checksum_algo);
let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
Ok(Some(reader))
}
Err(e) => Err(e),
@@ -116,7 +122,7 @@ mod tests {
let checksum_algo = HashAlgorithm::HighwayHash256;
let result =
create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo).await;
create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo, false).await;
assert!(result.is_ok());
assert!(result.unwrap().is_some());
@@ -127,7 +133,8 @@ mod tests {
let shard_size = 16;
let checksum_algo = HashAlgorithm::HighwayHash256;
let result = create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo).await;
let result =
create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false).await;
assert!(result.is_ok());
assert!(result.unwrap().is_none());