gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 10:17:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix bitrot readat bug

Browse Source
This commit is contained in:
weisd
2024-11-02 17:53:30 +08:00
parent 4a5443996f
commit 6ab538050d
3 changed files with 65 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/bitrot.rs
+46 -20
View File
@@ -3,6 +3,7 @@ use crate::{
erasure::{ReadAt, Write},
error::{Error, Result},
store_api::BitrotAlgorithm,
utils::crypto::hex,
};
use blake2::Blake2b512;
use blake2::Digest as _;
@@ -15,6 +16,7 @@ use std::{
collections::HashMap,
io::{Cursor, Read},
};
use tracing::warn;
use tokio::{
io::AsyncWriteExt,
@@ -516,7 +518,6 @@ impl Write for BitrotFileWriter {
self.hasher.reset();
self.hasher.update(&buf);
let hash_bytes = self.hasher.clone().finalize();
let _ = self.inner.write(&hash_bytes).await?;
let _ = self.inner.write(buf).await?;
@@ -524,32 +525,36 @@ impl Write for BitrotFileWriter {
}
}
pub fn new_bitrot_filewriter(inner: FileWriter, algo: BitrotAlgorithm, shard_size: usize) -> Result<BitrotWriter> {
Ok(Box::new(BitrotFileWriter::new(inner, algo, shard_size)))
pub fn new_bitrot_filewriter(inner: FileWriter, algo: BitrotAlgorithm, shard_size: usize) -> BitrotWriter {
Box::new(BitrotFileWriter::new(inner, algo, shard_size))
}
#[derive(Debug)]
struct BitrotFileReader {
pub inner: FileReader,
till_offset: usize,
// till_offset: usize,
curr_offset: usize,
hasher: Hasher,
shard_size: usize,
buf: Vec<u8>,
// buf: Vec<u8>,
hash_bytes: Vec<u8>,
}
// fn ceil(a: usize, b: usize) -> usize {
// (a + b - 1) / b
// }
impl BitrotFileReader {
pub fn new(inner: FileReader, algo: BitrotAlgorithm, till_offset: usize, shard_size: usize) -> Self {
pub fn new(inner: FileReader, algo: BitrotAlgorithm, _till_offset: usize, shard_size: usize) -> Self {
let hasher = algo.new();
Self {
inner,
till_offset: till_offset.div_ceil(shard_size) * hasher.size() + till_offset,
// till_offset: ceil(till_offset, shard_size) * hasher.size() + till_offset,
curr_offset: 0,
hash_bytes: Vec::with_capacity(hasher.size()),
hasher,
shard_size,
buf: Vec::new(),
// buf: Vec::new(),
}
}
}
@@ -560,22 +565,17 @@ impl ReadAt for BitrotFileReader {
if offset % self.shard_size != 0 {
return Err(Error::new(DiskError::Unexpected));
}
if self.buf.is_empty() {
self.curr_offset = offset;
let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset;
let buf_len = self.till_offset - stream_offset;
let (buf, _) = self.inner.read_at(stream_offset, buf_len).await?;
self.buf = buf;
}
if offset != self.curr_offset {
return Err(Error::new(DiskError::Unexpected));
}
self.hash_bytes = self.buf.drain(0..self.hash_bytes.capacity()).collect();
let buf = self.buf.drain(0..length).collect::<Vec<_>>();
let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset;
let buf_len = self.hasher.size() + length;
let (mut read_buf, _) = self.inner.read_at(stream_offset, buf_len).await?;
self.hash_bytes = read_buf.drain(0..self.hash_bytes.capacity()).collect();
let buf = read_buf.drain(0..length).collect::<Vec<_>>();
self.hasher.reset();
self.hasher.update(&buf);
let actual = self.hasher.clone().finalize();
if actual != self.hash_bytes {
return Err(Error::new(DiskError::FileCorrupt));
}
@@ -584,6 +584,32 @@ impl ReadAt for BitrotFileReader {
self.curr_offset += readed_len;
Ok((buf, readed_len))
// if self.buf.is_empty() {
// self.curr_offset = offset;
// let stream_offset = (offset / self.shard_size) * self.hasher.size() + offset;
// let buf_len = self.till_offset - stream_offset;
// let (buf, _) = self.inner.read_at(stream_offset, buf_len).await?;
// self.buf = buf;
// }
// if offset != self.curr_offset {
// return Err(Error::new(DiskError::Unexpected));
// }
// self.hash_bytes = self.buf.drain(0..self.hash_bytes.capacity()).collect();
// let buf = self.buf.drain(0..length).collect::<Vec<_>>();
// self.hasher.reset();
// self.hasher.update(&buf);
// let actual = self.hasher.clone().finalize();
// if actual != self.hash_bytes {
// return Err(Error::new(DiskError::FileCorrupt));
// }
// let readed_len = buf.len();
// self.curr_offset += readed_len;
// Ok((buf, readed_len))
}
}