mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 15:16:56 +00:00
refactor: Restructure project layout and clean up dependencies (#30)
This commit introduces a significant reorganization of the project structure to improve maintainability and clarity. Key changes include: - Adjusted the directory layout for a more logical module organization. - Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times. - Updated import paths and configuration files to reflect the structural changes.
This commit is contained in:
houseme
@@ -0,0 +1,482 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use bytes::Bytes;
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
pin_project! {
|
||||
/// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity.
|
||||
pub struct BitrotReader<R> {
|
||||
#[pin]
|
||||
inner: R,
|
||||
hash_algo: HashAlgorithm,
|
||||
shard_size: usize,
|
||||
buf: Vec<u8>,
|
||||
hash_buf: Vec<u8>,
|
||||
// hash_read: usize,
|
||||
// data_buf: Vec<u8>,
|
||||
// data_read: usize,
|
||||
// hash_checked: bool,
|
||||
id: Uuid,
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> BitrotReader<R>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
/// Create a new BitrotReader.
|
||||
pub fn new(inner: R, shard_size: usize, algo: HashAlgorithm) -> Self {
|
||||
let hash_size = algo.size();
|
||||
Self {
|
||||
inner,
|
||||
hash_algo: algo,
|
||||
shard_size,
|
||||
buf: Vec::new(),
|
||||
hash_buf: vec![0u8; hash_size],
|
||||
// hash_read: 0,
|
||||
// data_buf: Vec::new(),
|
||||
// data_read: 0,
|
||||
// hash_checked: false,
|
||||
id: Uuid::new_v4(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a single (hash+data) block, verify hash, and return the number of bytes read into `out`.
|
||||
/// Returns an error if hash verification fails or data exceeds shard_size.
|
||||
pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
|
||||
if out.len() > self.shard_size {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("data size {} exceeds shard size {}", out.len(), self.shard_size),
|
||||
));
|
||||
}
|
||||
|
||||
let hash_size = self.hash_algo.size();
|
||||
// Read hash
|
||||
|
||||
if hash_size > 0 {
|
||||
self.inner.read_exact(&mut self.hash_buf).await.map_err(|e| {
|
||||
error!("bitrot reader read hash error: {}", e);
|
||||
e
|
||||
})?;
|
||||
}
|
||||
|
||||
// Read data
|
||||
let mut data_len = 0;
|
||||
while data_len < out.len() {
|
||||
let n = self.inner.read(&mut out[data_len..]).await.map_err(|e| {
|
||||
error!("bitrot reader read data error: {}", e);
|
||||
e
|
||||
})?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
data_len += n;
|
||||
}
|
||||
|
||||
if hash_size > 0 {
|
||||
let actual_hash = self.hash_algo.hash_encode(&out[..data_len]);
|
||||
if actual_hash.as_ref() != self.hash_buf.as_slice() {
|
||||
error!("bitrot reader hash mismatch, id={} data_len={}, out_len={}", self.id, data_len, out.len());
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
}
|
||||
Ok(data_len)
|
||||
}
|
||||
}
|
||||
|
||||
pin_project! {
|
||||
/// BitrotWriter writes (hash+data) blocks to an async writer.
|
||||
pub struct BitrotWriter<W> {
|
||||
#[pin]
|
||||
inner: W,
|
||||
hash_algo: HashAlgorithm,
|
||||
shard_size: usize,
|
||||
buf: Vec<u8>,
|
||||
finished: bool,
|
||||
}
|
||||
}
|
||||
|
||||
impl<W> BitrotWriter<W>
|
||||
where
|
||||
W: AsyncWrite + Unpin + Send + Sync,
|
||||
{
|
||||
/// Create a new BitrotWriter.
|
||||
pub fn new(inner: W, shard_size: usize, algo: HashAlgorithm) -> Self {
|
||||
let hash_algo = algo;
|
||||
Self {
|
||||
inner,
|
||||
hash_algo,
|
||||
shard_size,
|
||||
buf: Vec::new(),
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_inner(self) -> W {
|
||||
self.inner
|
||||
}
|
||||
|
||||
/// Write a (hash+data) block. Returns the number of data bytes written.
|
||||
/// Returns an error if called after a short write or if data exceeds shard_size.
|
||||
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
if buf.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
if self.finished {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished"));
|
||||
}
|
||||
|
||||
if buf.len() > self.shard_size {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("data size {} exceeds shard size {}", buf.len(), self.shard_size),
|
||||
));
|
||||
}
|
||||
|
||||
if buf.len() < self.shard_size {
|
||||
self.finished = true;
|
||||
}
|
||||
|
||||
let hash_algo = &self.hash_algo;
|
||||
|
||||
if hash_algo.size() > 0 {
|
||||
let hash = hash_algo.hash_encode(buf);
|
||||
self.buf.extend_from_slice(hash.as_ref());
|
||||
}
|
||||
|
||||
self.buf.extend_from_slice(buf);
|
||||
|
||||
self.inner.write_all(&self.buf).await?;
|
||||
|
||||
// self.inner.flush().await?;
|
||||
|
||||
let n = buf.len();
|
||||
|
||||
self.buf.clear();
|
||||
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
pub async fn shutdown(&mut self) -> std::io::Result<()> {
|
||||
self.inner.shutdown().await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
|
||||
if algo != HashAlgorithm::HighwayHash256S {
|
||||
return size;
|
||||
}
|
||||
size.div_ceil(shard_size) * algo.size() + size
|
||||
}
|
||||
|
||||
pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
||||
mut r: R,
|
||||
want_size: usize,
|
||||
part_size: usize,
|
||||
algo: HashAlgorithm,
|
||||
_want: Bytes, // FIXME: useless parameter?
|
||||
mut shard_size: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let mut hash_buf = vec![0; algo.size()];
|
||||
let mut left = want_size;
|
||||
|
||||
if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) {
|
||||
return Err(std::io::Error::other("bitrot shard file size mismatch"));
|
||||
}
|
||||
|
||||
while left > 0 {
|
||||
let n = r.read_exact(&mut hash_buf).await?;
|
||||
left -= n;
|
||||
|
||||
if left < shard_size {
|
||||
shard_size = left;
|
||||
}
|
||||
|
||||
let mut buf = vec![0; shard_size];
|
||||
let read = r.read_exact(&mut buf).await?;
|
||||
|
||||
let actual_hash = algo.hash_encode(&buf);
|
||||
if actual_hash.as_ref() != &hash_buf[0..n] {
|
||||
return Err(std::io::Error::other("bitrot hash mismatch"));
|
||||
}
|
||||
|
||||
left -= read;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Custom writer enum that supports inline buffer storage
|
||||
pub enum CustomWriter {
|
||||
/// Inline buffer writer - stores data in memory
|
||||
InlineBuffer(Vec<u8>),
|
||||
/// Disk-based writer using tokio file
|
||||
Other(Box<dyn AsyncWrite + Unpin + Send + Sync>),
|
||||
}
|
||||
|
||||
impl CustomWriter {
|
||||
/// Create a new inline buffer writer
|
||||
pub fn new_inline_buffer() -> Self {
|
||||
Self::InlineBuffer(Vec::new())
|
||||
}
|
||||
|
||||
/// Create a new disk writer from any AsyncWrite implementation
|
||||
pub fn new_tokio_writer<W>(writer: W) -> Self
|
||||
where
|
||||
W: AsyncWrite + Unpin + Send + Sync + 'static,
|
||||
{
|
||||
Self::Other(Box::new(writer))
|
||||
}
|
||||
|
||||
/// Get the inline buffer data if this is an inline buffer writer
|
||||
pub fn get_inline_data(&self) -> Option<&[u8]> {
|
||||
match self {
|
||||
Self::InlineBuffer(data) => Some(data),
|
||||
Self::Other(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the inline buffer data, consuming the writer
|
||||
pub fn into_inline_data(self) -> Option<Vec<u8>> {
|
||||
match self {
|
||||
Self::InlineBuffer(data) => Some(data),
|
||||
Self::Other(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for CustomWriter {
|
||||
fn poll_write(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
match self.get_mut() {
|
||||
Self::InlineBuffer(data) => {
|
||||
data.extend_from_slice(buf);
|
||||
std::task::Poll::Ready(Ok(buf.len()))
|
||||
}
|
||||
Self::Other(writer) => {
|
||||
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||
pinned_writer.poll_write(cx, buf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
match self.get_mut() {
|
||||
Self::InlineBuffer(_) => std::task::Poll::Ready(Ok(())),
|
||||
Self::Other(writer) => {
|
||||
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||
pinned_writer.poll_flush(cx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
match self.get_mut() {
|
||||
Self::InlineBuffer(_) => std::task::Poll::Ready(Ok(())),
|
||||
Self::Other(writer) => {
|
||||
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||
pinned_writer.poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper around BitrotWriter that uses our custom writer
|
||||
pub struct BitrotWriterWrapper {
|
||||
bitrot_writer: BitrotWriter<CustomWriter>,
|
||||
writer_type: WriterType,
|
||||
}
|
||||
|
||||
/// Enum to track the type of writer we're using
|
||||
enum WriterType {
|
||||
InlineBuffer,
|
||||
Other,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BitrotWriterWrapper {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("BitrotWriterWrapper")
|
||||
.field(
|
||||
"writer_type",
|
||||
&match self.writer_type {
|
||||
WriterType::InlineBuffer => "InlineBuffer",
|
||||
WriterType::Other => "Other",
|
||||
},
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl BitrotWriterWrapper {
|
||||
/// Create a new BitrotWriterWrapper with custom writer
|
||||
pub fn new(writer: CustomWriter, shard_size: usize, checksum_algo: HashAlgorithm) -> Self {
|
||||
let writer_type = match &writer {
|
||||
CustomWriter::InlineBuffer(_) => WriterType::InlineBuffer,
|
||||
CustomWriter::Other(_) => WriterType::Other,
|
||||
};
|
||||
|
||||
Self {
|
||||
bitrot_writer: BitrotWriter::new(writer, shard_size, checksum_algo),
|
||||
writer_type,
|
||||
}
|
||||
}
|
||||
|
||||
/// Write data to the bitrot writer
|
||||
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.bitrot_writer.write(buf).await
|
||||
}
|
||||
|
||||
pub async fn shutdown(&mut self) -> std::io::Result<()> {
|
||||
self.bitrot_writer.shutdown().await
|
||||
}
|
||||
|
||||
/// Extract the inline buffer data, consuming the wrapper
|
||||
pub fn into_inline_data(self) -> Option<Vec<u8>> {
|
||||
match self.writer_type {
|
||||
WriterType::InlineBuffer => {
|
||||
let writer = self.bitrot_writer.into_inner();
|
||||
writer.into_inline_data()
|
||||
}
|
||||
WriterType::Other => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use super::BitrotReader;
|
||||
use super::BitrotWriter;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::io::Cursor;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bitrot_read_write_ok() {
|
||||
let data = b"hello world! this is a test shard.";
|
||||
let data_size = data.len();
|
||||
let shard_size = 8;
|
||||
|
||||
let buf: Vec<u8> = Vec::new();
|
||||
let writer = Cursor::new(buf);
|
||||
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
|
||||
|
||||
let mut n = 0;
|
||||
for chunk in data.chunks(shard_size) {
|
||||
n += bitrot_writer.write(chunk).await.unwrap();
|
||||
}
|
||||
assert_eq!(n, data.len());
|
||||
|
||||
// 读
|
||||
let reader = bitrot_writer.into_inner();
|
||||
let reader = Cursor::new(reader.into_inner());
|
||||
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
||||
let mut out = Vec::new();
|
||||
let mut n = 0;
|
||||
while n < data_size {
|
||||
let mut buf = vec![0u8; shard_size];
|
||||
let m = bitrot_reader.read(&mut buf).await.unwrap();
|
||||
assert_eq!(&buf[..m], &data[n..n + m]);
|
||||
|
||||
out.extend_from_slice(&buf[..m]);
|
||||
n += m;
|
||||
}
|
||||
|
||||
assert_eq!(n, data_size);
|
||||
assert_eq!(data, &out[..]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bitrot_read_hash_mismatch() {
|
||||
let data = b"test data for bitrot";
|
||||
let data_size = data.len();
|
||||
let shard_size = 8;
|
||||
let buf: Vec<u8> = Vec::new();
|
||||
let writer = Cursor::new(buf);
|
||||
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
|
||||
for chunk in data.chunks(shard_size) {
|
||||
let _ = bitrot_writer.write(chunk).await.unwrap();
|
||||
}
|
||||
let mut written = bitrot_writer.into_inner().into_inner();
|
||||
// change the last byte to make hash mismatch
|
||||
let pos = written.len() - 1;
|
||||
written[pos] ^= 0xFF;
|
||||
let reader = Cursor::new(written);
|
||||
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
|
||||
|
||||
let count = data_size.div_ceil(shard_size);
|
||||
|
||||
let mut idx = 0;
|
||||
let mut n = 0;
|
||||
while n < data_size {
|
||||
let mut buf = vec![0u8; shard_size];
|
||||
let res = bitrot_reader.read(&mut buf).await;
|
||||
|
||||
if idx == count - 1 {
|
||||
// 最后一个块，应该返回错误
|
||||
assert!(res.is_err());
|
||||
assert_eq!(res.unwrap_err().kind(), std::io::ErrorKind::InvalidData);
|
||||
break;
|
||||
}
|
||||
|
||||
let m = res.unwrap();
|
||||
|
||||
assert_eq!(&buf[..m], &data[n..n + m]);
|
||||
|
||||
n += m;
|
||||
idx += 1;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bitrot_read_write_none_hash() {
|
||||
let data = b"bitrot none hash test data!";
|
||||
let data_size = data.len();
|
||||
let shard_size = 8;
|
||||
|
||||
let buf: Vec<u8> = Vec::new();
|
||||
let writer = Cursor::new(buf);
|
||||
let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::None);
|
||||
|
||||
let mut n = 0;
|
||||
for chunk in data.chunks(shard_size) {
|
||||
n += bitrot_writer.write(chunk).await.unwrap();
|
||||
}
|
||||
assert_eq!(n, data.len());
|
||||
|
||||
let reader = bitrot_writer.into_inner();
|
||||
let reader = Cursor::new(reader.into_inner());
|
||||
let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None);
|
||||
let mut out = Vec::new();
|
||||
let mut n = 0;
|
||||
while n < data_size {
|
||||
let mut buf = vec![0u8; shard_size];
|
||||
let m = bitrot_reader.read(&mut buf).await.unwrap();
|
||||
assert_eq!(&buf[..m], &data[n..n + m]);
|
||||
out.extend_from_slice(&buf[..m]);
|
||||
n += m;
|
||||
}
|
||||
assert_eq!(n, data_size);
|
||||
assert_eq!(data, &out[..]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::BitrotReader;
|
||||
use super::Erasure;
|
||||
use crate::disk::error::Error;
|
||||
use crate::disk::error_reduce::reduce_errs;
|
||||
use futures::future::join_all;
|
||||
use pin_project_lite::pin_project;
|
||||
use std::io;
|
||||
use std::io::ErrorKind;
|
||||
use tokio::io::AsyncRead;
|
||||
use tokio::io::AsyncWrite;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tracing::error;
|
||||
|
||||
pin_project! {
|
||||
pub(crate) struct ParallelReader<R> {
|
||||
#[pin]
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
offset: usize,
|
||||
shard_size: usize,
|
||||
shard_file_size: usize,
|
||||
data_shards: usize,
|
||||
total_shards: usize,
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> ParallelReader<R>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
// readers传入前应处理disk错误，确保每个reader达到可用数量的BitrotReader
|
||||
pub fn new(readers: Vec<Option<BitrotReader<R>>>, e: Erasure, offset: usize, total_length: usize) -> Self {
|
||||
let shard_size = e.shard_size();
|
||||
let shard_file_size = e.shard_file_size(total_length as i64) as usize;
|
||||
|
||||
let offset = (offset / e.block_size) * shard_size;
|
||||
|
||||
// 确保offset不超过shard_file_size
|
||||
|
||||
ParallelReader {
|
||||
readers,
|
||||
offset,
|
||||
shard_size,
|
||||
shard_file_size,
|
||||
data_shards: e.data_shards,
|
||||
total_shards: e.data_shards + e.parity_shards,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> ParallelReader<R>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||
// if self.readers.len() != self.total_shards {
|
||||
// return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
|
||||
// }
|
||||
|
||||
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
|
||||
self.shard_file_size - self.offset
|
||||
} else {
|
||||
self.shard_size
|
||||
};
|
||||
|
||||
if shard_size == 0 {
|
||||
return (vec![None; self.readers.len()], vec![None; self.readers.len()]);
|
||||
}
|
||||
|
||||
// 使用并发读取所有分片
|
||||
let mut read_futs = Vec::with_capacity(self.readers.len());
|
||||
|
||||
for (i, opt_reader) in self.readers.iter_mut().enumerate() {
|
||||
let future = if let Some(reader) = opt_reader.as_mut() {
|
||||
Box::pin(async move {
|
||||
let mut buf = vec![0u8; shard_size];
|
||||
match reader.read(&mut buf).await {
|
||||
Ok(n) => {
|
||||
buf.truncate(n);
|
||||
(i, Ok(buf))
|
||||
}
|
||||
Err(e) => (i, Err(Error::from(e))),
|
||||
}
|
||||
}) as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
|
||||
} else {
|
||||
// reader是None时返回FileNotFound错误
|
||||
Box::pin(async move { (i, Err(Error::FileNotFound)) })
|
||||
as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
|
||||
};
|
||||
read_futs.push(future);
|
||||
}
|
||||
|
||||
let results = join_all(read_futs).await;
|
||||
|
||||
let mut shards: Vec<Option<Vec<u8>>> = vec![None; self.readers.len()];
|
||||
let mut errs = vec![None; self.readers.len()];
|
||||
|
||||
for (i, shard) in results.into_iter() {
|
||||
match shard {
|
||||
Ok(data) => {
|
||||
if !data.is_empty() {
|
||||
shards[i] = Some(data);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// error!("Error reading shard {}: {}", i, e);
|
||||
errs[i] = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.offset += shard_size;
|
||||
|
||||
(shards, errs)
|
||||
}
|
||||
|
||||
pub fn can_decode(&self, shards: &[Option<Vec<u8>>]) -> bool {
|
||||
shards.iter().filter(|s| s.is_some()).count() >= self.data_shards
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取数据块总长度
|
||||
fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
|
||||
let mut size = 0;
|
||||
for shard in shards.iter().take(data_blocks).flatten() {
|
||||
size += shard.len();
|
||||
}
|
||||
|
||||
size
|
||||
}
|
||||
|
||||
/// 将编码块中的数据块写入目标，支持 offset 和 length
|
||||
async fn write_data_blocks<W>(
|
||||
writer: &mut W,
|
||||
en_blocks: &[Option<Vec<u8>>],
|
||||
data_blocks: usize,
|
||||
mut offset: usize,
|
||||
length: usize,
|
||||
) -> std::io::Result<usize>
|
||||
where
|
||||
W: tokio::io::AsyncWrite + Send + Sync + Unpin,
|
||||
{
|
||||
if get_data_block_len(en_blocks, data_blocks) < length {
|
||||
error!("write_data_blocks get_data_block_len < length");
|
||||
return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
|
||||
}
|
||||
|
||||
let mut total_written = 0;
|
||||
let mut write_left = length;
|
||||
|
||||
for block_op in &en_blocks[..data_blocks] {
|
||||
if block_op.is_none() {
|
||||
error!("write_data_blocks block_op.is_none()");
|
||||
return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block"));
|
||||
}
|
||||
|
||||
let block = block_op.as_ref().unwrap();
|
||||
|
||||
if offset >= block.len() {
|
||||
offset -= block.len();
|
||||
continue;
|
||||
}
|
||||
|
||||
let block_slice = &block[offset..];
|
||||
offset = 0;
|
||||
|
||||
if write_left < block.len() {
|
||||
writer.write_all(&block_slice[..write_left]).await.map_err(|e| {
|
||||
error!("write_data_blocks write_all err: {}", e);
|
||||
e
|
||||
})?;
|
||||
|
||||
total_written += write_left;
|
||||
break;
|
||||
}
|
||||
|
||||
let n = block_slice.len();
|
||||
|
||||
writer.write_all(block_slice).await.map_err(|e| {
|
||||
error!("write_data_blocks write_all2 err: {}", e);
|
||||
e
|
||||
})?;
|
||||
|
||||
write_left -= n;
|
||||
|
||||
total_written += n;
|
||||
}
|
||||
|
||||
Ok(total_written)
|
||||
}
|
||||
|
||||
impl Erasure {
|
||||
pub async fn decode<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
total_length: usize,
|
||||
) -> (usize, Option<std::io::Error>)
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
if readers.len() != self.data_shards + self.parity_shards {
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
|
||||
}
|
||||
|
||||
if offset + length > total_length {
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
|
||||
}
|
||||
|
||||
let mut ret_err = None;
|
||||
|
||||
if length == 0 {
|
||||
return (0, ret_err);
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
|
||||
let mut reader = ParallelReader::new(readers, self.clone(), offset, total_length);
|
||||
|
||||
let start = offset / self.block_size;
|
||||
let end = (offset + length) / self.block_size;
|
||||
|
||||
for i in start..=end {
|
||||
let (block_offset, block_length) = if start == end {
|
||||
(offset % self.block_size, length)
|
||||
} else if i == start {
|
||||
(offset % self.block_size, self.block_size - (offset % self.block_size))
|
||||
} else if i == end {
|
||||
(0, (offset + length) % self.block_size)
|
||||
} else {
|
||||
(0, self.block_size)
|
||||
};
|
||||
|
||||
if block_length == 0 {
|
||||
// error!("erasure decode decode block_length == 0");
|
||||
break;
|
||||
}
|
||||
|
||||
let (mut shards, errs) = reader.read().await;
|
||||
|
||||
if ret_err.is_none() {
|
||||
if let (_, Some(err)) = reduce_errs(&errs, &[]) {
|
||||
if err == Error::FileNotFound || err == Error::FileCorrupt {
|
||||
ret_err = Some(err.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !reader.can_decode(&shards) {
|
||||
error!("erasure decode can_decode errs: {:?}", &errs);
|
||||
ret_err = Some(Error::ErasureReadQuorum.into());
|
||||
break;
|
||||
}
|
||||
|
||||
// Decode the shards
|
||||
if let Err(e) = self.decode_data(&mut shards) {
|
||||
error!("erasure decode decode_data err: {:?}", e);
|
||||
ret_err = Some(e);
|
||||
break;
|
||||
}
|
||||
|
||||
let n = match write_data_blocks(writer, &shards, self.data_shards, block_offset, block_length).await {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
error!("erasure decode write_data_blocks err: {:?}", e);
|
||||
ret_err = Some(e);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
written += n;
|
||||
}
|
||||
|
||||
if written < length {
|
||||
ret_err = Some(Error::LessData.into());
|
||||
}
|
||||
|
||||
(written, ret_err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::BitrotWriterWrapper;
|
||||
use super::Erasure;
|
||||
use crate::disk::error::Error;
|
||||
use crate::disk::error_reduce::count_errs;
|
||||
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use std::sync::Arc;
|
||||
use std::vec;
|
||||
use tokio::io::AsyncRead;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::error;
|
||||
|
||||
pub(crate) struct MultiWriter<'a> {
|
||||
writers: &'a mut [Option<BitrotWriterWrapper>],
|
||||
write_quorum: usize,
|
||||
errs: Vec<Option<Error>>,
|
||||
}
|
||||
|
||||
impl<'a> MultiWriter<'a> {
|
||||
pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
|
||||
let length = writers.len();
|
||||
MultiWriter {
|
||||
writers,
|
||||
write_quorum,
|
||||
errs: vec![None; length],
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: &Bytes) {
|
||||
match writer_opt {
|
||||
Some(writer) => {
|
||||
match writer.write(shard).await {
|
||||
Ok(n) => {
|
||||
if n < shard.len() {
|
||||
*err = Some(Error::ShortWrite);
|
||||
*writer_opt = None; // Mark as failed
|
||||
} else {
|
||||
*err = None;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
*err = Some(Error::from(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {
|
||||
*err = Some(Error::DiskNotFound);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn write(&mut self, data: Vec<Bytes>) -> std::io::Result<()> {
|
||||
assert_eq!(data.len(), self.writers.len());
|
||||
|
||||
{
|
||||
let mut futures = FuturesUnordered::new();
|
||||
for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(data.iter()) {
|
||||
if err.is_some() {
|
||||
continue; // Skip if we already have an error for this writer
|
||||
}
|
||||
futures.push(Self::write_shard(writer_opt, err, shard));
|
||||
}
|
||||
while let Some(()) = futures.next().await {}
|
||||
}
|
||||
|
||||
let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
|
||||
if nil_count >= self.write_quorum {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
|
||||
error!(
|
||||
"reduce_write_quorum_errs: {:?}, offline-disks={}/{}, errs={:?}",
|
||||
write_err,
|
||||
count_errs(&self.errs, &Error::DiskNotFound),
|
||||
self.writers.len(),
|
||||
self.errs
|
||||
);
|
||||
return Err(std::io::Error::other(format!(
|
||||
"Failed to write data: {} (offline-disks={}/{})",
|
||||
write_err,
|
||||
count_errs(&self.errs, &Error::DiskNotFound),
|
||||
self.writers.len()
|
||||
)));
|
||||
}
|
||||
|
||||
Err(std::io::Error::other(format!(
|
||||
"Failed to write data: (offline-disks={}/{}): {}",
|
||||
count_errs(&self.errs, &Error::DiskNotFound),
|
||||
self.writers.len(),
|
||||
self.errs
|
||||
.iter()
|
||||
.map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
)))
|
||||
}
|
||||
|
||||
pub async fn _shutdown(&mut self) -> std::io::Result<()> {
|
||||
for writer in self.writers.iter_mut().flatten() {
|
||||
writer.shutdown().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Erasure {
|
||||
pub async fn encode<R>(
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let block_size = self.block_size;
|
||||
let mut total = 0;
|
||||
let mut buf = vec![0u8; block_size];
|
||||
loop {
|
||||
match rustfs_utils::read_full(&mut reader, &mut buf).await {
|
||||
Ok(n) if n > 0 => {
|
||||
total += n;
|
||||
let res = self.encode_data(&buf[..n])?;
|
||||
if let Err(err) = tx.send(res).await {
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
}
|
||||
Ok(_) => break,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((reader, total))
|
||||
});
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
|
||||
while let Some(block) = rx.recv().await {
|
||||
if block.is_empty() {
|
||||
break;
|
||||
}
|
||||
writers.write(block).await?;
|
||||
}
|
||||
|
||||
let (reader, total) = task.await??;
|
||||
// writers.shutdown().await?;
|
||||
Ok((reader, total))
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,74 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::BitrotReader;
|
||||
use super::BitrotWriterWrapper;
|
||||
use super::decode::ParallelReader;
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::erasure_coding::encode::MultiWriter;
|
||||
use bytes::Bytes;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::info;
|
||||
|
||||
impl super::Erasure {
|
||||
pub async fn heal<R>(
|
||||
&self,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
total_length: usize,
|
||||
_prefer: &[bool],
|
||||
) -> Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
info!(
|
||||
"Erasure heal, writers len: {}, readers len: {}, total_length: {}",
|
||||
writers.len(),
|
||||
readers.len(),
|
||||
total_length
|
||||
);
|
||||
if writers.len() != self.parity_shards + self.data_shards {
|
||||
return Err(Error::other("invalid argument"));
|
||||
}
|
||||
let mut reader = ParallelReader::new(readers, self.clone(), 0, total_length);
|
||||
|
||||
let start_block = 0;
|
||||
let mut end_block = total_length / self.block_size;
|
||||
if total_length % self.block_size != 0 {
|
||||
end_block += 1;
|
||||
}
|
||||
|
||||
for _ in start_block..end_block {
|
||||
let (mut shards, errs) = reader.read().await;
|
||||
|
||||
if errs.iter().filter(|e| e.is_none()).count() < self.data_shards {
|
||||
return Err(Error::other(format!("can not reconstruct data: not enough data shards {errs:?}")));
|
||||
}
|
||||
|
||||
if self.parity_shards > 0 {
|
||||
self.decode_data(&mut shards)?;
|
||||
}
|
||||
|
||||
let shards = shards
|
||||
.into_iter()
|
||||
.map(|s| Bytes::from(s.unwrap_or_default()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let mut writers = MultiWriter::new(writers, self.data_shards);
|
||||
writers.write(shards).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod decode;
|
||||
pub mod encode;
|
||||
pub mod erasure;
|
||||
pub mod heal;
|
||||
|
||||
mod bitrot;
|
||||
pub use bitrot::*;
|
||||
|
||||
pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size};
|
||||
Reference in New Issue
Block a user