diff --git a/Cargo.lock b/Cargo.lock index a4baa981d..58472627b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -314,6 +314,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", + "crc32fast", "futures", "lazy_static", "netif", @@ -321,11 +322,14 @@ dependencies = [ "reed-solomon-erasure", "regex", "rmp-serde", + "s3s", "serde", "serde_json", + "siphasher", "thiserror", "time", "tokio", + "tokio-pipe", "tokio-util", "tracing", "tracing-error", @@ -1228,6 +1232,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.9" @@ -1387,6 +1397,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-pipe" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784" +dependencies = [ + "libc", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.11" diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 783c373c4..991375f38 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -30,6 +30,10 @@ path-absolutize = "3.1.1" time.workspace = true rmp-serde = "1.3.0" tokio-util = "0.7.11" +s3s = "0.10.0" +crc32fast = "1.4.2" +siphasher = "1.0.1" +tokio-pipe = "0.2.12" [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/ecstore/src/chunk_stream.rs b/ecstore/src/chunk_stream.rs index e89cfaddd..9afbdd387 100644 --- a/ecstore/src/chunk_stream.rs +++ b/ecstore/src/chunk_stream.rs @@ -1,3 +1,4 @@ +use anyhow::Error; use bytes::Bytes; use futures::pin_mut; use futures::stream::{Stream, StreamExt}; @@ -6,13 +7,11 @@ use std::pin::Pin; use std::task::{Context, Poll}; use transform_stream::AsyncTryStream; -use crate::error::StdError; - pub type SyncBoxFuture<'a, T> = Pin + Send + Sync + 'a>>; pub struct ChunkedStream { /// inner - inner: AsyncTryStream>>, + inner: AsyncTryStream>>, remaining_length: usize, } @@ -20,10 +19,10 @@ pub struct ChunkedStream { impl ChunkedStream { pub fn new(body: S, content_length: usize, chunk_size: usize, need_padding: bool) -> Self where - S: Stream> + Send + Sync + 'static, + S: Stream> + Send + Sync + 'static, { let inner = - AsyncTryStream::<_, _, SyncBoxFuture<'static, Result<(), StdError>>>::new(|mut y| { + AsyncTryStream::<_, _, SyncBoxFuture<'static, Result<(), Error>>>::new(|mut y| { #[allow(clippy::shadow_same)] // necessary for `pin_mut!` Box::pin(async move { pin_mut!(body); @@ -90,9 +89,9 @@ impl ChunkedStream { mut body: Pin<&mut S>, prev_bytes: Bytes, data_size: usize, - ) -> Option, Bytes), StdError>> + ) -> Option, Bytes), Error>> where - S: Stream> + Send + 'static, + S: Stream> + Send + 'static, { let mut bytes_buffer = Vec::new(); @@ -186,10 +185,7 @@ impl ChunkedStream { Some(Ok((bytes_buffer, remaining_bytes))) } - fn poll( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll>> { let ans = Pin::new(&mut self.inner).poll_next(cx); if let Poll::Ready(Some(Ok(ref bytes))) = ans { self.remaining_length = self.remaining_length.saturating_sub(bytes.len()); @@ -203,7 +199,7 @@ impl ChunkedStream { } impl Stream for ChunkedStream { - type Item = Result; + type Item = Result; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { self.poll(cx) diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index d2899b4ce..1e667b9cd 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -1,6 +1,14 @@ -use reed_solomon_erasure::{galois_8::ReedSolomon, Error}; +use tokio::io::AsyncWriteExt; -struct Erasure { +use anyhow::{Error, Result}; +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use reed_solomon_erasure::galois_8::ReedSolomon; +use tokio::io::AsyncWrite; + +use crate::chunk_stream::ChunkedStream; + +pub struct Erasure { data_shards: usize, parity_shards: usize, encoder: ReedSolomon, @@ -15,7 +23,50 @@ impl Erasure { } } - pub fn encode_data(&self, data: &[u8]) -> Result>, Error> { + pub async fn encode( + &self, + body: S, + writers: &mut Vec, + block_size: usize, + data_size: usize, + write_quorum: usize, + ) -> Result<()> + where + S: Stream> + Send + Sync + 'static, + W: AsyncWrite + Unpin, + { + let mut stream = ChunkedStream::new(body, data_size, block_size, true); + while let Some(result) = stream.next().await { + match result { + Ok(data) => { + let blocks = self.encode_data(data.as_ref())?; + + let mut errs = Vec::new(); + + for (i, w) in writers.iter_mut().enumerate() { + match w.write_all(blocks[i].as_ref()).await { + Ok(_) => errs.push(None), + Err(e) => errs.push(Some(e)), + } + } + + // TODO: reduceWriteQuorumErrs + } + Err(e) => return Err(e), + } + } + + Ok(()) + + // loop { + // match rd.next().await { + // Some(res) => todo!(), + // None => todo!(), + // } + // } + } + + pub fn encode_data(&self, data: &[u8]) -> Result>> { let (shard_size, total_size) = self.need_size(data.len()); let mut data_buffer = vec![0u8; total_size]; @@ -43,8 +94,9 @@ impl Erasure { Ok(shards) } - pub fn decode_data(&self, shards: &mut Vec>>) -> Result<(), Error> { - self.encoder.reconstruct(shards) + pub fn decode_data(&self, shards: &mut Vec>>) -> Result<()> { + self.encoder.reconstruct(shards)?; + Ok(()) } // 每个分片长度,所需要的总长度 diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 169b88bbe..33009d3e0 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -13,5 +13,4 @@ mod sets; pub mod store; mod store_api; mod store_init; -mod stream; mod utils; diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 2970978d7..63e97f3dc 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,25 +1,33 @@ +use std::sync::Arc; + use anyhow::Result; -use futures::StreamExt; + +use futures::AsyncWrite; +use time::OffsetDateTime; +use tokio_pipe::{PipeRead, PipeWrite}; use uuid::Uuid; use crate::{ - disk::DiskStore, + disk::{self, DiskStore}, endpoint::PoolEndpoints, - format::FormatV3, - store_api::{MakeBucketOptions, ObjectOptions, PutObjReader, StorageAPI}, + erasure::Erasure, + format::{DistributionAlgoVersion, FormatV3}, + store_api::{FileInfo, MakeBucketOptions, ObjectOptions, PutObjReader, StorageAPI}, + utils::hash, }; #[derive(Debug)] pub struct Sets { pub id: Uuid, // pub sets: Vec, - pub disk_indexs: Vec>>, // [set_count_idx][set_drive_count_idx] = disk_idx + pub disk_set: Vec>>, // [set_count_idx][set_drive_count_idx] = disk_idx pub pool_idx: usize, pub endpoints: PoolEndpoints, pub format: FormatV3, pub partiy_count: usize, pub set_count: usize, pub set_drive_count: usize, + pub distribution_algo: DistributionAlgoVersion, } impl Sets { @@ -33,7 +41,7 @@ impl Sets { let set_count = fm.erasure.sets.len(); let set_drive_count = fm.erasure.sets[0].len(); - let mut disk_indexs = Vec::with_capacity(set_count); + let mut disk_set = Vec::with_capacity(set_count); for i in 0..set_count { let mut set_drive = Vec::with_capacity(set_drive_count); @@ -47,25 +55,40 @@ impl Sets { } } - disk_indexs.push(set_drive); + disk_set.push(set_drive); } let sets = Self { id: fm.id.clone(), // sets: todo!(), - disk_indexs, + disk_set, pool_idx, endpoints: endpoints.clone(), format: fm.clone(), partiy_count, set_count, set_drive_count, + distribution_algo: fm.erasure.distribution_algo.clone(), }; Ok(sets) } pub fn get_disks(&self, set_idx: usize) -> Vec> { - self.disk_indexs[set_idx].clone() + self.disk_set[set_idx].clone() + } + + pub fn get_disks_by_key(&self, key: &str) -> Vec> { + self.get_disks(self.get_hashed_set_index(key)) + } + + fn get_hashed_set_index(&self, input: &str) -> usize { + match self.distribution_algo { + DistributionAlgoVersion::V1 => hash::crc_hash(input, self.disk_set.len()), + + DistributionAlgoVersion::V2 | DistributionAlgoVersion::V3 => { + hash::sip_hash(input, self.disk_set.len(), self.id.as_bytes()) + } + } } } @@ -92,7 +115,102 @@ impl StorageAPI for Sets { data: &PutObjReader, opts: &ObjectOptions, ) -> Result<()> { - // data.stream.next(); + let disks = self.get_disks_by_key(object); + + let mut parity_drives = self.partiy_count; + if opts.max_parity { + parity_drives = disks.len() / 2; + } + + let data_drives = disks.len() - parity_drives; + let mut write_quorum = data_drives; + if data_drives == parity_drives { + write_quorum += 1 + } + + let mut fi = FileInfo::new( + [bucket, object].join("/").as_str(), + data_drives, + parity_drives, + ); + + fi.data_dir = Uuid::new_v4().to_string(); + + let parts_metadata = vec![fi.clone(); disks.len()]; + + let (shuffle_disks, shuffle_parts_metadata) = + shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi); + + let mut writers = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + let (mut r, mut w) = tokio_pipe::pipe()?; + + writers.push(w); + } + + let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks); + + // erasure.encode( + // data.stream, + // &mut writers, + // fi.erasure.block_size, + // data.content_length, + // write_quorum, + // ); + unimplemented!() } } + +pub struct DiskWriter<'a> { + disk: &'a Option, + writer: PipeWrite, + reader: PipeRead, +} + +impl<'a> DiskWriter<'a> { + pub fn new(disk: &'a Option) -> Result { + let (mut reader, mut writer) = tokio_pipe::pipe()?; + Ok(Self { + disk, + reader, + writer, + }) + } + + pub fn wirter(&self) -> impl AsyncWrite {} +} + +// 打乱顺序 +fn shuffle_disks_and_parts_metadata( + disks: &Vec>, + parts_metadata: &Vec, + fi: &FileInfo, +) -> (Vec>, Vec) { + let init = fi.mod_time == OffsetDateTime::UNIX_EPOCH; + + let mut shuffled_disks = vec![None; disks.len()]; + let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()]; + let distribution = &fi.erasure.distribution; + + for (k, v) in disks.iter().enumerate() { + if v.is_none() { + continue; + } + + if !init && !parts_metadata[k].is_valid() { + continue; + } + + // if !init && fi.xlv1 != parts_metadata[k].xlv1 { + // continue; + // } + + let block_idx = distribution[k]; + shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone(); + shuffled_disks[block_idx - 1] = disks[k].clone(); + } + + (shuffled_disks, shuffled_parts_metadata) +} diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index b5e843c8a..0f8d80c20 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,8 +1,8 @@ -use std::{collections::HashMap, io::Cursor}; +use std::collections::HashMap; use anyhow::{Error, Result}; -use tokio_util::io::ReaderStream; +use s3s::Body; use uuid::Uuid; use crate::{ @@ -13,9 +13,7 @@ use crate::{ peer::{PeerS3Client, S3PeerSys}, sets::Sets, store_api::{MakeBucketOptions, ObjectOptions, PutObjReader, StorageAPI}, - store_init, - stream::into_dyn, - utils, + store_init, utils, }; #[derive(Debug)] @@ -121,19 +119,19 @@ impl StorageAPI for ECStore { let data = meta.marshal_msg()?; let file_path = meta.save_file_path(); - let stream = ReaderStream::new(Cursor::new(data)); - // TODO: wrap hash reader - // let reader = PutObjReader::new(stream); + let content_len = data.len() as u64; - // self.put_object( - // RUSTFS_META_BUCKET, - // &file_path, - // &reader, - // &ObjectOptions { max_parity: true }, - // ) - // .await?; + let reader = PutObjReader::new(Body::from(data), content_len); + + self.put_object( + RUSTFS_META_BUCKET, + &file_path, + &reader, + &ObjectOptions { max_parity: true }, + ) + .await?; // TODO: toObjectErr diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 7d6b4bcb0..3a43876c9 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -1,24 +1,136 @@ -use std::sync::Arc; +use std::{default, sync::Arc}; use anyhow::Result; use bytes::Bytes; use futures::Stream; +use s3s::Body; +use time::OffsetDateTime; -use crate::stream::DynByteStream; +pub const ERASURE_ALGORITHM: &str = "rs-vandermonde"; +pub const BLOCK_SIZE_V2: u64 = 1048576; // 1M + +#[derive(Debug, Clone)] +pub struct FileInfo { + pub erasure: ErasureInfo, + pub deleted: bool, + // DataDir of the file + pub data_dir: String, + pub mod_time: OffsetDateTime, +} + +impl Default for FileInfo { + fn default() -> Self { + Self { + erasure: Default::default(), + deleted: Default::default(), + data_dir: Default::default(), + mod_time: OffsetDateTime::UNIX_EPOCH, + } + } +} + +impl FileInfo { + pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self { + let indexs = { + let cardinality = data_blocks + parity_blocks; + let mut nums = vec![0; cardinality]; + let key_crc = crc32fast::hash(object.as_bytes()); + + let start = key_crc as usize % cardinality; + for i in 1..=cardinality { + nums[i - 1] = 1 + ((start + 1) % cardinality); + } + + nums + }; + Self { + erasure: ErasureInfo { + algorithm: ERASURE_ALGORITHM, + data_blocks: data_blocks, + parity_blocks: parity_blocks, + block_size: BLOCK_SIZE_V2, + distribution: indexs, + ..Default::default() + }, + ..Default::default() + } + } + + pub fn is_valid(&self) -> bool { + if self.deleted { + return true; + } + + let data_blocks = self.erasure.data_blocks; + let parity_blocks = self.erasure.parity_blocks; + + (data_blocks >= parity_blocks) + && (data_blocks > 0) + && (parity_blocks >= 0) + && (self.erasure.index > 0 + && self.erasure.index <= data_blocks + parity_blocks + && self.erasure.distribution.len() == (data_blocks + parity_blocks)) + } +} + +#[derive(Debug, Default, Clone)] +// ErasureInfo holds erasure coding and bitrot related information. +pub struct ErasureInfo { + // Algorithm is the String representation of erasure-coding-algorithm + pub algorithm: &'static str, + // DataBlocks is the number of data blocks for erasure-coding + pub data_blocks: usize, + // ParityBlocks is the number of parity blocks for erasure-coding + pub parity_blocks: usize, + // BlockSize is the size of one erasure-coded block + pub block_size: u64, + // Index is the index of the current disk + pub index: usize, + // Distribution is the distribution of the data and parity blocks + pub distribution: Vec, + // Checksums holds all bitrot checksums of all erasure encoded blocks + pub checksums: Vec, +} + +#[derive(Debug, Default, Clone)] +// ChecksumInfo - carries checksums of individual scattered parts per disk. +pub struct ChecksumInfo { + pub part_number: usize, + pub algorithm: BitrotAlgorithm, + pub hash: Vec, +} + +#[derive(Debug, Default, Clone)] +// BitrotAlgorithm specifies a algorithm used for bitrot protection. +pub enum BitrotAlgorithm { + // SHA256 represents the SHA-256 hash function + SHA256, + // HighwayHash256 represents the HighwayHash-256 hash function + HighwayHash256, + // HighwayHash256S represents the Streaming HighwayHash-256 hash function + #[default] + HighwayHash256S, + // BLAKE2b512 represents the BLAKE2b-512 hash function + BLAKE2b512, +} pub struct MakeBucketOptions { pub force_create: bool, } pub struct PutObjReader { - // pub stream: Box>, + pub stream: Body, + pub content_length: u64, } -// impl PutObjReader { -// pub fn new(stream: S) -> Self { -// PutObjReader { stream } -// } -// } +impl PutObjReader { + pub fn new(stream: Body, content_length: u64) -> Self { + PutObjReader { + stream, + content_length, + } + } +} pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files diff --git a/ecstore/src/stream.rs b/ecstore/src/stream.rs deleted file mode 100644 index cd643c4d1..000000000 --- a/ecstore/src/stream.rs +++ /dev/null @@ -1,174 +0,0 @@ -use anyhow::Error; -use tokio::io::AsyncRead; -use tokio_util::io::ReaderStream; - -use std::collections::VecDeque; -use std::fmt; -use std::pin::Pin; -use std::task::{Context, Poll}; - -use bytes::Bytes; -use futures::{pin_mut, Stream, StreamExt}; - -pub trait ByteStream: Stream { - fn remaining_length(&self) -> RemainingLength { - RemainingLength::unknown() - } -} - -pub type DynByteStream = - Pin> + Send + Sync + 'static>>; - -pub struct RemainingLength { - lower: usize, - upper: Option, -} - -impl RemainingLength { - /// Creates a new `RemainingLength` with the given lower and upper bounds. - /// - /// # Panics - /// This function asserts that `lower <= upper`. - #[must_use] - pub fn new(lower: usize, upper: Option) -> Self { - if let Some(upper) = upper { - assert!(lower <= upper); - } - Self { lower, upper } - } - - #[must_use] - pub fn unknown() -> Self { - Self { - lower: 0, - upper: None, - } - } - - #[must_use] - pub fn new_exact(n: usize) -> Self { - Self { - lower: n, - upper: Some(n), - } - } - - #[must_use] - pub fn exact(&self) -> Option { - self.upper.filter(|&upper| upper == self.lower) - } -} - -impl fmt::Debug for RemainingLength { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if let Some(exact) = self.exact() { - return write!(f, "{exact}"); - } - match self.upper { - Some(upper) => write!(f, "({}..={})", self.lower, upper), - None => write!(f, "({}..)", self.lower), - } - } -} - -pub(crate) fn into_dyn(s: S) -> DynByteStream -where - S: ByteStream> + Send + Sync + Unpin + 'static, - E: std::error::Error + Send + Sync + 'static, -{ - Box::pin(Wrapper(s)) -} - -struct Wrapper(S); - -impl Stream for Wrapper -where - S: ByteStream> + Send + Sync + Unpin + 'static, - E: std::error::Error + Send + Sync + 'static, -{ - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = Pin::new(&mut self.0); - this.poll_next(cx).map_err(|e| Error::new(e)) - } - - fn size_hint(&self) -> (usize, Option) { - self.0.size_hint() - } -} - -impl ByteStream for Wrapper -where - S: ByteStream> + Send + Sync + Unpin + 'static, - E: std::error::Error + Send + Sync + 'static, -{ - fn remaining_length(&self) -> RemainingLength { - self.0.remaining_length() - } -} - -// FIXME: unbounded memory allocation -pub(crate) async fn aggregate_unlimited(stream: S) -> Result, E> -where - S: ByteStream>, -{ - let mut vec = Vec::new(); - pin_mut!(stream); - while let Some(result) = stream.next().await { - vec.push(result?); - } - Ok(vec) -} - -pub(crate) struct VecByteStream { - queue: VecDeque, - remaining_bytes: usize, -} - -impl VecByteStream { - pub fn new(v: Vec) -> Self { - let total = v - .iter() - .map(Bytes::len) - .try_fold(0, usize::checked_add) - .expect("length overflow"); - - Self { - queue: v.into(), - remaining_bytes: total, - } - } - - pub fn exact_remaining_length(&self) -> usize { - self.remaining_bytes - } -} - -impl Stream for VecByteStream { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - let this = Pin::into_inner(self); - match this.queue.pop_front() { - Some(b) => { - this.remaining_bytes -= b.len(); - Poll::Ready(Some(Ok(b))) - } - None => Poll::Ready(None), - } - } - - fn size_hint(&self) -> (usize, Option) { - let cnt = self.queue.len(); - (cnt, Some(cnt)) - } -} - -impl ByteStream for VecByteStream { - fn remaining_length(&self) -> RemainingLength { - RemainingLength::new_exact(self.remaining_bytes) - } -} - -impl ByteStream for ReaderStream {} diff --git a/ecstore/src/utils/hash.rs b/ecstore/src/utils/hash.rs new file mode 100644 index 000000000..1aae48e13 --- /dev/null +++ b/ecstore/src/utils/hash.rs @@ -0,0 +1,21 @@ +use crc32fast::Hasher; +use siphasher::sip::SipHasher; + +pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize { + // 你的密钥,必须是16字节 + + // 计算字符串的SipHash值 + let result = SipHasher::new_with_key(id).hash(key.as_bytes()); + + result as usize % cardinality +} + +pub fn crc_hash(key: &str, cardinality: usize) -> usize { + let mut hasher = Hasher::new(); // 创建一个新的哈希器 + + hasher.update(key.as_bytes()); // 更新哈希状态,添加数据 + + let checksum = hasher.finalize(); + + checksum as usize % cardinality +} diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs index 9e4b95793..dc9cae50a 100644 --- a/ecstore/src/utils/mod.rs +++ b/ecstore/src/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod hash; pub mod net; pub mod path; pub mod string;