gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 03:22:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): add direct chunk GET fast path (#2351)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
This commit is contained in:
houseme
2026-04-07 08:33:46 +08:00
committed by GitHub
parent 8d27170ce4
commit 32bf8f5bf3
84 changed files with 15932 additions and 3592 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/data_movement.rs
+15 -8
View File
@@ -14,9 +14,11 @@
use crate::error::{Error, Result};
use crate::store::ECStore;
use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store_api::{
ChunkNativePutData, CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions,
};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_rio::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use std::io::Cursor;
use std::pin::Pin;
use std::sync::{
@@ -54,6 +56,11 @@ impl<R: AsyncRead + Unpin + Send + Sync> TryGetIndex for IndexedDataMovementRead
}
}
impl<R: AsyncRead + Unpin + Send + Sync> BlockReadable for IndexedDataMovementReader<R> {
fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
Box::pin(rustfs_utils::read_full(self, buf))
}
}
pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
let bytes = index?;
let mut decoded = Index::new();
@@ -64,7 +71,7 @@ pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
}
}
pub fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<PutObjReader> {
pub fn put_data_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<ChunkNativePutData> {
use sha2::{Digest, Sha256};
let sha256hex = if !chunk.is_empty() {
@@ -74,8 +81,8 @@ pub fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, in
};
let reader = IndexedDataMovementReader::new(Cursor::new(chunk), index);
let hash_reader = HashReader::from_stream(reader, size, actual_size, None, sha256hex, false)?;
Ok(PutObjReader::new(hash_reader))
let hash_reader = HashReader::from_reader(reader, size, actual_size, None, sha256hex, false)?;
Ok(ChunkNativePutData::new(hash_reader))
}
pub fn new_multipart_abort_flag() -> Arc<AtomicBool> {
@@ -172,7 +179,7 @@ pub(crate) async fn migrate_object(
let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
let index = decode_part_index(part.index.as_ref());
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
let mut data = put_data_from_chunk(chunk, part_size, part_actual_size, index)?;
let pi = match store
.put_object_part(
@@ -254,8 +261,8 @@ pub(crate) async fn migrate_object(
.first()
.and_then(|part| decode_part_index(part.index.as_ref()));
let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index);
let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let mut data = PutObjReader::new(hrd);
let hrd = HashReader::from_reader(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let mut data = ChunkNativePutData::new(hrd);
if let Err(err) = store
.put_object(