feat(storage): add direct chunk GET fast path (#2351)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
This commit is contained in:
houseme
2026-04-07 08:33:46 +08:00
committed by GitHub
parent 8d27170ce4
commit 32bf8f5bf3
84 changed files with 15932 additions and 3592 deletions
@@ -172,6 +172,52 @@ where
}
}
impl BitrotWriter<CustomWriter> {
fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
if self.finished {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished"));
}
if buf.len() > self.shard_size {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("data size {} exceeds shard size {}", buf.len(), self.shard_size),
));
}
if buf.len() < self.shard_size {
self.finished = true;
}
match &mut self.inner {
CustomWriter::InlineBuffer(data) => {
if self.hash_algo.size() > 0 {
let hash = self.hash_algo.hash_encode(buf);
if hash.as_ref().is_empty() {
error!("bitrot writer write hash error: hash is empty");
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty"));
}
data.extend_from_slice(hash.as_ref());
}
data.extend_from_slice(buf);
Ok(buf.len())
}
CustomWriter::Other(_) => Err(std::io::Error::other("inline sync write requires inline buffer writer")),
}
}
fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
match self.inner {
CustomWriter::InlineBuffer(_) => Ok(()),
CustomWriter::Other(_) => Err(std::io::Error::other("inline sync shutdown requires inline buffer writer")),
}
}
}
async fn write_all_vectored<W>(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()>
where
W: AsyncWrite + Unpin,
@@ -280,6 +326,10 @@ impl CustomWriter {
Self::Other(_) => None,
}
}
pub fn is_inline_buffer(&self) -> bool {
matches!(self, Self::InlineBuffer(_))
}
}
impl AsyncWrite for CustomWriter {
@@ -397,6 +447,24 @@ impl BitrotWriterWrapper {
self.bitrot_writer.shutdown().await
}
pub fn is_inline_buffer(&self) -> bool {
matches!(self.writer_type, WriterType::InlineBuffer)
}
pub fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if !self.is_inline_buffer() {
return Err(std::io::Error::other("inline sync write requires inline buffer writer"));
}
self.bitrot_writer.write_inline_sync(buf)
}
pub fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
if !self.is_inline_buffer() {
return Err(std::io::Error::other("inline sync shutdown requires inline buffer writer"));
}
self.bitrot_writer.shutdown_inline_sync()
}
/// Extract the inline buffer data, consuming the wrapper
pub fn into_inline_data(self) -> Option<Vec<u8>> {
match self.writer_type {