gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 03:46:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): harden HotPath profiling boundaries (#5555)

Browse Source 
* test(hotpath): gate mimalloc heap test by platform

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): attribute HotPath CPU measurements to impls

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(ecstore): trace raw shard I/O with HotPath

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): redact profiler and OPA endpoint diagnostics

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): settle encoded queue accounting

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): abort encoder producer on cancellation

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-01 16:49:48 +08:00
committed by GitHub
parent 62d44d10b8
commit d5c6ba99d5
19 changed files with 1480 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/disk/local.rs
+2 -2
View File
@@ -5078,7 +5078,7 @@ impl LocalDisk {
Ok((buf, mtime))
}
#[hotpath::measure]
#[hotpath::measure(impl_type = "LocalDisk")]
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
@@ -5121,7 +5121,7 @@ impl LocalDisk {
Ok((data, modtime))
}
#[hotpath::measure]
#[hotpath::measure(impl_type = "LocalDisk")]
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
// TODO: timeout support
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
crates/ecstore/src/erasure/coding/bitrot.rs
+2 -2
View File
@@ -103,7 +103,7 @@ where
/// or `out` is larger than one shard. On error `out`'s contents are
/// unspecified but never contain bytes that failed the hash check — the copy
/// happens only after verification.
#[hotpath::measure]
#[hotpath::measure(impl_type = "BitrotReader")]
pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
let want = out.len();
self.begin_read(want)?;
@@ -303,7 +303,7 @@ where
/// Write a (hash+data) block. Returns the number of data bytes written.
/// Returns an error if called after a short write or if data exceeds shard_size.
#[hotpath::measure(label = "BitrotWriter::write")]
#[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if buf.is_empty() {
return Ok(0);
crates/ecstore/src/erasure/coding/decode.rs
+2 -2
View File
@@ -691,7 +691,7 @@ impl<R> ParallelReader<R>
where
R: crate::erasure::coding::ShardSource,
{
#[hotpath::measure]
#[hotpath::measure(impl_type = "ParallelReader")]
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
// On the reconstruction-verifying GET path, read every live shard reader
// in lockstep so all readers advance one block per stripe and stay
@@ -1505,7 +1505,7 @@ where
}
impl Erasure {
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub async fn decode<W, R>(
&self,
writer: &mut W,
crates/ecstore/src/erasure/coding/encode.rs
+534 -56
View File
@@ -28,8 +28,11 @@ use std::vec;
use tokio::io::AsyncRead;
use tokio::runtime::RuntimeFlavor;
use tokio::sync::mpsc;
use tokio::task::{JoinError, JoinHandle};
use tracing::error;
/// Queue-capacity input for encoded blocks awaiting shard writers; it is not a
/// per-PUT or process-RSS memory limit.
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
const ENV_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: &str = "RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS";
const ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: &str = "RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST";
@@ -87,6 +90,33 @@ fn use_bytesmut_ingest() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST, DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST)
})
}
/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
/// task when its `JoinHandle` is dropped, so the producer must be aborted when
/// an upload is cancelled before the encode pipeline finishes.
struct AbortOnDropTask<T>(JoinHandle<T>);
impl<T> AbortOnDropTask<T> {
fn new(task: JoinHandle<T>) -> Self {
Self(task)
}
async fn abort_and_wait(&mut self) {
self.0.abort();
let _ = (&mut self.0).await;
}
async fn join(&mut self) -> Result<T, JoinError> {
(&mut self.0).await
}
}
impl<T> Drop for AbortOnDropTask<T> {
fn drop(&mut self) {
self.0.abort();
}
}
/// Read up to `limit` bytes into `buf`'s uninitialized spare capacity, appending after its
/// current length, and distinguish a clean EOF from a short read.
///
@@ -133,22 +163,65 @@ fn queued_block_bytes(block: &[Bytes]) -> usize {
block.iter().map(Bytes::len).sum()
}
async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
while let Some(block) = rx.recv().await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
/// Owns an encoded queue entry's gauge contribution until its consumer takes it.
struct QueuedInflightBytes {
bytes: usize,
}
impl QueuedInflightBytes {
fn new(bytes: usize) -> Self {
rustfs_io_metrics::add_ec_encode_inflight_bytes(bytes);
Self { bytes }
}
fn settle(&mut self) {
let bytes = std::mem::take(&mut self.bytes);
if bytes != 0 {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(bytes);
}
}
}
impl Drop for QueuedInflightBytes {
fn drop(&mut self) {
self.settle();
}
}
/// Couples an encoded block with its queue gauge contribution. Keeping the
/// guard in the queue entry also covers Tokio sends that complete through a
/// permit after the receiver has closed.
struct InflightEntry<T> {
entry: T,
accounting: QueuedInflightBytes,
}
impl<T> InflightEntry<T> {
fn new(entry: T, bytes: usize) -> Self {
Self {
entry,
accounting: QueuedInflightBytes::new(bytes),
}
}
fn into_inner(mut self) -> T {
self.accounting.settle();
self.entry
}
}
async fn send_queued<T>(
sender: &mpsc::Sender<InflightEntry<T>>,
entry: T,
bytes: usize,
) -> Result<(), mpsc::error::SendError<InflightEntry<T>>> {
sender.send(InflightEntry::new(entry, bytes)).await
}
fn queued_batch_bytes(batch: &[Vec<Bytes>]) -> usize {
batch.iter().map(|block| queued_block_bytes(block)).sum()
}
async fn drain_queued_batched_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Vec<Bytes>>>) {
while let Some(batch) = rx.recv().await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
}
}
fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
summary.dominant_error_label
}
@@ -504,7 +577,7 @@ impl Erasure {
Ok((reader, total))
}
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode<R>(
self: Arc<Self>,
reader: R,
@@ -540,13 +613,14 @@ impl Erasure {
));
}
// Bound queued encoded blocks by memory budget to avoid per-request spikes.
// Bound queued encoded blocks by a queue budget; this does not bound
// reader, encoder, writer, allocator, or process-RSS memory.
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Bytes>>>(inflight_blocks);
let task = tokio::spawn(async move {
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
let block_size = self.block_size;
let mut total = 0;
if use_bytesmut_ingest {
@@ -567,10 +641,8 @@ impl Erasure {
let res = self.clone().encode_block_bytes_mut(encode_buf, n).await?;
buf = BytesMut::with_capacity(ingest_capacity);
let queued_bytes = queued_block_bytes(&res);
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(res).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -598,10 +670,8 @@ impl Erasure {
let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
buf = returned_buf;
let queued_bytes = queued_block_bytes(&res);
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(res).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -626,7 +696,7 @@ impl Erasure {
}
Ok((reader, total))
});
}));
let mut writers = MultiWriter::new(writers, quorum);
@@ -638,11 +708,10 @@ impl Erasure {
break;
};
record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
let block = block.into_inner();
if block.is_empty() {
break;
}
let queued_bytes = queued_block_bytes(&block);
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
let write_stage_start = stage_timer_if_enabled();
if let Err(err) = writers.write(block).await {
write_err = Some(err);
@@ -652,9 +721,8 @@ impl Erasure {
}
if let Some(err) = write_err {
task.abort();
let _ = task.await;
drain_queued_inflight_bytes(&mut rx).await;
task.abort_and_wait().await;
drop(rx);
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -663,14 +731,14 @@ impl Erasure {
return Err(err);
}
let (reader, total) = task.await??;
let (reader, total) = task.join().await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
Ok((reader, total))
}
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode_batched<R>(
self: Arc<Self>,
mut reader: R,
@@ -692,9 +760,9 @@ impl Erasure {
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let batch_blocks = encode_batch_block_count().min(inflight_blocks);
let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
let (tx, mut rx) = mpsc::channel::<Vec<Vec<Bytes>>>(channel_capacity);
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Vec<Bytes>>>>(channel_capacity);
let task = tokio::spawn(async move {
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
let block_size = self.block_size;
let mut total = 0;
let mut buf = vec![0u8; block_size];
@@ -713,10 +781,8 @@ impl Erasure {
pending_batch.push(res);
if pending_batch.len() >= batch_blocks {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
@@ -742,17 +808,15 @@ impl Erasure {
}
if !pending_batch.is_empty() {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
}
Ok((reader, total))
});
}));
let mut writers = MultiWriter::new(writers, quorum);
let mut write_err = None;
@@ -763,7 +827,7 @@ impl Erasure {
break;
};
record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
let batch = batch.into_inner();
let write_stage_start = stage_timer_if_enabled();
for block in batch {
if let Err(err) = writers.write(block).await {
@@ -778,9 +842,8 @@ impl Erasure {
}
if let Some(err) = write_err {
task.abort();
let _ = task.await;
drain_queued_batched_inflight_bytes(&mut rx).await;
task.abort_and_wait().await;
drop(rx);
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -789,7 +852,7 @@ impl Erasure {
return Err(err);
}
let (reader, total) = task.await??;
let (reader, total) = task.join().await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
@@ -798,7 +861,7 @@ impl Erasure {
/// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
/// Reads all data, encodes directly, writes shards sequentially.
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode_inline_small<R>(
self: Arc<Self>,
reader: R,
@@ -813,7 +876,7 @@ impl Erasure {
/// Fast path for single-block non-inline objects: avoids the producer/consumer
/// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode_single_block_non_inline<R>(
self: Arc<Self>,
reader: R,
@@ -839,7 +902,104 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::Duration;
use tokio::io::{AsyncWrite, AsyncWriteExt};
use tokio::io::{AsyncWrite, AsyncWriteExt, ReadBuf};
use tokio::sync::oneshot;
struct PendingReader {
entered: Option<oneshot::Sender<()>>,
dropped: Option<oneshot::Sender<()>>,
}
impl PendingReader {
fn new() -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>) {
let (entered_tx, entered_rx) = oneshot::channel();
let (dropped_tx, dropped_rx) = oneshot::channel();
(
Self {
entered: Some(entered_tx),
dropped: Some(dropped_tx),
},
entered_rx,
dropped_rx,
)
}
}
impl AsyncRead for PendingReader {
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
if let Some(entered) = self.entered.take() {
let _ = entered.send(());
}
Poll::Pending
}
}
impl Drop for PendingReader {
fn drop(&mut self) {
if let Some(dropped) = self.dropped.take() {
let _ = dropped.send(());
}
}
}
struct BlocksThenPendingReader {
blocks_remaining: usize,
block: Vec<u8>,
blocked: Option<oneshot::Sender<()>>,
dropped: Option<oneshot::Sender<()>>,
final_block: Option<oneshot::Sender<()>>,
}
impl BlocksThenPendingReader {
fn new(
blocks_remaining: usize,
block_size: usize,
) -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>, oneshot::Receiver<()>) {
let (blocked_tx, blocked_rx) = oneshot::channel();
let (dropped_tx, dropped_rx) = oneshot::channel();
let (final_block_tx, final_block_rx) = oneshot::channel();
(
Self {
blocks_remaining,
block: vec![0x5a; block_size],
blocked: Some(blocked_tx),
dropped: Some(dropped_tx),
final_block: Some(final_block_tx),
},
blocked_rx,
dropped_rx,
final_block_rx,
)
}
}
impl AsyncRead for BlocksThenPendingReader {
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
if self.blocks_remaining == 0 {
if let Some(blocked) = self.blocked.take() {
let _ = blocked.send(());
}
return Poll::Pending;
}
if self.blocks_remaining == 1
&& let Some(final_block) = self.final_block.take()
{
let _ = final_block.send(());
}
self.blocks_remaining -= 1;
buf.put_slice(&self.block);
Poll::Ready(Ok(()))
}
}
impl Drop for BlocksThenPendingReader {
fn drop(&mut self) {
if let Some(dropped) = self.dropped.take() {
let _ = dropped.send(());
}
}
}
fn erasure_with_zero_block_size() -> Erasure {
let mut erasure = Erasure::default();
@@ -897,6 +1057,54 @@ mod tests {
}
}
struct FailAfterReaderBlocksWriter {
reader_blocked: oneshot::Receiver<()>,
writes: Arc<std::sync::atomic::AtomicUsize>,
}
impl AsyncWrite for FailAfterReaderBlocksWriter {
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
match Pin::new(&mut self.reader_blocked).poll(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(_) => {
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Poll::Ready(Err(std::io::Error::other("injected write failure after producer blocks")))
}
}
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
struct StallOnWriteWithSignal {
entered: Option<oneshot::Sender<()>>,
writes: Arc<std::sync::atomic::AtomicUsize>,
}
impl AsyncWrite for StallOnWriteWithSignal {
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if let Some(entered) = self.entered.take() {
let _ = entered.send(());
}
Poll::Pending
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[derive(Clone, Default)]
struct ShortWriteWriter;
@@ -1046,6 +1254,202 @@ mod tests {
BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::None)
}
#[derive(Clone, Copy)]
enum EncodePipeline {
Vec,
BytesMut,
Batched,
}
async fn aborting_encode_drops_blocked_producer(pipeline: EncodePipeline) {
const BLOCK_SIZE: usize = 16;
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let committed = Arc::new(Mutex::new(Vec::new()));
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE))];
let (reader, entered, dropped) = PendingReader::new();
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
let encode = match pipeline {
EncodePipeline::Vec => {
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
}
EncodePipeline::BytesMut => {
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
}
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
};
tokio::time::timeout(Duration::from_secs(1), entered)
.await
.expect("producer should enter the blocked reader before cancellation")
.expect("blocked reader should signal entry");
encode.abort();
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
tokio::time::timeout(Duration::from_secs(1), dropped)
.await
.expect("cancelling encode should drop the producer reader")
.expect("blocked reader should signal producer drop");
assert!(
committed.lock().expect("committed buffer should be lockable").is_empty(),
"cancelling before the first encoded block must not make data visible"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
gauge_baseline,
"cancelling the encode pipeline must preserve the inflight queue gauge"
);
}
async fn writer_error_aborts_blocked_producer(pipeline: EncodePipeline) {
const BLOCK_SIZE: usize = 16;
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let blocks_before_pending = match pipeline {
EncodePipeline::Batched => encode_batch_block_count(),
EncodePipeline::Vec | EncodePipeline::BytesMut => 1,
};
let (reader, reader_blocked, reader_dropped, _final_block) =
BlocksThenPendingReader::new(blocks_before_pending, BLOCK_SIZE);
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut writers = vec![Some(bitrot_writer(
FailAfterReaderBlocksWriter {
reader_blocked,
writes: writes.clone(),
},
BLOCK_SIZE,
))];
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
let result = match pipeline {
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
};
let err = match result {
Ok(_) => panic!("writer quorum failure should fail the encode pipeline"),
Err(err) => err,
};
assert!(err.to_string().contains("Failed to write data"));
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
.await
.expect("writer failure should abort the blocked producer")
.expect("blocked producer should signal reader drop");
assert_eq!(
writes.load(std::sync::atomic::Ordering::SeqCst),
1,
"writer failure must stop the pipeline before any additional shard write"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
gauge_baseline,
"writer failure must settle all queued and pending encoded bytes"
);
}
async fn aborting_full_queue_settles_pending_send() {
const BLOCK_SIZE: usize = 16;
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
let inflight_blocks = encode_channel_capacity(
erasure.shard_size().saturating_mul(erasure.total_shard_count()),
erasure_encode_max_inflight_bytes(),
);
let (reader, _reader_blocked, reader_dropped, final_block) =
BlocksThenPendingReader::new(inflight_blocks + 2, BLOCK_SIZE);
let (writer_entered_tx, writer_entered) = oneshot::channel();
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut writers = vec![Some(bitrot_writer(
StallOnWriteWithSignal {
entered: Some(writer_entered_tx),
writes: writes.clone(),
},
BLOCK_SIZE,
))];
let erasure_for_task = erasure.clone();
let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
tokio::time::timeout(Duration::from_secs(1), writer_entered)
.await
.expect("consumer should start the first writer call")
.expect("stalling writer should signal entry");
tokio::time::timeout(Duration::from_secs(1), final_block)
.await
.expect("producer should supply the block whose send fills the queue")
.expect("reader should signal final block");
let expected_queued_bytes =
u64::try_from((inflight_blocks + 1) * BLOCK_SIZE).expect("queued byte count should fit the gauge");
tokio::time::timeout(Duration::from_secs(1), async {
while rustfs_io_metrics::current_ec_encode_inflight_bytes() < gauge_baseline + expected_queued_bytes {
tokio::task::yield_now().await;
}
})
.await
.expect("producer should account for the pending send after the queue fills");
encode.abort();
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
.await
.expect("cancelling a full queue should abort its producer")
.expect("full-queue producer should signal reader drop");
assert_eq!(
writes.load(std::sync::atomic::Ordering::SeqCst),
1,
"cancellation must not resume the stalled writer"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
gauge_baseline,
"cancelling a full queue must settle queued and pending bytes"
);
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_vec_encode_drops_blocked_producer() {
aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await;
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_bytesmut_encode_drops_blocked_producer() {
aborting_encode_drops_blocked_producer(EncodePipeline::BytesMut).await;
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_batched_encode_drops_blocked_producer() {
aborting_encode_drops_blocked_producer(EncodePipeline::Batched).await;
}
#[tokio::test]
#[serial_test::serial]
async fn vec_writer_error_aborts_blocked_producer() {
writer_error_aborts_blocked_producer(EncodePipeline::Vec).await;
}
#[tokio::test]
#[serial_test::serial]
async fn bytesmut_writer_error_aborts_blocked_producer() {
writer_error_aborts_blocked_producer(EncodePipeline::BytesMut).await;
}
#[tokio::test]
#[serial_test::serial]
async fn batched_writer_error_aborts_blocked_producer() {
writer_error_aborts_blocked_producer(EncodePipeline::Batched).await;
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_full_queue_settles_pending_send() {
aborting_full_queue_settles_pending_send().await;
}
#[tokio::test]
async fn helper_writers_cover_flush_and_shutdown_paths() {
let mut failing_write = FailingWriteWriter;
@@ -1329,25 +1733,99 @@ mod tests {
}
#[tokio::test]
async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
let (tx, mut rx) = mpsc::channel(2);
tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
drop(tx);
#[serial_test::serial]
async fn queued_inflight_bytes_are_settled_on_all_queue_exit_paths() {
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let (tx, rx) = mpsc::channel(1);
let queued = vec![Bytes::from_static(b"queued")];
let queued_bytes = queued_block_bytes(&queued);
drain_queued_inflight_bytes(&mut rx).await;
send_queued(&tx, queued, queued_bytes)
.await
.expect("first queue entry should fit");
assert!(rx.recv().await.is_none());
let blocked = vec![Bytes::from_static(b"blocked")];
let blocked_bytes = queued_block_bytes(&blocked);
{
let pending_send = send_queued(&tx, blocked, blocked_bytes);
tokio::pin!(pending_send);
assert!(
futures::poll!(pending_send.as_mut()).is_pending(),
"full queue must suspend producer send"
);
}
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline + u64::try_from(queued_bytes).expect("queue bytes fit the gauge"),
"dropping a pending producer send must compensate its bytes"
);
drop(rx);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"dropping the receiver must settle every buffered entry"
);
let rejected = vec![Bytes::from_static(b"rejected")];
let rejected_bytes = queued_block_bytes(&rejected);
assert!(
send_queued(&tx, rejected, rejected_bytes).await.is_err(),
"closed receiver must reject a new send"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"failed sends must compensate their bytes"
);
}
#[tokio::test]
async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
let (tx, mut rx) = mpsc::channel(2);
tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
drop(tx);
#[serial_test::serial]
async fn queued_batch_entry_settles_bytes_before_handoff() {
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let (tx, rx) = mpsc::channel(2);
let mut rx = rx;
let batch = vec![vec![Bytes::from_static(b"queued")], vec![Bytes::from_static(b"batch")]];
let batch_bytes = queued_batch_bytes(&batch);
drain_queued_batched_inflight_bytes(&mut rx).await;
send_queued(&tx, batch, batch_bytes).await.expect("batch should be queued");
let batch = rx.recv().await.expect("queued batch should be received").into_inner();
assert_eq!(batch_bytes, queued_batch_bytes(&batch));
assert!(rx.recv().await.is_none());
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"receiving a batch must settle all contained block bytes before shard writes"
);
}
#[tokio::test]
#[serial_test::serial]
async fn queued_entry_settles_when_a_closed_receiver_accepts_an_outstanding_permit() {
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let (tx, mut rx) = mpsc::channel(1);
let permit = tx
.clone()
.reserve_owned()
.await
.expect("open receiver should reserve queue capacity");
rx.close();
assert!(
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"an outstanding permit must leave the closed queue observably empty"
);
let block = vec![Bytes::from_static(b"late-permit")];
let block_bytes = queued_block_bytes(&block);
permit.send(InflightEntry::new(block, block_bytes));
drop(rx);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"a queue entry sent through an outstanding permit must settle when Tokio drops it"
);
}
#[tokio::test]
crates/ecstore/src/erasure/coding/erasure.rs
+5 -5
View File
@@ -640,7 +640,7 @@ impl Erasure {
/// # Returns
/// A vector of encoded shards as `Bytes`.
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -688,7 +688,7 @@ impl Erasure {
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
/// Falls back to copying into a new buffer if zero-copy conversion fails.
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -752,7 +752,7 @@ impl Erasure {
/// block), the `resize(need_total_size)` below stays within capacity for every
/// `data_len <= block_size` — both shard-size formulas are monotone in
/// `data_len` — so this function never reallocates the buffer.
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -805,7 +805,7 @@ impl Erasure {
///
/// # Returns
/// Ok if reconstruction succeeds, error otherwise.
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if self.uses_legacy {
@@ -825,7 +825,7 @@ impl Erasure {
}
/// Decode and reconstruct missing data shards, then regenerate parity shards.
#[hotpath::measure]
#[hotpath::measure(impl_type = "Erasure")]
pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if self.uses_legacy {
crates/ecstore/src/io_support/bitrot.rs
+409 -22
View File
@@ -19,6 +19,8 @@ use crate::diagnostics::get::{
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
};
#[cfg(feature = "hotpath")]
use crate::disk::FileWriter;
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
use bytes::Bytes;
@@ -36,6 +38,11 @@ use std::time::Instant;
use tokio::io::{AsyncRead, ReadBuf};
use tracing::debug;
#[cfg(all(test, feature = "hotpath"))]
tokio::task_local! {
static FORCE_MMAP_COPY_FAILURE_FOR_TEST: ();
}
/// A shard source for the bitrot reader.
///
/// `InMemory` keeps the `Bytes` concrete instead of erasing it behind
@@ -360,10 +367,21 @@ async fn open_disk_reader(
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
});
match disk
.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
.await
{
let mmap_result = {
#[cfg(all(test, feature = "hotpath"))]
if FORCE_MMAP_COPY_FAILURE_FOR_TEST.try_with(|_| ()).is_ok() {
Err(DiskError::other("forced mmap-copy failure for test"))
} else {
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
.await
}
#[cfg(not(all(test, feature = "hotpath")))]
{
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
.await
}
};
match mmap_result {
Ok(bytes) => {
let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0;
@@ -398,7 +416,11 @@ async fn open_disk_reader(
}
return match stream_result {
Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)),
Ok(reader) => {
#[cfg(feature = "hotpath")]
let reader = instrument_raw_shard_reader(reader, disk.is_local());
Ok(wrap_first_read_metrics(reader, metrics_path))
}
Err(_) => Err(err),
};
}
@@ -407,6 +429,8 @@ async fn open_disk_reader(
let stream_start = stage_metrics_enabled.then(Instant::now);
let reader = disk.read_file_stream(bucket, path, offset, length).await?;
#[cfg(feature = "hotpath")]
let reader = instrument_raw_shard_reader(reader, disk.is_local());
if let Some(metrics_path) = metrics_path {
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
}
@@ -427,6 +451,37 @@ fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str
ShardReader::Stream(reader)
}
// The labels are deliberately fixed: object keys, disk paths, and remote hosts
// are all high-cardinality or sensitive and belong nowhere in a profiling report.
#[cfg(feature = "hotpath")]
const RAW_SHARD_READ_LOCAL_LABEL: &str = "EC raw shard read local";
#[cfg(feature = "hotpath")]
const RAW_SHARD_READ_REMOTE_LABEL: &str = "EC raw shard read remote";
#[cfg(feature = "hotpath")]
const RAW_SHARD_WRITE_LOCAL_LABEL: &str = "EC raw shard write local";
#[cfg(feature = "hotpath")]
const RAW_SHARD_WRITE_REMOTE_LABEL: &str = "EC raw shard write remote";
#[cfg(feature = "hotpath")]
fn instrument_raw_shard_reader(reader: FileReader, is_local: bool) -> FileReader {
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
if is_local {
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_LOCAL_LABEL))
} else {
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_REMOTE_LABEL))
}
}
#[cfg(feature = "hotpath")]
fn instrument_raw_shard_writer(writer: FileWriter, is_local: bool) -> FileWriter {
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
if is_local {
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_LOCAL_LABEL))
} else {
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_REMOTE_LABEL))
}
}
fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
(
offset.div_ceil(shard_size) * checksum_algo.size() + offset,
@@ -686,6 +741,8 @@ pub async fn create_bitrot_writer(
};
let file = disk.create_file("", volume, path, length).await?;
#[cfg(feature = "hotpath")]
let file = instrument_raw_shard_writer(file, disk.is_local());
CustomWriter::new_tokio_writer(file)
} else {
return Err(DiskError::DiskNotFound);
@@ -698,6 +755,182 @@ pub async fn create_bitrot_writer(
mod tests {
use super::*;
#[cfg(feature = "hotpath")]
use crate::cluster::rpc::RemoteDisk;
#[cfg(feature = "hotpath")]
use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, InternodeDataTransportCapabilities, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
};
#[cfg(feature = "hotpath")]
use crate::disk::{Disk, DiskOption, error::Result};
#[cfg(feature = "hotpath")]
#[derive(Debug, Clone, Default)]
struct TestRemoteDataTransport {
bytes: Arc<Mutex<Vec<u8>>>,
}
#[cfg(feature = "hotpath")]
impl TestRemoteDataTransport {
fn bytes(&self) -> Vec<u8> {
self.bytes
.lock()
.expect("test remote transport bytes lock should not be poisoned")
.clone()
}
}
#[cfg(feature = "hotpath")]
#[derive(Debug)]
struct TestRemoteWriter {
bytes: Arc<Mutex<Vec<u8>>>,
}
#[cfg(feature = "hotpath")]
impl tokio::io::AsyncWrite for TestRemoteWriter {
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
self.bytes
.lock()
.expect("test remote transport bytes lock should not be poisoned")
.extend_from_slice(buf);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[cfg(feature = "hotpath")]
#[async_trait::async_trait]
impl InternodeDataTransport for TestRemoteDataTransport {
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
Ok(Box::new(Cursor::new(self.bytes())))
}
async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
Ok(Box::new(TestRemoteWriter {
bytes: Arc::clone(&self.bytes),
}))
}
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
panic!("open_walk_dir must not be used by the raw shard I/O test")
}
fn name(&self) -> &'static str {
"bitrot-test-remote"
}
fn capabilities(&self) -> InternodeDataTransportCapabilities {
InternodeDataTransportCapabilities::tcp_http()
}
}
async fn local_test_disk() -> (DiskStore, tempfile::TempDir) {
use crate::disk::endpoint::Endpoint;
use crate::disk::{DiskOption, new_disk};
let dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("local disk should be created");
(disk, dir)
}
#[cfg(feature = "hotpath")]
async fn remote_test_disk() -> (DiskStore, TestRemoteDataTransport) {
use crate::disk::endpoint::Endpoint;
let endpoint = Endpoint {
url: url::Url::parse("http://remote-node:9000/data/rustfs0").expect("test remote endpoint should parse"),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
};
let transport = TestRemoteDataTransport::default();
let remote = RemoteDisk::new(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
Arc::new(transport.clone()),
)
.await
.expect("test remote disk should be created");
(Arc::new(Disk::Remote(Box::new(remote))), transport)
}
async fn round_trip_disk_bitrot(disk: &DiskStore, bucket: &str, path: &str, payload: &[u8], shard_size: usize) -> Vec<u8> {
disk.make_volume(bucket).await.expect("volume should be created");
let mut writer = create_bitrot_writer(
false,
Some(disk),
bucket,
path,
i64::try_from(payload.len()).expect("test payload length should fit i64"),
shard_size,
HashAlgorithm::None,
)
.await
.expect("disk bitrot writer should open the raw shard file");
for chunk in payload.chunks(shard_size) {
writer
.write(chunk)
.await
.expect("disk bitrot writer should preserve each shard block");
}
writer.shutdown().await.expect("disk bitrot writer should close cleanly");
let mut reader = create_bitrot_reader(
None,
Some(disk),
bucket,
path,
0,
payload.len(),
shard_size,
HashAlgorithm::None,
false,
false,
)
.await
.expect("disk bitrot reader should open the raw shard file")
.expect("disk bitrot reader should exist");
let mut actual = Vec::with_capacity(payload.len());
while actual.len() < payload.len() {
let remaining = payload.len() - actual.len();
let mut chunk = vec![0; remaining.min(shard_size)];
let read = reader
.read(&mut chunk)
.await
.expect("disk bitrot reader should return the complete shard body");
assert!(read > 0, "disk bitrot reader must not end before the expected shard body is complete");
actual.extend_from_slice(&chunk[..read]);
}
actual
}
#[test]
fn object_mmap_read_enabled_accepts_legacy_zero_copy_alias() {
temp_env::with_vars(
@@ -724,6 +957,165 @@ mod tests {
);
}
#[cfg(feature = "hotpath")]
#[test]
fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes() {
const CHILD_ENV: &str = "RUSTFS_HOTPATH_RAW_SHARD_IO_TEST_CHILD";
if std::env::var_os(CHILD_ENV).is_none() {
let status = std::process::Command::new(std::env::current_exe().expect("test executable path should be available"))
.arg("--exact")
.arg("io_support::bitrot::tests::raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes")
.arg("--nocapture")
.env(CHILD_ENV, "1")
.status()
.expect("isolated HotPath I/O test process should start");
assert!(status.success(), "isolated HotPath I/O test process should pass");
return;
}
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("test runtime should be created")
.block_on(raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process());
}
#[cfg(feature = "hotpath")]
async fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process() {
use hotpath::{Format, HotpathGuardBuilder, Section};
use tokio::io::AsyncReadExt;
let report_dir = tempfile::tempdir().expect("report tempdir should be created");
let report_path = report_dir.path().join("hotpath-io.json");
let guard = HotpathGuardBuilder::new("raw_shard_io_test")
.format(Format::Json)
.output_path(&report_path)
.sections(vec![Section::Io])
.build();
let (disk, _dir) = local_test_disk().await;
let bucket = "test-bucket";
let path = "obj/hotpath-part.1";
let payload = b"local shard bytes";
let shard_size = 4;
let local_read = round_trip_disk_bitrot(&disk, bucket, path, payload, shard_size).await;
assert_eq!(local_read, payload, "local raw shard I/O instrumentation must not alter stored bytes");
let fallback_path = "obj/hotpath-mmap-fallback-part.1";
let fallback_payload = b"mmap fallback shard bytes";
disk.write_all("test-bucket", fallback_path, Bytes::from_static(fallback_payload))
.await
.expect("fallback shard file should be written");
let mut fallback_reader = FORCE_MMAP_COPY_FAILURE_FOR_TEST
.scope((), open_disk_reader(&disk, bucket, fallback_path, 0, fallback_payload.len(), true, None))
.await
.expect("mmap-copy failure should fall back to a raw shard stream");
assert!(
matches!(fallback_reader, ShardReader::Stream(_)),
"forced mmap-copy failure must use the streaming fallback"
);
let mut fallback_read = Vec::new();
fallback_reader
.read_to_end(&mut fallback_read)
.await
.expect("mmap-copy fallback stream should preserve bytes");
assert_eq!(fallback_read, fallback_payload);
let (remote_disk, remote_transport) = remote_test_disk().await;
let remote_payload = b"remote shard bytes";
let mut remote_writer = create_bitrot_writer(
false,
Some(&remote_disk),
bucket,
"obj/hotpath-remote-part.1",
i64::try_from(remote_payload.len()).expect("remote payload length should fit i64"),
shard_size,
HashAlgorithm::None,
)
.await
.expect("remote bitrot writer should use the production raw writer path");
for chunk in remote_payload.chunks(shard_size) {
remote_writer
.write(chunk)
.await
.expect("remote bitrot writer should preserve bytes");
}
remote_writer
.shutdown()
.await
.expect("remote bitrot writer should close cleanly");
assert_eq!(remote_transport.bytes(), remote_payload);
let mut reader = create_bitrot_reader(
None,
Some(&remote_disk),
bucket,
"obj/hotpath-remote-part.1",
0,
remote_payload.len(),
shard_size,
HashAlgorithm::None,
false,
true,
)
.await
.expect("remote bitrot reader should use the production raw reader path")
.expect("remote bitrot reader should exist");
let mut remote_read = Vec::with_capacity(remote_payload.len());
while remote_read.len() < remote_payload.len() {
let remaining = remote_payload.len() - remote_read.len();
let mut chunk = vec![0; remaining.min(shard_size)];
let read = reader
.read(&mut chunk)
.await
.expect("remote bitrot reader should preserve bytes");
assert!(read > 0, "remote bitrot reader must not end before the expected shard body is complete");
remote_read.extend_from_slice(&chunk[..read]);
}
assert_eq!(remote_read, remote_payload);
drop(guard);
let report = std::fs::read_to_string(&report_path).expect("HotPath I/O report should be written");
let report: serde_json::Value = serde_json::from_str(&report).expect("HotPath I/O report should be valid JSON");
let entries = report["io"]["data"]
.as_array()
.expect("HotPath I/O report should include data rows");
let io_bytes = |label: &str, direction: &str| {
let byte_count: u64 = entries
.iter()
.filter(|entry| entry["label"].as_str().is_some_and(|entry_label| entry_label == label))
.filter_map(|entry| entry[direction]["bytes"].as_u64())
.sum();
assert!(byte_count > 0, "report must include fixed label {label}");
byte_count
};
let payload_len = u64::try_from(payload.len()).expect("test payload length should fit u64");
assert_eq!(
io_bytes(RAW_SHARD_READ_LOCAL_LABEL, "read"),
payload_len + u64::try_from(fallback_payload.len()).expect("fallback payload length should fit u64")
);
assert_eq!(io_bytes(RAW_SHARD_WRITE_LOCAL_LABEL, "write"), payload_len);
assert_eq!(
io_bytes(RAW_SHARD_READ_REMOTE_LABEL, "read"),
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
);
assert_eq!(
io_bytes(RAW_SHARD_WRITE_REMOTE_LABEL, "write"),
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
);
for label in [
RAW_SHARD_READ_LOCAL_LABEL,
RAW_SHARD_READ_REMOTE_LABEL,
RAW_SHARD_WRITE_LOCAL_LABEL,
RAW_SHARD_WRITE_REMOTE_LABEL,
] {
assert!(
!label.contains(['/', ':', '?', '@']),
"raw shard I/O labels must not carry a path, host, query, or credential delimiter: {label}"
);
}
}
#[test]
fn object_mmap_read_max_length_defaults_and_env_override() {
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
@@ -741,25 +1133,9 @@ mod tests {
// be materialized in memory by the mmap-copy path; over-cap reads stream.
#[tokio::test]
async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
use crate::disk::endpoint::Endpoint;
use crate::disk::{DiskOption, new_disk};
use tokio::io::AsyncReadExt;
let dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("local disk should be created");
let (disk, _dir) = local_test_disk().await;
let payload = vec![7u8; 4096];
disk.make_volume("test-bucket").await.expect("volume should be created");
@@ -814,6 +1190,17 @@ mod tests {
.await;
}
#[tokio::test]
async fn disk_bitrot_reader_and_writer_preserve_full_shard_body() {
let (disk, _dir) = local_test_disk().await;
let bucket = "test-bucket";
let path = "obj/wrapped-part.1";
let payload = b"wrapped shard body";
let actual = round_trip_disk_bitrot(&disk, bucket, path, payload, 4).await;
assert_eq!(actual, payload, "raw shard I/O instrumentation must not alter stored bytes");
}
#[tokio::test]
async fn test_create_bitrot_reader_with_inline_data() {
let test_data = b"hello world test data";
crates/ecstore/src/set_disk/read.rs
+6 -6
View File
@@ -199,7 +199,7 @@ impl SetDisks {
);
}
#[hotpath::measure]
#[hotpath::measure(impl_type = "SetDisks")]
pub async fn read_version_optimized(
&self,
bucket: &str,
@@ -238,7 +238,7 @@ impl SetDisks {
}
#[tracing::instrument(level = "debug", skip(self))]
#[hotpath::measure]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_fileinfo(
&self,
bucket: &str,
@@ -410,7 +410,7 @@ impl SetDisks {
Ok((fi, parts_metadata, op_online_disks))
}
#[hotpath::measure]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_info_and_quorum(
&self,
bucket: &str,
@@ -605,7 +605,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_with_fileinfo<W>(
// &self,
bucket: &str,
@@ -1140,7 +1140,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_decode_reader_with_fileinfo(
bucket: &str,
object: &str,
@@ -1296,7 +1296,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
#[hotpath::measure(impl_type = "SetDisks")]
async fn build_codec_streaming_part_reader(
bucket: &str,
object: &str,
crates/ecstore/src/store/multipart.rs
+1 -1
View File
@@ -238,7 +238,7 @@ impl ECStore {
}
#[instrument(skip(self, data))]
#[hotpath::measure]
#[hotpath::measure(impl_type = "ECStore")]
pub(super) async fn handle_put_object_part(
&self,
bucket: &str,
crates/ecstore/src/store/object.rs
+2 -2
View File
@@ -815,7 +815,7 @@ impl ECStore {
}
#[instrument(level = "debug", skip(self))]
#[hotpath::measure]
#[hotpath::measure(impl_type = "ECStore")]
pub(super) async fn handle_get_object_reader(
&self,
bucket: &str,
@@ -849,7 +849,7 @@ impl ECStore {
}
#[instrument(level = "debug", skip(self, data))]
#[hotpath::measure]
#[hotpath::measure(impl_type = "ECStore")]
pub(super) async fn handle_put_object(
&self,
bucket: &str,