feat(storage): extend PUT path tuning and observability (#3829)

* feat(storage): add multipart put stage metrics

* feat(scripts): add multipart put focus runner

* docs(operations): add multipart put server-path guides

* chore(scripts): add local rustfs restart helper

* docs(observability): add local metrics backend guide

* docs(observability): add localized multipart guides

* fix(ecstore): validate multipart batching path

* feat(obs): add erasure encode overlap metrics

* docs(ops): update overlap retest summary

* docs(ops): add batchblocks retest matrix

* docs(ops): extend overlap candidate summary

* docs(ops): capture 8-run overlap summary

* feat(storage): switch rename_data to msgpack map

* test(storage): add rename_data payload checks

* feat(object): add zero_copy_eager put path

* docs(ops): add zero_copy_eager put guide

* docs(ops): add deeper zero-copy next steps
This commit is contained in:
houseme
2026-06-25 19:24:35 +08:00
committed by GitHub
parent 3942186f3d
commit a30357d21e
24 changed files with 3037 additions and 35 deletions
+56 -14
View File
@@ -23,6 +23,7 @@ use bytes::Bytes;
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use std::sync::Arc;
use std::time::Instant;
use std::vec;
use tokio::io::AsyncRead;
use tokio::runtime::RuntimeFlavor;
@@ -40,6 +41,18 @@ const DEFAULT_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: usize = 4;
static CACHED_MAX_INFLIGHT_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
static CACHED_BATCH_BLOCKS: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
#[inline(always)]
fn stage_timer_if_enabled() -> Option<Instant> {
rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now)
}
#[inline(always)]
fn record_internal_stage_if_enabled(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
rustfs_io_metrics::record_stage_duration(stage, started_at.elapsed().as_secs_f64() * 1000.0);
}
}
fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
if expanded_block_bytes == 0 {
return 1;
@@ -57,6 +70,15 @@ fn encode_batch_block_count() -> usize {
})
}
fn erasure_encode_max_inflight_bytes() -> usize {
*CACHED_MAX_INFLIGHT_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
)
})
}
fn queued_block_bytes(block: &[Bytes]) -> usize {
block.iter().map(Bytes::len).sum()
}
@@ -237,6 +259,7 @@ impl<'a> MultiWriter<'a> {
impl Erasure {
async fn encode_block(self: Arc<Self>, encode_buf: Vec<u8>, len: usize) -> std::io::Result<(Vec<Bytes>, Vec<u8>)> {
let encode_stage_start = stage_timer_if_enabled();
let encode_once = move || {
let res = self.encode_data(&encode_buf[..len]);
(res, encode_buf)
@@ -252,6 +275,7 @@ impl Erasure {
.map_err(|err| std::io::Error::other(format!("EC encode task failed: {err}")))?,
};
record_internal_stage_if_enabled("erasure_encode_cpu", encode_stage_start);
Ok((res?, returned_buf))
}
@@ -316,12 +340,7 @@ impl Erasure {
// Bound queued encoded blocks by memory budget to avoid per-request spikes.
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = *CACHED_MAX_INFLIGHT_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
)
});
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
@@ -339,10 +358,12 @@ impl Erasure {
buf = returned_buf;
let queued_bytes = queued_block_bytes(&res);
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(res).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
}
Ok(None) => {
break;
@@ -369,30 +390,41 @@ impl Erasure {
let mut write_err = None;
while let Some(block) = rx.recv().await {
loop {
let recv_wait_stage_start = stage_timer_if_enabled();
let Some(block) = rx.recv().await else {
break;
};
record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
if block.is_empty() {
break;
}
let queued_bytes = queued_block_bytes(&block);
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
let write_stage_start = stage_timer_if_enabled();
if let Err(err) = writers.write(block).await {
write_err = Some(err);
break;
}
record_internal_stage_if_enabled("erasure_encode_write", write_stage_start);
}
if let Some(err) = write_err {
task.abort();
let _ = task.await;
drain_queued_inflight_bytes(&mut rx).await;
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
}
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
return Err(err);
}
let (reader, total) = task.await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
Ok((reader, total))
}
@@ -413,12 +445,7 @@ impl Erasure {
}
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = *CACHED_MAX_INFLIGHT_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
)
});
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let batch_blocks = encode_batch_block_count().min(inflight_blocks);
let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
@@ -444,10 +471,12 @@ impl Erasure {
if pending_batch.len() >= batch_blocks {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
pending_batch = Vec::with_capacity(batch_blocks);
pending_batch_bytes = 0;
}
@@ -471,10 +500,12 @@ impl Erasure {
if !pending_batch.is_empty() {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
}
Ok((reader, total))
@@ -483,14 +514,21 @@ impl Erasure {
let mut writers = MultiWriter::new(writers, quorum);
let mut write_err = None;
while let Some(batch) = rx.recv().await {
loop {
let recv_wait_stage_start = stage_timer_if_enabled();
let Some(batch) = rx.recv().await else {
break;
};
record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
let write_stage_start = stage_timer_if_enabled();
for block in batch {
if let Err(err) = writers.write(block).await {
write_err = Some(err);
break;
}
}
record_internal_stage_if_enabled("erasure_encode_batched_write", write_stage_start);
if write_err.is_some() {
break;
}
@@ -500,14 +538,18 @@ impl Erasure {
task.abort();
let _ = task.await;
drain_queued_batched_inflight_bytes(&mut rx).await;
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
}
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
return Err(err);
}
let (reader, total) = task.await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
Ok((reader, total))
}
+68 -1
View File
@@ -751,6 +751,12 @@ fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
Ok(serializer.into_inner())
}
fn encode_msgpack_named<T: Serialize>(value: &T) -> Result<Vec<u8>> {
let mut serializer = rmp_serde::Serializer::new(Vec::new()).with_struct_map();
value.serialize(&mut serializer)?;
Ok(serializer.into_inner())
}
fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Result<T> {
if !binary.is_empty() {
let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
@@ -1524,6 +1530,7 @@ impl DiskAPI for RemoteDisk {
"rename_data",
|| async {
let file_info = serde_json::to_string(&fi)?;
let file_info_bin = encode_msgpack_named(&fi)?;
let mut client = self
.get_client()
.await
@@ -1535,6 +1542,7 @@ impl DiskAPI for RemoteDisk {
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
file_info_bin: file_info_bin.into(),
});
let response = client.rename_data(request).await?.into_inner();
@@ -1543,7 +1551,8 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
let rename_data_resp =
decode_msgpack_or_json::<RenameDataResp>(&response.rename_data_resp_bin, &response.rename_data_resp)?;
Ok(rename_data_resp)
},
@@ -2373,6 +2382,64 @@ mod tests {
}
}
fn sample_rename_data_file_info() -> FileInfo {
FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
data_dir: Some(Uuid::new_v4()),
size: 64 * 1024,
mod_time: Some(::time::OffsetDateTime::UNIX_EPOCH + ::time::Duration::seconds(1)),
metadata: [
("etag".to_string(), "etag-value".to_string()),
("content-type".to_string(), "application/octet-stream".to_string()),
]
.into_iter()
.collect(),
erasure: rustfs_filemeta::ErasureInfo {
algorithm: rustfs_filemeta::ERASURE_ALGORITHM.to_string(),
data_blocks: 4,
parity_blocks: 2,
block_size: 1024 * 1024,
index: 1,
distribution: vec![1, 2, 3, 4, 5, 6],
..Default::default()
},
..Default::default()
}
}
#[test]
fn rename_data_file_info_named_msgpack_is_smaller_than_json() {
let file_info = sample_rename_data_file_info();
let json = serde_json::to_vec(&file_info).expect("file info json should encode");
let named_msgpack = encode_msgpack_named(&file_info).expect("file info named msgpack should encode");
assert!(
named_msgpack.len() < json.len(),
"expected named msgpack payload to be smaller than json (msgpack={}, json={})",
named_msgpack.len(),
json.len()
);
}
#[test]
fn rename_data_resp_named_msgpack_is_smaller_than_json() {
let response = RenameDataResp {
old_data_dir: Some(Uuid::new_v4()),
sign: Some(vec![1_u8; 32]),
};
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
let named_msgpack = encode_msgpack_named(&response).expect("rename data response named msgpack should encode");
assert!(
named_msgpack.len() < json.len(),
"expected named msgpack payload to be smaller than json (msgpack={}, json={})",
named_msgpack.len(),
json.len()
);
}
#[derive(Debug, Default)]
struct SinkTestWriter;
+91 -2
View File
@@ -152,6 +152,9 @@ const SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS: u128 = 5_000;
const ENV_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
const DEFAULT_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 64 * 1024 * 1024;
static CACHED_PUT_LARGE_BATCH_MIN_SIZE_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
const ENV_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
const DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 128 * 1024 * 1024;
static CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
@@ -827,6 +830,15 @@ impl SmallWritePath {
SmallWritePath::PipelineBatchedLarge => "write_pipeline_batched_large",
}
}
fn multipart_metric_label(&self) -> &'static str {
match self {
SmallWritePath::Inline => "multipart_write_inline",
SmallWritePath::SingleBlockNonInline => "multipart_write_single_block_non_inline",
SmallWritePath::Pipeline => "multipart_write_pipeline",
SmallWritePath::PipelineBatchedLarge => "multipart_write_pipeline_batched_large",
}
}
}
fn put_large_batch_min_size_bytes() -> usize {
@@ -835,6 +847,15 @@ fn put_large_batch_min_size_bytes() -> usize {
})
}
fn multipart_put_large_batch_min_size_bytes() -> usize {
*CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES,
DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES,
)
})
}
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
SmallWritePath::Inline
@@ -859,6 +880,17 @@ fn classify_put_write_path(is_inline_buffer: bool, object_size: i64, block_size:
}
}
fn classify_multipart_part_write_path(object_size: i64, block_size: usize) -> SmallWritePath {
if should_use_single_block_non_inline_fast_path(false, object_size, block_size) {
return SmallWritePath::SingleBlockNonInline;
}
match usize::try_from(object_size) {
Ok(size) if size >= multipart_put_large_batch_min_size_bytes() => SmallWritePath::PipelineBatchedLarge,
_ => SmallWritePath::Pipeline,
}
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for SetDisks {
type Error = Error;
@@ -3392,6 +3424,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let mut writers = Vec::with_capacity(shuffle_disks.len());
let mut errors = Vec::with_capacity(shuffle_disks.len());
@@ -3433,6 +3466,13 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
}
}
if let Some(stage_start) = writer_setup_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_set_disk_writer_setup",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
@@ -3442,12 +3482,16 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
return Err(Error::other(format!("not enough disks to write: {errors:?}")));
}
// Capture the original part size before swapping the stream out for encoding.
let multipart_part_size = data.size();
let stream = mem::replace(
&mut data.stream,
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let write_path = classify_small_write_path(false, data.size(), fi.erasure.block_size);
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
rustfs_io_metrics::record_put_object_path(write_path.multipart_metric_label());
let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let (reader, w_size) = match write_path {
SmallWritePath::SingleBlockNonInline => {
@@ -3455,11 +3499,19 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
.await?
}
SmallWritePath::Inline | SmallWritePath::Pipeline | SmallWritePath::PipelineBatchedLarge => {
SmallWritePath::PipelineBatchedLarge => Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await?,
SmallWritePath::Inline | SmallWritePath::Pipeline => {
Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?
}
}; // TODO: delete temporary directory on error
if let Some(stage_start) = encode_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_set_disk_encode",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
let _ = mem::replace(&mut data.stream, reader);
if (w_size as i64) < data.size() {
@@ -4337,6 +4389,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
);
}
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
self.cleanup_multipart_path(&parts).await;
let (online_disks, versions, op_old_dir, cleanup_disks) = Self::rename_data(
@@ -4355,6 +4408,13 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
.await?;
}
if let Some(stage_start) = complete_tail_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_complete_tail",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
drop(object_lock_guard); // drop object lock guard to release the lock
if let Some(versions) = versions {
@@ -7492,6 +7552,35 @@ mod tests {
));
}
#[test]
fn multipart_put_large_batch_path_only_applies_at_128m_and_above() {
assert!(matches!(
classify_multipart_part_write_path(128 * 1024 * 1024, 1024 * 1024),
SmallWritePath::PipelineBatchedLarge
));
assert!(matches!(
classify_multipart_part_write_path(64 * 1024 * 1024, 1024 * 1024),
SmallWritePath::Pipeline
));
assert!(matches!(
classify_multipart_part_write_path(1024 * 1024, 1024 * 1024),
SmallWritePath::SingleBlockNonInline
));
}
#[test]
fn multipart_write_paths_use_distinct_metric_labels() {
assert_eq!(SmallWritePath::Pipeline.multipart_metric_label(), "multipart_write_pipeline");
assert_eq!(
SmallWritePath::PipelineBatchedLarge.multipart_metric_label(),
"multipart_write_pipeline_batched_large"
);
assert_eq!(
SmallWritePath::SingleBlockNonInline.multipart_metric_label(),
"multipart_write_single_block_non_inline"
);
}
#[test]
fn test_is_cold_storage_class() {
// Test cold storage classes
@@ -3,7 +3,8 @@
use bytes::Bytes;
use rustfs_protos::proto_gen::node_service::{
ReadMultipleRequest, ReadMultipleResponse, ReadVersionResponse, ReadXlResponse, UpdateMetadataRequest, WriteMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadVersionResponse, ReadXlResponse, RenameDataRequest, RenameDataResponse,
UpdateMetadataRequest, WriteMetadataRequest,
};
fn expect_bytes(_: &Bytes) {}
@@ -17,6 +18,12 @@ fn protobuf_bytes_fields_use_bytes_consistently() {
let write = WriteMetadataRequest::default();
expect_bytes(&write.file_info_bin);
let rename_data = RenameDataRequest::default();
expect_bytes(&rename_data.file_info_bin);
let rename_data_response = RenameDataResponse::default();
expect_bytes(&rename_data_response.rename_data_resp_bin);
let version = ReadVersionResponse::default();
expect_bytes(&version.file_info_bin);
@@ -349,6 +349,8 @@ pub struct RenameDataRequest {
pub dst_volume: ::prost::alloc::string::String,
#[prost(string, tag = "6")]
pub dst_path: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "7")]
pub file_info_bin: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RenameDataResponse {
@@ -358,6 +360,8 @@ pub struct RenameDataResponse {
pub rename_data_resp: ::prost::alloc::string::String,
#[prost(message, optional, tag = "3")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "4")]
pub rename_data_resp_bin: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct MakeVolumesRequest {
+2
View File
@@ -253,12 +253,14 @@ message RenameDataRequest {
string file_info = 4;
string dst_volume = 5;
string dst_path = 6;
bytes file_info_bin = 7;
}
message RenameDataResponse {
bool success = 1;
string rename_data_resp = 2;
optional Error error = 3;
bytes rename_data_resp_bin = 4;
}
message MakeVolumesRequest {