feat(storage): extend PUT path tuning and observability (#3829)

* feat(storage): add multipart put stage metrics

* feat(scripts): add multipart put focus runner

* docs(operations): add multipart put server-path guides

* chore(scripts): add local rustfs restart helper

* docs(observability): add local metrics backend guide

* docs(observability): add localized multipart guides

* fix(ecstore): validate multipart batching path

* feat(obs): add erasure encode overlap metrics

* docs(ops): update overlap retest summary

* docs(ops): add batchblocks retest matrix

* docs(ops): extend overlap candidate summary

* docs(ops): capture 8-run overlap summary

* feat(storage): switch rename_data to msgpack map

* test(storage): add rename_data payload checks

* feat(object): add zero_copy_eager put path

* docs(ops): add zero_copy_eager put guide

* docs(ops): add deeper zero-copy next steps
This commit is contained in:
houseme
2026-06-25 19:24:35 +08:00
committed by GitHub
parent 3942186f3d
commit a30357d21e
24 changed files with 3037 additions and 35 deletions
+56 -14
View File
@@ -23,6 +23,7 @@ use bytes::Bytes;
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use std::sync::Arc;
use std::time::Instant;
use std::vec;
use tokio::io::AsyncRead;
use tokio::runtime::RuntimeFlavor;
@@ -40,6 +41,18 @@ const DEFAULT_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: usize = 4;
static CACHED_MAX_INFLIGHT_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
static CACHED_BATCH_BLOCKS: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
#[inline(always)]
fn stage_timer_if_enabled() -> Option<Instant> {
rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now)
}
#[inline(always)]
fn record_internal_stage_if_enabled(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
rustfs_io_metrics::record_stage_duration(stage, started_at.elapsed().as_secs_f64() * 1000.0);
}
}
fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
if expanded_block_bytes == 0 {
return 1;
@@ -57,6 +70,15 @@ fn encode_batch_block_count() -> usize {
})
}
fn erasure_encode_max_inflight_bytes() -> usize {
*CACHED_MAX_INFLIGHT_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
)
})
}
fn queued_block_bytes(block: &[Bytes]) -> usize {
block.iter().map(Bytes::len).sum()
}
@@ -237,6 +259,7 @@ impl<'a> MultiWriter<'a> {
impl Erasure {
async fn encode_block(self: Arc<Self>, encode_buf: Vec<u8>, len: usize) -> std::io::Result<(Vec<Bytes>, Vec<u8>)> {
let encode_stage_start = stage_timer_if_enabled();
let encode_once = move || {
let res = self.encode_data(&encode_buf[..len]);
(res, encode_buf)
@@ -252,6 +275,7 @@ impl Erasure {
.map_err(|err| std::io::Error::other(format!("EC encode task failed: {err}")))?,
};
record_internal_stage_if_enabled("erasure_encode_cpu", encode_stage_start);
Ok((res?, returned_buf))
}
@@ -316,12 +340,7 @@ impl Erasure {
// Bound queued encoded blocks by memory budget to avoid per-request spikes.
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = *CACHED_MAX_INFLIGHT_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
)
});
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
@@ -339,10 +358,12 @@ impl Erasure {
buf = returned_buf;
let queued_bytes = queued_block_bytes(&res);
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(res).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
}
Ok(None) => {
break;
@@ -369,30 +390,41 @@ impl Erasure {
let mut write_err = None;
while let Some(block) = rx.recv().await {
loop {
let recv_wait_stage_start = stage_timer_if_enabled();
let Some(block) = rx.recv().await else {
break;
};
record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
if block.is_empty() {
break;
}
let queued_bytes = queued_block_bytes(&block);
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
let write_stage_start = stage_timer_if_enabled();
if let Err(err) = writers.write(block).await {
write_err = Some(err);
break;
}
record_internal_stage_if_enabled("erasure_encode_write", write_stage_start);
}
if let Some(err) = write_err {
task.abort();
let _ = task.await;
drain_queued_inflight_bytes(&mut rx).await;
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
}
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
return Err(err);
}
let (reader, total) = task.await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
Ok((reader, total))
}
@@ -413,12 +445,7 @@ impl Erasure {
}
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = *CACHED_MAX_INFLIGHT_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
)
});
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let batch_blocks = encode_batch_block_count().min(inflight_blocks);
let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
@@ -444,10 +471,12 @@ impl Erasure {
if pending_batch.len() >= batch_blocks {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
pending_batch = Vec::with_capacity(batch_blocks);
pending_batch_bytes = 0;
}
@@ -471,10 +500,12 @@ impl Erasure {
if !pending_batch.is_empty() {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
}
Ok((reader, total))
@@ -483,14 +514,21 @@ impl Erasure {
let mut writers = MultiWriter::new(writers, quorum);
let mut write_err = None;
while let Some(batch) = rx.recv().await {
loop {
let recv_wait_stage_start = stage_timer_if_enabled();
let Some(batch) = rx.recv().await else {
break;
};
record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
let write_stage_start = stage_timer_if_enabled();
for block in batch {
if let Err(err) = writers.write(block).await {
write_err = Some(err);
break;
}
}
record_internal_stage_if_enabled("erasure_encode_batched_write", write_stage_start);
if write_err.is_some() {
break;
}
@@ -500,14 +538,18 @@ impl Erasure {
task.abort();
let _ = task.await;
drain_queued_batched_inflight_bytes(&mut rx).await;
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
}
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
return Err(err);
}
let (reader, total) = task.await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
Ok((reader, total))
}
+68 -1
View File
@@ -751,6 +751,12 @@ fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
Ok(serializer.into_inner())
}
fn encode_msgpack_named<T: Serialize>(value: &T) -> Result<Vec<u8>> {
let mut serializer = rmp_serde::Serializer::new(Vec::new()).with_struct_map();
value.serialize(&mut serializer)?;
Ok(serializer.into_inner())
}
fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str) -> Result<T> {
if !binary.is_empty() {
let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
@@ -1524,6 +1530,7 @@ impl DiskAPI for RemoteDisk {
"rename_data",
|| async {
let file_info = serde_json::to_string(&fi)?;
let file_info_bin = encode_msgpack_named(&fi)?;
let mut client = self
.get_client()
.await
@@ -1535,6 +1542,7 @@ impl DiskAPI for RemoteDisk {
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
file_info_bin: file_info_bin.into(),
});
let response = client.rename_data(request).await?.into_inner();
@@ -1543,7 +1551,8 @@ impl DiskAPI for RemoteDisk {
return Err(response.error.unwrap_or_default().into());
}
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
let rename_data_resp =
decode_msgpack_or_json::<RenameDataResp>(&response.rename_data_resp_bin, &response.rename_data_resp)?;
Ok(rename_data_resp)
},
@@ -2373,6 +2382,64 @@ mod tests {
}
}
fn sample_rename_data_file_info() -> FileInfo {
FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
data_dir: Some(Uuid::new_v4()),
size: 64 * 1024,
mod_time: Some(::time::OffsetDateTime::UNIX_EPOCH + ::time::Duration::seconds(1)),
metadata: [
("etag".to_string(), "etag-value".to_string()),
("content-type".to_string(), "application/octet-stream".to_string()),
]
.into_iter()
.collect(),
erasure: rustfs_filemeta::ErasureInfo {
algorithm: rustfs_filemeta::ERASURE_ALGORITHM.to_string(),
data_blocks: 4,
parity_blocks: 2,
block_size: 1024 * 1024,
index: 1,
distribution: vec![1, 2, 3, 4, 5, 6],
..Default::default()
},
..Default::default()
}
}
#[test]
fn rename_data_file_info_named_msgpack_is_smaller_than_json() {
let file_info = sample_rename_data_file_info();
let json = serde_json::to_vec(&file_info).expect("file info json should encode");
let named_msgpack = encode_msgpack_named(&file_info).expect("file info named msgpack should encode");
assert!(
named_msgpack.len() < json.len(),
"expected named msgpack payload to be smaller than json (msgpack={}, json={})",
named_msgpack.len(),
json.len()
);
}
#[test]
fn rename_data_resp_named_msgpack_is_smaller_than_json() {
let response = RenameDataResp {
old_data_dir: Some(Uuid::new_v4()),
sign: Some(vec![1_u8; 32]),
};
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
let named_msgpack = encode_msgpack_named(&response).expect("rename data response named msgpack should encode");
assert!(
named_msgpack.len() < json.len(),
"expected named msgpack payload to be smaller than json (msgpack={}, json={})",
named_msgpack.len(),
json.len()
);
}
#[derive(Debug, Default)]
struct SinkTestWriter;
+91 -2
View File
@@ -152,6 +152,9 @@ const SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS: u128 = 5_000;
const ENV_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
const DEFAULT_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 64 * 1024 * 1024;
static CACHED_PUT_LARGE_BATCH_MIN_SIZE_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
const ENV_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
const DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 128 * 1024 * 1024;
static CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
@@ -827,6 +830,15 @@ impl SmallWritePath {
SmallWritePath::PipelineBatchedLarge => "write_pipeline_batched_large",
}
}
fn multipart_metric_label(&self) -> &'static str {
match self {
SmallWritePath::Inline => "multipart_write_inline",
SmallWritePath::SingleBlockNonInline => "multipart_write_single_block_non_inline",
SmallWritePath::Pipeline => "multipart_write_pipeline",
SmallWritePath::PipelineBatchedLarge => "multipart_write_pipeline_batched_large",
}
}
}
fn put_large_batch_min_size_bytes() -> usize {
@@ -835,6 +847,15 @@ fn put_large_batch_min_size_bytes() -> usize {
})
}
fn multipart_put_large_batch_min_size_bytes() -> usize {
*CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES.get_or_init(|| {
rustfs_utils::get_env_usize(
ENV_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES,
DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES,
)
})
}
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
SmallWritePath::Inline
@@ -859,6 +880,17 @@ fn classify_put_write_path(is_inline_buffer: bool, object_size: i64, block_size:
}
}
fn classify_multipart_part_write_path(object_size: i64, block_size: usize) -> SmallWritePath {
if should_use_single_block_non_inline_fast_path(false, object_size, block_size) {
return SmallWritePath::SingleBlockNonInline;
}
match usize::try_from(object_size) {
Ok(size) if size >= multipart_put_large_batch_min_size_bytes() => SmallWritePath::PipelineBatchedLarge,
_ => SmallWritePath::Pipeline,
}
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for SetDisks {
type Error = Error;
@@ -3392,6 +3424,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let mut writers = Vec::with_capacity(shuffle_disks.len());
let mut errors = Vec::with_capacity(shuffle_disks.len());
@@ -3433,6 +3466,13 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
}
}
if let Some(stage_start) = writer_setup_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_set_disk_writer_setup",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
@@ -3442,12 +3482,16 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
return Err(Error::other(format!("not enough disks to write: {errors:?}")));
}
// Capture the original part size before swapping the stream out for encoding.
let multipart_part_size = data.size();
let stream = mem::replace(
&mut data.stream,
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let write_path = classify_small_write_path(false, data.size(), fi.erasure.block_size);
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
rustfs_io_metrics::record_put_object_path(write_path.multipart_metric_label());
let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let (reader, w_size) = match write_path {
SmallWritePath::SingleBlockNonInline => {
@@ -3455,11 +3499,19 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
.await?
}
SmallWritePath::Inline | SmallWritePath::Pipeline | SmallWritePath::PipelineBatchedLarge => {
SmallWritePath::PipelineBatchedLarge => Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await?,
SmallWritePath::Inline | SmallWritePath::Pipeline => {
Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?
}
}; // TODO: delete temporary directory on error
if let Some(stage_start) = encode_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_set_disk_encode",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
let _ = mem::replace(&mut data.stream, reader);
if (w_size as i64) < data.size() {
@@ -4337,6 +4389,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
);
}
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
self.cleanup_multipart_path(&parts).await;
let (online_disks, versions, op_old_dir, cleanup_disks) = Self::rename_data(
@@ -4355,6 +4408,13 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
.await?;
}
if let Some(stage_start) = complete_tail_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_complete_tail",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
drop(object_lock_guard); // drop object lock guard to release the lock
if let Some(versions) = versions {
@@ -7492,6 +7552,35 @@ mod tests {
));
}
#[test]
fn multipart_put_large_batch_path_only_applies_at_128m_and_above() {
assert!(matches!(
classify_multipart_part_write_path(128 * 1024 * 1024, 1024 * 1024),
SmallWritePath::PipelineBatchedLarge
));
assert!(matches!(
classify_multipart_part_write_path(64 * 1024 * 1024, 1024 * 1024),
SmallWritePath::Pipeline
));
assert!(matches!(
classify_multipart_part_write_path(1024 * 1024, 1024 * 1024),
SmallWritePath::SingleBlockNonInline
));
}
#[test]
fn multipart_write_paths_use_distinct_metric_labels() {
assert_eq!(SmallWritePath::Pipeline.multipart_metric_label(), "multipart_write_pipeline");
assert_eq!(
SmallWritePath::PipelineBatchedLarge.multipart_metric_label(),
"multipart_write_pipeline_batched_large"
);
assert_eq!(
SmallWritePath::SingleBlockNonInline.multipart_metric_label(),
"multipart_write_single_block_non_inline"
);
}
#[test]
fn test_is_cold_storage_class() {
// Test cold storage classes
@@ -3,7 +3,8 @@
use bytes::Bytes;
use rustfs_protos::proto_gen::node_service::{
ReadMultipleRequest, ReadMultipleResponse, ReadVersionResponse, ReadXlResponse, UpdateMetadataRequest, WriteMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadVersionResponse, ReadXlResponse, RenameDataRequest, RenameDataResponse,
UpdateMetadataRequest, WriteMetadataRequest,
};
fn expect_bytes(_: &Bytes) {}
@@ -17,6 +18,12 @@ fn protobuf_bytes_fields_use_bytes_consistently() {
let write = WriteMetadataRequest::default();
expect_bytes(&write.file_info_bin);
let rename_data = RenameDataRequest::default();
expect_bytes(&rename_data.file_info_bin);
let rename_data_response = RenameDataResponse::default();
expect_bytes(&rename_data_response.rename_data_resp_bin);
let version = ReadVersionResponse::default();
expect_bytes(&version.file_info_bin);
@@ -349,6 +349,8 @@ pub struct RenameDataRequest {
pub dst_volume: ::prost::alloc::string::String,
#[prost(string, tag = "6")]
pub dst_path: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "7")]
pub file_info_bin: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RenameDataResponse {
@@ -358,6 +360,8 @@ pub struct RenameDataResponse {
pub rename_data_resp: ::prost::alloc::string::String,
#[prost(message, optional, tag = "3")]
pub error: ::core::option::Option<Error>,
#[prost(bytes = "bytes", tag = "4")]
pub rename_data_resp_bin: ::prost::bytes::Bytes,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct MakeVolumesRequest {
+2
View File
@@ -253,12 +253,14 @@ message RenameDataRequest {
string file_info = 4;
string dst_volume = 5;
string dst_path = 6;
bytes file_info_bin = 7;
}
message RenameDataResponse {
bool success = 1;
string rename_data_resp = 2;
optional Error error = 3;
bytes rename_data_resp_bin = 4;
}
message MakeVolumesRequest {
@@ -0,0 +1,232 @@
# Issue #712 Local Queryable Metrics Backend Guide
## 1. Purpose
This guide is intended to unblock the third validation batch for `#712` by making multipart PUT stage metrics queryable from a local backend.
The current problem is not that multipart stage metrics are missing from the code path. The actual problem is that the local environment does not expose a queryable metrics backend:
1. `rustfs/admin/v3/metrics` is available, but it returns an admin-side JSON snapshot rather than the `metrics` crate histogram series
2. there is no local Prometheus or equivalent queryable metrics endpoint listening by default
3. therefore the following stage labels cannot be queried directly yet:
- `multipart_ingress_prepare`
- `multipart_set_disk_writer_setup`
- `multipart_set_disk_encode`
- `multipart_complete_tail`
The goal of this guide is to close that gap.
## 2. Recommended approach
Reuse the repository's existing observability stack:
1. `.docker/observability/docker-compose.yml`
Why this is the preferred path:
1. it is already maintained in-repo
2. it includes OTEL Collector, Prometheus, and Grafana
3. it can receive telemetry from RustFS through `RUSTFS_OBS_ENDPOINT`
## 3. Expected data flow
After startup, the intended flow is:
1. RustFS
- `RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318`
2. OTEL Collector
- receives OTLP/HTTP telemetry
3. Prometheus
- scrapes collector-exported metrics
4. Query surface
- `http://127.0.0.1:9090`
## 4. Startup steps
### 4.1 Start the observability stack
From the repository root:
```bash
cd .docker/observability
docker compose up -d
```
### 4.2 Wait for core services
Recommended checks:
```bash
curl -fsS http://127.0.0.1:9090/-/ready
curl -fsS http://127.0.0.1:3000/api/health
```
If you need container status:
```bash
docker compose ps
```
### 4.3 Point RustFS to the OTEL Collector
For local single-node multi-disk validation:
```bash
export RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
```
If you use the repository-local restart helper:
```bash
bash scripts/restart_local_single_node_multidisk_rustfs.sh
```
Make sure the final runtime environment really contains:
```bash
RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
```
## 5. Minimal query validation
### 5.1 Confirm Prometheus can see RustFS metrics
```bash
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=rustfs_s3_put_object_total'
```
### 5.2 Confirm stage labels exist
```bash
curl -fsS 'http://127.0.0.1:9090/api/v1/label/stage/values'
```
If the pipeline is working, the result should include:
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
### 5.3 Direct P95 query
```bash
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=histogram_quantile(0.95,sum by(stage,le)(rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~"multipart_.*"}[5m])))'
```
## 6. Recommended silent validation order
Once the backend is queryable, use this order for the third multipart validation batch:
1. start the observability stack
2. restart RustFS and confirm `RUSTFS_OBS_ENDPOINT` is active
3. run one multipart baseline:
- `1g-64m-pc4`
- `2g-128m-pc4`
4. ignore streaming benchmark logs and only keep:
- `summary.csv`
- Prometheus query outputs
5. summarize:
- throughput / reqps / average latency
- P95 / P99 for the four multipart stages
## 7. Recommended query set
### 7.1 Multipart stage P95
```promql
histogram_quantile(
0.95,
sum by (stage, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~"multipart_.*"}[5m])
)
)
```
### 7.2 Multipart stage P99
```promql
histogram_quantile(
0.99,
sum by (stage, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~"multipart_.*"}[5m])
)
)
```
### 7.3 Complete tail focus
```promql
histogram_quantile(
0.95,
sum by (instance, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage="multipart_complete_tail"}[5m])
)
)
```
### 7.4 Encode focus
```promql
histogram_quantile(
0.95,
sum by (instance, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage="multipart_set_disk_encode"}[5m])
)
)
```
## 8. Suggested result layout
Recommended layout:
```text
target/bench/
issue712-multipart-server-path-focus/
summary.csv
metrics-query.txt
promql/
multipart-stage-p95.txt
multipart-stage-p99.txt
multipart-complete-tail-p95.txt
multipart-encode-p95.txt
```
## 9. Troubleshooting
### 9.1 Prometheus does not start
Check:
```bash
cd .docker/observability
docker compose logs prometheus
```
### 9.2 RustFS does not export telemetry
Check:
1. `RUSTFS_OBS_ENDPOINT` really points to `http://host.docker.internal:4318`
2. the OTEL collector container is running
3. RustFS was restarted after the environment variable changed
### 9.3 Stage labels are missing
Check:
1. a multipart PUT workload really ran
2. `put_stage_metrics_enabled()` was enabled at runtime
3. the query window is not too short
## 10. Recommendation
When `#712` third-batch validation resumes, do not run the benchmark first and hunt for metrics later.
Use this order instead:
1. bring up a queryable backend
2. run the multipart baseline
3. read only:
- `summary.csv`
- Prometheus stage query outputs
@@ -0,0 +1,232 @@
# Issue #712 本地可查询 metrics backend 启动手册
## 1. 目的
本文用于打通 `#712` 第三批验证所需的本地可查询 metrics backend。
当前问题不是 multipart stage 指标没有打点,而是本地环境里没有可查询后端:
1. `rustfs/admin/v3/metrics` 可用,但返回的是管理侧 JSON 快照,不包含 `metrics` crate 的 histogram 指标
2. 本地默认没有 Prometheus / 可查询 metrics endpoint 在监听
3. 因此无法直接查询:
- `multipart_ingress_prepare`
- `multipart_set_disk_writer_setup`
- `multipart_set_disk_encode`
- `multipart_complete_tail`
本文的目标就是把这条链打通。
## 2. 推荐方案
推荐直接复用仓库内已有的 observability stack
1. `.docker/observability/docker-compose.yml`
这套 stack 的优点:
1. 已经是仓库现成维护的方案
2. 包含 OTEL Collector、Prometheus、Grafana
3. 可以直接承接 RustFS 的 `RUSTFS_OBS_ENDPOINT`
## 3. 核心链路
启动后,链路应该是:
1. RustFS
- `RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318`
2. OTEL Collector
- 接收 OTLP/HTTP
3. Prometheus
- 抓取 collector 暴露的 metrics
4. 查询
- `http://127.0.0.1:9090`
## 4. 启动步骤
### 4.1 启动 observability stack
在仓库根目录执行:
```bash
cd .docker/observability
docker compose up -d
```
### 4.2 等待组件就绪
建议检查:
```bash
curl -fsS http://127.0.0.1:9090/-/ready
curl -fsS http://127.0.0.1:3000/api/health
```
若需要查看容器:
```bash
docker compose ps
```
### 4.3 启动 RustFS 时显式指向 OTEL Collector
本地单机多盘场景建议:
```bash
export RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
```
如果使用我们现成的本地重启脚本:
```bash
bash scripts/restart_local_single_node_multidisk_rustfs.sh
```
请确保脚本最终生效的环境里包含:
```bash
RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
```
## 5. 最小查询验证
### 5.1 确认 Prometheus 能查询到 RustFS 指标
```bash
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=rustfs_s3_put_object_total'
```
### 5.2 查询 stage 指标是否存在
```bash
curl -fsS 'http://127.0.0.1:9090/api/v1/label/stage/values'
```
如果链路打通,返回结果中应能看到:
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
### 5.3 直接查询 P95
```bash
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=histogram_quantile(0.95,sum by(stage,le)(rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~"multipart_.*"}[5m])))'
```
## 6. 建议的静默验证顺序
打通 backend 后,建议按如下顺序做第三批:
1. 启动 observability stack
2. 重启 RustFS 并确认 `RUSTFS_OBS_ENDPOINT` 生效
3. 先跑一轮 multipart baseline
- `1g-64m-pc4`
- `2g-128m-pc4`
4. 不看过程日志,只保留:
- `summary.csv`
- Prometheus 查询结果
5. 最后整理:
- throughput / reqps / avg latency
- 4 个 multipart stage 的 P95 / P99
## 7. 推荐查询集合
### 7.1 multipart 阶段 P95
```promql
histogram_quantile(
0.95,
sum by (stage, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~"multipart_.*"}[5m])
)
)
```
### 7.2 multipart 阶段 P99
```promql
histogram_quantile(
0.99,
sum by (stage, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~"multipart_.*"}[5m])
)
)
```
### 7.3 complete tail 单独看
```promql
histogram_quantile(
0.95,
sum by (instance, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage="multipart_complete_tail"}[5m])
)
)
```
### 7.4 encode 单独看
```promql
histogram_quantile(
0.95,
sum by (instance, le) (
rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage="multipart_set_disk_encode"}[5m])
)
)
```
## 8. 结果目录建议
推荐目录:
```text
target/bench/
issue712-multipart-server-path-focus/
summary.csv
metrics-query.txt
promql/
multipart-stage-p95.txt
multipart-stage-p99.txt
multipart-complete-tail-p95.txt
multipart-encode-p95.txt
```
## 9. 失败排查
### 9.1 Prometheus 起不来
检查:
```bash
cd .docker/observability
docker compose logs prometheus
```
### 9.2 RustFS 没有上报
检查:
1. `RUSTFS_OBS_ENDPOINT` 是否真的是 `http://host.docker.internal:4318`
2. OTEL collector 是否运行
3. RustFS 重启后是否带上了新环境变量
### 9.3 查不到 stage label
检查:
1. 是否真的跑过 multipart PUT 请求
2. `put_stage_metrics_enabled()` 是否在运行期被开启
3. 查询窗口是否太短
## 10. 建议
下次推进 `#712` 第三批时,不要先跑 benchmark,再临时找 metrics。
正确顺序应是:
1. 先按本文把可查询 backend 起好
2. 再跑 baseline
3. 最后只读:
- `summary.csv`
- Prometheus stage 查询结果
@@ -0,0 +1,263 @@
# Issue #712 multipart PUT 分阶段指标 Dashboard / PromQL 指南
## 1. 目的
本文给 `#712` 的第一批 server-path 观测增强配套一份可执行的 Dashboard / PromQL 指南。
本批次新增的 multipart 阶段指标依然复用现有指标名:
1. `rustfs_s3_put_object_stage_duration_ms`
但新增了四个 stage label
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
## 2. 使用前提
这些阶段指标严格受全局开关控制:
1. `rustfs_io_metrics::put_stage_metrics_enabled() == true`
如果该开关没有开启:
1. 不会上报这些阶段指标
2. 也不会额外做阶段计时
## 3. 推荐直接复用现有 Grafana Row
当前 Dashboard 中已经有:
1. `Large PUT Stage Breakdown`
这意味着:
1. 不需要重新设计一套全新 row
2. 只需要在现有 row / stage 变量里选新的 multipart stage label 即可
## 4. 推荐 PromQL
### 4.1 multipart 阶段 P95
```promql
histogram_quantile(
0.95,
sum by (stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage=~"multipart_.*"
}[$__rate_interval]
)
)
)
```
### 4.2 multipart 阶段 P99
```promql
histogram_quantile(
0.99,
sum by (stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage=~"multipart_.*"
}[$__rate_interval]
)
)
)
```
### 4.3 单实例 multipart 阶段 P95
```promql
histogram_quantile(
0.95,
sum by (instance, stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
instance=~"$instance",
stage=~"multipart_.*"
}[$__rate_interval]
)
)
)
```
### 4.4 multipart 与 ordinary PUT encode 对比
```promql
histogram_quantile(
0.95,
sum by (stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage=~"set_disk_encode|multipart_set_disk_encode"
}[$__rate_interval]
)
)
)
```
### 4.5 multipart complete tail 重点盯盘
```promql
histogram_quantile(
0.95,
sum by (instance, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage="multipart_complete_tail",
instance=~"$instance"
}[$__rate_interval]
)
)
)
```
### 4.6 multipart path 命中计数
```promql
sum by (path) (
rustfs_s3_put_object_path_total{
path=~"multipart_.*"
}
)
```
用于回答:
1. 当前 run 是否真的命中了 `multipart_write_pipeline_batched_large`
2. batched gate 是否只是“代码存在”,还是“运行时实际生效”
### 4.7 erasure encode 内部阶段均值
```promql
sum by (stage) (
increase(
rustfs_internal_stage_duration_ms_sum{
stage=~"erasure_encode.*"
}[$__rate_interval]
)
)
/
sum by (stage) (
increase(
rustfs_internal_stage_duration_ms_count{
stage=~"erasure_encode.*"
}[$__rate_interval]
)
)
```
用于回答:
1. `multipart_set_disk_encode` 内部到底更偏 CPU encode,还是更偏 writer write
2. producer / consumer 之间是 encoder 在等 writer,还是 writer 在等 encoder
### 4.8 erasure encode 当前累计 counters
当窗口查询容易受到 scrape 周期影响时,可以直接看当前累计值:
```promql
rustfs_internal_stage_duration_ms_count{
stage=~"erasure_encode.*"
}
```
```promql
rustfs_internal_stage_duration_ms_sum{
stage=~"erasure_encode.*"
}
```
这在 focused 单 profile 验证里很有用,尤其适合“重启实例后只跑一轮”的场景。
## 5. 推荐看板顺序
当你在看 `>1GiB multipart PUT` 时,建议按下面顺序看:
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
5. `multipart_write_pipeline` vs `multipart_write_pipeline_batched_large`
6. `erasure_encode_*` / `erasure_encode_batched_*`
解释顺序:
1. 如果 ingress 先高,先看 part ingress buffer / request-body handling
2. 如果 writer setup 高,先看 bitrot writer / disk availability / shard_file_size path
3. 如果 encode 高,先看 multipart 是否需要独立 encode strategy
4. 如果 complete tail 高,优先看 `complete_multipart_upload()` 的 metadata / checksum / rename tail
5. 如果 batched 预期已打开,但 path 仍然只有 `multipart_write_pipeline`,优先检查 size gate 是否真正被命中
6. 如果 `erasure_encode_batched_send_wait` 很低、但 `erasure_encode_batched_recv_wait` 明显更高,优先怀疑当前 batch barrier 让 writer 侧在等下一批 encode 完成
## 6. 推荐结合看的辅助指标
建议和上面四个阶段一起看:
1. `rustfs_io_put_object_concurrent_requests`
2. `rustfs_ec_encode_inflight_bytes_current`
3. host CPU
4. per-instance disk write throughput
5. readiness / write quorum 异常计数
## 7. 典型解释模板
### 7.1 ingress 高
可能原因:
1. part body stream buffering 不合适
2. `part.size` 与 ingress buffer 不匹配
### 7.2 writer setup 高
可能原因:
1. bitrot writer 构建成本偏高
2. online disk / writer init 慢
3. shard_file_size 相关路径有额外成本
### 7.3 encode 高
可能原因:
1. multipart part 仍然借用了 ordinary PUT encode 行为
2. `part.size` 太大,单 part encode CPU 时间过长
3. batching / inflight 参数不合适
### 7.4 complete tail 高
可能原因:
1. complete 阶段 part metadata 处理放大
2. checksum combine 成本高
3. rename / cleanup / commit tail 成本高
## 8. 建议的截图 / 归档内容
每次 `>1GiB multipart PUT` 复测,建议固定归档:
1. multipart stage P95 截图
2. multipart stage P99 截图
3. `multipart_complete_tail` 单实例截图
4. CPU / disk write 辅助图
## 9. 当前阶段建议
下一次进入 `#712` 继续推进时:
1. 先开 `put_stage_metrics_enabled`
2. 先跑推荐 baseline
- `1GiB -> 64MiB / pc4`
- `2GiB -> 128MiB / pc4`
3. 先看 `multipart_complete_tail` 是否明显高于其他阶段
4. 再决定是先改 ingress / encode / writer setup / complete tail
@@ -0,0 +1,201 @@
# Issue #712 `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS` 候选值低噪声复测矩阵
## 1. 目标
本文用于下一轮更小面、更低噪声的复测。
本轮只回答一个问题:
1. `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2` 是否比 `4` 更稳地改善 `2g-128m-pc4` 的 multipart batched 路径
当前不回答:
1. 是否直接改代码默认值
2. 是否扩展到 ordinary PUT
3. 是否继续引入新的 encode 调度语义
## 2. 固定范围
只保留一个 profile
1. `2g-128m-pc4`
只保留两个候选值:
1. `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=4`
2. `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2`
不再混入:
1. `1g-64m-pc4`
2. `2g-256m-pc4`
3. 其他 batching / inflight 配置
## 3. 低噪声原则
本轮强制执行以下原则:
1. 每一轮都重启 RustFS 进程,避免 counters/sums 混轮
2. 每一轮之间固定冷却 `30s`
3. 每一轮都固定只跑 `2m`
4. 每一轮都只抓:
- `summary.csv`
- `path.json`
- `internal_count.json`
- `internal_sum.json`
5. 不看过程日志,不在中间临时扩项
## 4. 推荐矩阵
推荐使用交叉顺序,避免单边连续运行:
1. `b4-r1`
2. `b2-r1`
3. `b2-r2`
4. `b4-r2`
5. `b4-r3`
6. `b2-r3`
这是一个偏保守的 `ABBAAB` 顺序。
原因:
1. 不让 `2` 总是出现在后面
2. 不让 `4` 总是出现在后面
3. 能更快看出“只是后跑更快”还是“候选值真的更优”
## 5. 目录约定
建议统一使用:
```text
target/bench/issue712-batchblocks-candidate-runs/
b4-r1/
b2-r1/
b2-r2/
b4-r2/
b4-r3/
b2-r3/
```
每轮目录固定包含:
1. `summary.csv`
2. `path.json`
3. `internal_count.json`
4. `internal_sum.json`
## 6. 单轮执行模板
### 6.1 `batch_blocks=4`
启动:
```bash
env \
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
RUSTFS_ADDRESS=127.0.0.1:9000 \
RUSTFS_ACCESS_KEY=rustfsadmin \
RUSTFS_SECRET_KEY=rustfsadmin \
RUSTFS_RPC_SECRET=rustfs-rpc-secret \
RUSTFS_REGION=us-east-1 \
RUSTFS_CONSOLE_ENABLE=false \
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 \
RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=4 \
target/debug/rustfs server \
/private/tmp/issue708-single-node-multidisk/d1 \
/private/tmp/issue708-single-node-multidisk/d2 \
/private/tmp/issue708-single-node-multidisk/d3 \
/private/tmp/issue708-single-node-multidisk/d4
```
运行:
```bash
bash scripts/run_gt1g_multipart_put_server_path_focus.sh \
--host 127.0.0.1:9000 \
--access-key rustfsadmin \
--secret-key rustfsadmin \
--profiles 2g-128m-pc4 \
--duration 2m \
--out-dir target/bench/issue712-batchblocks-candidate-runs/b4-r1
```
采集:
```bash
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=rustfs_s3_put_object_path_total{path=~"multipart_.*"}' \
> target/bench/issue712-batchblocks-candidate-runs/b4-r1/path.json
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=rustfs_internal_stage_duration_ms_count{stage=~"erasure_encode.*"}' \
> target/bench/issue712-batchblocks-candidate-runs/b4-r1/internal_count.json
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=rustfs_internal_stage_duration_ms_sum{stage=~"erasure_encode.*"}' \
> target/bench/issue712-batchblocks-candidate-runs/b4-r1/internal_sum.json
```
### 6.2 `batch_blocks=2`
只改一个变量:
```bash
RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2
```
其余命令完全保持一致。
## 7. 每轮必须核对的点
### 7.1 path 命中
必须确认:
1. `multipart_write_pipeline_batched_large` 已命中
如果没有命中:
1. 该轮结果无效
2. 不要纳入比较
### 7.2 internal stages
至少比较以下 3 组:
1. `erasure_encode_batched_recv_wait`
2. `erasure_encode_batched_write`
3. `erasure_encode_cpu`
## 8. 推荐的人工比较方式
对每一轮,记录:
1. throughput
2. avg latency
3. `recv_wait_avg = internal_sum / internal_count`
4. `write_avg = internal_sum / internal_count`
5. `cpu_avg = internal_sum / internal_count`
重点不是单看 throughput,而是看:
1. `2` 是否更稳定地降低 `recv_wait_avg`
2. `2` 是否没有把 `write_avg``cpu_avg` 反向放大
## 9. 建议的判定门槛
只有同时满足下面两条,才建议继续往“默认值候选”方向推进:
1. 在至少 `3` 轮对照中,`batch_blocks=2` 的 throughput / latency 有 `>= 2` 轮优于 `4`
2. `recv_wait_avg` 的下降方向在大多数轮次都成立
如果只满足其中一条:
1. 保持它为候选 env 配置
2. 暂不改代码默认值
## 10. 当前建议
当前阶段最稳妥的动作顺序是:
1. 先按这份矩阵补足 `6`
2. 再做一次汇总表
3. 最后才决定是否让 multipart batched path 默认使用 `2`
@@ -0,0 +1,72 @@
# Issue #712 更深一层 zero-copy 下一阶段说明
## 1. 当前结论
当前分支已经具备:
1. `rename_data``msgpack named-map + JSON fallback`
2. ordinary PUT 的 `zero_copy_eager` 实验路径
其中 `zero_copy_eager` 已经是实际命中的业务路径,但它当前仍然不是端到端严格意义上的 zero-copy write path。
## 2. 当前 copy 还存在的层
当前 plain PUT 即使命中了 `zero_copy_eager`,仍然会在后续链路里发生复制,主要位置在:
1. `HashReader::from_stream(...)`
2. `Erasure::encode(...)` 的 block ingest
从当前代码 review 看,优先级更高的是:
1. `Erasure::encode`
而不是:
1. `HashReader`
## 3. 为什么先看 `Erasure::encode`
原因:
1. `HashReader` 主要负责包装 `AsyncRead` 与 checksum 语义,不是最直接的热点
2. `Erasure::encode` 目前仍然是每个 block 读入 `Vec<u8>` 后再进入编码
3. 这更像是 plain PUT 的下一层实际 copy 热点
## 4. 这轮已经尝试过但不保留的方向
本轮已经试过一个很小的局部改动:
1. 对 full block 优先走 `encode_data_owned(...)`
结果:
1. 对某些 ordinary PUT 面有正向迹象
2. 但不同 size 下收益不稳定
3. 因此当前不建议直接基于这个 patch 继续往前推
## 5. 下一阶段更稳妥的技术方向
如果要继续做更深一层 zero-copy,建议优先考虑:
1.`Erasure::encode` 设计更稳定的 block buffer 生命周期
2. 让 full block ingest 更接近 `Bytes` / owned buffer 复用
3. 避免在当前函数内继续做更多“局部替换一个调用”的小补丁
换句话说,下一阶段更适合做:
1. block buffer 生命周期设计
2. owned block ring / reusable block pool
3. encode/write 之间更明确的 buffer ownership
而不是先做:
1. `HashReader` 级别的大改
2. 更多无设计托底的局部 `Vec<u8>` 调整
## 6. 当前建议
当前最稳妥的推进顺序:
1. 先把当前 `zero_copy_eager` 路径继续作为实验性 ordinary PUT 路径保留
2. 如果要继续做 deeper zero-copy,单独开一个新阶段
3. 该阶段优先聚焦 `Erasure::encode` ingest / buffer lifecycle,而不是先改 `HashReader`
@@ -0,0 +1,284 @@
# Issue #712 encode / write overlap 观测小结
## 1. 目的
本文记录 `#712` 新一轮更细主线的第一步结果:
1. 不先冒进改 `encode.rs` 调度语义
2. 先把 `multipart_set_disk_encode` 内部拆成更细阶段
3. 用 focused benchmark 判断当前 overlap 更像卡在 encode 侧、write 侧,还是 batch barrier
## 2. 新增内部阶段
本轮在 `crates/ecstore/src/erasure_coding/encode.rs` 中补充了内部阶段观测,使用指标:
1. `rustfs_internal_stage_duration_ms`
新增阶段:
1. `erasure_encode_cpu`
2. `erasure_encode_send_wait`
3. `erasure_encode_recv_wait`
4. `erasure_encode_write`
5. `erasure_encode_shutdown`
6. `erasure_encode_batched_send_wait`
7. `erasure_encode_batched_recv_wait`
8. `erasure_encode_batched_write`
9. `erasure_encode_batched_shutdown`
## 3. focused baseline
profile
1. `2g-128m-pc4`
默认 batched 配置(`RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=4`)下,本轮 observed run 结果为:
1. Throughput: `351.96 MiB/s`
2. Avg latency: `5842.1ms`
3. path: `multipart_write_pipeline_batched_large`
从 raw sum / count 推出的内部均值近似为:
1. `erasure_encode_cpu`: `~2.95ms`
2. `erasure_encode_batched_send_wait`: `~0.01ms`
3. `erasure_encode_batched_recv_wait`: `~105.3ms`
4. `erasure_encode_batched_write`: `~75.4ms`
5. `erasure_encode_batched_shutdown`: `~0.90ms`
## 4. 第一轮判断
这组数据说明:
1. `send_wait` 几乎可以忽略,说明 encode producer 基本没有被 queue backpressure 卡住
2. `recv_wait` 明显高于 `write`,说明 consumer 侧更常见的是在等下一批 encode 结果,而不是 writer 太慢导致队列打满
3. `cpu` 本身并不大,真正放大的是 batched producer/consumer 之间的批次屏障
换句话说,这一轮更像是:
1. writer 在等 encoder / 等 batch 集齐
2. 而不是 encoder 在等 writer
## 5. 配置性验证
为了验证 batch barrier 假设,本轮只改一个变量:
1. `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2`
同样只跑:
1. `2g-128m-pc4`
结果:
1. Throughput: `368.57 MiB/s`
2. Avg latency: `5537.3ms`
3. path: `multipart_write_pipeline_batched_large`
raw sum / count 推导的内部均值近似为:
1. `erasure_encode_cpu`: `~2.94ms`
2. `erasure_encode_batched_send_wait`: `~0.01ms`
3. `erasure_encode_batched_recv_wait`: `~48.9ms`
4. `erasure_encode_batched_write`: `~35.9ms`
5. `erasure_encode_batched_shutdown`: `~0.36ms`
## 6. 当前结论
这轮可以先收敛出一个相对明确的方向:
1. 当前 batched 路径的第一优先问题不像是 encode CPU 绝对太重
2. 更像是 batch size 偏大,导致 writer 侧等待下一批 encode 结果的时间被放大
3.`RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS``4` 降到 `2` 后,结果明显好于默认值
## 7. 当前建议
下一步如果继续沿着 overlap 这条线推进,建议优先级如下:
1. 先把 `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2` 作为 batched 路径的候选值继续复测
2. 再决定是否要把 multipart batched path 的 batch size 做成与 ordinary PUT 分离
3. 在没有更多证据前,不要继续尝试更激进的 batch 内单次 blocking encode 调度改法
## 8. 第二轮更稳妥复测
为了减少热态偏差,本轮又按 `4 -> 2 -> 4 -> 2` 做了四轮受控复测。
profile 固定:
1. `2g-128m-pc4`
结果:
1. `b4-r1`: `317.20 MiB/s`, `6515.2ms`
2. `b2-r1`: `300.38 MiB/s`, `6842.8ms`
3. `b4-r2`: `334.58 MiB/s`, `5966.5ms`
4. `b2-r2`: `358.77 MiB/s`, `5748.2ms`
从这组数据看:
1. `batch_blocks=2` 不是每一轮都赢
2.`b2-r2` 明显优于同组前后的 `b4-r2`
3. 结果仍然存在不小波动,因此还不足以直接改代码默认值
## 9. 第二轮内部阶段对比
`b4-r2``b2-r2` 的 raw sum / count 做近似均值后,可以看到:
### `b4-r2`
1. `erasure_encode_batched_recv_wait`: `~107.4ms`
2. `erasure_encode_batched_write`: `~78.2ms`
3. `erasure_encode_cpu`: `~3.19ms`
### `b2-r2`
1. `erasure_encode_batched_recv_wait`: `~50.9ms`
2. `erasure_encode_batched_write`: `~37.7ms`
3. `erasure_encode_cpu`: `~3.06ms`
这说明:
1. `batch_blocks=2` 的主要收益方向仍然是降低 batched consumer 侧等待时间
2. `cpu` 本身没有发生决定性变化
3. 当前更像是在改善 batch barrier,而不是改变编码计算本体
## 10. 当前收敛结论
到这一轮为止,更稳妥的结论是:
1. `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2` 仍然值得保留为候选配置
2. 它对 `erasure_encode_batched_recv_wait` 的改善方向是清晰的
3. 但吞吐/延迟收益还不够稳定,当前不建议直接改默认值
4. 下一步更适合继续以环境变量方式复测,而不是马上把默认值写死到代码里
## 11. 第三轮补齐后的小结
按低噪声矩阵继续补到 `6` 轮之后,结果如下:
1. `b4-r1`: `317.20 MiB/s`, `6515.2ms`
2. `b2-r1`: `300.38 MiB/s`, `6842.8ms`
3. `b4-r2`: `334.58 MiB/s`, `5966.5ms`
4. `b2-r2`: `358.77 MiB/s`, `5748.2ms`
5. `b4-r3`: `323.35 MiB/s`, `6412.4ms`
6. `b2-r3`: `350.87 MiB/s`, `5827.6ms`
按组汇总:
### `batch_blocks=4`
1. Avg throughput: `325.04 MiB/s`
2. Median throughput: `323.35 MiB/s`
3. Avg latency: `6298.0ms`
4. Median latency: `6412.4ms`
### `batch_blocks=2`
1. Avg throughput: `336.67 MiB/s`
2. Median throughput: `350.87 MiB/s`
3. Avg latency: `6139.5ms`
4. Median latency: `5827.6ms`
这说明:
1. `batch_blocks=2` 经过 `6` 轮汇总后,组均值已经优于 `4`
2. 但单轮结果仍然存在明显波动,因此还不能把它视为“完全稳定结论”
## 12. 第三轮内部阶段补充
`b4-r3``b2-r3` 的 raw sum / count 看:
### `b4-r3`
1. `erasure_encode_batched_recv_wait`: `~115.55ms`
2. `erasure_encode_batched_write`: `~80.15ms`
3. `erasure_encode_cpu`: `~3.55ms`
### `b2-r3`
1. `erasure_encode_batched_recv_wait`: `~52.46ms`
2. `erasure_encode_batched_write`: `~37.99ms`
3. `erasure_encode_cpu`: `~3.12ms`
这与前一组 `b4-r2` / `b2-r2` 的方向一致,说明:
1. `batch_blocks=2` 主要还是在改善 batch barrier
2. 下降最明显的仍然是 consumer 侧等待时间
3. `cpu` 本体没有决定性变化
## 13. 当前更新后的建议
到这一步,建议可以进一步收敛为:
1. `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2` 仍然保留为 multipart batched 路径的强候选配置
2. 它已经具备“方向明确、组均值更优”的证据
3. 但由于单轮波动仍在,当前更合适的动作仍然是继续以 env 方式复测,而不是直接改代码默认值
## 14. 第四轮补齐后的更新判断
继续按同一矩阵补到 `8` 轮之后,新增结果为:
1. `b4-r4`: `296.34 MiB/s`, `6977.8ms`
2. `b2-r4`: `248.68 MiB/s`, `8310.8ms`
这样 `8` 轮完整结果为:
1. `b4-r1`: `317.20 MiB/s`, `6515.2ms`
2. `b2-r1`: `300.38 MiB/s`, `6842.8ms`
3. `b4-r2`: `334.58 MiB/s`, `5966.5ms`
4. `b2-r2`: `358.77 MiB/s`, `5748.2ms`
5. `b4-r3`: `323.35 MiB/s`, `6412.4ms`
6. `b2-r3`: `350.87 MiB/s`, `5827.6ms`
7. `b4-r4`: `296.34 MiB/s`, `6977.8ms`
8. `b2-r4`: `248.68 MiB/s`, `8310.8ms`
按组重新汇总:
### `batch_blocks=4`
1. Avg throughput: `317.87 MiB/s`
2. Median throughput: `320.27 MiB/s`
3. Avg latency: `6468.0ms`
4. Median latency: `6463.8ms`
### `batch_blocks=2`
1. Avg throughput: `314.68 MiB/s`
2. Median throughput: `325.62 MiB/s`
3. Avg latency: `6682.4ms`
4. Median latency: `6335.2ms`
## 15. 第四轮后的结论修正
补齐到 `8` 轮之后,需要把结论进一步收紧:
1. `batch_blocks=2` 仍然能稳定改善 `erasure_encode_batched_recv_wait`
2. 但吞吐/延迟层面的总收益并没有收敛成稳定优势
3. `b2-r4` 明显把组均值重新拉回,说明它目前还只是“有潜力的候选配置”,不是“已经证实优于 4 的配置”
`b4-r4``b2-r4` 的 raw sum / count 看:
### `b4-r4`
1. `erasure_encode_batched_recv_wait`: `~131.18ms`
2. `erasure_encode_batched_write`: `~82.04ms`
3. `erasure_encode_cpu`: `~3.95ms`
### `b2-r4`
1. `erasure_encode_batched_recv_wait`: `~81.49ms`
2. `erasure_encode_batched_write`: `~45.22ms`
3. `erasure_encode_cpu`: `~5.38ms`
这说明:
1. `batch_blocks=2` 对 wait/write 的改善方向依旧成立
2. 但这次同时伴随更差的整体 throughput/latency
3. 当前还不能把局部内部阶段改善直接视为端到端收益
## 16. 当前最稳妥的建议
到这一步,最稳妥的建议是:
1. `RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS=2` 继续保留为 env-only 候选配置
2. 当前不要改代码默认值
3. 后续如果继续验证,应优先排查为什么 `b2-r4` 会出现这种明显反向波动,而不是继续机械追加更多轮次
@@ -0,0 +1,213 @@
# Issue #712 multipart PUT server-path 静默验证 Runbook
## 1. 目的
本文给 `#712` 第二批工作提供一个只关注 multipart server path 的静默验证 runbook。
目标:
1. 不再扩散客户端参数矩阵
2. 固定当前推荐 baseline
3. 把注意力集中到 server-path 观测增强后的阶段结果
## 2. 固定 baseline
当前固定 baseline
1. `1GiB -> 64MiB part / pc4`
2. `2GiB -> 128MiB part / pc4`
可选补充:
1. `2GiB -> 256MiB part / pc4`
但默认不作为首选 baseline。
## 3. 推荐脚本
直接使用:
1. `scripts/run_gt1g_multipart_put_server_path_focus.sh`
该脚本默认只跑:
1. `1g-64m-pc4`
2. `2g-128m-pc4`
可选补充:
1. `2g-256m-pc4`
## 4. 静默执行命令
### 4.1 默认两组
```bash
bash scripts/run_gt1g_multipart_put_server_path_focus.sh \
--host 127.0.0.1:9000 \
--access-key rustfsadmin \
--secret-key rustfsadmin \
--bucket-prefix issue712-multipart-focus \
--duration 10m \
--out-dir target/bench/issue712-multipart-server-path-focus
```
### 4.2 加上 `2g-256m-pc4`
```bash
bash scripts/run_gt1g_multipart_put_server_path_focus.sh \
--host 127.0.0.1:9000 \
--access-key rustfsadmin \
--secret-key rustfsadmin \
--bucket-prefix issue712-multipart-focus \
--duration 10m \
--profiles 1g-64m-pc4,2g-128m-pc4,2g-256m-pc4 \
--out-dir target/bench/issue712-multipart-server-path-focus-wide
```
## 5. 强制要求
这轮 runbook 的要求是:
1. 静默跑
2. 只读 `summary.csv`
3. 如需解释异常,再去看 dashboard / 日志
## 6. 结果目录
建议统一:
```text
target/bench/
issue712-multipart-server-path-focus/
run_manifest.txt
commands.txt
summary.csv
logs/
benchdata/
```
## 7. 需要记录的指标
最终结果表之外,强制记录以下阶段:
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
同时建议固定记录 multipart path 计数:
1. `multipart_write_pipeline`
2. `multipart_write_pipeline_batched_large`
3. `multipart_write_single_block_non_inline`
## 8. 本轮的判断顺序
先看:
1. `summary.csv`
再看:
1. `multipart_complete_tail`
2. `multipart_set_disk_encode`
3. `multipart_set_disk_writer_setup`
4. `multipart_ingress_prepare`
## 9. 结果解释
### 9.1 `summary.csv` 先分出好坏组合
先回答:
1. `1GiB / 64MiB / pc4` 是否仍是最稳 baseline
2. `2GiB / 128MiB / pc4` 是否仍是最稳 baseline
### 9.2 再用 Dashboard 回答热点层
再回答:
1. `multipart_complete_tail` 是否最高
2. `multipart_set_disk_encode` 是否主导
3. `multipart_set_disk_writer_setup` 是否异常高
4. `multipart_ingress_prepare` 是否已经被 body buffering 放大
## 10. 下一步动作判定
### 如果 `multipart_set_disk_encode` 最高
下一步优先:
1. multipart part 专用 batching gate
2. multipart encode path 单独策略
### 如果 `multipart_complete_tail` 最高
下一步优先:
1. complete path metadata / checksum / rename tail 优化
### 如果 `multipart_set_disk_writer_setup` 最高
下一步优先:
1. writer init / bitrot writer path 优化
### 如果 `multipart_ingress_prepare` 最高
下一步优先:
1. part ingress buffer 分层
2. body read / HashReader 前的缓冲调整
## 11. `multipart_*` 指标为空时的排查顺序
如果本轮跑的是 multipart PUT,但 Prometheus 里查不到任何 `multipart_*` stage
1. 先不要直接判定“新打点无效”
2. 先查当前 `rustfs_s3_put_object_stage_duration_ms` 里到底有哪些 stage
3. 如果只看到了 ordinary PUT 的 `ingress_prepare` / `set_disk_writer_setup` / `set_disk_encode` / `set_disk_rename`,要优先怀疑当前 `127.0.0.1:9000` 上跑的不是预期的新二进制
推荐先查:
```promql
topk(
40,
count by (__name__, stage) (
{__name__=~"rustfs_s3_put_object_stage_duration_ms.*"}
)
)
```
如果结果里只有 ordinary stage,而没有:
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
则应优先检查:
1. 本轮 RustFS 进程是否确实来自当前 worktree 的 `target/debug/rustfs`
2. 重启脚本是否真的清掉了旧进程
3. `RUSTFS_OBS_ENDPOINT` 是否仍然指向当前可查询 backend
## 12. 已确认的假阴性根因样例
`2026-06-24` 的第三批继续验证中,出现过一次典型假阴性:
1. multipart benchmark 已经成功跑完
2. Prometheus 里却只有 ordinary PUT stage,没有任何 `multipart_*`
3. 根因并不是打点代码失效,而是 `127.0.0.1:9000` 上仍然挂着更早启动的旧 RustFS 进程
纠偏方式:
1. 用当前 worktree 的 `target/debug/rustfs` 前台直接拉起服务
2. 再跑最小化 focused smoke
3. 立刻查询 `multipart_*` stage
这次纠偏后的前台复测已经确认:
1. `multipart_*` 四个阶段可以正常上报
2. 当前热点仍然稳定落在 `multipart_set_disk_encode`
@@ -0,0 +1,109 @@
# Issue #712 multipart size-gated A/B 结果总结
## 1. 本轮目的
本轮不是继续扩面 benchmark,而是回答一个更窄的问题:
1. `set_disk.rs` 上的 multipart size-gated batching 是否真的在运行时生效
2. 如果生效,`2g-128m-pc4` 下是否优于纯 pipeline
## 2. 先修正的运行时问题
在继续验证中发现一个关键问题:
1. multipart 路径的 batching 分类使用了 `data.size()`
2. 但原逻辑是在 `data.stream` 被替换为空 reader 之后才读取这个 size
3. 这会导致 multipart size gate 在运行时无法按预期命中
本轮已在 multipart 路径上修正为:
1. 先捕获原始 `multipart_part_size`
2. 再执行 stream swap
3. 再基于原始 size 做 `classify_multipart_part_write_path(...)`
同时补充了 multipart path 计数标签:
1. `multipart_write_pipeline`
2. `multipart_write_pipeline_batched_large`
3. `multipart_write_single_block_non_inline`
## 3. focused 对照矩阵
本轮只保留受影响的 profile
1. `2g-128m-pc4`
并做两组对照:
1. pipeline 基线:
- `RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES=10737418240`
2. 默认门槛实验:
- 使用默认 `128MiB` 门槛
附加做了一组强制 batched 校验:
1. `RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES=1`
## 4. 结果
### 4.1 修正后的 pipeline 基线
结果目录:
1. `target/bench/issue712-multipart-server-path-fixed-pipeline/summary.csv`
结果:
1. Throughput: `364.83 MiB/s`
2. Avg latency: `5604.5ms`
3. Path counter: 仅出现 `multipart_write_pipeline`
4. `multipart_set_disk_encode` P95: `7348.88ms`
### 4.2 修正后的默认门槛实验
结果目录:
1. `target/bench/issue712-multipart-server-path-fixed-batched/summary.csv`
结果:
1. Throughput: `359.00 MiB/s`
2. Avg latency: `5734.9ms`
3. Path counter: 已出现 `multipart_write_pipeline_batched_large`
4. `multipart_set_disk_encode` P95: `7375ms`
说明:
1. batched 路径在修正后已经真正开始命中
2. 但当前结果并没有优于纯 pipeline
### 4.3 强制 batched 校验
结果目录:
1. `target/bench/issue712-multipart-server-path-forced-batched/summary.csv`
结果:
1. Throughput: `362.00 MiB/s`
2. Avg latency: `5664.3ms`
这组结果同样没有优于修正后的 pipeline 基线。
## 5. 本轮结论
本轮可以明确收敛出三点:
1. multipart size-gated batching 的运行时命中问题已经被定位并修正
2. batched path 修正后确实可以被 Prometheus path counter 观察到
3. 在当前单机多盘 `2g-128m-pc4` focused 验证下,默认 `128MiB` 门槛没有带来正收益,反而略逊于纯 pipeline
## 6. 当前建议
在当前证据下,不建议把 multipart batched path 作为默认推荐优化结论直接推进。
更稳妥的后续方向是:
1. 先保留这条路径为可控实验能力
2. 继续围绕 `multipart_set_disk_encode` 本身做更细粒度优化
3. 如果还要继续试 batching,应先解释为什么 `128MiB` part 在当前实现下没有优于 pipeline,而不是直接继续扩大默认启用范围
@@ -0,0 +1,201 @@
# Issue #712 multipart PUT 分阶段指标 Dashboard / PromQL 指南
## 1. 目的
本文给 `#712` 的第一批 server-path 观测增强配套一份可执行的 Dashboard / PromQL 指南。
本批次新增的 multipart 阶段指标依然复用现有指标名:
1. `rustfs_s3_put_object_stage_duration_ms`
但新增了四个 stage label
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
## 2. 使用前提
这些阶段指标严格受全局开关控制:
1. `rustfs_io_metrics::put_stage_metrics_enabled() == true`
如果该开关没有开启:
1. 不会上报这些阶段指标
2. 也不会额外做阶段计时
## 3. 推荐直接复用现有 Grafana Row
当前 Dashboard 中已经有:
1. `Large PUT Stage Breakdown`
这意味着:
1. 不需要重新设计一套全新 row
2. 只需要在现有 row / stage 变量里选新的 multipart stage label 即可
## 4. 推荐 PromQL
### 4.1 multipart 阶段 P95
```promql
histogram_quantile(
0.95,
sum by (stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage=~"multipart_.*"
}[$__rate_interval]
)
)
)
```
### 4.2 multipart 阶段 P99
```promql
histogram_quantile(
0.99,
sum by (stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage=~"multipart_.*"
}[$__rate_interval]
)
)
)
```
### 4.3 单实例 multipart 阶段 P95
```promql
histogram_quantile(
0.95,
sum by (instance, stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
instance=~"$instance",
stage=~"multipart_.*"
}[$__rate_interval]
)
)
)
```
### 4.4 multipart 与 ordinary PUT encode 对比
```promql
histogram_quantile(
0.95,
sum by (stage, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage=~"set_disk_encode|multipart_set_disk_encode"
}[$__rate_interval]
)
)
)
```
### 4.5 multipart complete tail 重点盯盘
```promql
histogram_quantile(
0.95,
sum by (instance, le) (
rate(
rustfs_s3_put_object_stage_duration_ms_bucket{
job=~"$job",
stage="multipart_complete_tail",
instance=~"$instance"
}[$__rate_interval]
)
)
)
```
## 5. 推荐看板顺序
当你在看 `>1GiB multipart PUT` 时,建议按下面顺序看:
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
解释顺序:
1. 如果 ingress 先高,先看 part ingress buffer / request-body handling
2. 如果 writer setup 高,先看 bitrot writer / disk availability / shard_file_size path
3. 如果 encode 高,先看 multipart 是否需要独立 encode strategy
4. 如果 complete tail 高,优先看 `complete_multipart_upload()` 的 metadata / checksum / rename tail
## 6. 推荐结合看的辅助指标
建议和上面四个阶段一起看:
1. `rustfs_io_put_object_concurrent_requests`
2. `rustfs_ec_encode_inflight_bytes_current`
3. host CPU
4. per-instance disk write throughput
5. readiness / write quorum 异常计数
## 7. 典型解释模板
### 7.1 ingress 高
可能原因:
1. part body stream buffering 不合适
2. `part.size` 与 ingress buffer 不匹配
### 7.2 writer setup 高
可能原因:
1. bitrot writer 构建成本偏高
2. online disk / writer init 慢
3. shard_file_size 相关路径有额外成本
### 7.3 encode 高
可能原因:
1. multipart part 仍然借用了 ordinary PUT encode 行为
2. `part.size` 太大,单 part encode CPU 时间过长
3. batching / inflight 参数不合适
### 7.4 complete tail 高
可能原因:
1. complete 阶段 part metadata 处理放大
2. checksum combine 成本高
3. rename / cleanup / commit tail 成本高
## 8. 建议的截图 / 归档内容
每次 `>1GiB multipart PUT` 复测,建议固定归档:
1. multipart stage P95 截图
2. multipart stage P99 截图
3. `multipart_complete_tail` 单实例截图
4. CPU / disk write 辅助图
## 9. 当前阶段建议
下一次进入 `#712` 继续推进时:
1. 先开 `put_stage_metrics_enabled`
2. 先跑推荐 baseline
- `1GiB -> 64MiB / pc4`
- `2GiB -> 128MiB / pc4`
3. 先看 `multipart_complete_tail` 是否明显高于其他阶段
4. 再决定是先改 ingress / encode / writer setup / complete tail
@@ -0,0 +1,107 @@
# Issue #712 第三批继续验证结果总结
## 1. 背景
`#712` 第三批的第一轮静默 baseline 已经确认:
1. `multipart_set_disk_encode` 是当前最主要热点
2. `multipart_complete_tail` 可见但不是第一瓶颈
在继续推进 multipart encode 优化线时,又出现了一次“指标空结果”的假阴性,需要单独记录,避免后续团队误判为打点无效。
## 2. 假阴性根因
现象:
1. multipart benchmark 成功完成
2. `summary.csv` 正常生成
3. Prometheus 中却只出现 ordinary PUT 的 stage
- `ingress_prepare`
- `set_disk_writer_setup`
- `set_disk_encode`
- `set_disk_rename`
4. 没有任何 `multipart_*` stage
根因:
1. 本轮 benchmark 实际命中了 `127.0.0.1:9000` 上残留的旧 RustFS 进程
2. 该旧进程不是当前 worktree 的新二进制
3. 因此即使 benchmark 是 multipart PUT,也不会产出本批新增的 multipart stage 标签
这次问题的本质不是“新打点代码无效”,而是“验证目标进程不对”。
## 3. 如何识别这类假阴性
推荐直接查询:
```promql
topk(
40,
count by (__name__, stage) (
{__name__=~"rustfs_s3_put_object_stage_duration_ms.*"}
)
)
```
如果你跑的是 multipart PUT,但结果里只有 ordinary PUT stage,而没有:
1. `multipart_ingress_prepare`
2. `multipart_set_disk_writer_setup`
3. `multipart_set_disk_encode`
4. `multipart_complete_tail`
则优先判断为“验证目标进程可能不对”,而不要先判断为“metrics 打点无效”。
## 4. 前台纠偏复测
为了排除旧进程干扰,本轮直接使用当前 worktree 的 `target/debug/rustfs` 前台拉起服务,再做最小化 focused smoke。
复测 profile
1. `2g-128m-pc4`
复测结果:
1. Throughput: `373.78 MiB/s`
2. Request rate: `2.92 obj/s`
3. Avg latency: `5471.4ms`
结果目录:
1. `target/bench/issue712-multipart-server-path-foreground-smoke/summary.csv`
## 5. 前台复测阶段指标
Prometheus 近窗口查询已经确认 multipart 四阶段都正常出现。
P95
1. `multipart_ingress_prepare`: `4.75ms`
2. `multipart_set_disk_writer_setup`: `4.86ms`
3. `multipart_set_disk_encode`: `7375ms`
4. `multipart_complete_tail`: `13ms`
同时还能看到 ordinary PUT 的阶段指标,但这不影响判断;关键是:
1. multipart stage 已经实际落盘到 TSDB
2. encode 依旧是最显著热点
## 6. 当前直接结论
这轮继续验证后的结论没有变化,反而更稳:
1. `multipart_set_disk_encode` 仍然是 `>1GiB multipart PUT` 当前最主要优化目标
2. `multipart_complete_tail` 仍然是次级问题,不应抢在 encode 之前
3. `multipart_ingress_prepare``multipart_set_disk_writer_setup` 暂时不是第一优先级
4. 后续优化应继续围绕 multipart encode path,而不是先转去 complete tail / ingress
## 7. 对后续验证的要求
后续再做 multipart server-path 对照验证时,建议强制执行:
1. 先确认服务进程确实来自当前 worktree 二进制
2. 先确认 `multipart_*` stage 能被 Prometheus 查询到
3. 再读取 `summary.csv`
4. 最后再判断 encode 优化是否真的有效
否则很容易再次出现“summary 正常,但阶段指标是旧进程数据”的假阴性。
@@ -0,0 +1,153 @@
# Issue #712 `zero_copy_eager` plain PUT 验证手册
## 1. 目的
本文用于记录当前 `zero_copy_eager` plain PUT 路径的实际使用方式、验证命令和当前边界。
这条路径当前的目标不是“端到端完全零拷贝”,而是先把 ordinary PUT 从“只有 zero-copy eligibility / metrics”推进到:
1. 真实存在的业务路径
2. 真实命中可观测的 `put_path`
3. 先减少请求体聚合阶段的额外复制
## 2. 当前启用条件
当前 `zero_copy_eager` 只在下面条件同时满足时才会命中:
1. 非加密
2. 非压缩
3. 非 extract 请求
4. 对象大小 `> 1MiB`
5. 对象大小 `<= 32MiB`
6. 请求长度已知
7. 不属于需要特殊处理的 aws-chunked 未知长度场景
## 3. 当前实现边界
这条路径已经是真实业务路径,但当前还不是端到端严格意义上的 zero-copy。
已经做到:
1. 请求体 chunk 以 `Bytes` 形式进入 `zero_copy_eager`
2. 不再先把整个对象拼成一个大 `Vec<u8>` 再进入后续写路径
3. 运行时会记录 `put_path=zero_copy_eager`
4. 会真实记录 `rustfs_zero_copy_write_total`
尚未做到:
1. `HashReader` 之后完全无复制
2. `Erasure::encode` 的 block ingest 完全无复制
3. shard write 全链路严格 zero-copy
因此当前最准确的描述是:
1. 这是 ordinary PUT 的真实 zero-copy eager ingress 路径
2. 不是完整的 end-to-end zero-copy write path
## 4. 推荐验证命令
### 4.1 启动本地单机多盘 RustFS
```bash
env \
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
RUSTFS_ADDRESS=127.0.0.1:9000 \
RUSTFS_ACCESS_KEY=rustfsadmin \
RUSTFS_SECRET_KEY=rustfsadmin \
RUSTFS_RPC_SECRET=rustfs-rpc-secret \
RUSTFS_REGION=us-east-1 \
RUSTFS_CONSOLE_ENABLE=false \
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 \
target/debug/rustfs server \
/private/tmp/issue708-single-node-multidisk/d1 \
/private/tmp/issue708-single-node-multidisk/d2 \
/private/tmp/issue708-single-node-multidisk/d3 \
/private/tmp/issue708-single-node-multidisk/d4
```
### 4.2 小面 ordinary PUT 验证
```bash
bash scripts/run_put_large_stage_breakdown.sh \
--endpoint http://127.0.0.1:9000 \
--access-key rustfsadmin \
--secret-key rustfsadmin \
--sizes 16MiB,32MiB \
--concurrencies 16 \
--duration 60s \
--rounds 1 \
--retry-per-round 1 \
--retry-sleep-secs 2 \
--cooldown-secs 15 \
--out-dir target/bench/issue712-zero-copy-eager-put-verify
```
## 5. 运行时必须核对的指标
### 5.1 PUT path 命中
必须确认:
```promql
rustfs_s3_put_object_path_total{
path=~"zero_copy_eager|small_eager|streaming|stream_compressed"
}
```
如果看不到 `zero_copy_eager`
1. 说明本轮没有真正命中这条路径
2. 不能据此评价它的收益
### 5.2 zero-copy write 指标
建议同时看:
```promql
rustfs_zero_copy_write_total
```
```promql
rustfs_zero_copy_write_size_bytes_sum
```
### 5.3 普通 PUT summary
最终固定读取:
1. `aggregate_median_summary.csv`
## 6. 当前已知结果
当前分支上已经验证过一次小面 ordinary PUT
1. `16MiB, c16`: `318.77 MiB/s`, `851.3ms`
2. `32MiB, c16`: `276.38 MiB/s`, `1846.5ms`
并确认运行时命中了:
1. `put_path=zero_copy_eager`
这说明:
1. 这条路径已经不只是 eligibility / metrics
2. 它已经是真实参与 ordinary PUT 的运行时路径
## 7. 当前最稳妥的使用建议
当前阶段建议把这条路径当作:
1. 实验性 ordinary PUT 优化路径
2. 需要继续压测验证的真实实现
当前不建议把它描述成:
1. 已完成的端到端 zero-copy write path
## 8. 下一步方向
如果要继续把收益往下穿透,下一阶段优先顺序建议是:
1. 先继续验证 `zero_copy_eager` 的普通 PUT 命中率和端到端收益
2. 再评估是否要继续下钻 `Erasure::encode` 的 block ingest
3. `HashReader` 层暂时不是第一优先级
+8
View File
@@ -800,6 +800,7 @@ impl DefaultMultipartUsecase {
.map_err(ApiError::from)?;
let mut size = size.ok_or_else(|| s3_error!(UnexpectedContent))?;
let ingress_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(std::time::Instant::now);
// Apply adaptive buffer sizing based on part size for optimal streaming performance.
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
@@ -931,6 +932,13 @@ impl DefaultMultipartUsecase {
let mut reader = PutObjReader::new(reader);
if let Some(stage_start) = ingress_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_ingress_prepare",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
let info = store
.put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &opts)
.await
+164 -1
View File
@@ -430,6 +430,43 @@ impl AsyncRead for PooledBufferReader {
}
}
struct ChunkedBytesReader {
chunks: Vec<Bytes>,
chunk_index: usize,
chunk_offset: usize,
}
impl ChunkedBytesReader {
fn new(chunks: Vec<Bytes>) -> Self {
Self {
chunks,
chunk_index: 0,
chunk_offset: 0,
}
}
}
impl AsyncRead for ChunkedBytesReader {
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
while self.chunk_index < self.chunks.len() {
let chunk = &self.chunks[self.chunk_index];
if self.chunk_offset >= chunk.len() {
self.chunk_index += 1;
self.chunk_offset = 0;
continue;
}
let remaining = &chunk[self.chunk_offset..];
let to_read = remaining.len().min(buf.remaining());
buf.put_slice(&remaining[..to_read]);
self.chunk_offset += to_read;
return Poll::Ready(Ok(()));
}
Poll::Ready(Ok(()))
}
}
/// Determine if zero-copy write should be used for this PutObject operation.
///
/// Zero-copy is beneficial for large objects without encryption or compression.
@@ -485,6 +522,34 @@ fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool {
true
}
fn should_use_zero_copy_eager_put_path(
size: i64,
headers: &HeaderMap,
server_side_encryption_requested: bool,
should_compress: bool,
is_extract: bool,
) -> bool {
const ZERO_COPY_EAGER_PUT_MAX_SIZE: i64 = 32 * 1024 * 1024;
if is_extract || should_compress || server_side_encryption_requested {
return false;
}
if size <= 0 || size > ZERO_COPY_EAGER_PUT_MAX_SIZE {
return false;
}
if !should_use_zero_copy(size, headers) {
return false;
}
if request_uses_aws_chunked(headers) && decoded_content_length_from_headers(headers).ok().flatten().is_none() {
return false;
}
true
}
fn has_put_sse_request_headers(headers: &HeaderMap) -> bool {
headers.get(AMZ_SERVER_SIDE_ENCRYPTION).is_some()
|| headers.get(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM).is_some()
@@ -584,6 +649,41 @@ where
Ok(std::io::Cursor::new(buf))
}
async fn read_zero_copy_put_body_exact<S, E>(mut body: S, size: usize) -> S3Result<ChunkedBytesReader>
where
S: futures::Stream<Item = std::result::Result<Bytes, E>> + Unpin,
E: std::fmt::Display,
{
let mut chunks = Vec::new();
let mut filled = 0usize;
while filled < size {
let Some(chunk) = body.next().await else {
return Err(s3_error!(IncompleteBody));
};
let chunk = chunk.map_err(|err| ApiError::from(StorageError::other(err.to_string())))?;
if chunk.is_empty() {
continue;
}
if filled.saturating_add(chunk.len()) > size {
return Err(s3_error!(UnexpectedContent));
}
rustfs_io_metrics::record_zero_copy_buffer_operation("put_chunk", chunk.len());
filled += chunk.len();
chunks.push(chunk);
}
while let Some(chunk) = body.next().await {
let chunk = chunk.map_err(|err| ApiError::from(StorageError::other(err.to_string())))?;
if !chunk.is_empty() {
return Err(s3_error!(UnexpectedContent));
}
}
Ok(ChunkedBytesReader::new(chunks))
}
fn object_seek_support_threshold() -> usize {
static OBJECT_SEEK_SUPPORT_THRESHOLD: OnceLock<usize> = OnceLock::new();
*OBJECT_SEEK_SUPPORT_THRESHOLD.get_or_init(|| {
@@ -2076,8 +2176,12 @@ impl DefaultObjectUsecase {
let use_small_eager_put_path =
should_use_small_eager_put_path(size, &req.headers, server_side_encryption_requested, should_compress, false);
let use_zero_copy_eager_put_path =
should_use_zero_copy_eager_put_path(size, &req.headers, server_side_encryption_requested, should_compress, false);
let put_path = if should_compress {
"stream_compressed"
} else if use_zero_copy_eager_put_path {
"zero_copy_eager"
} else if use_small_eager_put_path {
"small_eager"
} else {
@@ -2229,7 +2333,12 @@ impl DefaultObjectUsecase {
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
if use_small_eager_put_path {
if use_zero_copy_eager_put_path {
let zero_copy_start = std::time::Instant::now();
let eager_body = read_zero_copy_put_body_exact(body, actual_size as usize).await?;
rustfs_io_metrics::record_zero_copy_write(actual_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
} else if use_small_eager_put_path {
if (actual_size as usize) <= POOL_BYPASS_MAX_SIZE {
// Bypass BytesPool for very small objects to avoid Small-tier
// Mutex contention under high concurrency. Direct allocation
@@ -5486,6 +5595,24 @@ mod tests {
assert!(!should_use_small_eager_put_path(1024 * 1024 + 1, &headers, false, false, false));
}
#[test]
fn should_use_zero_copy_eager_put_path_allows_large_plain_objects_within_cap() {
let headers = HeaderMap::new();
assert!(should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, false, false, false));
assert!(should_use_zero_copy_eager_put_path(32 * 1024 * 1024, &headers, false, false, false));
assert!(!should_use_zero_copy_eager_put_path(32 * 1024 * 1024 + 1, &headers, false, false, false));
}
#[test]
fn should_use_zero_copy_eager_put_path_rejects_compression_sse_and_extract() {
let headers = HeaderMap::new();
assert!(!should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, true, false, false));
assert!(!should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, false, true, false));
assert!(!should_use_zero_copy_eager_put_path(2 * 1024 * 1024, &headers, false, false, true));
}
#[tokio::test]
async fn read_small_put_body_exact_pooled_reads_exact_bytes() {
let pool = get_concurrency_manager().bytes_pool();
@@ -5511,6 +5638,42 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
}
#[tokio::test]
async fn read_zero_copy_put_body_exact_reads_chunked_body() {
use tokio::io::AsyncReadExt;
let body = futures::stream::iter(vec![
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"hello ")),
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"world")),
]);
let mut reader = read_zero_copy_put_body_exact(body, 11)
.await
.expect("zero-copy eager body read should succeed");
let mut out = Vec::new();
reader
.read_to_end(&mut out)
.await
.expect("chunked bytes reader should be readable");
assert_eq!(out, b"hello world");
}
#[tokio::test]
async fn read_zero_copy_put_body_exact_rejects_extra_bytes() {
let body = futures::stream::iter(vec![
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"hello")),
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"!")),
]);
let err = match read_zero_copy_put_body_exact(body, 5).await {
Ok(_) => panic!("extra bytes should fail"),
Err(err) => err,
};
assert_eq!(err.code(), &S3ErrorCode::UnexpectedContent);
}
#[tokio::test]
async fn pooled_buffer_reader_keeps_buffer_alive_until_consumed() {
use tokio::io::AsyncReadExt;
+34 -16
View File
@@ -38,6 +38,14 @@ fn encode_msgpack<T: serde::Serialize>(value: &T, value_name: &str) -> std::resu
Ok(serializer.into_inner())
}
fn encode_msgpack_named<T: serde::Serialize>(value: &T, value_name: &str) -> std::result::Result<Vec<u8>, DiskError> {
let mut serializer = rmp_serde::Serializer::new(Vec::new()).with_struct_map();
value
.serialize(&mut serializer)
.map_err(|err| DiskError::other(format!("encode {value_name} named msgpack failed: {err}")))?;
Ok(serializer.into_inner())
}
impl NodeService {
pub(super) async fn handle_disk_info(&self, request: Request<DiskInfoRequest>) -> Result<Response<DiskInfoResponse>, Status> {
let request = request.into_inner();
@@ -629,12 +637,13 @@ impl NodeService {
) -> Result<Response<RenameDataResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
let file_info = match serde_json::from_str::<FileInfo>(&request.file_info) {
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
Ok(file_info) => file_info,
Err(err) => {
return Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
}));
}
@@ -644,25 +653,33 @@ impl NodeService {
.await
{
Ok(rename_data_resp) => {
let rename_data_resp = match serde_json::to_string(&rename_data_resp) {
Ok(file_info) => file_info,
Err(err) => {
return Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
}));
}
};
Ok(Response::new(RenameDataResponse {
success: true,
rename_data_resp,
error: None,
}))
let rename_data_resp_json = serde_json::to_string(&rename_data_resp);
let rename_data_resp_bin = encode_msgpack_named(&rename_data_resp, "RenameDataResp");
match (rename_data_resp_json, rename_data_resp_bin) {
(Ok(rename_data_resp), Ok(rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse {
success: true,
rename_data_resp,
rename_data_resp_bin: rename_data_resp_bin.into(),
error: None,
})),
(Err(err), _) => Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
(_, Err(err)) => Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
})),
}
}
Err(err) => Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(err.into()),
})),
}
@@ -670,6 +687,7 @@ impl NodeService {
Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other("can not find disk".to_string()).into()),
}))
}
+2
View File
@@ -1598,6 +1598,7 @@ mod tests {
dst_volume: "dst-volume".to_string(),
dst_path: "dst-path".to_string(),
file_info: "{}".to_string(),
file_info_bin: Vec::new().into(),
});
let response = service.rename_data(request).await;
@@ -1619,6 +1620,7 @@ mod tests {
dst_volume: "dst-volume".to_string(),
dst_path: "dst-path".to_string(),
file_info: "invalid json".to_string(),
file_info_bin: Vec::new().into(),
});
let response = service.rename_data(request).await;
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -euo pipefail
PIDFILE=/private/tmp/issue708-single-node-multidisk/rustfs.pid
LOGFILE=/private/tmp/issue708-single-node-multidisk/rustfs.log
RUSTFS_CMD_PATTERN='target/debug/rustfs server /private/tmp/issue708-single-node-multidisk/d1 /private/tmp/issue708-single-node-multidisk/d2 /private/tmp/issue708-single-node-multidisk/d3 /private/tmp/issue708-single-node-multidisk/d4'
mkdir -p /private/tmp/issue708-single-node-multidisk/{d1,d2,d3,d4}
if [[ -f "$PIDFILE" ]]; then
old_pid="$(cat "$PIDFILE")"
if kill -0 "$old_pid" >/dev/null 2>&1; then
kill "$old_pid" >/dev/null 2>&1 || true
sleep 2
fi
fi
lingering_pids="$(pgrep -f "$RUSTFS_CMD_PATTERN" || true)"
if [[ -n "$lingering_pids" ]]; then
while IFS= read -r lingering_pid; do
[[ -z "$lingering_pid" ]] && continue
kill "$lingering_pid" >/dev/null 2>&1 || true
done <<< "$lingering_pids"
sleep 2
fi
if [[ -n "${RUSTFS_TUNING_ENV_FILE:-}" && -f "${RUSTFS_TUNING_ENV_FILE:-}" ]]; then
set -a
source "$RUSTFS_TUNING_ENV_FILE"
set +a
fi
launch_rustfs() {
if command -v setsid >/dev/null 2>&1; then
setsid env \
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
RUSTFS_ADDRESS=127.0.0.1:9000 \
RUSTFS_ACCESS_KEY=rustfsadmin \
RUSTFS_SECRET_KEY=rustfsadmin \
RUSTFS_RPC_SECRET=rustfs-rpc-secret \
RUSTFS_REGION=us-east-1 \
RUSTFS_CONSOLE_ENABLE=false \
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 \
target/debug/rustfs server \
/private/tmp/issue708-single-node-multidisk/d1 \
/private/tmp/issue708-single-node-multidisk/d2 \
/private/tmp/issue708-single-node-multidisk/d3 \
/private/tmp/issue708-single-node-multidisk/d4 \
</dev/null >"$LOGFILE" 2>&1 &
else
nohup env \
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
RUSTFS_ADDRESS=127.0.0.1:9000 \
RUSTFS_ACCESS_KEY=rustfsadmin \
RUSTFS_SECRET_KEY=rustfsadmin \
RUSTFS_RPC_SECRET=rustfs-rpc-secret \
RUSTFS_REGION=us-east-1 \
RUSTFS_CONSOLE_ENABLE=false \
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 \
target/debug/rustfs server \
/private/tmp/issue708-single-node-multidisk/d1 \
/private/tmp/issue708-single-node-multidisk/d2 \
/private/tmp/issue708-single-node-multidisk/d3 \
/private/tmp/issue708-single-node-multidisk/d4 \
</dev/null >"$LOGFILE" 2>&1 &
fi
}
launch_rustfs
new_pid=$!
sleep 5
actual_pid="$(pgrep -f "$RUSTFS_CMD_PATTERN" | head -n 1 || true)"
if [[ -z "$actual_pid" ]]; then
echo "failed to locate restarted rustfs process; launcher pid was $new_pid" >&2
exit 1
fi
echo "$actual_pid" > "$PIDFILE"
curl -fsS http://127.0.0.1:9000/health >/dev/null
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
HOST="${HOST:-127.0.0.1:9000}"
ACCESS_KEY="${ACCESS_KEY:-}"
SECRET_KEY="${SECRET_KEY:-}"
BUCKET_PREFIX="${BUCKET_PREFIX:-rustfs-gt1g-multipart}"
LOOKUP="${LOOKUP:-path}"
DURATION="${DURATION:-10m}"
WARP_BIN="${WARP_BIN:-warp}"
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/gt1g-multipart-put-$(date +%Y%m%d-%H%M%S)}"
DRY_RUN=false
DEFAULT_PROFILES="1g-64m-pc4,1g-128m-pc4,2g-128m-pc4,2g-256m-pc4"
PROFILES="${PROFILES:-$DEFAULT_PROFILES}"
usage() {
cat <<'USAGE'
Usage:
scripts/run_gt1g_multipart_put_matrix.sh --access-key <ak> --secret-key <sk> [options]
Required:
--access-key <ak>
--secret-key <sk>
Options:
--host <host:port> Default: 127.0.0.1:9000
--bucket-prefix <prefix> Default: rustfs-gt1g-multipart
--lookup <path|dns> Default: path
--duration <dur> Default: 10m
--profiles <csv> Default: 1g-64m-pc4,1g-128m-pc4,2g-128m-pc4,2g-256m-pc4
--warp-bin <path> Default: warp
--out-dir <dir> Default: target/bench/gt1g-multipart-put-<timestamp>
--dry-run
-h, --help
Built-in profiles:
1g-64m-pc4 concurrent=4 parts=16 part.size=64MiB part.concurrent=4
1g-128m-pc4 concurrent=4 parts=8 part.size=128MiB part.concurrent=4
2g-128m-pc4 concurrent=4 parts=16 part.size=128MiB part.concurrent=4
2g-256m-pc4 concurrent=4 parts=8 part.size=256MiB part.concurrent=4
1g-64m-pc8 concurrent=4 parts=16 part.size=64MiB part.concurrent=8
2g-128m-pc8 concurrent=4 parts=16 part.size=128MiB part.concurrent=8
2g-256m-pc8 concurrent=4 parts=8 part.size=256MiB part.concurrent=8
USAGE
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "ERROR: command not found: $1" >&2
exit 1
fi
}
trim() {
echo "$1" | awk '{$1=$1;print}'
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--host) HOST="$2"; shift 2 ;;
--access-key) ACCESS_KEY="$2"; shift 2 ;;
--secret-key) SECRET_KEY="$2"; shift 2 ;;
--bucket-prefix) BUCKET_PREFIX="$2"; shift 2 ;;
--lookup) LOOKUP="$2"; shift 2 ;;
--duration) DURATION="$2"; shift 2 ;;
--profiles) PROFILES="$2"; shift 2 ;;
--warp-bin) WARP_BIN="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
validate_args() {
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
echo "ERROR: --access-key and --secret-key are required" >&2
exit 1
fi
}
setup_output() {
mkdir -p "$OUT_DIR/logs" "$OUT_DIR/benchdata"
SUMMARY_CSV="$OUT_DIR/summary.csv"
COMMANDS_TXT="$OUT_DIR/commands.txt"
MANIFEST_TXT="$OUT_DIR/run_manifest.txt"
echo "profile,host,bucket,duration,concurrent,parts,part_size,part_concurrent,throughput_human,reqps,latency_human,log_file,benchdata_file,status" > "$SUMMARY_CSV"
: > "$COMMANDS_TXT"
}
write_manifest() {
{
echo "created_at=$(date +%Y-%m-%dT%H:%M:%S%z)"
echo "host=${HOST}"
echo "bucket_prefix=${BUCKET_PREFIX}"
echo "lookup=${LOOKUP}"
echo "duration=${DURATION}"
echo "profiles=${PROFILES}"
echo "warp_bin=${WARP_BIN}"
echo "dry_run=${DRY_RUN}"
echo "access_key=REDACTED"
echo "secret_key=REDACTED"
} > "$MANIFEST_TXT"
}
profile_spec() {
case "$1" in
1g-64m-pc4) echo "4|16|64MiB|4" ;;
1g-128m-pc4) echo "4|8|128MiB|4" ;;
2g-128m-pc4) echo "4|16|128MiB|4" ;;
2g-256m-pc4) echo "4|8|256MiB|4" ;;
1g-64m-pc8) echo "4|16|64MiB|8" ;;
2g-128m-pc8) echo "4|16|128MiB|8" ;;
2g-256m-pc8) echo "4|8|256MiB|8" ;;
*)
echo "ERROR: unknown profile: $1" >&2
exit 1
;;
esac
}
extract_first() {
local regex="$1"
local file="$2"
rg -o "$regex" "$file" | head -n1 || true
}
extract_metrics() {
local log_file="$1"
local throughput reqps latency
throughput="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")"
reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")"
latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')"
throughput="$(trim "${throughput:-N/A}")"
reqps="$(trim "${reqps:-N/A}")"
latency="$(trim "${latency:-N/A}")"
reqps="$(echo "$reqps" | awk '{print $1}')"
echo "${throughput},${reqps:-N/A},${latency}"
}
run_profile() {
local profile="$1"
local spec concurrent parts part_size part_concurrent
local bucket log_file benchdata_file status metrics throughput reqps latency
spec="$(profile_spec "$profile")"
IFS='|' read -r concurrent parts part_size part_concurrent <<< "$spec"
bucket="$(echo "${BUCKET_PREFIX}-${profile}-$(date +%Y%m%d%H%M%S)" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/-{2,}/-/g; s/^-+//; s/-+$//')"
bucket="${bucket:0:63}"
log_file="$OUT_DIR/logs/${profile}.log"
benchdata_file="$OUT_DIR/benchdata/${profile}.csv.zst"
local -a cmd=(
"$WARP_BIN" multipart-put
--host "$HOST"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--bucket "$bucket"
--lookup "$LOOKUP"
--duration "$DURATION"
--concurrent "$concurrent"
--parts "$parts"
--part.size "$part_size"
--part.concurrent "$part_concurrent"
--benchdata "$benchdata_file"
--analyze.v
--no-color
)
printf '%q ' "${cmd[@]}" >> "$COMMANDS_TXT"
printf '\n' >> "$COMMANDS_TXT"
status="ok"
if [[ "$DRY_RUN" == "true" ]]; then
printf '[DRY-RUN] %q ' "${cmd[@]}"
printf '\n'
: > "$log_file"
else
if ! "${cmd[@]}" >"$log_file" 2>&1; then
status="failed"
fi
fi
metrics="$(extract_metrics "$log_file")"
throughput="$(echo "$metrics" | cut -d',' -f1)"
reqps="$(echo "$metrics" | cut -d',' -f2)"
latency="$(echo "$metrics" | cut -d',' -f3)"
echo "${profile},${HOST},${bucket},${DURATION},${concurrent},${parts},${part_size},${part_concurrent},${throughput},${reqps},${latency},${log_file},${benchdata_file},${status}" >> "$SUMMARY_CSV"
}
main() {
parse_args "$@"
validate_args
require_cmd "$WARP_BIN"
require_cmd rg
setup_output
write_manifest
echo "Output dir: $OUT_DIR"
echo "Profiles: $PROFILES"
IFS=',' read -r -a profiles <<< "$PROFILES"
for raw in "${profiles[@]}"; do
profile="$(trim "$raw")"
[[ -z "$profile" ]] && continue
run_profile "$profile"
done
echo
echo "Summary:"
cat "$SUMMARY_CSV"
}
main "$@"
@@ -0,0 +1,222 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
HOST="${HOST:-127.0.0.1:9000}"
ACCESS_KEY="${ACCESS_KEY:-}"
SECRET_KEY="${SECRET_KEY:-}"
BUCKET_PREFIX="${BUCKET_PREFIX:-issue712-gt1g-multipart-focus}"
LOOKUP="${LOOKUP:-path}"
DURATION="${DURATION:-10m}"
WARP_BIN="${WARP_BIN:-warp}"
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/gt1g-multipart-server-path-$(date +%Y%m%d-%H%M%S)}"
DRY_RUN=false
DEFAULT_PROFILES="1g-64m-pc4,2g-128m-pc4"
PROFILES="${PROFILES:-$DEFAULT_PROFILES}"
usage() {
cat <<'USAGE'
Usage:
scripts/run_gt1g_multipart_put_server_path_focus.sh --access-key <ak> --secret-key <sk> [options]
Required:
--access-key <ak>
--secret-key <sk>
Options:
--host <host:port> Default: 127.0.0.1:9000
--bucket-prefix <prefix> Default: issue712-gt1g-multipart-focus
--lookup <path|dns> Default: path
--duration <dur> Default: 10m
--profiles <csv> Default: 1g-64m-pc4,2g-128m-pc4
--warp-bin <path> Default: warp
--out-dir <dir> Default: target/bench/gt1g-multipart-server-path-<timestamp>
--dry-run
-h, --help
Profiles:
1g-64m-pc4 concurrent=4 parts=16 part.size=64MiB part.concurrent=4
2g-128m-pc4 concurrent=4 parts=16 part.size=128MiB part.concurrent=4
2g-256m-pc4 concurrent=4 parts=8 part.size=256MiB part.concurrent=4
USAGE
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "ERROR: command not found: $1" >&2
exit 1
fi
}
trim() {
echo "$1" | awk '{$1=$1;print}'
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--host) HOST="$2"; shift 2 ;;
--access-key) ACCESS_KEY="$2"; shift 2 ;;
--secret-key) SECRET_KEY="$2"; shift 2 ;;
--bucket-prefix) BUCKET_PREFIX="$2"; shift 2 ;;
--lookup) LOOKUP="$2"; shift 2 ;;
--duration) DURATION="$2"; shift 2 ;;
--profiles) PROFILES="$2"; shift 2 ;;
--warp-bin) WARP_BIN="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
validate_args() {
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
echo "ERROR: --access-key and --secret-key are required" >&2
exit 1
fi
}
setup_output() {
mkdir -p "$OUT_DIR/logs" "$OUT_DIR/benchdata"
SUMMARY_CSV="$OUT_DIR/summary.csv"
COMMANDS_TXT="$OUT_DIR/commands.txt"
MANIFEST_TXT="$OUT_DIR/run_manifest.txt"
echo "profile,host,bucket,duration,concurrent,parts,part_size,part_concurrent,throughput_human,reqps,latency_human,log_file,benchdata_file,status" > "$SUMMARY_CSV"
: > "$COMMANDS_TXT"
}
write_manifest() {
{
echo "created_at=$(date +%Y-%m-%dT%H:%M:%S%z)"
echo "host=${HOST}"
echo "bucket_prefix=${BUCKET_PREFIX}"
echo "lookup=${LOOKUP}"
echo "duration=${DURATION}"
echo "profiles=${PROFILES}"
echo "warp_bin=${WARP_BIN}"
echo "dry_run=${DRY_RUN}"
echo "access_key=REDACTED"
echo "secret_key=REDACTED"
} > "$MANIFEST_TXT"
}
profile_spec() {
case "$1" in
1g-64m-pc4) echo "4|16|64MiB|4" ;;
2g-128m-pc4) echo "4|16|128MiB|4" ;;
2g-256m-pc4) echo "4|8|256MiB|4" ;;
*)
echo "ERROR: unknown profile: $1" >&2
exit 1
;;
esac
}
extract_first() {
local regex="$1"
local file="$2"
rg -o "$regex" "$file" | head -n1 || true
}
extract_metrics() {
local log_file="$1"
local throughput reqps latency
throughput="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")"
reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")"
latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')"
throughput="$(trim "${throughput:-N/A}")"
reqps="$(trim "${reqps:-N/A}")"
latency="$(trim "${latency:-N/A}")"
reqps="$(echo "$reqps" | awk '{print $1}')"
echo "${throughput},${reqps:-N/A},${latency}"
}
run_profile() {
local profile="$1"
local spec concurrent parts part_size part_concurrent
local bucket log_file benchdata_file status metrics throughput reqps latency
spec="$(profile_spec "$profile")"
IFS='|' read -r concurrent parts part_size part_concurrent <<< "$spec"
bucket="$(echo "${BUCKET_PREFIX}-${profile}-$(date +%Y%m%d%H%M%S)" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/-{2,}/-/g; s/^-+//; s/-+$//')"
bucket="${bucket:0:63}"
log_file="$OUT_DIR/logs/${profile}.log"
benchdata_file="$OUT_DIR/benchdata/${profile}.csv.zst"
local -a cmd=(
"$WARP_BIN" multipart-put
--host "$HOST"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--bucket "$bucket"
--lookup "$LOOKUP"
--duration "$DURATION"
--concurrent "$concurrent"
--parts "$parts"
--part.size "$part_size"
--part.concurrent "$part_concurrent"
--benchdata "$benchdata_file"
--analyze.v
--no-color
)
printf '%q ' "${cmd[@]}" >> "$COMMANDS_TXT"
printf '\n' >> "$COMMANDS_TXT"
status="ok"
if [[ "$DRY_RUN" == "true" ]]; then
printf '[DRY-RUN] %q ' "${cmd[@]}"
printf '\n'
: > "$log_file"
else
if ! "${cmd[@]}" >"$log_file" 2>&1; then
status="failed"
fi
fi
metrics="$(extract_metrics "$log_file")"
throughput="$(echo "$metrics" | cut -d',' -f1)"
reqps="$(echo "$metrics" | cut -d',' -f2)"
latency="$(echo "$metrics" | cut -d',' -f3)"
echo "${profile},${HOST},${bucket},${DURATION},${concurrent},${parts},${part_size},${part_concurrent},${throughput},${reqps},${latency},${log_file},${benchdata_file},${status}" >> "$SUMMARY_CSV"
}
main() {
parse_args "$@"
validate_args
require_cmd "$WARP_BIN"
require_cmd rg
setup_output
write_manifest
echo "Output dir: $OUT_DIR"
echo "Profiles: $PROFILES"
IFS=',' read -r -a profiles <<< "$PROFILES"
for raw in "${profiles[@]}"; do
profile="$(trim "$raw")"
[[ -z "$profile" ]] && continue
run_profile "$profile"
done
echo
echo "Summary:"
cat "$SUMMARY_CSV"
}
main "$@"