feat(get): add codec engine ab matrix (#3940)

* upgrade version

* feat(get): add codec engine ab matrix

* chore(get): drop unrelated dependency drift

* upgrade version

* upgrade version

* fix cargo deny
This commit is contained in:
houseme
2026-06-27 13:27:16 +08:00
committed by GitHub
parent 996e58fd9a
commit 58b76a3d45
8 changed files with 440 additions and 141 deletions
Generated
+17 -17
View File
@@ -260,9 +260,9 @@ dependencies = [
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.102" version = "1.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
[[package]] [[package]]
name = "apache-avro" name = "apache-avro"
@@ -3607,7 +3607,7 @@ dependencies = [
"libc", "libc",
"option-ext", "option-ext",
"redox_users", "redox_users",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -3945,7 +3945,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -5350,7 +5350,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi",
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -6584,7 +6584,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -6757,7 +6757,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.22.1",
"chrono", "chrono",
"getrandom 0.2.17", "getrandom 0.2.17",
"http 1.4.2", "http 1.4.2",
@@ -7019,9 +7019,9 @@ dependencies = [
[[package]] [[package]]
name = "opentelemetry-semantic-conventions" name = "opentelemetry-semantic-conventions"
version = "0.32.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47"
[[package]] [[package]]
name = "opentelemetry-stdout" name = "opentelemetry-stdout"
@@ -7927,7 +7927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
dependencies = [ dependencies = [
"heck", "heck",
"itertools 0.10.5", "itertools 0.14.0",
"log", "log",
"multimap", "multimap",
"once_cell", "once_cell",
@@ -7947,7 +7947,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
dependencies = [ dependencies = [
"heck", "heck",
"itertools 0.10.5", "itertools 0.14.0",
"log", "log",
"multimap", "multimap",
"petgraph 0.8.3", "petgraph 0.8.3",
@@ -7968,7 +7968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"itertools 0.10.5", "itertools 0.14.0",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.118", "syn 2.0.118",
@@ -7981,7 +7981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"itertools 0.10.5", "itertools 0.14.0",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.118", "syn 2.0.118",
@@ -10232,7 +10232,7 @@ dependencies = [
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.12.1", "linux-raw-sys 0.12.1",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -10306,7 +10306,7 @@ dependencies = [
"security-framework", "security-framework",
"security-framework-sys", "security-framework-sys",
"webpki-root-certs", "webpki-root-certs",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -11484,7 +11484,7 @@ dependencies = [
"getrandom 0.4.3", "getrandom 0.4.3",
"once_cell", "once_cell",
"rustix 1.1.4", "rustix 1.1.4",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -12610,7 +12610,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
+2 -2
View File
@@ -206,7 +206,7 @@ tokio-postgres = { version = "0.7.18", default-features = false, features = ["ru
tokio-postgres-rustls = "0.14.0" tokio-postgres-rustls = "0.14.0"
# Utilities and Tools # Utilities and Tools
anyhow = "1.0.102" anyhow = "1.0.103"
arc-swap = "1.9.1" arc-swap = "1.9.1"
astral-tokio-tar = "0.6.3" astral-tokio-tar = "0.6.3"
atoi = "3.1.0" atoi = "3.1.0"
@@ -312,7 +312,7 @@ opentelemetry = { version = "0.32.0" }
opentelemetry-appender-tracing = { version = "0.32.0", features = ["experimental_span_attributes", "experimental_metadata_attributes"] } opentelemetry-appender-tracing = { version = "0.32.0", features = ["experimental_span_attributes", "experimental_metadata_attributes"] }
opentelemetry-otlp = { version = "0.32.0", features = ["gzip-http", "reqwest-rustls"] } opentelemetry-otlp = { version = "0.32.0", features = ["gzip-http", "reqwest-rustls"] }
opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio"] } opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio"] }
opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } opentelemetry-semantic-conventions = { version = "0.32.1", features = ["semconv_experimental"] }
opentelemetry-stdout = { version = "0.32.0" } opentelemetry-stdout = { version = "0.32.0" }
pyroscope = { version = "=2.0.5", features = ["backend-pprof-rs", "backend-jemalloc"] } pyroscope = { version = "=2.0.5", features = ["backend-pprof-rs", "backend-jemalloc"] }
+4
View File
@@ -17,6 +17,8 @@ use crate::error::StorageError;
use std::io; use std::io;
pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING: &str = "codec_streaming"; pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING: &str = "codec_streaming";
pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE: &str = "codec_streaming_legacy_engine";
pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE: &str = "codec_streaming_rustfs_engine";
pub(crate) const GET_OBJECT_PATH_EMPTY: &str = "empty"; pub(crate) const GET_OBJECT_PATH_EMPTY: &str = "empty";
pub(crate) const GET_OBJECT_PATH_LEGACY_DUPLEX: &str = "legacy_duplex"; pub(crate) const GET_OBJECT_PATH_LEGACY_DUPLEX: &str = "legacy_duplex";
pub(crate) const GET_OBJECT_PATH_REMOTE_TRANSITION: &str = "remote_transition"; pub(crate) const GET_OBJECT_PATH_REMOTE_TRANSITION: &str = "remote_transition";
@@ -216,6 +218,8 @@ mod tests {
assert_eq!(GetObjectFailureReason::DecodeError.as_str(), "decode_error"); assert_eq!(GetObjectFailureReason::DecodeError.as_str(), "decode_error");
assert_eq!(GET_READER_BUFFER_OUTPUT, "output"); assert_eq!(GET_READER_BUFFER_OUTPUT, "output");
assert_eq!(GET_READER_BUFFER_PREFETCH, "prefetch"); assert_eq!(GET_READER_BUFFER_PREFETCH, "prefetch");
assert_eq!(GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, "codec_streaming_legacy_engine");
assert_eq!(GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, "codec_streaming_rustfs_engine");
assert_eq!(GET_READER_PREFETCH_DIRECT, "direct"); assert_eq!(GET_READER_PREFETCH_DIRECT, "direct");
assert_eq!(GET_READER_PREFETCH_STORED, "stored"); assert_eq!(GET_READER_PREFETCH_STORED, "stored");
assert_eq!(GET_READER_PREFETCH_EOF, "eof"); assert_eq!(GET_READER_PREFETCH_EOF, "eof");
@@ -43,6 +43,7 @@ pub(crate) struct ErasureDecodeReader<S, E>
where where
E: ErasureDecodeEngine, E: ErasureDecodeEngine,
{ {
metrics_path: &'static str,
source: Option<S>, source: Option<S>,
engine: E, engine: E,
workspace: Option<E::Workspace>, workspace: Option<E::Workspace>,
@@ -62,6 +63,15 @@ where
E: ErasureDecodeEngine + Clone + Send + Sync + 'static, E: ErasureDecodeEngine + Clone + Send + Sync + 'static,
{ {
pub(crate) fn new(source: S, engine: E, total_length: usize) -> io::Result<Self> { pub(crate) fn new(source: S, engine: E, total_length: usize) -> io::Result<Self> {
Self::new_with_metrics_path(source, engine, total_length, GET_OBJECT_PATH_CODEC_STREAMING)
}
pub(crate) fn new_with_metrics_path(
source: S,
engine: E,
total_length: usize,
metrics_path: &'static str,
) -> io::Result<Self> {
if engine.data_shards() == 0 { if engine.data_shards() == 0 {
return Err(io::Error::new(ErrorKind::InvalidInput, "erasure reader requires data shards")); return Err(io::Error::new(ErrorKind::InvalidInput, "erasure reader requires data shards"));
} }
@@ -73,6 +83,7 @@ where
let workspace = engine.prepare_workspace(shard_len)?; let workspace = engine.prepare_workspace(shard_len)?;
Ok(Self { Ok(Self {
metrics_path,
source: Some(source), source: Some(source),
engine, engine,
workspace: Some(workspace), workspace: Some(workspace),
@@ -97,25 +108,26 @@ where
}; };
let engine = self.engine.clone(); let engine = self.engine.clone();
let metrics_path = self.metrics_path;
let remaining = self.remaining; let remaining = self.remaining;
self.fill = Some(tokio::spawn(async move { self.fill = Some(tokio::spawn(async move {
let fill_stage_start = Instant::now(); let fill_stage_start = Instant::now();
let stripe_read_stage_start = Instant::now(); let stripe_read_stage_start = Instant::now();
let state = source.read_next_stripe().await; let state = source.read_next_stripe().await;
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_STRIPE_READ, GET_STAGE_STRIPE_READ,
stripe_read_stage_start.elapsed().as_secs_f64(), stripe_read_stage_start.elapsed().as_secs_f64(),
); );
let decode_stage_start = Instant::now(); let decode_stage_start = Instant::now();
let result = decode_stripe(&engine, &mut workspace, state, remaining); let result = decode_stripe(metrics_path, &engine, &mut workspace, state, remaining);
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_DECODE, GET_STAGE_DECODE,
decode_stage_start.elapsed().as_secs_f64(), decode_stage_start.elapsed().as_secs_f64(),
); );
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_FILL, GET_STAGE_FILL,
fill_stage_start.elapsed().as_secs_f64(), fill_stage_start.elapsed().as_secs_f64(),
); );
@@ -153,8 +165,8 @@ where
if buf.is_empty() && self.remaining > 0 { if buf.is_empty() && self.remaining > 0 {
return Poll::Ready(Err(DiskError::LessData.into())); return Poll::Ready(Err(DiskError::LessData.into()));
} }
rustfs_io_metrics::record_get_object_reader_stripe(GET_OBJECT_PATH_CODEC_STREAMING); rustfs_io_metrics::record_get_object_reader_stripe(self.metrics_path);
rustfs_io_metrics::record_get_object_reader_bytes(GET_OBJECT_PATH_CODEC_STREAMING, buf.len()); rustfs_io_metrics::record_get_object_reader_bytes(self.metrics_path, buf.len());
self.remaining -= buf.len(); self.remaining -= buf.len();
Poll::Ready(Ok(Some(buf))) Poll::Ready(Ok(Some(buf)))
} }
@@ -182,7 +194,7 @@ where
Poll::Ready(result) => { Poll::Ready(result) => {
if let Some(started_at) = self.prefetch_wait_started_at.take() { if let Some(started_at) = self.prefetch_wait_started_at.take() {
rustfs_io_metrics::record_get_object_reader_prefetch_wait( rustfs_io_metrics::record_get_object_reader_prefetch_wait(
GET_OBJECT_PATH_CODEC_STREAMING, self.metrics_path,
started_at.elapsed().as_secs_f64(), started_at.elapsed().as_secs_f64(),
); );
} }
@@ -194,48 +206,28 @@ where
match fill { match fill {
Ok(Some(buf)) => { Ok(Some(buf)) => {
if self.output_pos < self.output_buf.len() { if self.output_pos < self.output_buf.len() {
rustfs_io_metrics::record_get_object_reader_prefetch( rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_STORED);
GET_OBJECT_PATH_CODEC_STREAMING, rustfs_io_metrics::record_get_object_reader_buffer(self.metrics_path, GET_READER_BUFFER_PREFETCH, buf.len());
GET_READER_PREFETCH_STORED,
);
rustfs_io_metrics::record_get_object_reader_buffer(
GET_OBJECT_PATH_CODEC_STREAMING,
GET_READER_BUFFER_PREFETCH,
buf.len(),
);
self.prefetched_buf = Some(buf); self.prefetched_buf = Some(buf);
} else { } else {
rustfs_io_metrics::record_get_object_reader_prefetch( rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_DIRECT);
GET_OBJECT_PATH_CODEC_STREAMING, rustfs_io_metrics::record_get_object_reader_buffer(self.metrics_path, GET_READER_BUFFER_OUTPUT, buf.len());
GET_READER_PREFETCH_DIRECT,
);
rustfs_io_metrics::record_get_object_reader_buffer(
GET_OBJECT_PATH_CODEC_STREAMING,
GET_READER_BUFFER_OUTPUT,
buf.len(),
);
self.output_buf = buf; self.output_buf = buf;
self.output_pos = 0; self.output_pos = 0;
} }
Poll::Ready(Ok(())) Poll::Ready(Ok(()))
} }
Ok(None) => { Ok(None) => {
rustfs_io_metrics::record_get_object_reader_prefetch(GET_OBJECT_PATH_CODEC_STREAMING, GET_READER_PREFETCH_EOF); rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_EOF);
Poll::Ready(Ok(())) Poll::Ready(Ok(()))
} }
Err(err) => { Err(err) => {
if self.output_pos < self.output_buf.len() { if self.output_pos < self.output_buf.len() {
rustfs_io_metrics::record_get_object_reader_prefetch( rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_ERROR_DEFERRED);
GET_OBJECT_PATH_CODEC_STREAMING,
GET_READER_PREFETCH_ERROR_DEFERRED,
);
self.prefetch_error = Some(err); self.prefetch_error = Some(err);
Poll::Ready(Ok(())) Poll::Ready(Ok(()))
} else { } else {
rustfs_io_metrics::record_get_object_reader_prefetch( rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_ERROR_IMMEDIATE);
GET_OBJECT_PATH_CODEC_STREAMING,
GET_READER_PREFETCH_ERROR_IMMEDIATE,
);
Poll::Ready(Err(err)) Poll::Ready(Err(err))
} }
} }
@@ -281,7 +273,7 @@ where
self.output_pos += copy_len; self.output_pos += copy_len;
if copy_len > 0 { if copy_len > 0 {
rustfs_io_metrics::record_get_object_reader_copy( rustfs_io_metrics::record_get_object_reader_copy(
GET_OBJECT_PATH_CODEC_STREAMING, self.metrics_path,
copy_len, copy_len,
read_buf_remaining_before, read_buf_remaining_before,
output_remaining_before, output_remaining_before,
@@ -292,11 +284,7 @@ where
} }
if let Some(next_buf) = self.prefetched_buf.take() { if let Some(next_buf) = self.prefetched_buf.take() {
rustfs_io_metrics::record_get_object_reader_buffer( rustfs_io_metrics::record_get_object_reader_buffer(self.metrics_path, GET_READER_BUFFER_OUTPUT, next_buf.len());
GET_OBJECT_PATH_CODEC_STREAMING,
GET_READER_BUFFER_OUTPUT,
next_buf.len(),
);
self.output_buf = next_buf; self.output_buf = next_buf;
self.output_pos = 0; self.output_pos = 0;
continue; continue;
@@ -380,6 +368,7 @@ where
} }
fn decode_stripe<E>( fn decode_stripe<E>(
metrics_path: &'static str,
engine: &E, engine: &E,
workspace: &mut E::Workspace, workspace: &mut E::Workspace,
state: StripeReadState, state: StripeReadState,
@@ -398,14 +387,14 @@ where
let reconstruct_stage_start = Instant::now(); let reconstruct_stage_start = Instant::now();
if state.data_shards_complete(engine.data_shards()) { if state.data_shards_complete(engine.data_shards()) {
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_RECONSTRUCT, GET_STAGE_RECONSTRUCT,
reconstruct_stage_start.elapsed().as_secs_f64(), reconstruct_stage_start.elapsed().as_secs_f64(),
); );
let emit_stage_start = Instant::now(); let emit_stage_start = Instant::now();
let output = emit_data_shards(&state, engine.data_shards(), engine.block_size(), remaining)?; let output = emit_data_shards(&state, engine.data_shards(), engine.block_size(), remaining)?;
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_EMIT, GET_STAGE_EMIT,
emit_stage_start.elapsed().as_secs_f64(), emit_stage_start.elapsed().as_secs_f64(),
); );
@@ -415,14 +404,14 @@ where
let (mut shards, _errs) = state.into_parts(); let (mut shards, _errs) = state.into_parts();
if let Err(err) = engine.reconstruct_into(&mut shards, workspace) { if let Err(err) = engine.reconstruct_into(&mut shards, workspace) {
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_RECONSTRUCT, GET_STAGE_RECONSTRUCT,
reconstruct_stage_start.elapsed().as_secs_f64(), reconstruct_stage_start.elapsed().as_secs_f64(),
); );
return Err(err); return Err(err);
} }
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_RECONSTRUCT, GET_STAGE_RECONSTRUCT,
reconstruct_stage_start.elapsed().as_secs_f64(), reconstruct_stage_start.elapsed().as_secs_f64(),
); );
@@ -446,11 +435,7 @@ where
let copy_len = shard.len().min(remaining - output.len()); let copy_len = shard.len().min(remaining - output.len());
output.extend_from_slice(&shard[..copy_len]); output.extend_from_slice(&shard[..copy_len]);
} }
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(metrics_path, GET_STAGE_EMIT, emit_stage_start.elapsed().as_secs_f64());
GET_OBJECT_PATH_CODEC_STREAMING,
GET_STAGE_EMIT,
emit_stage_start.elapsed().as_secs_f64(),
);
Ok(Some(output)) Ok(Some(output))
} }
+11 -2
View File
@@ -24,8 +24,10 @@ use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl}; use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::cluster::rpc::heal_bucket_local_on_disks;
use crate::diagnostics::get::{ use crate::diagnostics::get::{
GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE,
GET_STAGE_EMIT, GET_STAGE_METADATA, classify_storage_error, record_get_object_pipeline_failure, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_STAGE_EMIT, GET_STAGE_METADATA, classify_storage_error,
record_get_object_pipeline_failure,
}; };
use crate::disk::error_reduce::{ use crate::disk::error_reduce::{
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs, BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
@@ -410,6 +412,13 @@ fn build_get_codec_streaming_decode_engine(erasure: coding::Erasure) -> std::io:
} }
} }
fn get_codec_streaming_metrics_path() -> &'static str {
match get_codec_streaming_engine() {
GetCodecStreamingEngine::Legacy => GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE,
GetCodecStreamingEngine::Rustfs => GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE,
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GetCodecStreamingDecision { enum GetCodecStreamingDecision {
Use, Use,
+28 -9
View File
@@ -2244,8 +2244,9 @@ impl SetDisks {
BitrotReaderSetupMode::VerifyReconstruction, BitrotReaderSetupMode::VerifyReconstruction,
) )
.await; .await;
let metrics_path = get_codec_streaming_metrics_path();
rustfs_io_metrics::record_get_object_stage_duration( rustfs_io_metrics::record_get_object_stage_duration(
GET_OBJECT_PATH_CODEC_STREAMING, metrics_path,
GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP,
reader_setup_stage_start.elapsed().as_secs_f64(), reader_setup_stage_start.elapsed().as_secs_f64(),
); );
@@ -2254,14 +2255,10 @@ impl SetDisks {
if available_shards < erasure.data_shards { if available_shards < erasure.data_shards {
if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) { if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
let reason = classify_disk_error(&read_err); let reason = classify_disk_error(&read_err);
record_get_object_pipeline_failure_for_path(GET_OBJECT_PATH_CODEC_STREAMING, GET_STAGE_READER_SETUP, reason); record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason);
return Err(to_object_err(read_err.into(), vec![bucket, object])); return Err(to_object_err(read_err.into(), vec![bucket, object]));
} }
record_get_object_pipeline_failure_for_path( record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum);
GET_OBJECT_PATH_CODEC_STREAMING,
GET_STAGE_READER_SETUP,
GetObjectFailureReason::ReadQuorum,
);
return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors))); return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
} }
@@ -2287,11 +2284,16 @@ impl SetDisks {
erasure.clone(), erasure.clone(),
0, 0,
part_size, part_size,
Some(GET_OBJECT_PATH_CODEC_STREAMING), Some(metrics_path),
read_costs, read_costs,
); );
let engine = build_get_codec_streaming_decode_engine(erasure)?; let engine = build_get_codec_streaming_decode_engine(erasure)?;
let reader = crate::erasure::coding::decode_reader::ErasureDecodeReader::new(source, engine, part_length)?; let reader = crate::erasure::coding::decode_reader::ErasureDecodeReader::new_with_metrics_path(
source,
engine,
part_length,
metrics_path,
)?;
Ok(Box::new(crate::erasure::coding::decode_reader::SyncErasureDecodeReader::new(reader))) Ok(Box::new(crate::erasure::coding::decode_reader::SyncErasureDecodeReader::new(reader)))
} }
} }
@@ -3362,6 +3364,23 @@ mod tests {
}); });
} }
#[test]
fn codec_streaming_metrics_path_matches_selected_engine() {
temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, None::<&str>, || {
assert_eq!(
get_codec_streaming_metrics_path(),
crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE
);
});
temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS), || {
assert_eq!(
get_codec_streaming_metrics_path(),
crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE
);
});
}
#[test] #[test]
fn codec_streaming_fallback_metric_labels_are_stable() { fn codec_streaming_fallback_metric_labels_are_stable() {
assert_eq!(GetCodecStreamingFallbackReason::Disabled.as_str(), "disabled"); assert_eq!(GetCodecStreamingFallbackReason::Disabled.as_str(), "disabled");
+1
View File
@@ -39,6 +39,7 @@ allow-git = [
# Custom S3 server library with minio compatibility patches not yet upstreamed. # Custom S3 server library with minio compatibility patches not yet upstreamed.
# Pinned to a specific commit in workspace Cargo.toml. # Pinned to a specific commit in workspace Cargo.toml.
"https://github.com/rustfs/s3s", "https://github.com/rustfs/s3s",
"https://github.com/apache/datafusion.git",
] ]
[bans] [bans]
+343 -62
View File
@@ -24,6 +24,7 @@ ROUNDS=3
RETRY_PER_ROUND=1 RETRY_PER_ROUND=1
ROUND_COOLDOWN_SECS=0 ROUND_COOLDOWN_SECS=0
MODE="both" MODE="both"
CODEC_ENGINES="legacy"
OUT_DIR="" OUT_DIR=""
RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs" RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs"
WARP_BIN="warp" WARP_BIN="warp"
@@ -49,6 +50,7 @@ Purpose:
Core options: Core options:
--mode <legacy|codec|both> Which profile(s) to run (default: both) --mode <legacy|codec|both> Which profile(s) to run (default: both)
--codec-engine <csv> Codec engine(s) for codec profiles (default: legacy)
--address <host:port> RustFS listen address (default: 127.0.0.1:19030) --address <host:port> RustFS listen address (default: 127.0.0.1:19030)
--bucket <name> Benchmark bucket (default: rustfs-get-codec-smoke) --bucket <name> Benchmark bucket (default: rustfs-get-codec-smoke)
--sizes <csv> Object sizes (default: 1MiB,4MiB,10MiB) --sizes <csv> Object sizes (default: 1MiB,4MiB,10MiB)
@@ -76,13 +78,17 @@ Credentials:
Output: Output:
<out-dir>/legacy/warp/median_summary.csv <out-dir>/legacy/warp/median_summary.csv
<out-dir>/codec/warp/median_summary.csv <out-dir>/codec-legacy/warp/median_summary.csv
<out-dir>/codec/warp/baseline_compare.csv when --mode both <out-dir>/codec-rustfs/warp/median_summary.csv
<out-dir>/compat_summary.csv when --mode both <out-dir>/engine_compare.csv when legacy and codec profiles both run
<out-dir>/compat_summary.csv when legacy and codec profiles both run
<out-dir>/metrics_summary.csv
<out-dir>/body_sha256_legacy.txt when legacy profile runs <out-dir>/body_sha256_legacy.txt when legacy profile runs
<out-dir>/body_sha256_new.txt when codec profile runs <out-dir>/body_sha256_codec_legacy.txt when codec-legacy profile runs
<out-dir>/body_sha256_codec_rustfs.txt when codec-rustfs profile runs
<out-dir>/response_headers_legacy.json when legacy profile runs <out-dir>/response_headers_legacy.json when legacy profile runs
<out-dir>/response_headers_new.json when codec profile runs <out-dir>/response_headers_codec_legacy.json when codec-legacy profile runs
<out-dir>/response_headers_codec_rustfs.json when codec-rustfs profile runs
<out-dir>/<profile>/manifest.env <out-dir>/<profile>/manifest.env
<out-dir>/<profile>/metrics_summary.csv <out-dir>/<profile>/metrics_summary.csv
<out-dir>/<profile>/compat/compat_summary.csv <out-dir>/<profile>/compat/compat_summary.csv
@@ -92,7 +98,7 @@ Output:
Example: Example:
scripts/run_get_codec_streaming_smoke.sh \ scripts/run_get_codec_streaming_smoke.sh \
--mode both --sizes 1MiB,4MiB,10MiB --concurrency 64 --duration 30s --mode both --codec-engine legacy,rustfs --sizes 1MiB,4MiB,10MiB --concurrency 64 --duration 30s
USAGE USAGE
} }
@@ -131,6 +137,7 @@ parse_args() {
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
case "$1" in case "$1" in
--mode) MODE="$2"; shift 2 ;; --mode) MODE="$2"; shift 2 ;;
--codec-engine) CODEC_ENGINES="$2"; shift 2 ;;
--address) ADDRESS="$2"; shift 2 ;; --address) ADDRESS="$2"; shift 2 ;;
--bucket) BUCKET="$2"; shift 2 ;; --bucket) BUCKET="$2"; shift 2 ;;
--sizes) SIZES="$2"; shift 2 ;; --sizes) SIZES="$2"; shift 2 ;;
@@ -166,6 +173,17 @@ validate_args() {
*) die "--mode must be legacy, codec, or both" ;; *) die "--mode must be legacy, codec, or both" ;;
esac esac
local raw engine
IFS=',' read -r -a engines <<< "$CODEC_ENGINES"
[[ "${#engines[@]}" -gt 0 ]] || die "--codec-engine must not be empty"
for raw in "${engines[@]}"; do
engine="${raw//[[:space:]]/}"
case "$engine" in
legacy|rustfs) ;;
*) die "--codec-engine supports only legacy,rustfs, got: $engine" ;;
esac
done
[[ -n "$ADDRESS" ]] || die "--address must not be empty" [[ -n "$ADDRESS" ]] || die "--address must not be empty"
[[ -n "$ACCESS_KEY" ]] || die "--access-key must not be empty" [[ -n "$ACCESS_KEY" ]] || die "--access-key must not be empty"
[[ -n "$SECRET_KEY" ]] || die "--secret-key must not be empty" [[ -n "$SECRET_KEY" ]] || die "--secret-key must not be empty"
@@ -210,7 +228,26 @@ profile_codec_enabled() {
local profile="$1" local profile="$1"
case "$profile" in case "$profile" in
legacy) echo "false" ;; legacy) echo "false" ;;
codec) echo "true" ;; codec-legacy|codec-rustfs) echo "true" ;;
*) die "unknown profile: $profile" ;;
esac
}
profile_codec_engine() {
local profile="$1"
case "$profile" in
legacy|codec-legacy) echo "legacy" ;;
codec-rustfs) echo "rustfs" ;;
*) die "unknown profile: $profile" ;;
esac
}
profile_metrics_path() {
local profile="$1"
case "$profile" in
legacy) echo "legacy_duplex" ;;
codec-legacy) echo "codec_streaming_legacy_engine" ;;
codec-rustfs) echo "codec_streaming_rustfs_engine" ;;
*) die "unknown profile: $profile" ;; *) die "unknown profile: $profile" ;;
esac esac
} }
@@ -234,6 +271,8 @@ write_manifest() {
local profile_dir="$2" local profile_dir="$2"
local codec_enabled="$3" local codec_enabled="$3"
local volumes="$4" local volumes="$4"
local codec_engine="$5"
local metrics_path="$6"
local git_head git_dirty_count local git_head git_dirty_count
git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)" git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)"
@@ -261,7 +300,9 @@ compat_object_key=${COMPAT_OBJECT_KEY}
compat_object_size=${COMPAT_OBJECT_SIZE} compat_object_size=${COMPAT_OBJECT_SIZE}
rustfs_volumes=${volumes} rustfs_volumes=${volumes}
RUSTFS_GET_CODEC_STREAMING_ENABLE=${codec_enabled} RUSTFS_GET_CODEC_STREAMING_ENABLE=${codec_enabled}
RUSTFS_GET_CODEC_STREAMING_ENGINE=${codec_engine}
RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE} RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE}
metrics_path=${metrics_path}
RUSTFS_SCANNER_ENABLED=false RUSTFS_SCANNER_ENABLED=false
RUSTFS_SCANNER_START_DELAY_SECS=3600 RUSTFS_SCANNER_START_DELAY_SECS=3600
RUSTFS_SCANNER_CYCLE=3600 RUSTFS_SCANNER_CYCLE=3600
@@ -307,15 +348,19 @@ start_server() {
local data_root local data_root
local volumes local volumes
local codec_enabled local codec_enabled
local codec_engine
local metrics_path
local rustfs_log local rustfs_log
data_root="$(profile_data_root "$profile")" data_root="$(profile_data_root "$profile")"
volumes="$(profile_volumes "$data_root")" volumes="$(profile_volumes "$data_root")"
codec_enabled="$(profile_codec_enabled "$profile")" codec_enabled="$(profile_codec_enabled "$profile")"
codec_engine="$(profile_codec_engine "$profile")"
metrics_path="$(profile_metrics_path "$profile")"
rustfs_log="${profile_dir}/rustfs.log" rustfs_log="${profile_dir}/rustfs.log"
mkdir -p "${data_root}/disk1" "${data_root}/disk2" "${data_root}/disk3" "${data_root}/disk4" mkdir -p "${data_root}/disk1" "${data_root}/disk2" "${data_root}/disk3" "${data_root}/disk4"
write_manifest "$profile" "$profile_dir" "$codec_enabled" "$volumes" write_manifest "$profile" "$profile_dir" "$codec_enabled" "$volumes" "$codec_engine" "$metrics_path"
if [[ "$DRY_RUN" == "true" ]]; then if [[ "$DRY_RUN" == "true" ]]; then
log "[DRY-RUN] start RustFS profile=${profile} endpoint=$(endpoint_url)" log "[DRY-RUN] start RustFS profile=${profile} endpoint=$(endpoint_url)"
@@ -332,6 +377,7 @@ start_server() {
export RUSTFS_VOLUMES="$volumes" export RUSTFS_VOLUMES="$volumes"
export RUSTFS_CONSOLE_ENABLE=false export RUSTFS_CONSOLE_ENABLE=false
export RUSTFS_GET_CODEC_STREAMING_ENABLE="$codec_enabled" export RUSTFS_GET_CODEC_STREAMING_ENABLE="$codec_enabled"
export RUSTFS_GET_CODEC_STREAMING_ENGINE="$codec_engine"
export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE" export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE"
export RUSTFS_REGION="$REGION" export RUSTFS_REGION="$REGION"
export RUSTFS_RPC_SECRET="rustfs-get-codec-smoke-rpc-secret" export RUSTFS_RPC_SECRET="rustfs-get-codec-smoke-rpc-secret"
@@ -440,6 +486,7 @@ import http.client
import json import json
import pathlib import pathlib
import sys import sys
from typing import Dict, List, Optional, Tuple
import urllib.parse import urllib.parse
endpoint, access_key, secret_key, region, bucket, object_key, object_size_raw, out_dir_raw, profile = sys.argv[1:] endpoint, access_key, secret_key, region, bucket, object_key, object_size_raw, out_dir_raw, profile = sys.argv[1:]
@@ -481,7 +528,7 @@ def canonical_uri(path: str) -> str:
return "/".join(urllib.parse.quote(part, safe="-_.~/") for part in path.split("/")) or "/" return "/".join(urllib.parse.quote(part, safe="-_.~/") for part in path.split("/")) or "/"
def signed_headers(method: str, path: str, body: bytes, extra_headers: list[tuple[str, str]]) -> dict[str, str]: def signed_headers(method: str, path: str, body: bytes, extra_headers: List[Tuple[str, str]]) -> Dict[str, str]:
parsed = urllib.parse.urlsplit(endpoint) parsed = urllib.parse.urlsplit(endpoint)
amz_date = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") amz_date = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
date_stamp = amz_date[:8] date_stamp = amz_date[:8]
@@ -509,7 +556,7 @@ def signed_headers(method: str, path: str, body: bytes, extra_headers: list[tupl
return result return result
def request(method: str, path: str, body: bytes = b"", extra_headers: list[tuple[str, str]] | None = None): def request(method: str, path: str, body: bytes = b"", extra_headers: Optional[List[Tuple[str, str]]] = None):
parsed = urllib.parse.urlsplit(endpoint) parsed = urllib.parse.urlsplit(endpoint)
headers = signed_headers(method, path, body, extra_headers or []) headers = signed_headers(method, path, body, extra_headers or [])
if body and "content-length" not in headers: if body and "content-length" not in headers:
@@ -529,8 +576,8 @@ def request(method: str, path: str, body: bytes = b"", extra_headers: list[tuple
return snapshot, response_body return snapshot, response_body
def header_map(headers: list[list[str] | tuple[str, str]]) -> dict[str, list[str]]: def header_map(headers) -> Dict[str, List[str]]:
result: dict[str, list[str]] = {} result: Dict[str, List[str]] = {}
for name, value in headers: for name, value in headers:
result.setdefault(name.lower(), []).append(value) result.setdefault(name.lower(), []).append(value)
for value in result.values(): for value in result.values():
@@ -609,22 +656,33 @@ copy_profile_compat_artifacts() {
fi fi
} }
write_ab_compat_summary() { write_root_compat_summary() {
local legacy_snapshot="${OUT_DIR}/legacy/compat/snapshot.json" local legacy_snapshot="${OUT_DIR}/legacy/compat/snapshot.json"
local codec_snapshot="${OUT_DIR}/codec/compat/snapshot.json" local codec_snapshots=()
local profile snapshot
if [[ ! -f "$legacy_snapshot" || ! -f "$codec_snapshot" ]]; then [[ -f "$legacy_snapshot" ]] || return
return
fi
"$PYTHON_BIN" - "$legacy_snapshot" "$codec_snapshot" "${OUT_DIR}/compat_summary.csv" <<'PY' for profile in "$@"; do
[[ "$profile" == legacy ]] && continue
snapshot="${OUT_DIR}/${profile}/compat/snapshot.json"
if [[ -f "$snapshot" ]]; then
codec_snapshots+=("$snapshot")
fi
done
[[ "${#codec_snapshots[@]}" -gt 0 ]] || return 0
"$PYTHON_BIN" - "${OUT_DIR}/compat_summary.csv" "$legacy_snapshot" "${codec_snapshots[@]}" <<'PY'
import csv import csv
import json import json
import sys import sys
legacy_path, codec_path, out_csv = sys.argv[1:] out_csv = sys.argv[1]
legacy_path = sys.argv[2]
codec_paths = sys.argv[3:]
legacy = json.load(open(legacy_path, encoding="utf-8")) legacy = json.load(open(legacy_path, encoding="utf-8"))
codec = json.load(open(codec_path, encoding="utf-8")) codecs = [json.load(open(path, encoding="utf-8")) for path in codec_paths]
def values(snapshot, name): def values(snapshot, name):
@@ -636,23 +694,6 @@ def prefixed(snapshot, prefix):
return {key: headers[key] for key in sorted(headers) if key.startswith(prefix)} return {key: headers[key] for key in sorted(headers) if key.startswith(prefix)}
body_match = legacy.get("body_sha256") == codec.get("body_sha256")
content_length_match = values(legacy, "content-length") == values(codec, "content-length")
etag_match = values(legacy, "etag") == values(codec, "etag")
content_range_match = values(legacy, "content-range") == values(codec, "content-range")
checksum_headers_match = prefixed(legacy, "x-amz-checksum") == prefixed(codec, "x-amz-checksum")
sse_headers_match = prefixed(legacy, "x-amz-server-side-encryption") == prefixed(codec, "x-amz-server-side-encryption")
status_code_match = legacy.get("get_status") == codec.get("get_status") and legacy.get("head_status") == codec.get("head_status")
checks = [
body_match,
content_length_match,
etag_match,
content_range_match,
checksum_headers_match,
sse_headers_match,
status_code_match,
]
with open(out_csv, "w", encoding="utf-8", newline="") as handle: with open(out_csv, "w", encoding="utf-8", newline="") as handle:
writer = csv.writer(handle) writer = csv.writer(handle)
writer.writerow( writer.writerow(
@@ -673,24 +714,239 @@ with open(out_csv, "w", encoding="utf-8", newline="") as handle:
"new_status_code", "new_status_code",
] ]
) )
writer.writerow( for codec in codecs:
[ body_match = legacy.get("body_sha256") == codec.get("body_sha256")
codec.get("size", legacy.get("size", "")), content_length_match = values(legacy, "content-length") == values(codec, "content-length")
"codec_streaming", etag_match = values(legacy, "etag") == values(codec, "etag")
str(body_match).lower(), content_range_match = values(legacy, "content-range") == values(codec, "content-range")
str(content_length_match).lower(), checksum_headers_match = prefixed(legacy, "x-amz-checksum") == prefixed(codec, "x-amz-checksum")
str(etag_match).lower(), sse_headers_match = prefixed(legacy, "x-amz-server-side-encryption") == prefixed(codec, "x-amz-server-side-encryption")
str(content_range_match).lower(), status_code_match = legacy.get("get_status") == codec.get("get_status") and legacy.get("head_status") == codec.get("head_status")
str(checksum_headers_match).lower(), checks = [
str(sse_headers_match).lower(), body_match,
str(status_code_match).lower(), content_length_match,
sum(1 for check in checks if not check), etag_match,
legacy.get("body_sha256", ""), content_range_match,
codec.get("body_sha256", ""), checksum_headers_match,
legacy.get("get_status", ""), sse_headers_match,
codec.get("get_status", ""), status_code_match,
] ]
writer.writerow(
[
codec.get("size", legacy.get("size", "")),
codec.get("path", ""),
str(body_match).lower(),
str(content_length_match).lower(),
str(etag_match).lower(),
str(content_range_match).lower(),
str(checksum_headers_match).lower(),
str(sse_headers_match).lower(),
str(status_code_match).lower(),
sum(1 for check in checks if not check),
legacy.get("body_sha256", ""),
codec.get("body_sha256", ""),
legacy.get("get_status", ""),
codec.get("get_status", ""),
]
)
PY
}
write_root_metrics_summary() {
local profile_dirs=()
local profile
for profile in "$@"; do
profile_dirs+=("${OUT_DIR}/${profile}")
done
[[ "${#profile_dirs[@]}" -gt 0 ]] || return
"$PYTHON_BIN" - "${OUT_DIR}/metrics_summary.csv" "${OUT_DIR}/engine_compare.csv" "${profile_dirs[@]}" <<'PY'
import csv
import pathlib
import sys
metrics_csv = pathlib.Path(sys.argv[1])
engine_compare_csv = pathlib.Path(sys.argv[2])
profile_dirs = [pathlib.Path(value) for value in sys.argv[3:]]
def load_manifest(path):
data = {}
for line in path.read_text(encoding="utf-8").splitlines():
if "=" not in line:
continue
key, value = line.split("=", 1)
data[key] = value
return data
def parse_float(value):
if value in ("", "N/A", None):
return None
return float(value)
def delta_pct(new_value, baseline_value):
if new_value is None or baseline_value in (None, 0.0):
return ""
return f"{((new_value - baseline_value) / baseline_value) * 100:.2f}"
def performance_note(throughput_delta, latency_delta):
if throughput_delta == "" or latency_delta == "":
return ""
throughput_delta = float(throughput_delta)
latency_delta = float(latency_delta)
if throughput_delta < 0 and latency_delta > 0:
return "slower_than_legacy_and_higher_latency"
if throughput_delta < 0:
return "slower_than_legacy"
if latency_delta > 0:
return "higher_latency_than_legacy"
return "at_or_better_than_legacy"
rows = []
by_profile = {}
for profile_dir in profile_dirs:
manifest = load_manifest(profile_dir / "manifest.env")
profile = manifest["profile"]
with open(profile_dir / "warp" / "median_summary.csv", encoding="utf-8", newline="") as handle:
profile_rows = list(csv.DictReader(handle))
by_profile[profile] = profile_rows
for row in profile_rows:
rows.append(
{
"profile": profile,
"read_path": manifest.get("metrics_path", profile),
"codec_streaming_enabled": manifest.get("RUSTFS_GET_CODEC_STREAMING_ENABLE", ""),
"codec_engine": manifest.get("RUSTFS_GET_CODEC_STREAMING_ENGINE", ""),
"size": row["size"],
"tool": row["tool"],
"concurrency": row["concurrency"],
"successful_rounds": row["successful_rounds"],
"failed_rounds": row["failed_rounds"],
"median_throughput_bps": row["median_throughput_bps"],
"median_reqps": row["median_reqps"],
"median_latency_ms": row["median_latency_ms"],
}
)
legacy_by_size = {row["size"]: row for row in by_profile.get("legacy", [])}
for row in rows:
baseline = legacy_by_size.get(row["size"])
if baseline is None or row["profile"] == "legacy":
row["baseline_profile"] = ""
row["delta_throughput_pct_vs_legacy"] = ""
row["delta_reqps_pct_vs_legacy"] = ""
row["delta_latency_pct_vs_legacy"] = ""
row["performance_note"] = ""
continue
throughput_delta = delta_pct(parse_float(row["median_throughput_bps"]), parse_float(baseline["median_throughput_bps"]))
reqps_delta = delta_pct(parse_float(row["median_reqps"]), parse_float(baseline["median_reqps"]))
latency_delta = delta_pct(parse_float(row["median_latency_ms"]), parse_float(baseline["median_latency_ms"]))
row["baseline_profile"] = "legacy"
row["delta_throughput_pct_vs_legacy"] = throughput_delta
row["delta_reqps_pct_vs_legacy"] = reqps_delta
row["delta_latency_pct_vs_legacy"] = latency_delta
row["performance_note"] = performance_note(throughput_delta, latency_delta)
fieldnames = [
"profile",
"read_path",
"codec_streaming_enabled",
"codec_engine",
"size",
"tool",
"concurrency",
"successful_rounds",
"failed_rounds",
"median_throughput_bps",
"median_reqps",
"median_latency_ms",
"baseline_profile",
"delta_throughput_pct_vs_legacy",
"delta_reqps_pct_vs_legacy",
"delta_latency_pct_vs_legacy",
"performance_note",
]
with open(metrics_csv, "w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
codec_legacy_by_size = {row["size"]: row for row in by_profile.get("codec-legacy", [])}
codec_rustfs_by_size = {row["size"]: row for row in by_profile.get("codec-rustfs", [])}
if not legacy_by_size:
raise SystemExit(0)
compare_fields = [
"size",
"legacy_median_throughput_bps",
"codec_legacy_median_throughput_bps",
"codec_rustfs_median_throughput_bps",
"legacy_median_reqps",
"codec_legacy_median_reqps",
"codec_rustfs_median_reqps",
"legacy_median_latency_ms",
"codec_legacy_median_latency_ms",
"codec_rustfs_median_latency_ms",
"codec_legacy_delta_throughput_pct_vs_legacy",
"codec_rustfs_delta_throughput_pct_vs_legacy",
"codec_rustfs_delta_throughput_pct_vs_codec_legacy",
"codec_legacy_delta_latency_pct_vs_legacy",
"codec_rustfs_delta_latency_pct_vs_legacy",
"codec_rustfs_delta_latency_pct_vs_codec_legacy",
]
compare_rows = []
for size, legacy in legacy_by_size.items():
codec_legacy = codec_legacy_by_size.get(size, {})
codec_rustfs = codec_rustfs_by_size.get(size, {})
compare_rows.append(
{
"size": size,
"legacy_median_throughput_bps": legacy.get("median_throughput_bps", ""),
"codec_legacy_median_throughput_bps": codec_legacy.get("median_throughput_bps", ""),
"codec_rustfs_median_throughput_bps": codec_rustfs.get("median_throughput_bps", ""),
"legacy_median_reqps": legacy.get("median_reqps", ""),
"codec_legacy_median_reqps": codec_legacy.get("median_reqps", ""),
"codec_rustfs_median_reqps": codec_rustfs.get("median_reqps", ""),
"legacy_median_latency_ms": legacy.get("median_latency_ms", ""),
"codec_legacy_median_latency_ms": codec_legacy.get("median_latency_ms", ""),
"codec_rustfs_median_latency_ms": codec_rustfs.get("median_latency_ms", ""),
"codec_legacy_delta_throughput_pct_vs_legacy": delta_pct(
parse_float(codec_legacy.get("median_throughput_bps", "")),
parse_float(legacy.get("median_throughput_bps", "")),
),
"codec_rustfs_delta_throughput_pct_vs_legacy": delta_pct(
parse_float(codec_rustfs.get("median_throughput_bps", "")),
parse_float(legacy.get("median_throughput_bps", "")),
),
"codec_rustfs_delta_throughput_pct_vs_codec_legacy": delta_pct(
parse_float(codec_rustfs.get("median_throughput_bps", "")),
parse_float(codec_legacy.get("median_throughput_bps", "")),
),
"codec_legacy_delta_latency_pct_vs_legacy": delta_pct(
parse_float(codec_legacy.get("median_latency_ms", "")),
parse_float(legacy.get("median_latency_ms", "")),
),
"codec_rustfs_delta_latency_pct_vs_legacy": delta_pct(
parse_float(codec_rustfs.get("median_latency_ms", "")),
parse_float(legacy.get("median_latency_ms", "")),
),
"codec_rustfs_delta_latency_pct_vs_codec_legacy": delta_pct(
parse_float(codec_rustfs.get("median_latency_ms", "")),
parse_float(codec_legacy.get("median_latency_ms", "")),
),
}
) )
with open(engine_compare_csv, "w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=compare_fields)
writer.writeheader()
writer.writerows(compare_rows)
PY PY
} }
@@ -722,27 +978,52 @@ main() {
trap stop_server EXIT INT TERM trap stop_server EXIT INT TERM
log "Output dir: $OUT_DIR" log "Output dir: $OUT_DIR"
local codec_profiles=()
local profiles=()
local raw engine profile
IFS=',' read -r -a engines <<< "$CODEC_ENGINES"
for raw in "${engines[@]}"; do
engine="${raw//[[:space:]]/}"
[[ -n "$engine" ]] || continue
profile="codec-${engine}"
codec_profiles+=("$profile")
done
case "$MODE" in case "$MODE" in
legacy) legacy)
run_profile legacy profiles=("legacy")
copy_profile_compat_artifacts legacy legacy
;; ;;
codec) codec)
run_profile codec profiles=("${codec_profiles[@]}")
copy_profile_compat_artifacts codec new
;; ;;
both) both)
run_profile legacy profiles=("legacy" "${codec_profiles[@]}")
copy_profile_compat_artifacts legacy legacy
run_profile codec "${OUT_DIR}/legacy/warp/median_summary.csv"
copy_profile_compat_artifacts codec new
write_ab_compat_summary
;; ;;
esac esac
local legacy_baseline_csv=""
for profile in "${profiles[@]}"; do
if [[ "$profile" == "legacy" ]]; then
run_profile "$profile"
legacy_baseline_csv="${OUT_DIR}/legacy/warp/median_summary.csv"
else
run_profile "$profile" "$legacy_baseline_csv"
fi
copy_profile_compat_artifacts "$profile" "${profile//-/_}"
done
write_root_metrics_summary "${profiles[@]}"
write_root_compat_summary "${profiles[@]}"
if [[ -f "${OUT_DIR}/compat_summary.csv" ]]; then if [[ -f "${OUT_DIR}/compat_summary.csv" ]]; then
log "Compatibility compare: ${OUT_DIR}/compat_summary.csv" log "Compatibility compare: ${OUT_DIR}/compat_summary.csv"
fi fi
if [[ -f "${OUT_DIR}/engine_compare.csv" ]]; then
log "Engine compare: ${OUT_DIR}/engine_compare.csv"
fi
if [[ -f "${OUT_DIR}/metrics_summary.csv" ]]; then
log "Metrics summary: ${OUT_DIR}/metrics_summary.csv"
fi
log "GET codec streaming smoke finished." log "GET codec streaming smoke finished."
} }