From 46d7f9e1f2915c13b7be85d6313a818d286950f5 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 28 Jun 2026 11:20:21 +0800 Subject: [PATCH] feat(get): harden codec streaming rollout (#3981) * feat(get): consolidate GET performance optimization Consolidated implementation of all GET performance optimizations into a single, well-organized commit replacing the previous patch-on-patch approach. ## Changes ### Configuration (set_disk/mod.rs) - Consolidated all GET optimization flags into a single organized section - Enabled by default: codec streaming, metadata early-stop, page cache reclaim - Added codec streaming multipart flag (default: disabled) - Added version-aware early-stop flag (default: disabled) - Added adaptive duplex buffer sizing based on object size - All flags use OnceLock caching with rollout percentage support ### Metadata Early-Stop (set_disk/read.rs) - Delete marker early-stop when quorum agrees - Version-aware early-stop for versioned GET requests - MetadataQuorumAccumulator enhanced with: - delete_marker_votes tracking - requested_version_id and matching_version_votes tracking - version_early_stop_decision() method - 6 new tests for version early-stop scenarios ### Codec Streaming (erasure/coding/decode_reader.rs) - DualInFlight (2-stripe lookahead) enabled by default ### Decode Pipeline (erasure/coding/decode.rs) - Stripe prefetch count configuration - Bitrot-decode overlap configuration ### Disk Layer (disk/local.rs) - O_DIRECT read configuration constants (preparation) ### Metrics (io-metrics/lib.rs) - BytesPool acquisition/return metrics - Metadata phase duration with early-stop label - Total duration with reader_path label ### Diagnostics (diagnostics/) - Early-stop reason constants - Pool tier/outcome label constants ### Observability (.docker/observability/) - 3 Grafana dashboards for GET optimization monitoring - Prometheus alert rules (6 alerts: 3 critical, 3 warning) - Updated README.md and README_ZH.md with usage docs ### Config (config/src/constants/runtime.rs) - Page cache reclaim read enabled by default ## Environment Variables | Variable | Default | Description | |----------|---------|-------------| | RUSTFS_GET_CODEC_STREAMING_ENABLE | true | Codec streaming base flag | | RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT | 100 | Codec streaming rollout % | | RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE | false | Multipart codec streaming | | RUSTFS_GET_METADATA_EARLY_STOP_ENABLE | true | Early-stop base flag | | RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT | 100 | Early-stop rollout % | | RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE | false | Version-aware early-stop | | RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE | true | Page cache reclaim | | RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE | false | O_DIRECT (preparation) | | RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT | 1 | Stripe prefetch | | RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE | false | Bitrot-decode overlap | | RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT | 2 | DualInFlight stripes | ## Rollback All optimizations can be disabled via environment variables: RUSTFS_GET_CODEC_STREAMING_ENABLE=false RUSTFS_GET_METADATA_EARLY_STOP_ENABLE=false RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE=false Co-Authored-By: heihutu * test(get): add stress test scripts for GET optimization validation - quick-validate-get-optimization.sh: Quick 5-minute validation - stress-test-get-optimization.sh: Full 30+ minute stress test - README-stress-test.md: Usage documentation Co-Authored-By: heihutu * test(ecstore): align file cache reclaim defaults * chore(deps): update redis and erasure codec * test(ecstore): align decode fill policy default * fix(get): wire codec streaming rollout gate * perf(get): skip metrics-off codec timers * test(get): capture codec streaming diagnostics * test(get): add multipart fallback probe * test(get): add encrypted fallback probe * test(get): add compressed fallback probe * test(get): add degraded read fallback probe * test(get): cover remote fallback probe * test(get): report warp request p99 * test(get): capture OTLP metric deltas * perf(get): align codec streaming inflight default * perf(get): reuse codec reader output buffers * test(get): count codec reader fill starts * perf(get): reuse codec reader fill worker * perf(get): lazy init rustfs codec reconstruct * test(get): cover rustfs codec source faults * docs(get): record rustfs codec fallback scope * feat(get): add multipart codec reader opt-in * test(get): add multipart codec smoke option * test(get): cover multipart codec degraded fallback * perf(get): bound multipart codec eager setup * test(get): satisfy codec hardening PR gate --------- Co-authored-by: heihutu --- crates/ecstore/src/diagnostics/get.rs | 13 + crates/ecstore/src/erasure/codec/bridge.rs | 160 +- crates/ecstore/src/erasure/coding/decode.rs | 93 +- .../src/erasure/coding/decode_reader.rs | 676 ++++++-- crates/ecstore/src/set_disk/mod.rs | 84 +- crates/ecstore/src/set_disk/read.rs | 461 +++++- crates/io-metrics/src/lib.rs | 38 + .../src/app/lifecycle_transition_api_test.rs | 90 ++ rustfs/src/app/storage_api.rs | 4 +- scripts/run_get_codec_streaming_smoke.sh | 1398 ++++++++++++++++- scripts/run_object_batch_bench_enhanced.sh | 59 +- 11 files changed, 2767 insertions(+), 309 deletions(-) diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index c92473a41..0ff69e21e 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -15,6 +15,7 @@ use crate::disk::error::DiskError; use crate::error::StorageError; use std::io; +use std::time::Instant; pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING: &str = "codec_streaming"; pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE: &str = "codec_streaming_legacy_engine"; @@ -173,6 +174,18 @@ pub(crate) fn record_get_object_pipeline_failure_for_path( rustfs_io_metrics::record_get_object_pipeline_failure_for_path(path, stage, reason.as_str()); } +#[inline(always)] +pub(crate) fn get_stage_timer_if_enabled(stage_metrics_enabled: bool) -> Option { + stage_metrics_enabled.then(Instant::now) +} + +#[inline(always)] +pub(crate) fn record_get_stage_duration_if_enabled(path: &'static str, stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + rustfs_io_metrics::record_get_object_stage_duration(path, stage, started_at.elapsed().as_secs_f64()); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ecstore/src/erasure/codec/bridge.rs b/crates/ecstore/src/erasure/codec/bridge.rs index aa10c66fa..8224af8c1 100644 --- a/crates/ecstore/src/erasure/codec/bridge.rs +++ b/crates/ecstore/src/erasure/codec/bridge.rs @@ -16,9 +16,14 @@ use crate::erasure::codec::workspace::RustfsCodecDecodeWorkspace; use crate::erasure::coding::Erasure; use reed_solomon_erasure::galois_8::ReedSolomon; use std::io; +use std::sync::{Arc, OnceLock}; pub(crate) const GET_CODEC_STREAMING_ENGINE_LEGACY: &str = "legacy"; pub(crate) const GET_CODEC_STREAMING_ENGINE_RUSTFS: &str = "rustfs"; +pub(crate) const GET_RECONSTRUCT_OUTCOME_LEGACY_CALLED: &str = "legacy_called"; +pub(crate) const GET_RECONSTRUCT_OUTCOME_RUSTFS_CALLED: &str = "rustfs_called"; +pub(crate) const GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE: &str = "skip_data_complete"; +pub(crate) const GET_RECONSTRUCT_OUTCOME_SKIP_EMPTY_PAYLOAD: &str = "skip_empty_payload"; pub(crate) trait DecodeWorkspace: Send + Sync + 'static { fn shard_len(&self) -> usize; @@ -30,13 +35,14 @@ pub(crate) trait ErasureDecodeEngine: Send + Sync + 'static { fn data_shards(&self) -> usize; fn parity_shards(&self) -> usize; fn block_size(&self) -> usize; + fn engine_name(&self) -> &'static str; fn supports_progressive_decode(&self) -> bool; fn supports_aligned_shards(&self) -> bool; fn prepare_workspace(&self, shard_len: usize) -> io::Result; - fn reconstruct_into(&self, shards: &mut [Option>], workspace: &mut Self::Workspace) -> io::Result<()>; + fn reconstruct_into(&self, shards: &mut [Option>], workspace: &mut Self::Workspace) -> io::Result<&'static str>; } fn data_shards_complete(shards: &[Option>], data_shards: usize) -> bool { @@ -126,6 +132,10 @@ impl ErasureDecodeEngine for LegacyEcDecodeEngine { self.erasure.block_size } + fn engine_name(&self) -> &'static str { + GET_CODEC_STREAMING_ENGINE_LEGACY + } + fn supports_progressive_decode(&self) -> bool { false } @@ -138,12 +148,13 @@ impl ErasureDecodeEngine for LegacyEcDecodeEngine { Ok(LegacyDecodeWorkspace::new(shard_len)) } - fn reconstruct_into(&self, shards: &mut [Option>], _workspace: &mut Self::Workspace) -> io::Result<()> { + fn reconstruct_into(&self, shards: &mut [Option>], _workspace: &mut Self::Workspace) -> io::Result<&'static str> { if data_shards_complete(shards, self.erasure.data_shards) { - return Ok(()); + return Ok(GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE); } - self.erasure.decode_data_with_reconstruction_verification(shards) + self.erasure.decode_data_with_reconstruction_verification(shards)?; + Ok(GET_RECONSTRUCT_OUTCOME_LEGACY_CALLED) } } @@ -152,26 +163,74 @@ pub(crate) struct RustfsCodecDecodeEngine { data_shards: usize, parity_shards: usize, block_size: usize, - codec: Option, + codec: OnceLock>, } impl RustfsCodecDecodeEngine { pub(crate) fn new(erasure: &Erasure) -> io::Result { - let codec = if erasure.parity_shards > 0 { - ReedSolomon::new(erasure.data_shards, erasure.parity_shards) - .map_err(|err| io::Error::other(format!("Failed to create RustFS codec decode engine: {err:?}"))) - .map(Some)? - } else { - None - }; - Ok(Self { data_shards: erasure.data_shards, parity_shards: erasure.parity_shards, block_size: erasure.block_size, - codec, + codec: OnceLock::new(), }) } + + fn codec(&self) -> io::Result>> { + if self.parity_shards == 0 { + return Ok(None); + } + + if let Some(codec) = self.codec.get() { + return Ok(Some(Arc::clone(codec))); + } + + let codec = Arc::new( + ReedSolomon::new(self.data_shards, self.parity_shards) + .map_err(|err| io::Error::other(format!("Failed to create RustFS codec decode engine: {err:?}")))?, + ); + if self.codec.set(Arc::clone(&codec)).is_err() { + return Ok(self.codec.get().map(Arc::clone).or(Some(codec))); + } + Ok(Some(codec)) + } + + fn needs_source_parity_verification(&self, shards: &[Option>]) -> bool { + let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none()); + let available_shards = shards.iter().filter(|shard| shard.is_some()).count(); + missing_data_source && available_shards > self.data_shards + } + + fn verify_source_parity(&self, codec: &ReedSolomon, shards: &[Option>], needs_verification: bool) -> io::Result<()> { + if !needs_verification { + return Ok(()); + } + + let mut shard_refs = Vec::with_capacity(self.data_shards + self.parity_shards); + for (index, shard) in shards.iter().enumerate() { + let shard = shard.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("missing shard {index} after RustFS codec reconstruction"), + ) + })?; + shard_refs.push(shard.as_slice()); + } + + let valid = codec + .verify(&shard_refs) + .map_err(|err| io::Error::other(format!("RustFS codec verify failed: {err:?}")))?; + if !valid { + return Err(io::Error::new(io::ErrorKind::InvalidData, "inconsistent read source shards")); + } + + Ok(()) + } + + #[cfg(test)] + fn codec_is_initialized(&self) -> bool { + self.codec.get().is_some() + } } impl ErasureDecodeEngine for RustfsCodecDecodeEngine { @@ -189,6 +248,10 @@ impl ErasureDecodeEngine for RustfsCodecDecodeEngine { self.block_size } + fn engine_name(&self) -> &'static str { + GET_CODEC_STREAMING_ENGINE_RUSTFS + } + fn supports_progressive_decode(&self) -> bool { false } @@ -201,21 +264,23 @@ impl ErasureDecodeEngine for RustfsCodecDecodeEngine { Ok(RustfsCodecDecodeWorkspace::new(shard_len)) } - fn reconstruct_into(&self, shards: &mut [Option>], _workspace: &mut Self::Workspace) -> io::Result<()> { + fn reconstruct_into(&self, shards: &mut [Option>], _workspace: &mut Self::Workspace) -> io::Result<&'static str> { if data_shards_complete(shards, self.data_shards) { - return Ok(()); + return Ok(GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE); } if recover_empty_payload_data_shards(shards, self.data_shards, self.parity_shards)? { - return Ok(()); + return Ok(GET_RECONSTRUCT_OUTCOME_SKIP_EMPTY_PAYLOAD); } - if let Some(codec) = &self.codec { + if let Some(codec) = self.codec()? { + let needs_source_parity_verification = self.needs_source_parity_verification(shards); codec .reconstruct_data_opt(shards) - .map_err(|err| io::Error::other(format!("RustFS codec reconstruct failed: {err:?}"))) - } else { - Ok(()) + .map_err(|err| io::Error::other(format!("RustFS codec reconstruct failed: {err:?}")))?; + self.verify_source_parity(&codec, shards, needs_source_parity_verification)?; } + + Ok(GET_RECONSTRUCT_OUTCOME_RUSTFS_CALLED) } } @@ -273,6 +338,13 @@ impl ErasureDecodeEngine for CodecStreamingDecodeEngine { } } + fn engine_name(&self) -> &'static str { + match self { + Self::Legacy(engine) => engine.engine_name(), + Self::Rustfs(engine) => engine.engine_name(), + } + } + fn supports_progressive_decode(&self) -> bool { match self { Self::Legacy(engine) => engine.supports_progressive_decode(), @@ -294,7 +366,7 @@ impl ErasureDecodeEngine for CodecStreamingDecodeEngine { } } - fn reconstruct_into(&self, shards: &mut [Option>], workspace: &mut Self::Workspace) -> io::Result<()> { + fn reconstruct_into(&self, shards: &mut [Option>], workspace: &mut Self::Workspace) -> io::Result<&'static str> { match (self, workspace) { (Self::Legacy(engine), CodecStreamingDecodeWorkspace::Legacy(workspace)) => { engine.reconstruct_into(shards, workspace) @@ -325,7 +397,7 @@ mod tests { E: ErasureDecodeEngine, { let mut workspace = engine.prepare_workspace(4)?; - engine.reconstruct_into(shards, &mut workspace) + engine.reconstruct_into(shards, &mut workspace).map(|_| ()) } #[test] @@ -393,8 +465,10 @@ mod tests { let before = shards.clone(); let engine = RustfsCodecDecodeEngine::new(&erasure).expect("engine should be created"); + assert!(!engine.codec_is_initialized()); reconstruct_with(&engine, &mut shards).expect("complete data shards should not reconstruct"); + assert!(!engine.codec_is_initialized()); assert_eq!(shards, before); } @@ -446,6 +520,46 @@ mod tests { assert!(reconstruct_with(&rustfs, &mut rustfs_shards).is_err()); } + #[test] + fn rustfs_codec_decode_engine_rejects_inconsistent_reconstruction_sources() { + let erasure = Erasure::new(2, 2, 32); + let encoded = erasure + .encode_data(&(0u8..64u8).collect::>()) + .expect("test stripe should encode"); + let mut shards = encoded.into_iter().map(|shard| Some(shard.to_vec())).collect::>(); + shards[0] = None; + let Some(corrupt_parity) = shards[erasure.data_shards].as_mut() else { + panic!("test parity shard should be present"); + }; + corrupt_parity[0] ^= 0x80; + + let engine = RustfsCodecDecodeEngine::new(&erasure).expect("engine should be created"); + let err = reconstruct_with(&engine, &mut shards).expect_err("rustfs codec should reject inconsistent sources"); + + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert!(err.to_string().contains("inconsistent read source shards")); + } + + #[test] + fn rustfs_codec_decode_engine_rejects_stale_data_source() { + let erasure = Erasure::new(4, 2, 32); + let encoded = erasure + .encode_data(&(0u8..128u8).collect::>()) + .expect("test stripe should encode"); + let mut shards = encoded.into_iter().map(|shard| Some(shard.to_vec())).collect::>(); + shards[0] = None; + let Some(stale_data) = shards[1].as_mut() else { + panic!("test data shard should be present"); + }; + stale_data[0] ^= 0x40; + + let engine = RustfsCodecDecodeEngine::new(&erasure).expect("engine should be created"); + let err = reconstruct_with(&engine, &mut shards).expect_err("rustfs codec should reject stale data sources"); + + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert!(err.to_string().contains("inconsistent read source shards")); + } + #[test] fn rustfs_codec_decode_engine_recovers_empty_data_shard() { let erasure = Erasure::new(4, 2, 16); diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index c74538a75..1cd993c81 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -16,7 +16,8 @@ use crate::diagnostics::get::{ GET_OBJECT_PATH_LEGACY_DUPLEX, GET_SHARD_READ_ERROR_MISSING, GET_SHARD_READ_ERROR_NONE, GET_SHARD_READ_OUTCOME_ERROR, GET_SHARD_READ_OUTCOME_MISSING, GET_SHARD_READ_OUTCOME_SUCCESS, GET_SHARD_ROLE_DATA, GET_SHARD_ROLE_PARITY, GET_STAGE_EMIT, GET_STAGE_RANGE, GET_STAGE_RECONSTRUCT, GET_STAGE_STRIPE_READ, GET_STAGE_STRIPE_READ_FIRST_SHARD, - GET_STAGE_STRIPE_READ_QUORUM, GetObjectFailureReason, classify_io_error, record_get_object_pipeline_failure, + GET_STAGE_STRIPE_READ_QUORUM, GetObjectFailureReason, classify_io_error, get_stage_timer_if_enabled, + record_get_object_pipeline_failure, record_get_stage_duration_if_enabled, }; use crate::disk::disk_store::get_object_disk_read_timeout; use crate::disk::error::Error; @@ -182,7 +183,7 @@ where Box::pin(async move { let mut buf = recycled_buf.unwrap_or_else(|| vec![0; shard_size]); debug_assert_eq!(buf.len(), shard_size); - let read_start = Instant::now(); + let read_start = metrics_path.map(|_| Instant::now()); let read_result = if read_timeout.is_zero() { reader.read(&mut buf).await } else { @@ -200,7 +201,7 @@ where GET_SHARD_READ_OUTCOME_ERROR, error_class, 0, - read_start.elapsed().as_secs_f64(), + read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()), reader.last_verify_duration().as_secs_f64(), ); } @@ -221,7 +222,7 @@ where GET_SHARD_READ_OUTCOME_SUCCESS, GET_SHARD_READ_ERROR_NONE, n, - read_start.elapsed().as_secs_f64(), + read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()), reader.last_verify_duration().as_secs_f64(), ); } @@ -240,7 +241,7 @@ where GET_SHARD_READ_OUTCOME_ERROR, error_class, 0, - read_start.elapsed().as_secs_f64(), + read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()), verify_duration_secs, ); } @@ -450,6 +451,7 @@ where read_timeout: Duration, verify_reconstruction: bool, ) -> Self { + let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled()); let shard_size = e.shard_size(); let shard_file_size = e.shard_file_size(total_length as i64) as usize; @@ -615,7 +617,7 @@ where let mut reader_iter = ReaderLaunchIter::new(&mut self.readers, read_costs, locality_preference_enabled); let mut sets = FuturesUnordered::new(); let mut active_readers = vec![false; num_readers]; - let stripe_read_start = Instant::now(); + let stripe_read_start = self.metrics_path.map(|_| Instant::now()); let mut scheduled = 0usize; for _ in 0..self.data_shards { if let Some((i, reader)) = reader_iter.next() { @@ -722,11 +724,7 @@ where completed += 1; if !first_shard_recorded { if let Some(path) = self.metrics_path { - rustfs_io_metrics::record_get_object_stage_duration( - path, - GET_STAGE_STRIPE_READ_FIRST_SHARD, - stripe_read_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_FIRST_SHARD, stripe_read_start); } first_shard_recorded = true; } @@ -876,11 +874,7 @@ where } if let Some(path) = self.metrics_path { - rustfs_io_metrics::record_get_object_stage_duration( - path, - GET_STAGE_STRIPE_READ_QUORUM, - stripe_read_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_QUORUM, stripe_read_start); rustfs_io_metrics::record_get_object_shard_read_fanout(path, scheduled, completed, success, failed); if locality_preference_enabled { let remote_avoided = remote_available.saturating_sub(remote_scheduled); @@ -1038,9 +1032,11 @@ where offset = 0; let write_len = write_left.min(block_slice.len()); - let write_stage_start = Instant::now(); + let write_stage_start = get_stage_timer_if_enabled(rustfs_io_metrics::get_stage_metrics_enabled()); if let Err(e) = writer.write_all(&block_slice[..write_len]).await { - rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64()); + if let Some(write_stage_start) = write_stage_start { + rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64()); + } let reason = classify_io_error(&e); record_get_object_pipeline_failure(GET_STAGE_EMIT, reason); error!( @@ -1054,7 +1050,9 @@ where ); return Err(e); } - rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64()); + if let Some(write_stage_start) = write_stage_start { + rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64()); + } total_written += write_len; write_left -= write_len; @@ -1180,13 +1178,10 @@ impl Erasure { break; } - let stripe_read_stage_start = Instant::now(); + let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let (mut shards, errs) = reader.read().await; - rustfs_io_metrics::record_get_object_stage_duration( - "legacy_duplex", - "stripe_read", - stripe_read_stage_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_STRIPE_READ, stripe_read_stage_start); if ret_err.is_none() && let (_, Some(err)) = reduce_errs(&errs, &[]) @@ -1216,12 +1211,12 @@ impl Erasure { // Decode the shards. If this stripe needed parity to reconstruct a // missing data shard and an extra source shard was available, verify // the reconstructed data against that source before streaming bytes. - let reconstruct_stage_start = Instant::now(); + let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); if let Err(e) = self.decode_data_with_reconstruction_verification(&mut shards) { - rustfs_io_metrics::record_get_object_stage_duration( - "legacy_duplex", - "reconstruct", - reconstruct_stage_start.elapsed().as_secs_f64(), + record_get_stage_duration_if_enabled( + GET_OBJECT_PATH_LEGACY_DUPLEX, + GET_STAGE_RECONSTRUCT, + reconstruct_stage_start, ); let reason = GetObjectFailureReason::DecodeError; error!( @@ -1238,28 +1233,16 @@ impl Erasure { ret_err = Some(e); break; } - rustfs_io_metrics::record_get_object_stage_duration( - "legacy_duplex", - "reconstruct", - reconstruct_stage_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start); - let emit_stage_start = Instant::now(); + let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let n = match write_data_blocks(writer, &shards, self.data_shards, block_offset, block_length).await { Ok(n) => { - rustfs_io_metrics::record_get_object_stage_duration( - "legacy_duplex", - "emit", - emit_stage_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start); n } Err(e) => { - rustfs_io_metrics::record_get_object_stage_duration( - "legacy_duplex", - "emit", - emit_stage_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start); error!( block_offset, block_length, @@ -1624,6 +1607,24 @@ mod tests { assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 2, 0, 4]); } + #[test] + #[serial_test::serial] + fn parallel_reader_drops_metrics_path_when_stage_metrics_disabled() { + let erasure = Erasure::new(2, 1, 32); + let readers: Vec>>>> = vec![None, None, None]; + + rustfs_io_metrics::set_get_stage_metrics_enabled(false); + let reader = ParallelReader::new_with_metrics_path(readers, erasure.clone(), 0, 1, Some(GET_OBJECT_PATH_LEGACY_DUPLEX)); + assert_eq!(reader.metrics_path, None); + + let readers: Vec>>>> = vec![None, None, None]; + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + let reader = ParallelReader::new_with_metrics_path(readers, erasure, 0, 1, Some(GET_OBJECT_PATH_LEGACY_DUPLEX)); + assert_eq!(reader.metrics_path, Some(GET_OBJECT_PATH_LEGACY_DUPLEX)); + + rustfs_io_metrics::set_get_stage_metrics_enabled(false); + } + #[tokio::test] #[serial_test::serial] async fn test_parallel_reader_local_first_avoids_remote_when_local_quorum_exists() { diff --git a/crates/ecstore/src/erasure/coding/decode_reader.rs b/crates/ecstore/src/erasure/coding/decode_reader.rs index 528113dff..d9f58c9bf 100644 --- a/crates/ecstore/src/erasure/coding/decode_reader.rs +++ b/crates/ecstore/src/erasure/coding/decode_reader.rs @@ -17,10 +17,10 @@ use crate::diagnostics::get::{ GET_READER_POLL_READY_DATA, GET_READER_POLL_READY_EMPTY, GET_READER_POLL_READY_ERROR, GET_READER_PREFETCH_DIRECT, GET_READER_PREFETCH_EOF, GET_READER_PREFETCH_ERROR_DEFERRED, GET_READER_PREFETCH_ERROR_IMMEDIATE, GET_READER_PREFETCH_STORED, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_FILL, GET_STAGE_OUTPUT_LOCK_WAIT, GET_STAGE_OUTPUT_POLL, GET_STAGE_RECONSTRUCT, - GET_STAGE_STRIPE_READ, + GET_STAGE_STRIPE_READ, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled, }; use crate::disk::error::Error as DiskError; -use crate::erasure::codec::bridge::ErasureDecodeEngine; +use crate::erasure::codec::bridge::{ErasureDecodeEngine, GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE}; use crate::set_disk::shard_source::{ShardStripeSource, StripeReadState}; use std::collections::VecDeque; use std::io; @@ -30,6 +30,7 @@ use std::sync::Mutex; use std::task::{Context, Poll, ready}; use std::time::Instant; use tokio::io::{AsyncRead, ReadBuf}; +use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; const ENV_RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT: &str = "RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT"; @@ -37,13 +38,23 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT: usize = 2; const FILL_POLICY_SINGLE_INFLIGHT: &str = "single_inflight"; const FILL_POLICY_DUAL_INFLIGHT: &str = "dual_inflight"; -type FillTask = JoinHandle>; +type FillTask = oneshot::Receiver; -struct FillResult { - source: S, - workspace: W, +struct FillWorker { + tx: mpsc::Sender, + task: JoinHandle<()>, +} + +struct FillRequest { + remaining: usize, + reusable_buffers: Vec>, + response: oneshot::Sender, +} + +struct FillResult { result: io::Result>>, queued_buffers: VecDeque>, + reusable_buffers: Vec>, deferred_error: Option, } @@ -88,19 +99,22 @@ where E: ErasureDecodeEngine, { metrics_path: &'static str, + stage_metrics_enabled: bool, fill_policy: FillPolicy, source: Option, - engine: E, + engine: Option, workspace: Option, + worker: Option, output_buf: Vec, output_pos: usize, + reusable_output_bufs: Vec>, prefetched_bufs: VecDeque>, prefetch_error: Option, prefetch_wait_started_at: Option, output_wait_started_at: Option, remaining: usize, // Bounded lookahead controlled by `FillPolicy`. - fill: Option>, + fill: Option, } impl ErasureDecodeReader @@ -140,12 +154,15 @@ where Ok(Self { metrics_path, + stage_metrics_enabled: rustfs_io_metrics::get_stage_metrics_enabled(), fill_policy, source: Some(source), - engine, + engine: Some(engine), workspace: Some(workspace), + worker: None, output_buf: Vec::new(), output_pos: 0, + reusable_output_bufs: Vec::new(), prefetched_bufs: VecDeque::new(), prefetch_error: None, prefetch_wait_started_at: None, @@ -155,6 +172,69 @@ where }) } + fn max_reusable_output_bufs(&self) -> usize { + self.fill_policy.max_inflight() + 1 + } + + fn push_reusable_output_buf(&mut self, mut buf: Vec) { + if buf.capacity() == 0 || self.reusable_output_bufs.len() >= self.max_reusable_output_bufs() { + return; + } + buf.clear(); + self.reusable_output_bufs.push(buf); + } + + fn recycle_drained_output_buf(&mut self) { + if self.output_pos < self.output_buf.len() { + return; + } + + let buf = std::mem::take(&mut self.output_buf); + self.output_pos = 0; + self.push_reusable_output_buf(buf); + } + + fn extend_reusable_output_bufs(&mut self, bufs: Vec>) { + for buf in bufs { + self.push_reusable_output_buf(buf); + } + } + + fn fill_worker_tx(&mut self) -> io::Result> { + if self.worker.is_none() { + let Some(source) = self.source.take() else { + return Err(io::Error::new(ErrorKind::BrokenPipe, "erasure reader source missing")); + }; + let Some(engine) = self.engine.take() else { + self.source = Some(source); + return Err(io::Error::new(ErrorKind::BrokenPipe, "erasure reader engine missing")); + }; + let Some(workspace) = self.workspace.take() else { + self.source = Some(source); + self.engine = Some(engine); + return Err(io::Error::new(ErrorKind::BrokenPipe, "erasure reader workspace missing")); + }; + + let (tx, rx) = mpsc::channel(1); + rustfs_io_metrics::record_get_object_fill_worker_started(self.metrics_path, self.fill_policy.as_str()); + let task = tokio::spawn(run_fill_worker( + source, + engine, + workspace, + self.fill_policy, + self.metrics_path, + self.stage_metrics_enabled, + rx, + )); + self.worker = Some(FillWorker { tx, task }); + } + + self.worker + .as_ref() + .map(|worker| worker.tx.clone()) + .ok_or_else(|| io::Error::new(ErrorKind::BrokenPipe, "erasure reader fill worker missing")) + } + #[cfg(test)] fn new_with_fill_policy( source: S, @@ -168,87 +248,31 @@ where fn poll_fill_result(&mut self, cx: &mut Context<'_>) -> Poll>>> { if self.fill.is_none() { - let Some(mut source) = self.source.take() else { - return Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "erasure reader source missing"))); + let fill_worker_tx = match self.fill_worker_tx() { + Ok(tx) => tx, + Err(err) => return Poll::Ready(Err(err)), }; - let Some(mut workspace) = self.workspace.take() else { - self.source = Some(source); - return Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "erasure reader workspace missing"))); - }; - - let engine = self.engine.clone(); let metrics_path = self.metrics_path; let fill_policy = self.fill_policy; let remaining = self.remaining; - self.fill = Some(tokio::spawn(async move { - let mut queued_buffers = VecDeque::new(); - let mut deferred_error = None; - let fill_stage_start = Instant::now(); - let stripe_read_stage_start = Instant::now(); - let state = source.read_next_stripe().await; - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_STRIPE_READ, - stripe_read_stage_start.elapsed().as_secs_f64(), - ); - let decode_stage_start = Instant::now(); - let result = decode_stripe(metrics_path, &engine, &mut workspace, state, remaining); - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_DECODE, - decode_stage_start.elapsed().as_secs_f64(), - ); - if let Ok(Some(first_buf)) = result.as_ref() { - let mut remaining_after_first = remaining.saturating_sub(first_buf.len()); - for _ in 0..fill_policy.additional_queued_buffers() { - if remaining_after_first == 0 { - break; - } - let stripe_read_stage_start = Instant::now(); - let state = source.read_next_stripe().await; - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_STRIPE_READ, - stripe_read_stage_start.elapsed().as_secs_f64(), - ); - let decode_stage_start = Instant::now(); - let queued_result = decode_stripe(metrics_path, &engine, &mut workspace, state, remaining_after_first); - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_DECODE, - decode_stage_start.elapsed().as_secs_f64(), - ); - match queued_result { - Ok(Some(buf)) => { - remaining_after_first = remaining_after_first.saturating_sub(buf.len()); - queued_buffers.push_back(buf); - } - Ok(None) => { - if remaining_after_first > 0 { - deferred_error = Some(DiskError::LessData.into()); - } - break; - } - Err(err) => { - deferred_error = Some(err); - break; - } - } - } - } - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_FILL, - fill_stage_start.elapsed().as_secs_f64(), - ); - FillResult { - source, - workspace, - result, - queued_buffers, - deferred_error, - } - })); + let reusable_buffers = std::mem::take(&mut self.reusable_output_bufs); + let (response, fill) = oneshot::channel(); + let request = FillRequest { + remaining, + reusable_buffers, + response, + }; + + if let Err(err) = fill_worker_tx.try_send(request) { + self.extend_reusable_output_bufs(err.into_inner().reusable_buffers); + return Poll::Ready(Err(io::Error::new( + ErrorKind::BrokenPipe, + "erasure reader fill worker request queue is closed or full", + ))); + } + + rustfs_io_metrics::record_get_object_fill_started(metrics_path, fill_policy.as_str()); + self.fill = Some(fill); } let fill = self @@ -257,22 +281,20 @@ where .ok_or_else(|| io::Error::new(ErrorKind::BrokenPipe, "erasure reader fill future missing"))?; let fill_result = ready!(Pin::new(fill).poll(cx)); let FillResult { - source, - workspace, result, queued_buffers, + reusable_buffers, deferred_error, } = match fill_result { Ok(result) => result, Err(err) => { self.fill = None; - return Poll::Ready(Err(io::Error::other(format!("erasure reader fill task failed: {err}")))); + return Poll::Ready(Err(io::Error::other(format!("erasure reader fill worker stopped: {err}")))); } }; - self.source = Some(source); - self.workspace = Some(workspace); self.fill = None; + self.extend_reusable_output_bufs(reusable_buffers); if let Some(deferred_error) = deferred_error { self.prefetch_error = Some(deferred_error); } @@ -304,13 +326,15 @@ where return Poll::Ready(Ok(())); } - if self.prefetch_wait_started_at.is_none() { + if self.stage_metrics_enabled && self.prefetch_wait_started_at.is_none() { self.prefetch_wait_started_at = Some(Instant::now()); } let fill = match self.poll_fill_result(cx) { Poll::Ready(result) => { - if let Some(started_at) = self.prefetch_wait_started_at.take() { + if self.stage_metrics_enabled + && let Some(started_at) = self.prefetch_wait_started_at.take() + { rustfs_io_metrics::record_get_object_reader_prefetch_wait( self.metrics_path, started_at.elapsed().as_secs_f64(), @@ -385,14 +409,146 @@ where } } +async fn run_fill_worker( + mut source: S, + engine: E, + mut workspace: E::Workspace, + fill_policy: FillPolicy, + metrics_path: &'static str, + stage_metrics_enabled: bool, + mut rx: mpsc::Receiver, +) where + S: ShardStripeSource + Send + 'static, + E: ErasureDecodeEngine + Send + Sync + 'static, +{ + while let Some(request) = rx.recv().await { + let response = request.response; + let result = run_fill_request(FillRequestWork { + source: &mut source, + engine: &engine, + workspace: &mut workspace, + fill_policy, + metrics_path, + stage_metrics_enabled, + remaining: request.remaining, + reusable_buffers: request.reusable_buffers, + }) + .await; + let _ = response.send(result); + } +} + +struct FillRequestWork<'a, S, E> +where + E: ErasureDecodeEngine, +{ + source: &'a mut S, + engine: &'a E, + workspace: &'a mut E::Workspace, + fill_policy: FillPolicy, + metrics_path: &'static str, + stage_metrics_enabled: bool, + remaining: usize, + reusable_buffers: Vec>, +} + +async fn run_fill_request(work: FillRequestWork<'_, S, E>) -> FillResult +where + S: ShardStripeSource + Send, + E: ErasureDecodeEngine, +{ + let FillRequestWork { + source, + engine, + workspace, + fill_policy, + metrics_path, + stage_metrics_enabled, + remaining, + mut reusable_buffers, + } = work; + let mut queued_buffers = VecDeque::new(); + let mut deferred_error = None; + let fill_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let state = source.read_next_stripe().await; + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_STRIPE_READ, stripe_read_stage_start); + let decode_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let mut output_buf = reusable_buffers.pop().unwrap_or_default(); + let result = + match decode_stripe_into(metrics_path, stage_metrics_enabled, engine, workspace, state, remaining, &mut output_buf) { + Ok(true) => Ok(Some(output_buf)), + Ok(false) => { + reusable_buffers.push(output_buf); + Ok(None) + } + Err(err) => { + reusable_buffers.push(output_buf); + Err(err) + } + }; + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_DECODE, decode_stage_start); + if let Ok(Some(first_buf)) = result.as_ref() { + let mut remaining_after_first = remaining.saturating_sub(first_buf.len()); + for _ in 0..fill_policy.additional_queued_buffers() { + if remaining_after_first == 0 { + break; + } + let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let state = source.read_next_stripe().await; + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_STRIPE_READ, stripe_read_stage_start); + let decode_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + let mut queued_buf = reusable_buffers.pop().unwrap_or_default(); + let queued_result = decode_stripe_into( + metrics_path, + stage_metrics_enabled, + engine, + workspace, + state, + remaining_after_first, + &mut queued_buf, + ); + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_DECODE, decode_stage_start); + match queued_result { + Ok(true) => { + remaining_after_first = remaining_after_first.saturating_sub(queued_buf.len()); + queued_buffers.push_back(queued_buf); + } + Ok(false) => { + reusable_buffers.push(queued_buf); + if remaining_after_first > 0 { + deferred_error = Some(DiskError::LessData.into()); + } + break; + } + Err(err) => { + reusable_buffers.push(queued_buf); + deferred_error = Some(err); + break; + } + } + } + } + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_FILL, fill_stage_start); + + FillResult { + result, + queued_buffers, + reusable_buffers, + deferred_error, + } +} + impl Drop for ErasureDecodeReader where E: ErasureDecodeEngine, { fn drop(&mut self) { - if let Some(fill) = self.fill.take() { + if self.fill.take().is_some() { rustfs_io_metrics::record_get_object_fill_cancelled_on_drop(self.metrics_path, self.fill_policy.as_str()); - fill.abort(); + } + if let Some(worker) = self.worker.take() { + worker.task.abort(); } } } @@ -421,10 +577,12 @@ where let read_buf_remaining_before = buf.remaining(); let output_remaining_before = available.len(); let copy_len = available.len().min(buf.remaining()); - let copy_start = Instant::now(); + let copy_start = get_stage_timer_if_enabled(self.stage_metrics_enabled); buf.put_slice(&available[..copy_len]); self.output_pos += copy_len; - if copy_len > 0 { + if copy_len > 0 + && let Some(copy_start) = copy_start + { rustfs_io_metrics::record_get_object_reader_copy( self.metrics_path, copy_len, @@ -439,6 +597,8 @@ where continue; } + self.recycle_drained_output_buf(); + if let Some(next_buf) = self.prefetched_bufs.pop_front() { rustfs_io_metrics::record_get_object_reader_buffer(self.metrics_path, GET_READER_BUFFER_OUTPUT, next_buf.len()); self.output_buf = next_buf; @@ -458,14 +618,16 @@ where } if self.output_wait_started_at.is_none() { - self.output_wait_started_at = Some(Instant::now()); + self.output_wait_started_at = get_stage_timer_if_enabled(self.stage_metrics_enabled); } let prefetch = match self.poll_prefetch(cx) { Poll::Ready(result) => result, Poll::Pending if buf.filled().len() > filled_before_poll => return Poll::Ready(Ok(())), Poll::Pending => return Poll::Pending, }; - if let Some(started_at) = self.output_wait_started_at.take() { + if self.stage_metrics_enabled + && let Some(started_at) = self.output_wait_started_at.take() + { rustfs_io_metrics::record_get_object_fill_waited_by_output( self.metrics_path, self.fill_policy.as_str(), @@ -480,6 +642,7 @@ where pub(crate) struct SyncErasureDecodeReader { inner: Mutex, metrics_path: &'static str, + stage_metrics_enabled: bool, } impl SyncErasureDecodeReader { @@ -491,6 +654,7 @@ impl SyncErasureDecodeReader { Self { inner: Mutex::new(inner), metrics_path, + stage_metrics_enabled: rustfs_io_metrics::get_stage_metrics_enabled(), } } } @@ -500,97 +664,90 @@ where R: AsyncRead + Unpin + Send, { fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let lock_wait_start = Instant::now(); + let stage_metrics_enabled = self.stage_metrics_enabled; + let lock_wait_start = get_stage_timer_if_enabled(stage_metrics_enabled); let mut inner = match self.inner.lock() { Ok(inner) => { - rustfs_io_metrics::record_get_object_stage_duration( - self.metrics_path, - GET_STAGE_OUTPUT_LOCK_WAIT, - lock_wait_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(self.metrics_path, GET_STAGE_OUTPUT_LOCK_WAIT, lock_wait_start); inner } Err(_) => { - rustfs_io_metrics::record_get_object_stage_duration( - self.metrics_path, - GET_STAGE_OUTPUT_LOCK_WAIT, - lock_wait_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(self.metrics_path, GET_STAGE_OUTPUT_LOCK_WAIT, lock_wait_start); return Poll::Ready(Err(io::Error::other("erasure decode reader lock poisoned"))); } }; - let read_buf_remaining_before = buf.remaining(); - let filled_before = buf.filled().len(); - let poll_start = Instant::now(); + let read_buf_remaining_before = stage_metrics_enabled.then(|| buf.remaining()); + let filled_before = stage_metrics_enabled.then(|| buf.filled().len()); + let poll_start = get_stage_timer_if_enabled(stage_metrics_enabled); let result = Pin::new(&mut *inner).poll_read(cx, buf); - let poll_duration = poll_start.elapsed().as_secs_f64(); - let filled_bytes = buf.filled().len().saturating_sub(filled_before); - let poll_outcome = match &result { - Poll::Ready(Ok(())) if filled_bytes > 0 => GET_READER_POLL_READY_DATA, - Poll::Ready(Ok(())) => GET_READER_POLL_READY_EMPTY, - Poll::Ready(Err(_)) => GET_READER_POLL_READY_ERROR, - Poll::Pending => GET_READER_POLL_PENDING, - }; - rustfs_io_metrics::record_get_object_stage_duration(self.metrics_path, GET_STAGE_OUTPUT_POLL, poll_duration); - rustfs_io_metrics::record_get_object_reader_poll( - self.metrics_path, - poll_outcome, - read_buf_remaining_before, - filled_bytes, - poll_duration, - ); + if let (Some(read_buf_remaining_before), Some(filled_before), Some(poll_start)) = + (read_buf_remaining_before, filled_before, poll_start) + { + let poll_duration = poll_start.elapsed().as_secs_f64(); + let filled_bytes = buf.filled().len().saturating_sub(filled_before); + let poll_outcome = match &result { + Poll::Ready(Ok(())) if filled_bytes > 0 => GET_READER_POLL_READY_DATA, + Poll::Ready(Ok(())) => GET_READER_POLL_READY_EMPTY, + Poll::Ready(Err(_)) => GET_READER_POLL_READY_ERROR, + Poll::Pending => GET_READER_POLL_PENDING, + }; + rustfs_io_metrics::record_get_object_stage_duration(self.metrics_path, GET_STAGE_OUTPUT_POLL, poll_duration); + rustfs_io_metrics::record_get_object_reader_poll( + self.metrics_path, + poll_outcome, + read_buf_remaining_before, + filled_bytes, + poll_duration, + ); + } result } } -fn decode_stripe( +fn decode_stripe_into( metrics_path: &'static str, + stage_metrics_enabled: bool, engine: &E, workspace: &mut E::Workspace, state: StripeReadState, remaining: usize, -) -> io::Result>> + output: &mut Vec, +) -> io::Result where E: ErasureDecodeEngine, { + output.clear(); if state.slots().is_empty() { - return Ok(None); + return Ok(false); } if !state.can_decode() { return Err(DiskError::ErasureReadQuorum.into()); } - let reconstruct_stage_start = Instant::now(); + let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); if state.data_shards_complete(engine.data_shards()) { - rustfs_io_metrics::record_get_object_stage_duration( + rustfs_io_metrics::record_get_object_reconstruct_outcome( metrics_path, - GET_STAGE_RECONSTRUCT, - reconstruct_stage_start.elapsed().as_secs_f64(), + engine.engine_name(), + GET_RECONSTRUCT_OUTCOME_SKIP_DATA_COMPLETE, ); - let emit_stage_start = Instant::now(); - let output = emit_data_shards(&state, engine.data_shards(), engine.block_size(), remaining)?; - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_EMIT, - emit_stage_start.elapsed().as_secs_f64(), - ); - return Ok(Some(output)); + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_RECONSTRUCT, reconstruct_stage_start); + let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + emit_data_shards_into(&state, engine.data_shards(), engine.block_size(), remaining, output)?; + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_EMIT, emit_stage_start); + return Ok(true); } let (mut shards, _errs) = state.into_parts(); - if let Err(err) = engine.reconstruct_into(&mut shards, workspace) { - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_RECONSTRUCT, - reconstruct_stage_start.elapsed().as_secs_f64(), - ); - return Err(err); - } - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_RECONSTRUCT, - reconstruct_stage_start.elapsed().as_secs_f64(), - ); + let reconstruct_outcome = match engine.reconstruct_into(&mut shards, workspace) { + Ok(outcome) => outcome, + Err(err) => { + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_RECONSTRUCT, reconstruct_stage_start); + return Err(err); + } + }; + rustfs_io_metrics::record_get_object_reconstruct_outcome(metrics_path, engine.engine_name(), reconstruct_outcome); + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_RECONSTRUCT, reconstruct_stage_start); if shards.len() < engine.data_shards() { return Err(io::Error::new( @@ -599,8 +756,8 @@ where )); } - let emit_stage_start = Instant::now(); - let mut output = Vec::with_capacity(engine.block_size().min(remaining)); + let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); + reserve_output_capacity(output, engine.block_size().min(remaining)); for shard in shards.iter().take(engine.data_shards()) { if output.len() >= remaining { break; @@ -611,13 +768,32 @@ where let copy_len = shard.len().min(remaining - output.len()); output.extend_from_slice(&shard[..copy_len]); } - rustfs_io_metrics::record_get_object_stage_duration(metrics_path, GET_STAGE_EMIT, emit_stage_start.elapsed().as_secs_f64()); + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_EMIT, emit_stage_start); - Ok(Some(output)) + Ok(true) } fn emit_data_shards(state: &StripeReadState, data_shards: usize, block_size: usize, remaining: usize) -> io::Result> { - let mut output = Vec::with_capacity(block_size.min(remaining)); + let mut output = Vec::new(); + emit_data_shards_into(state, data_shards, block_size, remaining, &mut output)?; + Ok(output) +} + +fn reserve_output_capacity(output: &mut Vec, target_capacity: usize) { + if output.capacity() < target_capacity { + output.reserve(target_capacity - output.capacity()); + } +} + +fn emit_data_shards_into( + state: &StripeReadState, + data_shards: usize, + block_size: usize, + remaining: usize, + output: &mut Vec, +) -> io::Result<()> { + output.clear(); + reserve_output_capacity(output, block_size.min(remaining)); for index in 0..data_shards { if output.len() >= remaining { break; @@ -631,7 +807,7 @@ fn emit_data_shards(state: &StripeReadState, data_shards: usize, block_size: usi let copy_len = shard.len().min(remaining - output.len()); output.extend_from_slice(&shard[..copy_len]); } - Ok(output) + Ok(()) } #[cfg(test)] @@ -641,7 +817,7 @@ mod tests { CodecStreamingDecodeEngine, ErasureDecodeEngine, LegacyEcDecodeEngine, RustfsCodecDecodeEngine, }; use crate::erasure::coding::decode::ParallelReader; - use crate::erasure::coding::{BitrotReader, Erasure}; + use crate::erasure::coding::{BitrotReader, BitrotWriter, Erasure}; use crate::set_disk::shard_source::{ShardSlot, StripeReadState}; use rustfs_utils::HashAlgorithm; use std::collections::VecDeque; @@ -757,6 +933,40 @@ mod tests { decode_all_with_engine(&erasure, engine, data, missing_indexes).await } + async fn bitrot_readers_from_encoded( + erasure: &Erasure, + data: &[u8], + missing_indexes: &[usize], + corrupt_indexes: &[usize], + hash_algo: HashAlgorithm, + ) -> Vec>>>> { + let shard_size = erasure.shard_size(); + let mut readers = Vec::with_capacity(erasure.data_shards + erasure.parity_shards); + for (index, shard) in erasure + .encode_data(data) + .expect("test stripe should encode") + .into_iter() + .enumerate() + { + if missing_indexes.contains(&index) { + readers.push(None); + continue; + } + + let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()); + writer.write(&shard).await.expect("test shard should write with bitrot hash"); + let mut encoded = writer.into_inner().into_inner(); + if corrupt_indexes.contains(&index) { + let data_offset = hash_algo.size(); + if let Some(byte) = encoded.get_mut(data_offset) { + *byte ^= 0x80; + } + } + readers.push(Some(BitrotReader::new(Cursor::new(encoded), shard_size, hash_algo.clone(), false))); + } + readers + } + #[test] fn fill_policy_defaults_to_dual_inflight() { with_var(ENV_RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT, None::<&str>, || { @@ -772,6 +982,41 @@ mod tests { }); } + #[test] + #[serial_test::serial] + fn erasure_decode_reader_caches_stage_metrics_enabled_at_construction() { + let erasure = Erasure::new(4, 2, 32); + let data = b"metrics switch cache"; + + rustfs_io_metrics::set_get_stage_metrics_enabled(false); + let source = source_from_data(&erasure, data, &[]); + let engine = LegacyEcDecodeEngine::new(erasure.clone()); + let reader = ErasureDecodeReader::new_with_fill_policy( + source, + engine, + data.len(), + GET_OBJECT_PATH_CODEC_STREAMING, + FillPolicy::SingleInFlight, + ) + .expect("reader should be constructed"); + assert!(!reader.stage_metrics_enabled); + + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + let source = source_from_data(&erasure, data, &[]); + let engine = LegacyEcDecodeEngine::new(erasure); + let reader = ErasureDecodeReader::new_with_fill_policy( + source, + engine, + data.len(), + GET_OBJECT_PATH_CODEC_STREAMING, + FillPolicy::SingleInFlight, + ) + .expect("reader should be constructed"); + assert!(reader.stage_metrics_enabled); + + rustfs_io_metrics::set_get_stage_metrics_enabled(false); + } + #[tokio::test] async fn erasure_decode_reader_reads_single_stripe() { let erasure = Erasure::new(4, 2, 64); @@ -1012,6 +1257,39 @@ mod tests { assert!(decoded.is_empty()); } + #[tokio::test] + async fn erasure_decode_reader_rustfs_engine_recovers_after_bitrot_source_mismatch() { + let erasure = Erasure::new(4, 2, 64); + let data = (0..64u16) + .map(|value| value.wrapping_mul(29).to_le_bytes()[0]) + .collect::>(); + let readers = bitrot_readers_from_encoded(&erasure, &data, &[0], &[1], HashAlgorithm::HighwayHash256).await; + let source = ParallelReader::new_with_metrics_path_and_reconstruction_verification( + readers, + erasure.clone(), + 0, + data.len(), + Some(GET_OBJECT_PATH_CODEC_STREAMING), + ); + let engine = RustfsCodecDecodeEngine::new(&erasure).expect("engine should be created"); + let mut reader = ErasureDecodeReader::new_with_fill_policy( + source, + engine, + data.len(), + GET_OBJECT_PATH_CODEC_STREAMING, + FillPolicy::SingleInFlight, + ) + .expect("reader should be constructed"); + let mut decoded = Vec::new(); + + reader + .read_to_end(&mut decoded) + .await + .expect("rustfs reader should reconstruct from clean shards after bitrot mismatch"); + + assert_eq!(decoded, data); + } + #[tokio::test] async fn erasure_decode_reader_verifying_parallel_source_rejects_inconsistent_reconstruction_sources() { const DATA_SHARDS: usize = 2; @@ -1176,4 +1454,74 @@ mod tests { .await .expect("reader should start reading the next stripe before the current output buffer is fully consumed"); } + + #[tokio::test] + async fn erasure_decode_reader_reuses_output_buffers_after_drain() { + let erasure = Erasure::new(4, 2, 32); + let data = (0..128u16) + .map(|value| value.wrapping_mul(5).to_le_bytes()[0]) + .collect::>(); + let source = source_from_data(&erasure, &data, &[]); + let engine = LegacyEcDecodeEngine::new(erasure.clone()); + let mut reader = ErasureDecodeReader::new_with_fill_policy( + source, + engine, + data.len(), + GET_OBJECT_PATH_CODEC_STREAMING, + FillPolicy::SingleInFlight, + ) + .expect("reader should be constructed"); + let mut decoded = Vec::new(); + + reader + .read_to_end(&mut decoded) + .await + .expect("reader should decode all stripes"); + + assert_eq!(decoded, data); + assert!(reader.reusable_output_bufs.len() <= reader.max_reusable_output_bufs()); + assert!( + reader + .reusable_output_bufs + .iter() + .any(|buf| buf.capacity() >= erasure.block_size), + "drained stripe output buffers should be available for reuse" + ); + } + + #[tokio::test] + async fn erasure_decode_reader_reuses_single_fill_worker_across_fills() { + let erasure = Erasure::new(4, 2, 32); + let data = (0..128u16) + .map(|value| value.wrapping_mul(7).to_le_bytes()[0]) + .collect::>(); + let source = source_from_data(&erasure, &data, &[]); + let engine = LegacyEcDecodeEngine::new(erasure); + let mut reader = ErasureDecodeReader::new_with_fill_policy( + source, + engine, + data.len(), + GET_OBJECT_PATH_CODEC_STREAMING, + FillPolicy::SingleInFlight, + ) + .expect("reader should be constructed"); + let mut decoded = Vec::new(); + let mut first_read = [0u8; 1]; + + let read = reader.read(&mut first_read).await.expect("first read should succeed"); + decoded.extend_from_slice(&first_read[..read]); + + assert!(reader.worker.is_some()); + assert!(reader.source.is_none()); + assert!(reader.engine.is_none()); + assert!(reader.workspace.is_none()); + + reader + .read_to_end(&mut decoded) + .await + .expect("reader should continue using the fill worker"); + + assert_eq!(decoded, data); + assert!(reader.worker.is_some()); + } } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 921d85d0e..b3722938b 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -355,6 +355,9 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT: u32 = 100; const ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE"; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE: bool = false; +const ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS: &str = "RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS"; +const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS: usize = 256; + // --- Metadata Early-Stop Configuration --- const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE"; @@ -472,13 +475,43 @@ fn is_get_codec_streaming_enabled() -> bool { /// When enabled, multipart objects use per-part codec streaming /// instead of falling back to the legacy duplex path. fn is_codec_streaming_multipart_enabled() -> bool { - static CACHED: OnceLock = OnceLock::new(); - *CACHED.get_or_init(|| { + #[cfg(test)] + { rustfs_utils::get_env_bool( ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, ) - }) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_bool( + ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, + ) + }) + } +} + +fn get_codec_streaming_multipart_max_parts() -> usize { + #[cfg(test)] + { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS, + ) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_usize( + ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS, + DEFAULT_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS, + ) + }) + } } /// Check if metadata early-stop is enabled (base flag). @@ -506,10 +539,17 @@ fn is_version_early_stop_enabled() -> bool { // --- Rollout Percentage Functions --- fn get_codec_streaming_rollout_pct() -> u32 { - static CACHED: OnceLock = OnceLock::new(); - *CACHED.get_or_init(|| { + #[cfg(test)] + { rustfs_utils::get_env_u32(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, DEFAULT_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT) - }) + } + #[cfg(not(test))] + { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_u32(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, DEFAULT_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT) + }) + } } fn get_metadata_early_stop_rollout_pct() -> u32 { @@ -545,7 +585,6 @@ fn is_optimization_enabled_for_request(base_enabled: bool, rollout_pct: u32, buc (hash as u32) < rollout_pct } - /// Should this specific request use codec streaming? pub fn should_use_codec_streaming(bucket: &str, object: &str) -> bool { let base = is_get_codec_streaming_enabled(); @@ -633,6 +672,7 @@ impl GetCodecStreamingRollout { enum GetCodecStreamingFallbackReason { Disabled, RolloutNotOptedIn, + RolloutPctNotSelected, BodyCompatibilityUnconfirmed, HeaderCompatibilityUnconfirmed, LockOptimizationDisabled, @@ -644,6 +684,7 @@ enum GetCodecStreamingFallbackReason { Multipart, InvalidMinSize, ReadQuorumNotSafe, + MultipartPartLimit, } impl GetCodecStreamingFallbackReason { @@ -651,6 +692,7 @@ impl GetCodecStreamingFallbackReason { match self { Self::Disabled => "disabled", Self::RolloutNotOptedIn => "rollout_not_opted_in", + Self::RolloutPctNotSelected => "rollout_pct_not_selected", Self::BodyCompatibilityUnconfirmed => "body_compatibility_unconfirmed", Self::HeaderCompatibilityUnconfirmed => "header_compatibility_unconfirmed", Self::LockOptimizationDisabled => "lock_optimization_disabled", @@ -662,6 +704,7 @@ impl GetCodecStreamingFallbackReason { Self::Multipart => "multipart", Self::InvalidMinSize => "invalid_min_size", Self::ReadQuorumNotSafe => "read_quorum_not_safe", + Self::MultipartPartLimit => "multipart_part_limit", } } } @@ -732,6 +775,8 @@ fn classify_get_codec_streaming_object_class( } fn get_codec_streaming_reader_gate( + bucket: &str, + object: &str, range: &Option, object_info: &ObjectInfo, fi: &FileInfo, @@ -751,6 +796,12 @@ fn get_codec_streaming_reader_gate( decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::RolloutNotOptedIn), }; } + if !should_use_codec_streaming(bucket, object) { + return GetCodecStreamingGate { + object_class, + decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::RolloutPctNotSelected), + }; + } if !is_get_codec_streaming_body_compat_confirmed() { return GetCodecStreamingGate { object_class, @@ -807,10 +858,18 @@ fn get_codec_streaming_reader_gate( }; } if object_class == GetCodecStreamingObjectClass::Multipart { - return GetCodecStreamingGate { - object_class, - decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Multipart), - }; + if !is_codec_streaming_multipart_enabled() { + return GetCodecStreamingGate { + object_class, + decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Multipart), + }; + } + if fi.parts.len() > get_codec_streaming_multipart_max_parts() { + return GetCodecStreamingGate { + object_class, + decision: GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::MultipartPartLimit), + }; + } } GetCodecStreamingGate { @@ -1557,7 +1616,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { return Ok(reader); } - let codec_streaming_gate = get_codec_streaming_reader_gate(&range, &object_info, &fi, lock_optimization_enabled); + let codec_streaming_gate = + get_codec_streaming_reader_gate(bucket, object, &range, &object_info, &fi, lock_optimization_enabled); if object_info.is_remote() { if let GetCodecStreamingDecision::Fallback(reason) = codec_streaming_gate.decision { diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 6c4158de1..5ab0d1d7f 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -22,7 +22,8 @@ use crate::diagnostics::get::{ GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_DECODE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GetObjectFailureReason, - classify_disk_error, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, + classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, + record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::create_deferred_bitrot_reader; @@ -31,13 +32,14 @@ use futures::stream::{FuturesUnordered, StreamExt}; use metrics::counter; use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE}; use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, future::Future, pin::Pin, sync::OnceLock, + task::{Context, Poll}, time::{Duration, Instant}, }; -use tokio::io::AsyncRead; +use tokio::io::{AsyncRead, ReadBuf}; use tokio::sync::RwLock; use tokio::task::JoinSet; @@ -53,6 +55,39 @@ pub(super) enum GetCodecStreamingReaderBuildOutcome { Fallback(GetCodecStreamingFallbackReason), } +struct MultipartCodecStreamingReader { + readers: VecDeque>, +} + +impl MultipartCodecStreamingReader { + fn new(readers: Vec>) -> Self { + Self { + readers: VecDeque::from(readers), + } + } +} + +impl AsyncRead for MultipartCodecStreamingReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + loop { + let Some(reader) = self.readers.front_mut() else { + return Poll::Ready(Ok(())); + }; + let filled_before = buf.filled().len(); + match Pin::new(reader).poll_read(cx, buf) { + Poll::Ready(Ok(())) if buf.filled().len() == filled_before => { + self.readers.pop_front(); + } + result => return result, + } + } + } +} + pub(super) fn codec_streaming_reader_setup_fallback_reason(missing_shards: usize) -> Option { (missing_shards > 0).then_some(GetCodecStreamingFallbackReason::ReadQuorumNotSafe) } @@ -2252,9 +2287,6 @@ impl SetDisks { skip_verify_bitrot: bool, ) -> Result { let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi); - if fi.parts.len() != 1 { - return Err(Error::other("codec streaming reader only supports single-part plain objects")); - } let erasure = crate::erasure::coding::Erasure::new_with_options( fi.erasure.data_blocks, @@ -2262,14 +2294,94 @@ impl SetDisks { fi.erasure.block_size, fi.uses_legacy_checksum, ); - let part = &fi.parts[0]; - let part_number = part.number; - let part_size = part.size; - let part_length = usize::try_from(fi.size).map_err(|_| Error::other("codec streaming reader object size is invalid"))?; + + if fi.parts.len() == 1 { + let part = &fi.parts[0]; + let part_length = + usize::try_from(fi.size).map_err(|_| Error::other("codec streaming reader object size is invalid"))?; + return Self::build_codec_streaming_part_reader( + bucket, + object, + fi, + &files, + &disks, + &erasure, + part.number, + 0, + part_length, + part.size, + skip_verify_bitrot, + ) + .await; + } + + if !is_codec_streaming_multipart_enabled() { + return Ok(GetCodecStreamingReaderBuildOutcome::Fallback(GetCodecStreamingFallbackReason::Multipart)); + } + if fi.parts.len() > get_codec_streaming_multipart_max_parts() { + return Ok(GetCodecStreamingReaderBuildOutcome::Fallback( + GetCodecStreamingFallbackReason::MultipartPartLimit, + )); + } + + let object_length = + usize::try_from(fi.size).map_err(|_| Error::other("codec streaming reader object size is invalid"))?; + let mut total_part_size = 0usize; + for part in &fi.parts { + total_part_size = total_part_size + .checked_add(part.size) + .ok_or_else(|| Error::other("codec streaming multipart part sizes overflow"))?; + } + if total_part_size != object_length { + return Err(Error::other("codec streaming multipart part sizes do not match object size")); + } + + let mut readers = Vec::with_capacity(fi.parts.len()); + for part in &fi.parts { + match Self::build_codec_streaming_part_reader( + bucket, + object, + fi, + &files, + &disks, + &erasure, + part.number, + 0, + part.size, + part.size, + skip_verify_bitrot, + ) + .await? + { + GetCodecStreamingReaderBuildOutcome::Reader(reader) => readers.push(reader), + GetCodecStreamingReaderBuildOutcome::Fallback(reason) => { + return Ok(GetCodecStreamingReaderBuildOutcome::Fallback(reason)); + } + } + } + + Ok(GetCodecStreamingReaderBuildOutcome::Reader(Box::new(MultipartCodecStreamingReader::new( + readers, + )))) + } + + #[allow(clippy::too_many_arguments)] + async fn build_codec_streaming_part_reader( + bucket: &str, + object: &str, + fi: &FileInfo, + files: &[FileInfo], + disks: &[Option], + erasure: &crate::erasure::coding::Erasure, + part_number: usize, + part_offset: usize, + part_length: usize, + part_size: usize, + skip_verify_bitrot: bool, + ) -> Result { if part_length > part_size { return Err(Error::other("codec streaming reader part length exceeds part size")); } - let checksum_info = fi.erasure.get_checksum_info(part_number); let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { @@ -2278,21 +2390,24 @@ impl SetDisks { checksum_info.algorithm }; let use_mmap_read = rustfs_utils::get_env_bool(ENV_OBJECT_ZERO_COPY_ENABLE, DEFAULT_OBJECT_ZERO_COPY_ENABLE); - let till_offset = erasure.shard_file_offset(0, part_length, part_size); + let till_offset = erasure.shard_file_offset(part_offset, part_length, part_size); + let read_offset = (part_offset / erasure.block_size) * erasure.shard_size(); + let read_length = till_offset.saturating_sub(read_offset); - let reader_setup_stage_start = Instant::now(); + let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let reader_setup_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let read_costs = disks .iter() .map(|disk| shard_read_cost_for_disk(disk.as_ref())) .collect::>(); let reader_setup = create_bitrot_readers_until_quorum( - &files, - &disks, + files, + disks, bucket, object, part_number, - 0, - till_offset, + read_offset, + read_length, erasure.shard_size(), checksum_algo, skip_verify_bitrot, @@ -2303,11 +2418,7 @@ impl SetDisks { ) .await; let metrics_path = get_codec_streaming_metrics_path(); - rustfs_io_metrics::record_get_object_stage_duration( - metrics_path, - GET_STAGE_READER_SETUP, - reader_setup_stage_start.elapsed().as_secs_f64(), - ); + record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_SETUP, reader_setup_stage_start); let available_shards = reader_setup.available_shards(); if available_shards < erasure.data_shards { @@ -2330,12 +2441,12 @@ impl SetDisks { crate::erasure::coding::decode::ParallelReader::new_with_metrics_path_read_costs_and_reconstruction_verification( readers, erasure.clone(), - 0, + part_offset, part_size, Some(metrics_path), read_costs, ); - let engine = build_get_codec_streaming_decode_engine(erasure)?; + let engine = build_get_codec_streaming_decode_engine(erasure.clone())?; let reader = crate::erasure::coding::decode_reader::ErasureDecodeReader::new_with_metrics_path( source, engine, @@ -2706,6 +2817,15 @@ mod metadata_cache_tests { #[cfg(test)] mod tests { use super::*; + use std::io::Cursor; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use tokio::io::AsyncReadExt; + + const CODEC_STREAMING_TEST_BUCKET: &str = "bucket"; + const CODEC_STREAMING_TEST_OBJECT: &str = "object"; fn metadata_fanout_test_fileinfo(object: &str) -> FileInfo { let mut fi = FileInfo::new(object, 2, 2); @@ -3257,7 +3377,147 @@ mod tests { } fn codec_streaming_test_object_info(fi: &FileInfo) -> ObjectInfo { - ObjectInfo::from_file_info(fi, "bucket", "object", false) + ObjectInfo::from_file_info(fi, CODEC_STREAMING_TEST_BUCKET, CODEC_STREAMING_TEST_OBJECT, false) + } + + fn codec_streaming_reader_gate_for_test( + range: &Option, + object_info: &ObjectInfo, + fi: &FileInfo, + lock_optimization_enabled: bool, + ) -> GetCodecStreamingGate { + get_codec_streaming_reader_gate( + CODEC_STREAMING_TEST_BUCKET, + CODEC_STREAMING_TEST_OBJECT, + range, + object_info, + fi, + lock_optimization_enabled, + ) + } + + #[tokio::test] + async fn multipart_codec_streaming_reader_reads_parts_in_order() { + let readers: Vec> = vec![ + Box::new(Cursor::new(b"hello ".to_vec())), + Box::new(Cursor::new(b"multipart".to_vec())), + ]; + let mut reader = MultipartCodecStreamingReader::new(readers); + let mut output = Vec::new(); + + reader + .read_to_end(&mut output) + .await + .expect("multipart codec reader should read all parts"); + + assert_eq!(output, b"hello multipart"); + } + + struct OneByteAsyncReader { + data: Vec, + position: usize, + } + + impl OneByteAsyncReader { + fn new(data: &'static [u8]) -> Self { + Self { + data: data.to_vec(), + position: 0, + } + } + } + + impl AsyncRead for OneByteAsyncReader { + fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.position >= self.data.len() || buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + buf.put_slice(&self.data[self.position..self.position + 1]); + self.position += 1; + Poll::Ready(Ok(())) + } + } + + #[tokio::test] + async fn multipart_codec_streaming_reader_crosses_part_boundaries_with_short_reads() { + let readers: Vec> = vec![ + Box::new(OneByteAsyncReader::new(b"abc")), + Box::new(OneByteAsyncReader::new(b"def")), + ]; + let mut reader = MultipartCodecStreamingReader::new(readers); + let mut first = [0u8; 5]; + let mut second = Vec::new(); + + reader + .read_exact(&mut first) + .await + .expect("multipart codec reader should cross part boundaries"); + reader + .read_to_end(&mut second) + .await + .expect("multipart codec reader should drain the final part"); + + assert_eq!(&first, b"abcde"); + assert_eq!(second, b"f"); + } + + struct DropCountingReader { + data: Vec, + position: usize, + drops: Arc, + } + + impl DropCountingReader { + fn new(data: &'static [u8], drops: Arc) -> Self { + Self { + data: data.to_vec(), + position: 0, + drops, + } + } + } + + impl AsyncRead for DropCountingReader { + fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.position >= self.data.len() || buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + let available = self.data.len() - self.position; + let count = available.min(buf.remaining()); + let end = self.position + count; + buf.put_slice(&self.data[self.position..end]); + self.position = end; + Poll::Ready(Ok(())) + } + } + + impl Drop for DropCountingReader { + fn drop(&mut self) { + self.drops.fetch_add(1, Ordering::SeqCst); + } + } + + #[tokio::test] + async fn multipart_codec_streaming_reader_drops_remaining_parts_on_abort() { + let drops = Arc::new(AtomicUsize::new(0)); + { + let readers: Vec> = vec![ + Box::new(DropCountingReader::new(b"abc", Arc::clone(&drops))), + Box::new(DropCountingReader::new(b"def", Arc::clone(&drops))), + ]; + let mut reader = MultipartCodecStreamingReader::new(readers); + let mut first = [0u8; 1]; + + reader + .read_exact(&mut first) + .await + .expect("multipart codec reader should support partial reads"); + assert_eq!(&first, b"a"); + } + + assert_eq!(drops.load(Ordering::SeqCst), 2); } fn inline_reader_setup_fileinfo(data: Option<&'static [u8]>) -> FileInfo { @@ -3405,12 +3665,12 @@ mod tests { let fi = codec_streaming_test_fileinfo(1024, 1); let object_info = codec_streaming_test_object_info(&fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, GetCodecStreamingDecision::Use ); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, false).decision, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, false).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::LockOptimizationDisabled) ); @@ -3420,14 +3680,14 @@ mod tests { end: 1, }); assert_eq!( - get_codec_streaming_reader_gate(&range, &object_info, &fi, true).decision, + codec_streaming_reader_gate_for_test(&range, &object_info, &fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Range) ); let multipart_fi = codec_streaming_test_fileinfo(1024, 2); let multipart_object_info = codec_streaming_test_object_info(&multipart_fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &multipart_object_info, &multipart_fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &multipart_object_info, &multipart_fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Multipart) ); @@ -3437,7 +3697,7 @@ mod tests { .insert("x-amz-server-side-encryption".to_string(), "AES256".to_string()); let encrypted = codec_streaming_test_object_info(&encrypted_fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &encrypted, &encrypted_fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &encrypted, &encrypted_fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Encrypted) ); @@ -3445,14 +3705,14 @@ mod tests { insert_str(&mut compressed_fi.metadata, SUFFIX_COMPRESSION, "lz4".to_string()); let compressed = codec_streaming_test_object_info(&compressed_fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &compressed, &compressed_fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &compressed, &compressed_fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Compressed) ); let small_fi = codec_streaming_test_fileinfo(0, 1); let small_object_info = codec_streaming_test_object_info(&small_fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &small_object_info, &small_fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &small_object_info, &small_fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize) ); @@ -3460,7 +3720,7 @@ mod tests { remote_fi.transition_status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(); let remote = codec_streaming_test_object_info(&remote_fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &remote, &remote_fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &remote, &remote_fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Remote) ); }, @@ -3498,13 +3758,86 @@ mod tests { let object_info = codec_streaming_test_object_info(&fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Disabled) ); }, ); } + #[test] + fn codec_streaming_reader_gate_allows_multipart_when_explicitly_enabled() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + ], + || { + let fi = codec_streaming_test_fileinfo(1024, 2); + let object_info = codec_streaming_test_object_info(&fi); + let gate = codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true); + + assert_eq!(gate.object_class, GetCodecStreamingObjectClass::Multipart); + assert_eq!(gate.decision, GetCodecStreamingDecision::Use); + }, + ); + } + + #[test] + fn codec_streaming_reader_gate_keeps_multipart_default_off() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, None), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + ], + || { + let fi = codec_streaming_test_fileinfo(1024, 2); + let object_info = codec_streaming_test_object_info(&fi); + let gate = codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true); + + assert_eq!(gate.object_class, GetCodecStreamingObjectClass::Multipart); + assert_eq!( + gate.decision, + GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Multipart) + ); + }, + ); + } + + #[test] + fn codec_streaming_reader_gate_limits_multipart_part_count() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS, Some("1")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + ], + || { + let fi = codec_streaming_test_fileinfo(1024, 2); + let object_info = codec_streaming_test_object_info(&fi); + let gate = codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true); + + assert_eq!(gate.object_class, GetCodecStreamingObjectClass::Multipart); + assert_eq!( + gate.decision, + GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::MultipartPartLimit) + ); + }, + ); + } + #[test] fn codec_streaming_decode_engine_builder_selects_rustfs() { temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS), || { @@ -3536,6 +3869,10 @@ mod tests { fn codec_streaming_fallback_metric_labels_are_stable() { assert_eq!(GetCodecStreamingFallbackReason::Disabled.as_str(), "disabled"); assert_eq!(GetCodecStreamingFallbackReason::RolloutNotOptedIn.as_str(), "rollout_not_opted_in"); + assert_eq!( + GetCodecStreamingFallbackReason::RolloutPctNotSelected.as_str(), + "rollout_pct_not_selected" + ); assert_eq!( GetCodecStreamingFallbackReason::BodyCompatibilityUnconfirmed.as_str(), "body_compatibility_unconfirmed" @@ -3556,6 +3893,7 @@ mod tests { assert_eq!(GetCodecStreamingFallbackReason::Multipart.as_str(), "multipart"); assert_eq!(GetCodecStreamingFallbackReason::InvalidMinSize.as_str(), "invalid_min_size"); assert_eq!(GetCodecStreamingFallbackReason::ReadQuorumNotSafe.as_str(), "read_quorum_not_safe"); + assert_eq!(GetCodecStreamingFallbackReason::MultipartPartLimit.as_str(), "multipart_part_limit"); assert_eq!(GetCodecStreamingObjectClass::PlainSinglePart.as_str(), "plain_single_part"); assert_eq!(GetCodecStreamingObjectClass::Range.as_str(), "range"); assert_eq!(GetCodecStreamingObjectClass::Encrypted.as_str(), "encrypted"); @@ -3579,7 +3917,7 @@ mod tests { let object_info = codec_streaming_test_object_info(&fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Disabled) ); }, @@ -3601,7 +3939,7 @@ mod tests { let object_info = codec_streaming_test_object_info(&fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::RolloutNotOptedIn) ); }, @@ -3620,7 +3958,7 @@ mod tests { let object_info = codec_streaming_test_object_info(&fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BodyCompatibilityUnconfirmed) ); }, @@ -3639,13 +3977,56 @@ mod tests { let object_info = codec_streaming_test_object_info(&fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, true).decision, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::HeaderCompatibilityUnconfirmed) ); }, ); } + #[test] + fn codec_streaming_reader_gate_honors_rollout_percentage() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("0")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + ], + || { + let fi = codec_streaming_test_fileinfo(1024, 1); + let object_info = codec_streaming_test_object_info(&fi); + + assert_eq!( + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, + GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::RolloutPctNotSelected) + ); + }, + ); + + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("100")), + (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + ], + || { + let fi = codec_streaming_test_fileinfo(1024, 1); + let object_info = codec_streaming_test_object_info(&fi); + + assert_eq!( + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision, + GetCodecStreamingDecision::Use + ); + }, + ); + } + #[test] fn codec_streaming_reader_gate_records_object_classes() { temp_env::with_vars( @@ -3660,7 +4041,7 @@ mod tests { let fi = codec_streaming_test_fileinfo(1024, 1); let object_info = codec_streaming_test_object_info(&fi); assert_eq!( - get_codec_streaming_reader_gate(&None, &object_info, &fi, true).object_class, + codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).object_class, GetCodecStreamingObjectClass::PlainSinglePart ); @@ -3670,7 +4051,7 @@ mod tests { end: 1, }); assert_eq!( - get_codec_streaming_reader_gate(&range, &object_info, &fi, true).object_class, + codec_streaming_reader_gate_for_test(&range, &object_info, &fi, true).object_class, GetCodecStreamingObjectClass::Range ); }, diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index b20ecc6e7..9da20255a 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -555,6 +555,21 @@ pub fn record_get_object_reconstruct_duration(path: &'static str, duration_secs: record_get_object_stage_duration(path, "reconstruct", duration_secs); } +/// Record the reconstruction outcome for a GetObject reader path. +#[inline(always)] +pub fn record_get_object_reconstruct_outcome(path: &'static str, engine: &'static str, outcome: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!( + "rustfs_io_get_object_reconstruct_outcome_total", + "path" => path, + "engine" => engine, + "outcome" => outcome + ) + .increment(1); +} + /// Record GetObject emit duration. #[inline(always)] pub fn record_get_object_emit_duration(path: &'static str, duration_secs: f64) { @@ -715,6 +730,24 @@ pub fn record_get_object_fill_queued(path: &'static str, policy: &'static str, q histogram!("rustfs_io_get_object_fill_queued", "path" => path, "policy" => policy).record(usize_to_f64(queued)); } +/// Record that a background fill task was started for a GetObject reader path. +#[inline(always)] +pub fn record_get_object_fill_started(path: &'static str, policy: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_fill_started_total", "path" => path, "policy" => policy).increment(1); +} + +/// Record that a persistent fill worker was started for a GetObject reader path. +#[inline(always)] +pub fn record_get_object_fill_worker_started(path: &'static str, policy: &'static str) { + if !get_stage_metrics_enabled() { + return; + } + counter!("rustfs_io_get_object_fill_worker_started_total", "path" => path, "policy" => policy).increment(1); +} + /// Record that a fill completed while the current output buffer still had unread bytes. #[inline(always)] pub fn record_get_object_fill_completed_before_output_drained(path: &'static str, policy: &'static str) { @@ -1675,6 +1708,7 @@ mod tests { record_get_object_first_shard_read_duration("codec_streaming", 0.004); record_get_object_bitrot_verify_duration("codec_streaming", 0.005); record_get_object_reconstruct_duration("codec_streaming", 0.006); + record_get_object_reconstruct_outcome("codec_streaming", "legacy", "skip_data_complete"); record_get_object_emit_duration("codec_streaming", 0.007); record_get_object_first_byte_latency("s3_handler", 0.008); record_get_object_full_body_latency("s3_handler", 0.009); @@ -1696,6 +1730,8 @@ mod tests { #[test] fn test_record_get_object_fill_metrics() { record_get_object_fill_queued("codec_streaming", "single_inflight", 1); + record_get_object_fill_started("codec_streaming", "single_inflight"); + record_get_object_fill_worker_started("codec_streaming", "single_inflight"); record_get_object_fill_completed_before_output_drained("codec_streaming", "single_inflight"); record_get_object_fill_waited_by_output("codec_streaming", "single_inflight", 0.0003); record_get_object_fill_cancelled_on_drop("codec_streaming", "single_inflight"); @@ -1770,6 +1806,7 @@ mod tests { record_get_object_first_shard_read_duration("codec_streaming", 0.004); record_get_object_bitrot_verify_duration("codec_streaming", 0.005); record_get_object_reconstruct_duration("codec_streaming", 0.006); + record_get_object_reconstruct_outcome("codec_streaming", "legacy", "legacy_called"); record_get_object_emit_duration("codec_streaming", 0.007); record_get_object_first_byte_latency("s3_handler", 0.008); record_get_object_full_body_latency("s3_handler", 0.009); @@ -1813,6 +1850,7 @@ mod tests { record_get_object_first_shard_read_duration("codec_streaming", 0.004); record_get_object_bitrot_verify_duration("codec_streaming", 0.005); record_get_object_reconstruct_duration("codec_streaming", 0.006); + record_get_object_reconstruct_outcome("codec_streaming", "rustfs", "rustfs_called"); record_get_object_emit_duration("codec_streaming", 0.007); record_get_object_first_byte_latency("s3_handler", 0.008); record_get_object_full_body_latency("s3_handler", 0.009); diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 7ba3bb6c2..7a58346d9 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -61,6 +61,11 @@ use uuid::Uuid; static GLOBAL_ENV: OnceLock<(Vec, Arc)> = OnceLock::new(); static INIT: Once = Once::new(); const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15); +const ENV_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_ENABLE"; +const ENV_GET_CODEC_STREAMING_ROLLOUT: &str = "RUSTFS_GET_CODEC_STREAMING_ROLLOUT"; +const ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED"; +const ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED"; +const ENV_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE"; fn init_tracing() { INIT.call_once(|| {}); @@ -405,6 +410,31 @@ where } } +async fn with_get_codec_streaming_remote_probe_env(test_fn: F) +where + F: FnOnce() -> Fut, + Fut: std::future::Future, +{ + let metrics_was_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + let result = std::panic::AssertUnwindSafe(temp_env::async_with_vars( + [ + (ENV_GET_CODEC_STREAMING_ENABLE, Some("true")), + (ENV_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), + (ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (ENV_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + ], + test_fn(), + )) + .catch_unwind() + .await; + rustfs_io_metrics::set_get_stage_metrics_enabled(metrics_was_enabled); + if let Err(err) = result { + std::panic::resume_unwind(err); + } +} + async fn wait_for_remote_absence(backend: &MockWarmBackend, object: &str, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; @@ -911,6 +941,66 @@ async fn complete_multipart_upload_transitions_immediately_via_usecase() { assert!(backend.objects.lock().await.contains_key(&info.transitioned_object.name)); } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[ignore = "requires isolated global object layer state"] +async fn get_transitioned_object_uses_remote_codec_fallback_path() { + with_get_codec_streaming_remote_probe_env(|| async { + let (_disk_paths, ecstore) = setup_test_env().await; + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&tier_name).await; + + let bucket = format!("test-api-get-remote-{}", &Uuid::new_v4().simple().to_string()[..8]); + let object = "test/remote-codec-fallback.txt"; + let payload: Vec = (0..(1024 * 1024)) + .map(|index| u8::try_from(index % 251).expect("payload byte fits in u8")) + .collect(); + + create_test_bucket(&ecstore, bucket.as_str()).await; + set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name) + .await + .expect("Failed to set lifecycle configuration"); + + let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await; + let transition_opts = ObjectOptions { + transition: lifecycle::lifecycle_contract::TransitionOptions { + status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(), + tier: tier_name.clone(), + etag: uploaded.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: uploaded.version_id.map(|version| version.to_string()), + versioned: true, + mod_time: uploaded.mod_time, + ..Default::default() + }; + ecstore + .transition_object(bucket.as_str(), object, &transition_opts) + .await + .expect("Failed to transition object directly"); + + let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT) + .await + .expect("object should transition before remote fallback GET"); + + assert_eq!(transitioned.transitioned_object.status, "complete"); + assert_eq!(transitioned.transitioned_object.tier, tier_name); + assert!(!transitioned.transitioned_object.name.is_empty()); + assert!( + backend + .objects + .lock() + .await + .contains_key(&transitioned.transitioned_object.name) + ); + + let actual = read_object_bytes(&ecstore, bucket.as_str(), object).await; + assert_eq!(actual, payload); + }) + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial] #[ignore = "requires isolated global object layer state"] diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index fb403f835..d3975fb47 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -382,7 +382,9 @@ pub(crate) mod bucket { pub(crate) const TRANSITION_COMPLETE: &str = crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; - + #[cfg(test)] + pub(crate) const TRANSITION_PENDING: &str = + crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::TRANSITION_PENDING; pub(crate) fn expected_expiry_time(mod_time: time::OffsetDateTime, days: i32) -> time::OffsetDateTime { crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, days) } diff --git a/scripts/run_get_codec_streaming_smoke.sh b/scripts/run_get_codec_streaming_smoke.sh index 48d99e11b..cba5a89bd 100755 --- a/scripts/run_get_codec_streaming_smoke.sh +++ b/scripts/run_get_codec_streaming_smoke.sh @@ -26,9 +26,24 @@ ROUND_COOLDOWN_SECS=20 MODE="both" CODEC_ENGINES="legacy" CODEC_MAX_INFLIGHT=1 +CODEC_MULTIPART="off" +CODEC_MULTIPART_MAX_PARTS=256 METADATA_EARLY_STOP="off" SHARD_LOCALITY_PREFERENCE="off" OUTPUT_HANDOFF_ATTRIBUTION=false +DIAGNOSTIC_METRICS=false +DIAGNOSTIC_METRICS_URL="http://127.0.0.1:8889/metrics" +DIAGNOSTIC_PROMETHEUS_QUERY_URL="" +DIAGNOSTIC_PROMETHEUS_QUERY='{__name__=~"rustfs_io_get_object_.*"}' +DIAGNOSTIC_METRICS_SETTLE_SECS="${RUSTFS_DIAGNOSTIC_METRICS_SETTLE_SECS:-2}" +DIAGNOSTIC_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}" +DIAGNOSTIC_OBS_METRIC_ENDPOINT="${RUSTFS_OBS_METRIC_ENDPOINT:-}" +DIAGNOSTIC_OBS_METER_INTERVAL="${RUSTFS_OBS_METER_INTERVAL:-1}" +DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX="${RUSTFS_OBS_SERVICE_NAME:-RustFS-get-codec}" +SERVICE_METRIC_PREFIX="rustfs_io_get_object_" +COMPRESSED_FALLBACK_PROBE=false +COMPRESSED_PROBE_EXTENSION=".compressed-probe.txt" +COMPRESSED_PROBE_MIME_TYPE="text/plain" OUT_DIR="" RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs" WARP_BIN="warp" @@ -63,7 +78,34 @@ Core options: --shard-locality-preference Enable shard locality preference env (default: off) --codec-max-inflight RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT (default: 1) + --codec-multipart Enable multipart codec streaming opt-in for codec profiles + (default: off) + --codec-multipart-max-parts + RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS + (default: 256) --handoff-attribution Enable output handoff attribution metrics + --diagnostic-metrics Enable observability metrics export and capture server-side metrics + --diagnostic-metrics-url Prometheus scrape URL for diagnostic captures + (default: http://127.0.0.1:8889/metrics) + --diagnostic-prometheus-query-url + Prometheus HTTP API query endpoint for OTLP-exported metrics, + e.g. http://localhost:9090/api/v1/query + --diagnostic-prometheus-query + PromQL used with --diagnostic-prometheus-query-url + (default: {__name__=~"rustfs_io_get_object_.*"}) + --diagnostic-metrics-settle-secs + Seconds to wait before the after snapshot so OTLP periodic + metrics can export probe counters (default: 2) + --diagnostic-obs-endpoint + RUSTFS_OBS_ENDPOINT passed to RustFS during diagnostic runs + --diagnostic-obs-metric-endpoint + RUSTFS_OBS_METRIC_ENDPOINT passed to RustFS during diagnostic runs + --diagnostic-obs-meter-interval + RUSTFS_OBS_METER_INTERVAL passed during diagnostic runs (default: 1) + --diagnostic-obs-service-name-prefix + Prefix used for unique per-profile RUSTFS_OBS_SERVICE_NAME + (default: RustFS-get-codec, or existing RUSTFS_OBS_SERVICE_NAME) + --compressed-fallback-probe Enable disk compression only for the compressed fallback probe object --address RustFS listen address (default: 127.0.0.1:19030) --bucket Benchmark bucket (default: rustfs-get-codec-smoke) --sizes Object sizes (default: 1MiB,4MiB,10MiB) @@ -104,6 +146,9 @@ Output: /engine_compare.csv when legacy and codec profiles both run /compat_summary.csv when legacy and codec profiles both run /metrics_summary.csv + /service_metrics_summary.csv when --diagnostic-metrics is set + /service_metrics_acceptance.csv when --diagnostic-metrics is set + /fallback_probe_summary.csv /body_sha256_legacy.txt when legacy profile runs /body_sha256_codec_legacy.txt when codec-legacy profile runs /body_sha256_codec_rustfs.txt when codec-rustfs profile runs @@ -112,7 +157,10 @@ Output: /response_headers_codec_rustfs.json when codec-rustfs profile runs //manifest.env //metrics_summary.csv + //service_metrics_summary.csv + //service-metrics/*.prom before/after snapshots when --diagnostic-metrics is set //compat/compat_summary.csv + //compat/fallback_probe_summary.csv //compat/response_headers.json //compat/body_sha256.txt //rustfs.log @@ -163,7 +211,19 @@ parse_args() { --metadata-early-stop) METADATA_EARLY_STOP="$2"; shift 2 ;; --shard-locality-preference) SHARD_LOCALITY_PREFERENCE="$2"; shift 2 ;; --codec-max-inflight) CODEC_MAX_INFLIGHT="$2"; shift 2 ;; + --codec-multipart) CODEC_MULTIPART="$2"; shift 2 ;; + --codec-multipart-max-parts) CODEC_MULTIPART_MAX_PARTS="$2"; shift 2 ;; --handoff-attribution) OUTPUT_HANDOFF_ATTRIBUTION=true; shift ;; + --diagnostic-metrics) DIAGNOSTIC_METRICS=true; shift ;; + --diagnostic-metrics-url) DIAGNOSTIC_METRICS_URL="$2"; shift 2 ;; + --diagnostic-prometheus-query-url) DIAGNOSTIC_PROMETHEUS_QUERY_URL="$2"; shift 2 ;; + --diagnostic-prometheus-query) DIAGNOSTIC_PROMETHEUS_QUERY="$2"; shift 2 ;; + --diagnostic-metrics-settle-secs) DIAGNOSTIC_METRICS_SETTLE_SECS="$2"; shift 2 ;; + --diagnostic-obs-endpoint) DIAGNOSTIC_OBS_ENDPOINT="$2"; shift 2 ;; + --diagnostic-obs-metric-endpoint) DIAGNOSTIC_OBS_METRIC_ENDPOINT="$2"; shift 2 ;; + --diagnostic-obs-meter-interval) DIAGNOSTIC_OBS_METER_INTERVAL="$2"; shift 2 ;; + --diagnostic-obs-service-name-prefix) DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX="$2"; shift 2 ;; + --compressed-fallback-probe) COMPRESSED_FALLBACK_PROBE=true; shift ;; --address) ADDRESS="$2"; shift 2 ;; --bucket) BUCKET="$2"; shift 2 ;; --sizes) SIZES="$2"; shift 2 ;; @@ -209,6 +269,11 @@ validate_args() { *) die "--shard-locality-preference must be on or off" ;; esac + case "$CODEC_MULTIPART" in + on|off) ;; + *) die "--codec-multipart must be on or off" ;; + esac + local raw engine IFS=',' read -r -a engines <<< "$CODEC_ENGINES" [[ "${#engines[@]}" -gt 0 ]] || die "--codec-engine must not be empty" @@ -227,13 +292,18 @@ validate_args() { [[ -n "$SIZES" ]] || die "--sizes must not be empty" validate_positive_int "$CONCURRENCY" "--concurrency" validate_positive_int "$CODEC_MAX_INFLIGHT" "--codec-max-inflight" + validate_positive_int "$CODEC_MULTIPART_MAX_PARTS" "--codec-multipart-max-parts" validate_positive_int "$ROUNDS" "--rounds" validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" validate_non_negative_int "$ROUND_COOLDOWN_SECS" "--round-cooldown-secs" validate_positive_int "$CODEC_MIN_SIZE" "--codec-min-size" validate_positive_int "$COMPAT_OBJECT_SIZE" "--compat-object-size" validate_positive_int "$HEALTH_TIMEOUT_SECS" "--health-timeout-secs" + validate_non_negative_int "$DIAGNOSTIC_METRICS_SETTLE_SECS" "--diagnostic-metrics-settle-secs" + validate_positive_int "$DIAGNOSTIC_OBS_METER_INTERVAL" "--diagnostic-obs-meter-interval" [[ -n "$COMPAT_OBJECT_KEY" ]] || die "--compat-object-key must not be empty" + [[ -n "$DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX" ]] || die "--diagnostic-obs-service-name-prefix must not be empty" + [[ -n "$DIAGNOSTIC_METRICS_URL" ]] || die "--diagnostic-metrics-url must not be empty" [[ -x "$ENHANCED_BENCH" ]] || die "enhanced benchmark script is not executable: $ENHANCED_BENCH" require_cmd curl @@ -260,10 +330,31 @@ bool_from_on_off() { esac } +dry_run_multipart_expected_reason() { + if [[ "$CODEC_MULTIPART" != "on" ]]; then + echo "multipart" + elif [[ "$CODEC_MULTIPART_MAX_PARTS" -lt 2 ]]; then + echo "multipart_part_limit" + else + echo "none" + fi +} + command_line_string() { printf '%q ' "${BASH_SOURCE[0]}" "${ORIGINAL_ARGS[@]}" } +sanitize_metric_label_value() { + printf '%s' "$1" | tr -c '[:alnum:]_-' '-' +} + +diagnostic_service_name() { + local profile="$1" + local run_id + run_id="$(sanitize_metric_label_value "$(basename "$OUT_DIR")")" + echo "${DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX}-${run_id}-${profile}" +} + detect_cpu_summary() { if command -v sysctl >/dev/null 2>&1; then local brand logical @@ -332,6 +423,13 @@ rustfs/backlog#724 rustfs/backlog#725 rustfs/backlog#726 rustfs/backlog#727 +rustfs/backlog#758 +rustfs/backlog#759 +rustfs/backlog#760 +rustfs/backlog#761 +rustfs/backlog#762 +rustfs/backlog#763 +rustfs/backlog#764 EOF } @@ -343,6 +441,7 @@ codec_streaming rollout fallback coverage proof Static gate reasons: - disabled - rollout_not_opted_in +- rollout_pct_not_selected - body_compatibility_unconfirmed - header_compatibility_unconfirmed - lock_optimization_disabled @@ -352,6 +451,7 @@ Static gate reasons: - compressed - remote - multipart +- multipart_part_limit - invalid_min_size - read_quorum_not_safe @@ -361,10 +461,19 @@ Relevant focused proof points: - set_disk::read::tests::codec_streaming_reader_gate_requires_explicit_rollout_and_compat_confirmation - set_disk::read::tests::codec_streaming_reader_gate_is_conservative - set_disk::read::tests::codec_streaming_reader_build_falls_back_when_read_quorum_is_not_safe +- erasure::codec::bridge::tests::rustfs_codec_decode_engine_rejects_inconsistent_reconstruction_sources +- erasure::codec::bridge::tests::rustfs_codec_decode_engine_rejects_stale_data_source +- erasure::coding::decode_reader::tests::erasure_decode_reader_rustfs_engine_recovers_after_bitrot_source_mismatch +- rustfs_io_get_object_reconstruct_outcome_total{path,engine,outcome} Conclusion: -- range / encrypted / compressed / multipart / remote objects remain on legacy fallback paths +- range / encrypted / compressed / remote objects remain on legacy fallback paths +- multipart remains a legacy fallback by default and can be separately enabled with RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE=true +- multipart codec streaming is bounded by RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS - degraded reader setup remains opt-out via read_quorum_not_safe +- codec-rustfs remains explicit opt-in through RUSTFS_GET_CODEC_STREAMING_ENGINE=rustfs +- healthy codec-rustfs reads should report skip_data_complete instead of rustfs_called +- rustfs_called is reserved for explicit reconstruction test/probe coverage and must not be required for healthy-read rollout - env kill switch remains available through RUSTFS_GET_CODEC_STREAMING_ENABLE=false EOF } @@ -390,7 +499,22 @@ codec_max_inflight=${CODEC_MAX_INFLIGHT} codec_rollout_codec_profile=benchmark codec_body_compat_confirmed_codec_profile=true codec_header_compat_confirmed_codec_profile=true +codec_multipart=${CODEC_MULTIPART} +codec_multipart_max_parts=${CODEC_MULTIPART_MAX_PARTS} output_handoff_attribution=${OUTPUT_HANDOFF_ATTRIBUTION} +diagnostic_metrics_enabled=${DIAGNOSTIC_METRICS} +diagnostic_metrics_url=${DIAGNOSTIC_METRICS_URL} +diagnostic_prometheus_query_url=${DIAGNOSTIC_PROMETHEUS_QUERY_URL} +diagnostic_prometheus_query=${DIAGNOSTIC_PROMETHEUS_QUERY} +diagnostic_metrics_settle_secs=${DIAGNOSTIC_METRICS_SETTLE_SECS} +diagnostic_obs_endpoint=${DIAGNOSTIC_OBS_ENDPOINT} +diagnostic_obs_metric_endpoint=${DIAGNOSTIC_OBS_METRIC_ENDPOINT} +diagnostic_obs_meter_interval=${DIAGNOSTIC_OBS_METER_INTERVAL} +diagnostic_obs_service_name_prefix=${DIAGNOSTIC_OBS_SERVICE_NAME_PREFIX} +service_metric_prefix=${SERVICE_METRIC_PREFIX} +compressed_fallback_probe=${COMPRESSED_FALLBACK_PROBE} +compressed_probe_extension=${COMPRESSED_PROBE_EXTENSION} +compressed_probe_mime_type=${COMPRESSED_PROBE_MIME_TYPE} address=${ADDRESS} bucket=${BUCKET} region=${REGION} @@ -497,11 +621,12 @@ write_manifest() { local volumes="$4" local codec_engine="$5" local metrics_path="$6" - local rollout_target body_compat_confirmed header_compat_confirmed + local rollout_target body_compat_confirmed header_compat_confirmed obs_service_name local git_head git_dirty_count rollout_target="$(profile_rollout_target "$profile")" body_compat_confirmed="$(profile_body_compat_confirmed "$profile")" header_compat_confirmed="$(profile_header_compat_confirmed "$profile")" + obs_service_name="$(diagnostic_service_name "$profile")" git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)" git_dirty_count="$(git -C "$PROJECT_ROOT" status --porcelain | awk 'END { print NR + 0 }')" @@ -535,8 +660,24 @@ RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED=${header_compat_confirmed} RUSTFS_GET_METADATA_EARLY_STOP_ENABLE=$(bool_from_on_off "$METADATA_EARLY_STOP") RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE=$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE") RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT=${CODEC_MAX_INFLIGHT} +RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE=$(bool_from_on_off "$CODEC_MULTIPART") +RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS=${CODEC_MULTIPART_MAX_PARTS} RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=${OUTPUT_HANDOFF_ATTRIBUTION} RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE} +RUSTFS_OBS_METRICS_EXPORT_ENABLED=${DIAGNOSTIC_METRICS} +RUSTFS_OBS_ENDPOINT=${DIAGNOSTIC_OBS_ENDPOINT} +RUSTFS_OBS_METRIC_ENDPOINT=${DIAGNOSTIC_OBS_METRIC_ENDPOINT} +RUSTFS_OBS_METER_INTERVAL=${DIAGNOSTIC_OBS_METER_INTERVAL} +RUSTFS_OBS_SERVICE_NAME=${obs_service_name} +RUSTFS_COMPRESSION_ENABLED=${COMPRESSED_FALLBACK_PROBE} +RUSTFS_COMPRESSION_EXTENSIONS=${COMPRESSED_PROBE_EXTENSION} +RUSTFS_COMPRESSION_MIME_TYPES=${COMPRESSED_PROBE_MIME_TYPE} +diagnostic_metrics_enabled=${DIAGNOSTIC_METRICS} +diagnostic_metrics_url=${DIAGNOSTIC_METRICS_URL} +diagnostic_prometheus_query_url=${DIAGNOSTIC_PROMETHEUS_QUERY_URL} +diagnostic_prometheus_query=${DIAGNOSTIC_PROMETHEUS_QUERY} +diagnostic_metrics_settle_secs=${DIAGNOSTIC_METRICS_SETTLE_SECS} +service_metric_prefix=${SERVICE_METRIC_PREFIX} metrics_path=${metrics_path} RUSTFS_SCANNER_ENABLED=false RUSTFS_SCANNER_START_DELAY_SECS=3600 @@ -597,6 +738,7 @@ start_server() { local rollout_target local body_compat_confirmed local header_compat_confirmed + local obs_service_name local rustfs_log data_root="$(profile_data_root "$profile")" @@ -607,6 +749,7 @@ start_server() { rollout_target="$(profile_rollout_target "$profile")" body_compat_confirmed="$(profile_body_compat_confirmed "$profile")" header_compat_confirmed="$(profile_header_compat_confirmed "$profile")" + obs_service_name="$(diagnostic_service_name "$profile")" rustfs_log="${profile_dir}/rustfs.log" mkdir -p "${data_root}/disk1" "${data_root}/disk2" "${data_root}/disk3" "${data_root}/disk4" @@ -634,8 +777,24 @@ start_server() { export RUSTFS_GET_METADATA_EARLY_STOP_ENABLE="$(bool_from_on_off "$METADATA_EARLY_STOP")" export RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE="$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE")" export RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT="$CODEC_MAX_INFLIGHT" + export RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE="$(bool_from_on_off "$CODEC_MULTIPART")" + export RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS="$CODEC_MULTIPART_MAX_PARTS" export RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE="$OUTPUT_HANDOFF_ATTRIBUTION" export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE" + export RUSTFS_OBS_METRICS_EXPORT_ENABLED="$DIAGNOSTIC_METRICS" + if [[ "$DIAGNOSTIC_METRICS" == "true" ]]; then + export RUSTFS_OBS_METER_INTERVAL="$DIAGNOSTIC_OBS_METER_INTERVAL" + export RUSTFS_OBS_SERVICE_NAME="$obs_service_name" + if [[ -n "$DIAGNOSTIC_OBS_ENDPOINT" ]]; then + export RUSTFS_OBS_ENDPOINT="$DIAGNOSTIC_OBS_ENDPOINT" + fi + if [[ -n "$DIAGNOSTIC_OBS_METRIC_ENDPOINT" ]]; then + export RUSTFS_OBS_METRIC_ENDPOINT="$DIAGNOSTIC_OBS_METRIC_ENDPOINT" + fi + fi + export RUSTFS_COMPRESSION_ENABLED="$COMPRESSED_FALLBACK_PROBE" + export RUSTFS_COMPRESSION_EXTENSIONS="$COMPRESSED_PROBE_EXTENSION" + export RUSTFS_COMPRESSION_MIME_TYPES="$COMPRESSED_PROBE_MIME_TYPE" export RUSTFS_REGION="$REGION" export RUSTFS_RPC_SECRET="rustfs-get-codec-smoke-rpc-secret" export RUSTFS_SCANNER_ENABLED=false @@ -696,6 +855,619 @@ write_metrics_summary() { fi } +service_metrics_dir() { + local profile="$1" + echo "${OUT_DIR}/${profile}/service-metrics" +} + +capture_prometheus_query_snapshot() { + local phase="$1" + local snapshot_file="$2" + local status_file="$3" + local service_name="$4" + + "$PYTHON_BIN" - "$DIAGNOSTIC_PROMETHEUS_QUERY_URL" "$DIAGNOSTIC_PROMETHEUS_QUERY" "$phase" "$snapshot_file" "$status_file" "$service_name" <<'PY' +import json +import pathlib +import sys +import urllib.parse +import urllib.request + +query_url, query, phase, snapshot_raw, status_raw, service_name = sys.argv[1:] +snapshot_path = pathlib.Path(snapshot_raw) +status_path = pathlib.Path(status_raw) + + +def write_status(status): + status_path.write_text( + "\n".join( + [ + f"phase={phase}", + f"status={status}", + f"url={query_url}", + f"query={query}", + f"service_name={service_name}", + "", + ] + ), + encoding="utf-8", + ) + + +def escape_label(value): + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def sample_to_line(sample): + metric = dict(sample.get("metric", {})) + name = metric.pop("__name__", "") + if not name: + return None + labels = ",".join(f'{key}="{escape_label(value)}"' for key, value in sorted(metric.items())) + value = sample.get("value", [None, None])[1] + if value is None: + return None + if labels: + return f"{name}{{{labels}}} {value}" + return f"{name} {value}" + + +try: + separator = "&" if "?" in query_url else "?" + url = f"{query_url}{separator}{urllib.parse.urlencode({'query': query})}" + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=5) as response: + payload = json.loads(response.read().decode("utf-8")) +except Exception as err: # noqa: BLE001 - shell harness reports the failure in status files. + snapshot_path.write_text(f"# prometheus_query_failed error={err}\n", encoding="utf-8") + write_status("query_failed") + raise SystemExit(0) + +if payload.get("status") != "success": + snapshot_path.write_text(f"# prometheus_query_failed payload_status={payload.get('status')}\n", encoding="utf-8") + write_status("query_failed") + raise SystemExit(0) + +samples = [ + sample + for sample in payload.get("data", {}).get("result", []) + if not service_name or sample.get("metric", {}).get("service.name") == service_name +] +lines = [line for sample in samples if (line := sample_to_line(sample))] +if not lines: + snapshot_path.write_text(f"# no_matching_metrics service_name={service_name}\n", encoding="utf-8") + write_status("no_matching_metrics") + raise SystemExit(0) + +snapshot_path.write_text("\n".join(lines) + "\n", encoding="utf-8") +write_status("ok") +PY +} + +capture_service_metrics_snapshot() { + local profile="$1" + local phase="$2" + local metrics_dir snapshot_file status_file + metrics_dir="$(service_metrics_dir "$profile")" + snapshot_file="${metrics_dir}/${phase}.prom" + status_file="${metrics_dir}/${phase}.status" + + if [[ "$DIAGNOSTIC_METRICS" != "true" ]]; then + return 0 + fi + + mkdir -p "$metrics_dir" + + if [[ "$DRY_RUN" == "true" ]]; then + : >"$snapshot_file" + cat >"$status_file" <"$snapshot_file"; then + if [[ ! -s "$snapshot_file" ]]; then + cat >"$status_file" <"$status_file" <"$snapshot_file" + cat >"$status_file" <"$out_csv" <"$out_csv" <"$out_csv" <"$out_csv" + + for profile in "$@"; do + summary="${OUT_DIR}/${profile}/service_metrics_summary.csv" + [[ -f "$summary" ]] || continue + if [[ "$wrote_header" == "false" ]]; then + cat "$summary" >>"$out_csv" + wrote_header=true + else + tail -n +2 "$summary" >>"$out_csv" + fi + done + + if [[ "$wrote_header" == "false" ]]; then + cat >"$out_csv" <<'EOF' +profile,status,metric,labels,before,after,delta,classification +N/A,missing,N/A,N/A,N/A,N/A,N/A,no_profile_service_metrics_summary +EOF + fi +} + +write_service_metrics_acceptance() { + local out_csv="${OUT_DIR}/service_metrics_acceptance.csv" + local service_csv="${OUT_DIR}/service_metrics_summary.csv" + + "$PYTHON_BIN" - "$out_csv" "$service_csv" "$DIAGNOSTIC_METRICS" "$MODE" "$CODEC_ENGINES" \ + "$COMPRESSED_FALLBACK_PROBE" "${OUT_DIR}/metrics_summary.csv" "$CODEC_MULTIPART" \ + "$CODEC_MULTIPART_MAX_PARTS" <<'PY' +import csv +import pathlib +import sys + +out_path = pathlib.Path(sys.argv[1]) +service_path = pathlib.Path(sys.argv[2]) +diagnostic_enabled = sys.argv[3] == "true" +mode = sys.argv[4] +codec_engines = [item.strip() for item in sys.argv[5].split(",") if item.strip()] +compressed_fallback_probe_enabled = sys.argv[6] == "true" +metrics_path = pathlib.Path(sys.argv[7]) +codec_multipart_enabled = sys.argv[8] == "on" +codec_multipart_max_parts = int(sys.argv[9]) + + +def expected_profiles(): + codec_profiles = [f"codec-{engine}" for engine in codec_engines] + if mode == "legacy": + return ["legacy"] + if mode == "codec": + return codec_profiles + return ["legacy", *codec_profiles] + + +rows = [] +if service_path.exists(): + with service_path.open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + +metrics_rows = [] +if metrics_path.exists(): + with metrics_path.open(encoding="utf-8", newline="") as handle: + metrics_rows = list(csv.DictReader(handle)) + + +def metric_matches(actual, expected): + return actual == expected or actual == f"{expected}_total" + + +def delta_for(profile, metric, *label_fragments): + total = 0.0 + for row in rows: + if row.get("profile") != profile or row.get("status") != "ok": + continue + if not metric_matches(row.get("metric", ""), metric): + continue + labels = row.get("labels", "") + if all(fragment in labels for fragment in label_fragments): + try: + total += float(row.get("delta", "0") or 0) + except ValueError: + pass + return total + + +def delta_prefix(profile, metric_prefix, *label_fragments): + total = 0.0 + for row in rows: + if row.get("profile") != profile or row.get("status") != "ok": + continue + if not row.get("metric", "").startswith(metric_prefix): + continue + labels = row.get("labels", "") + if all(fragment in labels for fragment in label_fragments): + try: + total += float(row.get("delta", "0") or 0) + except ValueError: + pass + return total + + +def status_for(delta): + return "pass" if delta > 0 else "fail" + + +def add(profile, check, expected, delta, note): + rows_out.append( + { + "profile": profile, + "check": check, + "expected": expected, + "status": status_for(delta), + "observed_delta": f"{delta:.12g}", + "note": note, + } + ) + + +def p99_available(): + for row in metrics_rows: + value = row.get("median_req_p99_ms", "") + if value and value != "N/A": + return True + return False + + +rows_out = [] +if not diagnostic_enabled: + rows_out.append( + { + "profile": "N/A", + "check": "diagnostic_metrics_enabled", + "expected": "true", + "status": "skipped", + "observed_delta": "N/A", + "note": "performance run: observability metrics export intentionally disabled", + } + ) +elif rows and all(row.get("status") == "not_run_dry_run" for row in rows): + rows_out.append( + { + "profile": "N/A", + "check": "diagnostic_metrics_enabled", + "expected": "non-dry-run service metrics snapshots", + "status": "skipped", + "observed_delta": "N/A", + "note": "dry-run only validates artifact wiring; runtime deltas require a real server run", + } + ) +elif rows and all(row.get("status") == "snapshot_missing" for row in rows): + rows_out.append( + { + "profile": "N/A", + "check": "diagnostic_metrics_capture", + "expected": "non-empty service metrics snapshots", + "status": "fail", + "observed_delta": "N/A", + "note": "metrics endpoint returned empty or missing snapshots; verify --diagnostic-metrics-url and exporter setup", + } + ) +else: + for profile in expected_profiles(): + if profile == "legacy": + add( + profile, + "reader_path", + 'rustfs_io_get_object_reader_path_total{path="legacy_duplex"} delta > 0', + delta_for(profile, "rustfs_io_get_object_reader_path_total", 'path="legacy_duplex"'), + "proves legacy_duplex path was exercised", + ) + add( + profile, + "fallback_disabled", + 'rustfs_io_get_object_codec_streaming_fallback_total{reason="disabled"} delta > 0', + delta_for(profile, "rustfs_io_get_object_codec_streaming_fallback_total", 'reason="disabled"'), + "proves kill-switch fallback reason at runtime", + ) + else: + engine = profile[len("codec-") :] if profile.startswith("codec-") else profile + engine_path = "codec_streaming_rustfs_engine" if engine == "rustfs" else "codec_streaming_legacy_engine" + add( + profile, + "codec_decision_use", + 'rustfs_io_get_object_codec_streaming_decision_total{outcome="use",reason="none"} delta > 0', + delta_for( + profile, + "rustfs_io_get_object_codec_streaming_decision_total", + 'outcome="use"', + 'reason="none"', + ), + "proves eligible GETs selected codec streaming", + ) + add( + profile, + "reader_path", + 'rustfs_io_get_object_reader_path_total{path="codec_streaming"} delta > 0', + delta_for(profile, "rustfs_io_get_object_reader_path_total", 'path="codec_streaming"'), + "proves response reader used the codec streaming path", + ) + add( + profile, + "engine_stage", + f'rustfs_io_get_object_stage_duration_seconds*{{path="{engine_path}"}} delta > 0', + delta_prefix(profile, "rustfs_io_get_object_stage_duration_seconds", f'path="{engine_path}"'), + "proves engine-specific codec path stage attribution", + ) + add( + profile, + "reader_bytes", + f'rustfs_io_get_object_reader_bytes_total{{path="{engine_path}"}} delta > 0', + delta_for(profile, "rustfs_io_get_object_reader_bytes_total", f'path="{engine_path}"'), + "proves emitted reader bytes were attributed to the selected engine", + ) + add( + profile, + "fallback_range", + 'rustfs_io_get_object_codec_streaming_fallback_total{reason="range"} delta > 0', + delta_for(profile, "rustfs_io_get_object_codec_streaming_fallback_total", 'reason="range"'), + "proves runtime Range GET fallback reason delta", + ) + if codec_multipart_enabled: + if codec_multipart_max_parts < 2: + add( + profile, + "multipart_fallback_part_limit", + 'rustfs_io_get_object_codec_streaming_decision_total{outcome="fallback",object_class="multipart",reason="multipart_part_limit"} delta > 0', + delta_for( + profile, + "rustfs_io_get_object_codec_streaming_decision_total", + 'outcome="fallback"', + 'object_class="multipart"', + 'reason="multipart_part_limit"', + ), + "proves multipart codec streaming falls back when part count exceeds the configured guard", + ) + else: + add( + profile, + "multipart_codec_decision_use", + 'rustfs_io_get_object_codec_streaming_decision_total{outcome="use",object_class="multipart",reason="none"} delta > 0', + delta_for( + profile, + "rustfs_io_get_object_codec_streaming_decision_total", + 'outcome="use"', + 'object_class="multipart"', + 'reason="none"', + ), + "proves runtime multipart GET selected codec streaming under explicit opt-in", + ) + add( + profile, + "multipart_fallback_read_quorum_not_safe", + 'rustfs_io_get_object_codec_streaming_decision_total{outcome="fallback",object_class="multipart",reason="read_quorum_not_safe"} delta > 0', + delta_for( + profile, + "rustfs_io_get_object_codec_streaming_decision_total", + 'outcome="fallback"', + 'object_class="multipart"', + 'reason="read_quorum_not_safe"', + ), + "proves unsafe multipart part setup falls back before returning a codec reader", + ) + else: + add( + profile, + "fallback_multipart", + 'rustfs_io_get_object_codec_streaming_fallback_total{reason="multipart"} delta > 0', + delta_for(profile, "rustfs_io_get_object_codec_streaming_fallback_total", 'reason="multipart"'), + "proves runtime multipart object fallback reason delta", + ) + add( + profile, + "fallback_encrypted", + 'rustfs_io_get_object_codec_streaming_fallback_total{reason="encrypted"} delta > 0', + delta_for(profile, "rustfs_io_get_object_codec_streaming_fallback_total", 'reason="encrypted"'), + "proves runtime encrypted object fallback reason delta", + ) + add( + profile, + "fallback_read_quorum_not_safe", + 'rustfs_io_get_object_codec_streaming_fallback_total{reason="read_quorum_not_safe"} delta > 0', + delta_for( + profile, + "rustfs_io_get_object_codec_streaming_fallback_total", + 'reason="read_quorum_not_safe"', + ), + "proves degraded-but-readable shard setup uses the legacy fallback path", + ) + compressed_delta = delta_for( + profile, + "rustfs_io_get_object_codec_streaming_fallback_total", + 'reason="compressed"', + ) + rows_out.append( + { + "profile": profile, + "check": "fallback_compressed", + "expected": 'rustfs_io_get_object_codec_streaming_fallback_total{reason="compressed"} delta > 0', + "status": "pass" + if compressed_delta > 0 + else ("fail" if compressed_fallback_probe_enabled else "unavailable"), + "observed_delta": f"{compressed_delta:.12g}", + "note": "Enable --compressed-fallback-probe to generate a runtime compressed fallback delta" + if not compressed_fallback_probe_enabled + else "proves runtime disk-compressed object fallback reason delta", + } + ) + below_delta = delta_for( + profile, + "rustfs_io_get_object_codec_streaming_fallback_total", + 'reason="below_min_size"', + ) + rows_out.append( + { + "profile": profile, + "check": "fallback_below_min_size", + "expected": 'rustfs_io_get_object_codec_streaming_fallback_total{reason="below_min_size"} delta > 0', + "status": "pass" if below_delta > 0 else "unavailable", + "observed_delta": f"{below_delta:.12g}", + "note": "Set --codec-min-size greater than 1 to force this runtime fallback probe when unavailable", + } + ) + + rows_out.append( + { + "profile": "all", + "check": "diagnostic_vs_performance_separation", + "expected": "diagnostic artifacts are separate from performance conclusions", + "status": "pass", + "observed_delta": "N/A", + "note": "use --diagnostic-metrics for path proof; omit it for throughput acceptance runs", + } + ) + p99_status = p99_available() + rows_out.append( + { + "profile": "all", + "check": "p95_p99", + "expected": "p99 available from warp request percentile output; p95 explicitly marked unavailable when warp does not emit it", + "status": "pass" if p99_status else "unavailable", + "observed_delta": "p99_available" if p99_status else "N/A", + "note": "median_req_p99_ms is aggregated in metrics_summary.csv; p95 remains unavailable because current warp text output emits 90% and 99%, not 95%", + } + ) + +with out_path.open("w", encoding="utf-8", newline="") as handle: + fieldnames = ["profile", "check", "expected", "status", "observed_delta", "note"] + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows_out) +PY +} + start_server_sampler() { local pid="$1" local profile_dir="$2" @@ -795,6 +1567,26 @@ ${COMPAT_OBJECT_SIZE},${profile},true,true,true,true,true,true,true,0,DRY_RUN,20 EOF printf '%s\n' "DRY_RUN" >"${compat_dir}/body_sha256.txt" printf '{}\n' >"${compat_dir}/response_headers.json" + { + cat <"${compat_dir}/fallback_probe_summary.csv" cat >"${compat_dir}/snapshot.json" < Dict[str, List[str]]: return dict(sorted(result.items())) +def first_header(snapshot, name: str) -> str: + name = name.lower() + for header_name, value in snapshot["headers"]: + if header_name.lower() == name: + return value + return "" + + +def find_xml_text(body: bytes, name: str) -> Optional[str]: + try: + root = ET.fromstring(body) + except ET.ParseError: + return None + for elem in root.iter(): + if elem.tag.rsplit("}", 1)[-1] == name: + return elem.text + return None + + bucket_path = "/" + urllib.parse.quote(bucket, safe="") object_path = bucket_path + "/" + urllib.parse.quote(object_key, safe="/-_.~") body = payload(object_size) @@ -943,6 +1780,399 @@ get_object, get_body = request("GET", object_path) if get_object["status"] != 200: raise SystemExit(f"get object failed: {get_object['status']} {get_object['reason']}") +range_object, range_body = request("GET", object_path, extra_headers=[("range", "bytes=0-0")]) +fallback_rows = [ + { + "profile": profile, + "probe": "range", + "object_key": object_key, + "status": "ok" if range_object["status"] == 206 else "unexpected_status", + "status_code": range_object["status"], + "body_len": len(range_body), + "expected_runtime_fallback_reason": "range", + "note": "Range GET should stay on the legacy fallback path for codec profiles", + } +] + +if codec_min_size > 1: + small_size = max(1, min(codec_min_size - 1, object_size)) + small_key = object_key + ".below-min-size" + small_path = bucket_path + "/" + urllib.parse.quote(small_key, safe="/-_.~") + small_body = payload(small_size) + small_put, _ = request("PUT", small_path, small_body, [("content-type", "application/octet-stream")]) + if small_put["status"] not in (200, 204): + fallback_rows.append( + { + "profile": profile, + "probe": "below_min_size", + "object_key": small_key, + "status": "put_failed", + "status_code": small_put["status"], + "body_len": 0, + "expected_runtime_fallback_reason": "below_min_size", + "note": "below-min-size probe upload failed", + } + ) + else: + small_get, small_get_body = request("GET", small_path) + fallback_rows.append( + { + "profile": profile, + "probe": "below_min_size", + "object_key": small_key, + "status": "ok" if small_get["status"] == 200 else "unexpected_status", + "status_code": small_get["status"], + "body_len": len(small_get_body), + "expected_runtime_fallback_reason": "below_min_size", + "note": "Object smaller than codec-min-size should stay on the legacy fallback path for codec profiles", + } + ) +else: + fallback_rows.append( + { + "profile": profile, + "probe": "below_min_size", + "object_key": object_key + ".below-min-size", + "status": "skipped", + "status_code": "N/A", + "body_len": "N/A", + "expected_runtime_fallback_reason": "below_min_size", + "note": "Set --codec-min-size greater than 1 to generate a runtime below_min_size fallback delta", + } + ) + +compressed_key = object_key + compressed_probe_extension +compressed_path = bucket_path + "/" + urllib.parse.quote(compressed_key, safe="/-_.~") +if compressed_fallback_probe: + compressed_body = (b"RustFS compressed fallback probe\n" * 4096)[:128 * 1024] + compressed_put, _ = request( + "PUT", + compressed_path, + compressed_body, + [("content-type", compressed_probe_mime_type)], + ) + if compressed_put["status"] not in (200, 204): + fallback_rows.append( + { + "profile": profile, + "probe": "compressed", + "object_key": compressed_key, + "status": "put_failed", + "status_code": compressed_put["status"], + "body_len": 0, + "expected_runtime_fallback_reason": "compressed", + "note": "compressed probe upload failed", + } + ) + else: + compressed_get, compressed_get_body = request("GET", compressed_path) + compressed_ok = compressed_get["status"] == 200 and sha256_hex(compressed_get_body) == sha256_hex(compressed_body) + fallback_rows.append( + { + "profile": profile, + "probe": "compressed", + "object_key": compressed_key, + "status": "ok" if compressed_ok else "unexpected_status_or_body", + "status_code": compressed_get["status"], + "body_len": len(compressed_get_body), + "expected_runtime_fallback_reason": "compressed", + "note": "Disk-compressed object GET should stay on the legacy fallback path for codec profiles", + } + ) +else: + fallback_rows.append( + { + "profile": profile, + "probe": "compressed", + "object_key": compressed_key, + "status": "skipped", + "status_code": "N/A", + "body_len": "N/A", + "expected_runtime_fallback_reason": "compressed", + "note": "Enable --compressed-fallback-probe to generate a runtime compressed fallback delta", + } + ) + +degraded_key = object_key + ".degraded-read" +degraded_path = bucket_path + "/" + urllib.parse.quote(degraded_key, safe="/-_.~") +degraded_body = payload(max(object_size, 8 * 1024 * 1024)) +degraded_put, _ = request("PUT", degraded_path, degraded_body, [("content-type", "application/octet-stream")]) +if degraded_put["status"] not in (200, 204): + fallback_rows.append( + { + "profile": profile, + "probe": "read_quorum_not_safe", + "object_key": degraded_key, + "status": "put_failed", + "status_code": degraded_put["status"], + "body_len": 0, + "expected_runtime_fallback_reason": "read_quorum_not_safe", + "note": "degraded-read probe upload failed", + } + ) +else: + profile_dir = out_dir.parent + data_root = profile_dir / "data" + degraded_rel = pathlib.Path(*degraded_key.split("/")) + part_candidates = [] + for disk_dir in sorted(data_root.glob("disk*")): + object_dir = disk_dir / bucket / degraded_rel + part_candidates.extend(sorted(object_dir.glob("*/part.1"))) + + if not part_candidates: + fallback_rows.append( + { + "profile": profile, + "probe": "read_quorum_not_safe", + "object_key": degraded_key, + "status": "part_file_missing", + "status_code": "N/A", + "body_len": 0, + "expected_runtime_fallback_reason": "read_quorum_not_safe", + "note": "degraded-read probe could not find an on-disk part.1 shard to move", + } + ) + else: + degraded_part = part_candidates[0] + missing_dir = out_dir / "degraded-missing-shards" + missing_dir.mkdir(parents=True, exist_ok=True) + missing_part = missing_dir / f"{degraded_part.parent.name}-{degraded_part.name}" + if missing_part.exists(): + missing_part.unlink() + degraded_part.rename(missing_part) + try: + degraded_part.parent.rmdir() + except OSError: + pass + degraded_get, degraded_get_body = request("GET", degraded_path) + degraded_ok = degraded_get["status"] == 200 and sha256_hex(degraded_get_body) == sha256_hex(degraded_body) + fallback_rows.append( + { + "profile": profile, + "probe": "read_quorum_not_safe", + "object_key": degraded_key, + "status": "ok" if degraded_ok else "unexpected_status_or_body", + "status_code": degraded_get["status"], + "body_len": len(degraded_get_body), + "expected_runtime_fallback_reason": "read_quorum_not_safe", + "note": f"Moved one shard to {missing_part}; degraded GET should stay on the legacy fallback path for codec profiles", + } + ) + +multipart_key = object_key + ".multipart" +multipart_path = bucket_path + "/" + urllib.parse.quote(multipart_key, safe="/-_.~") +multipart_part_one = payload(5 * 1024 * 1024) +multipart_part_two = payload(1) +multipart_expected_reason = "multipart" +if codec_multipart_enabled and profile.startswith("codec-"): + multipart_expected_reason = "multipart_part_limit" if codec_multipart_max_parts < 2 else "none" +multipart_note = ( + "Multipart object GET should use the codec streaming path for codec profiles" + if multipart_expected_reason == "none" + else "Multipart object GET should exceed the codec streaming part-count guard" + if multipart_expected_reason == "multipart_part_limit" + else "Multipart object GET should stay on the legacy fallback path for codec profiles" +) +initiate_multipart, initiate_multipart_body = request("POST", multipart_path + "?uploads") +if initiate_multipart["status"] not in (200, 201): + fallback_rows.append( + { + "profile": profile, + "probe": "multipart", + "object_key": multipart_key, + "status": "initiate_failed", + "status_code": initiate_multipart["status"], + "body_len": 0, + "expected_runtime_fallback_reason": multipart_expected_reason, + "note": "multipart initiate failed", + } + ) +else: + upload_id = find_xml_text(initiate_multipart_body, "UploadId") + if not upload_id: + fallback_rows.append( + { + "profile": profile, + "probe": "multipart", + "object_key": multipart_key, + "status": "upload_id_missing", + "status_code": initiate_multipart["status"], + "body_len": 0, + "expected_runtime_fallback_reason": multipart_expected_reason, + "note": "multipart initiate response did not include UploadId", + } + ) + else: + upload_id_query = urllib.parse.quote(upload_id, safe="") + part_one, _ = request("PUT", f"{multipart_path}?partNumber=1&uploadId={upload_id_query}", multipart_part_one) + part_two, _ = request("PUT", f"{multipart_path}?partNumber=2&uploadId={upload_id_query}", multipart_part_two) + etag_one = first_header(part_one, "etag") + etag_two = first_header(part_two, "etag") + if part_one["status"] not in (200, 201) or part_two["status"] not in (200, 201) or not etag_one or not etag_two: + fallback_rows.append( + { + "profile": profile, + "probe": "multipart", + "object_key": multipart_key, + "status": "upload_part_failed", + "status_code": f"{part_one['status']};{part_two['status']}", + "body_len": 0, + "expected_runtime_fallback_reason": multipart_expected_reason, + "note": "multipart part upload failed or ETag was missing", + } + ) + request("DELETE", f"{multipart_path}?uploadId={upload_id_query}") + else: + complete_body = ( + '' + "" + f"1{xml_escape(etag_one)}" + f"2{xml_escape(etag_two)}" + "" + ).encode() + complete_multipart, _ = request( + "POST", + f"{multipart_path}?uploadId={upload_id_query}", + complete_body, + [("content-type", "application/xml")], + ) + if complete_multipart["status"] not in (200, 201): + fallback_rows.append( + { + "profile": profile, + "probe": "multipart", + "object_key": multipart_key, + "status": "complete_failed", + "status_code": complete_multipart["status"], + "body_len": 0, + "expected_runtime_fallback_reason": multipart_expected_reason, + "note": "multipart complete failed", + } + ) + request("DELETE", f"{multipart_path}?uploadId={upload_id_query}") + else: + multipart_get, multipart_get_body = request("GET", multipart_path) + expected_multipart_len = len(multipart_part_one) + len(multipart_part_two) + expected_multipart_body_hash = sha256_hex(multipart_part_one + multipart_part_two) + multipart_ok = ( + multipart_get["status"] == 200 + and len(multipart_get_body) == expected_multipart_len + and sha256_hex(multipart_get_body) == expected_multipart_body_hash + ) + fallback_rows.append( + { + "profile": profile, + "probe": "multipart", + "object_key": multipart_key, + "status": "ok" if multipart_ok else "unexpected_status_or_length", + "status_code": multipart_get["status"], + "body_len": len(multipart_get_body), + "expected_runtime_fallback_reason": multipart_expected_reason, + "note": multipart_note, + } + ) + if multipart_expected_reason == "none": + profile_dir = out_dir.parent + data_root = profile_dir / "data" + multipart_rel = pathlib.Path(*multipart_key.split("/")) + part_candidates = [] + for disk_dir in sorted(data_root.glob("disk*")): + object_dir = disk_dir / bucket / multipart_rel + part_candidates.extend(sorted(object_dir.glob("*/part.1"))) + + if not part_candidates: + fallback_rows.append( + { + "profile": profile, + "probe": "multipart_degraded_read", + "object_key": multipart_key, + "status": "part_file_missing", + "status_code": "N/A", + "body_len": 0, + "expected_runtime_fallback_reason": "read_quorum_not_safe", + "note": "multipart degraded-read probe could not find an on-disk part.1 shard to move", + } + ) + else: + multipart_part = part_candidates[0] + missing_dir = out_dir / "multipart-degraded-missing-shards" + missing_dir.mkdir(parents=True, exist_ok=True) + missing_part = missing_dir / f"{multipart_part.parent.name}-{multipart_part.name}" + if missing_part.exists(): + missing_part.unlink() + multipart_part.rename(missing_part) + degraded_multipart_get, degraded_multipart_body = request("GET", multipart_path) + degraded_multipart_ok = ( + degraded_multipart_get["status"] == 200 + and len(degraded_multipart_body) == expected_multipart_len + and sha256_hex(degraded_multipart_body) == expected_multipart_body_hash + ) + fallback_rows.append( + { + "profile": profile, + "probe": "multipart_degraded_read", + "object_key": multipart_key, + "status": "ok" if degraded_multipart_ok else "unexpected_status_or_body", + "status_code": degraded_multipart_get["status"], + "body_len": len(degraded_multipart_body), + "expected_runtime_fallback_reason": "read_quorum_not_safe", + "note": f"Moved one multipart shard to {missing_part}; unsafe part setup should fall back before returning a codec reader", + } + ) + else: + fallback_rows.append( + { + "profile": profile, + "probe": "multipart_degraded_read", + "object_key": multipart_key, + "status": "skipped", + "status_code": "N/A", + "body_len": "N/A", + "expected_runtime_fallback_reason": "read_quorum_not_safe", + "note": "Enable --codec-multipart on with a multipart max-parts value of at least 2 for degraded-read fallback proof", + } + ) + +encrypted_key = object_key + ".encrypted" +encrypted_path = bucket_path + "/" + urllib.parse.quote(encrypted_key, safe="/-_.~") +ssec_key = b"0123456789abcdef0123456789abcdef" +ssec_key_b64 = base64.b64encode(ssec_key).decode() +ssec_key_md5_b64 = base64.b64encode(hashlib.md5(ssec_key).digest()).decode() +ssec_headers = [ + ("x-amz-server-side-encryption-customer-algorithm", "AES256"), + ("x-amz-server-side-encryption-customer-key", ssec_key_b64), + ("x-amz-server-side-encryption-customer-key-md5", ssec_key_md5_b64), +] +encrypted_put, _ = request("PUT", encrypted_path, body, [("content-type", "application/octet-stream"), *ssec_headers]) +if encrypted_put["status"] not in (200, 204): + fallback_rows.append( + { + "profile": profile, + "probe": "encrypted", + "object_key": encrypted_key, + "status": "put_failed", + "status_code": encrypted_put["status"], + "body_len": 0, + "expected_runtime_fallback_reason": "encrypted", + "note": "SSE-C encrypted object upload failed", + } + ) +else: + encrypted_get, encrypted_body = request("GET", encrypted_path, extra_headers=ssec_headers) + encrypted_ok = encrypted_get["status"] == 200 and sha256_hex(encrypted_body) == sha256_hex(body) + fallback_rows.append( + { + "profile": profile, + "probe": "encrypted", + "object_key": encrypted_key, + "status": "ok" if encrypted_ok else "unexpected_status_or_body", + "status_code": encrypted_get["status"], + "body_len": len(encrypted_body), + "expected_runtime_fallback_reason": "encrypted", + "note": "SSE-C encrypted object GET should stay on the legacy fallback path for codec profiles", + } + ) + body_sha = sha256_hex(get_body) expected_sha = sha256_hex(body) body_match = body_sha == expected_sha @@ -969,6 +2199,20 @@ snapshot = { (out_dir / "response_headers.json").write_text(json.dumps(headers_report, indent=2, sort_keys=True) + "\n") (out_dir / "snapshot.json").write_text(json.dumps(snapshot, indent=2, sort_keys=True) + "\n") (out_dir / "body_sha256.txt").write_text(body_sha + "\n") +with (out_dir / "fallback_probe_summary.csv").open("w", encoding="utf-8", newline="") as handle: + fieldnames = [ + "profile", + "probe", + "object_key", + "status", + "status_code", + "body_len", + "expected_runtime_fallback_reason", + "note", + ] + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(fallback_rows) get_headers = snapshot["get_headers"] content_length = ",".join(get_headers.get("content-length", [])) @@ -1093,6 +2337,30 @@ with open(out_csv, "w", encoding="utf-8", newline="") as handle: PY } +write_root_fallback_probe_summary() { + local out_csv="${OUT_DIR}/fallback_probe_summary.csv" + local profile summary wrote_header=false + : >"$out_csv" + + for profile in "$@"; do + summary="${OUT_DIR}/${profile}/compat/fallback_probe_summary.csv" + [[ -f "$summary" ]] || continue + if [[ "$wrote_header" == "false" ]]; then + cat "$summary" >>"$out_csv" + wrote_header=true + else + tail -n +2 "$summary" >>"$out_csv" + fi + done + + if [[ "$wrote_header" == "false" ]]; then + cat >"$out_csv" <<'EOF' +profile,probe,object_key,status,status_code,body_len,expected_runtime_fallback_reason,note +N/A,missing,N/A,missing,N/A,N/A,N/A,no_profile_fallback_probe_summary +EOF + fi +} + write_root_metrics_summary() { local profile_dirs=() local profile @@ -1150,6 +2418,10 @@ def performance_note(throughput_delta, latency_delta): return "at_or_better_than_legacy" +def row_value(row, key): + return row.get(key, "") + + rows = [] by_profile = {} for profile_dir in profile_dirs: @@ -1179,6 +2451,8 @@ for profile_dir in profile_dirs: "median_throughput_bps": row["median_throughput_bps"], "median_reqps": row["median_reqps"], "median_latency_ms": row["median_latency_ms"], + "median_req_p90_ms": row_value(row, "median_req_p90_ms"), + "median_req_p99_ms": row_value(row, "median_req_p99_ms"), } ) @@ -1195,10 +2469,14 @@ for row in rows: throughput_delta = delta_pct(parse_float(row["median_throughput_bps"]), parse_float(baseline["median_throughput_bps"])) reqps_delta = delta_pct(parse_float(row["median_reqps"]), parse_float(baseline["median_reqps"])) latency_delta = delta_pct(parse_float(row["median_latency_ms"]), parse_float(baseline["median_latency_ms"])) + p90_delta = delta_pct(parse_float(row.get("median_req_p90_ms", "")), parse_float(baseline.get("median_req_p90_ms", ""))) + p99_delta = delta_pct(parse_float(row.get("median_req_p99_ms", "")), parse_float(baseline.get("median_req_p99_ms", ""))) row["baseline_profile"] = "legacy" row["delta_throughput_pct_vs_legacy"] = throughput_delta row["delta_reqps_pct_vs_legacy"] = reqps_delta row["delta_latency_pct_vs_legacy"] = latency_delta + row["delta_req_p90_pct_vs_legacy"] = p90_delta + row["delta_req_p99_pct_vs_legacy"] = p99_delta row["performance_note"] = performance_note(throughput_delta, latency_delta) fieldnames = [ @@ -1220,10 +2498,14 @@ fieldnames = [ "median_throughput_bps", "median_reqps", "median_latency_ms", + "median_req_p90_ms", + "median_req_p99_ms", "baseline_profile", "delta_throughput_pct_vs_legacy", "delta_reqps_pct_vs_legacy", "delta_latency_pct_vs_legacy", + "delta_req_p90_pct_vs_legacy", + "delta_req_p99_pct_vs_legacy", "performance_note", ] with open(metrics_csv, "w", encoding="utf-8", newline="") as handle: @@ -1254,6 +2536,12 @@ baseline_fields = [ "new_median_latency_ms", "baseline_median_latency_ms", "delta_latency_pct_vs_legacy", + "new_median_req_p90_ms", + "baseline_median_req_p90_ms", + "delta_req_p90_pct_vs_legacy", + "new_median_req_p99_ms", + "baseline_median_req_p99_ms", + "delta_req_p99_pct_vs_legacy", "new_median_throughput_bps", "baseline_median_throughput_bps", "delta_throughput_pct_vs_legacy", @@ -1290,6 +2578,12 @@ for row in rows: "new_median_latency_ms": row["median_latency_ms"], "baseline_median_latency_ms": baseline.get("median_latency_ms", ""), "delta_latency_pct_vs_legacy": row["delta_latency_pct_vs_legacy"], + "new_median_req_p90_ms": row.get("median_req_p90_ms", ""), + "baseline_median_req_p90_ms": baseline.get("median_req_p90_ms", ""), + "delta_req_p90_pct_vs_legacy": row.get("delta_req_p90_pct_vs_legacy", ""), + "new_median_req_p99_ms": row.get("median_req_p99_ms", ""), + "baseline_median_req_p99_ms": baseline.get("median_req_p99_ms", ""), + "delta_req_p99_pct_vs_legacy": row.get("delta_req_p99_pct_vs_legacy", ""), "new_median_throughput_bps": row["median_throughput_bps"], "baseline_median_throughput_bps": baseline.get("median_throughput_bps", ""), "delta_throughput_pct_vs_legacy": row["delta_throughput_pct_vs_legacy"], @@ -1318,12 +2612,24 @@ compare_fields = [ "legacy_median_latency_ms", "codec_legacy_median_latency_ms", "codec_rustfs_median_latency_ms", + "legacy_median_req_p90_ms", + "codec_legacy_median_req_p90_ms", + "codec_rustfs_median_req_p90_ms", + "legacy_median_req_p99_ms", + "codec_legacy_median_req_p99_ms", + "codec_rustfs_median_req_p99_ms", "codec_legacy_delta_throughput_pct_vs_legacy", "codec_rustfs_delta_throughput_pct_vs_legacy", "codec_rustfs_delta_throughput_pct_vs_codec_legacy", "codec_legacy_delta_latency_pct_vs_legacy", "codec_rustfs_delta_latency_pct_vs_legacy", "codec_rustfs_delta_latency_pct_vs_codec_legacy", + "codec_legacy_delta_req_p90_pct_vs_legacy", + "codec_rustfs_delta_req_p90_pct_vs_legacy", + "codec_rustfs_delta_req_p90_pct_vs_codec_legacy", + "codec_legacy_delta_req_p99_pct_vs_legacy", + "codec_rustfs_delta_req_p99_pct_vs_legacy", + "codec_rustfs_delta_req_p99_pct_vs_codec_legacy", ] compare_rows = [] for size, legacy in legacy_by_size.items(): @@ -1341,6 +2647,12 @@ for size, legacy in legacy_by_size.items(): "legacy_median_latency_ms": legacy.get("median_latency_ms", ""), "codec_legacy_median_latency_ms": codec_legacy.get("median_latency_ms", ""), "codec_rustfs_median_latency_ms": codec_rustfs.get("median_latency_ms", ""), + "legacy_median_req_p90_ms": legacy.get("median_req_p90_ms", ""), + "codec_legacy_median_req_p90_ms": codec_legacy.get("median_req_p90_ms", ""), + "codec_rustfs_median_req_p90_ms": codec_rustfs.get("median_req_p90_ms", ""), + "legacy_median_req_p99_ms": legacy.get("median_req_p99_ms", ""), + "codec_legacy_median_req_p99_ms": codec_legacy.get("median_req_p99_ms", ""), + "codec_rustfs_median_req_p99_ms": codec_rustfs.get("median_req_p99_ms", ""), "codec_legacy_delta_throughput_pct_vs_legacy": delta_pct( parse_float(codec_legacy.get("median_throughput_bps", "")), parse_float(legacy.get("median_throughput_bps", "")), @@ -1365,6 +2677,30 @@ for size, legacy in legacy_by_size.items(): parse_float(codec_rustfs.get("median_latency_ms", "")), parse_float(codec_legacy.get("median_latency_ms", "")), ), + "codec_legacy_delta_req_p90_pct_vs_legacy": delta_pct( + parse_float(codec_legacy.get("median_req_p90_ms", "")), + parse_float(legacy.get("median_req_p90_ms", "")), + ), + "codec_rustfs_delta_req_p90_pct_vs_legacy": delta_pct( + parse_float(codec_rustfs.get("median_req_p90_ms", "")), + parse_float(legacy.get("median_req_p90_ms", "")), + ), + "codec_rustfs_delta_req_p90_pct_vs_codec_legacy": delta_pct( + parse_float(codec_rustfs.get("median_req_p90_ms", "")), + parse_float(codec_legacy.get("median_req_p90_ms", "")), + ), + "codec_legacy_delta_req_p99_pct_vs_legacy": delta_pct( + parse_float(codec_legacy.get("median_req_p99_ms", "")), + parse_float(legacy.get("median_req_p99_ms", "")), + ), + "codec_rustfs_delta_req_p99_pct_vs_legacy": delta_pct( + parse_float(codec_rustfs.get("median_req_p99_ms", "")), + parse_float(legacy.get("median_req_p99_ms", "")), + ), + "codec_rustfs_delta_req_p99_pct_vs_codec_legacy": delta_pct( + parse_float(codec_rustfs.get("median_req_p99_ms", "")), + parse_float(codec_legacy.get("median_req_p99_ms", "")), + ), } ) @@ -1407,10 +2743,11 @@ write_default_switch_readiness_report() { local compat_summary_path="${OUT_DIR}/compat_summary.csv" local cpu_rss_notes_path="${OUT_DIR}/cpu_rss_notes.txt" local fallback_coverage_path="${OUT_DIR}/fallback_coverage.txt" + local service_metrics_acceptance_path="${OUT_DIR}/service_metrics_acceptance.csv" local tracking_issues_path="${OUT_DIR}/tracking_issues.txt" local stable not_worse improved_over_five two_sizes_gt_five - local headers_compatible p95_p99_ok cpu_rss_ok fallback_proven kill_switch_verified correctness_matrix_ok + local headers_compatible p95_p99_ok cpu_rss_ok runtime_metrics_ok fallback_proven kill_switch_verified correctness_matrix_ok local all_pass decision if [[ -f "$baseline_compare_path" ]]; then @@ -1431,7 +2768,7 @@ write_default_switch_readiness_report() { if ($19 == "" || $22 == "" || ($19 + 0.0) < 0.0 || ($22 + 0.0) > 0.0) { not_worse = 0 } - if ($25 != "" && ($25 + 0.0) > 5.0) { + if ($31 != "" && ($31 + 0.0) > 5.0) { improved++ } } @@ -1464,6 +2801,25 @@ write_default_switch_readiness_report() { p95_p99_ok="fail" correctness_matrix_ok="fail" + runtime_metrics_ok="fail" + + if [[ "$DIAGNOSTIC_METRICS" == "true" && -f "$service_metrics_acceptance_path" ]] \ + && "$PYTHON_BIN" - "$service_metrics_acceptance_path" <<'PY' +import csv +import sys + +with open(sys.argv[1], encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + +statuses = [row.get("status") for row in rows] +if not rows or "fail" in statuses or "pass" not in statuses: + raise SystemExit(1) +PY + then + runtime_metrics_ok="pass" + elif [[ "$DIAGNOSTIC_METRICS" != "true" ]]; then + runtime_metrics_ok="skipped" + fi if [[ -f "$cpu_rss_notes_path" ]] && grep -q 'cpu_rss_acceptability=acceptable' "$cpu_rss_notes_path"; then cpu_rss_ok="pass" @@ -1473,7 +2829,7 @@ write_default_switch_readiness_report() { if [[ -f "$fallback_coverage_path" ]] \ && grep -q 'read_quorum_not_safe' "$fallback_coverage_path" \ - && grep -q 'range / encrypted / compressed / multipart / remote' "$fallback_coverage_path"; then + && grep -q 'range / encrypted / compressed / remote' "$fallback_coverage_path"; then fallback_proven="pass" else fallback_proven="fail" @@ -1490,6 +2846,7 @@ write_default_switch_readiness_report() { && "$two_sizes_gt_five" == "pass" \ && "$p95_p99_ok" == "pass" \ && "$cpu_rss_ok" == "pass" \ + && "$runtime_metrics_ok" == "pass" \ && "$correctness_matrix_ok" == "pass" \ && "$headers_compatible" == "pass" \ && "$fallback_proven" == "pass" \ @@ -1513,11 +2870,12 @@ write_default_switch_readiness_report() { echo "| multi-round cooled A/B stable | ${stable} | expected rounds per target size: ${ROUNDS} |" echo "| 1MiB / 4MiB / 10MiB not worse than legacy | ${not_worse} | derived from baseline_compare.csv |" echo "| at least two sizes improve by more than 5% | ${two_sizes_gt_five} | qualifying sizes: ${improved_over_five} |" - echo "| p95 / p99 do not regress | ${p95_p99_ok} | current harness retains raw output paths but does not aggregate p95/p99 yet |" + echo "| p95 / p99 do not regress | ${p95_p99_ok} | p99 is aggregated in metrics_summary.csv; p95 remains unavailable in current warp text output |" echo "| CPU and RSS acceptable | ${cpu_rss_ok} | see cpu_rss_notes.txt |" + echo "| runtime server metrics prove path and fallback deltas | ${runtime_metrics_ok} | see service_metrics_acceptance.csv; skipped for observability-off performance runs |" echo "| correctness matrix passes | ${correctness_matrix_ok} | compat smoke passes, but full correctness matrix is not fully evidenced by this harness alone |" echo "| response headers compatible | ${headers_compatible} | see compat_summary.csv |" - echo "| range / encrypted / compressed / multipart / remote fallback proven | ${fallback_proven} | see fallback_coverage.txt |" + echo "| range / encrypted / compressed / remote fallback proven | ${fallback_proven} | see fallback_coverage.txt |" echo "| kill switch verified | ${kill_switch_verified} | see fallback_coverage.txt |" echo echo "## Evidence Files" @@ -1525,6 +2883,9 @@ write_default_switch_readiness_report() { echo "- \`baseline_compare.csv\`" echo "- \`compat_summary.csv\`" echo "- \`metrics_summary.csv\`" + echo "- \`service_metrics_summary.csv\`" + echo "- \`service_metrics_acceptance.csv\`" + echo "- \`fallback_probe_summary.csv\`" echo "- \`cpu_rss_notes.txt\`" echo "- \`fallback_coverage.txt\`" echo "- \`tracking_issues.txt\`" @@ -1547,6 +2908,7 @@ run_profile() { stop_server start_server "$profile" + capture_service_metrics_snapshot "$profile" before if run_bench "$profile" "$baseline_csv"; then : else @@ -1554,11 +2916,17 @@ run_profile() { fi write_metrics_summary "$profile" run_compat_probe "$profile" + if [[ "$DIAGNOSTIC_METRICS" == "true" && "$DIAGNOSTIC_METRICS_SETTLE_SECS" -gt 0 ]]; then + sleep "$DIAGNOSTIC_METRICS_SETTLE_SECS" + fi + capture_service_metrics_snapshot "$profile" after + write_profile_service_metrics_summary "$profile" stop_server write_profile_cpu_rss_notes "$profile" log "Median summary: ${OUT_DIR}/${profile}/warp/median_summary.csv" log "Metrics summary: ${OUT_DIR}/${profile}/metrics_summary.csv" + log "Service metrics summary: ${OUT_DIR}/${profile}/service_metrics_summary.csv" log "Compatibility summary: ${OUT_DIR}/${profile}/compat/compat_summary.csv" if [[ -f "${OUT_DIR}/${profile}/warp/baseline_compare.csv" ]]; then log "Baseline compare: ${OUT_DIR}/${profile}/warp/baseline_compare.csv" @@ -1623,7 +2991,10 @@ main() { done write_root_metrics_summary "${profiles[@]}" + write_root_service_metrics_summary "${profiles[@]}" + write_service_metrics_acceptance write_root_compat_summary "${profiles[@]}" + write_root_fallback_probe_summary "${profiles[@]}" write_root_cpu_rss_notes write_fallback_coverage write_tracking_issues @@ -1639,6 +3010,15 @@ main() { if [[ -f "${OUT_DIR}/metrics_summary.csv" ]]; then log "Metrics summary: ${OUT_DIR}/metrics_summary.csv" fi + if [[ -f "${OUT_DIR}/service_metrics_summary.csv" ]]; then + log "Service metrics summary: ${OUT_DIR}/service_metrics_summary.csv" + fi + if [[ -f "${OUT_DIR}/service_metrics_acceptance.csv" ]]; then + log "Service metrics acceptance: ${OUT_DIR}/service_metrics_acceptance.csv" + fi + if [[ -f "${OUT_DIR}/fallback_probe_summary.csv" ]]; then + log "Fallback probe summary: ${OUT_DIR}/fallback_probe_summary.csv" + fi if [[ -f "${OUT_DIR}/baseline_compare.csv" ]]; then log "Baseline compare: ${OUT_DIR}/baseline_compare.csv" fi diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh index 04eac1ba3..38d06bedb 100755 --- a/scripts/run_object_batch_bench_enhanced.sh +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -220,8 +220,8 @@ setup_output() { MEDIAN_CSV="$OUT_DIR/median_summary.csv" COMPARE_CSV="$OUT_DIR/baseline_compare.csv" - echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file" > "$ROUND_CSV" - echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms" > "$MEDIAN_CSV" + echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file,req_p90_human,req_p90_ms,req_p99_human,req_p99_ms" > "$ROUND_CSV" + echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,median_req_p90_ms,median_req_p99_ms" > "$MEDIAN_CSV" } trim() { @@ -288,13 +288,32 @@ extract_first() { rg -o "$regex" "$file" | head -n1 || true } +normalize_duration_metric() { + local value="$1" + value="$(trim "${value:-N/A}")" + + if [[ -z "$value" || "$value" == "N/A" ]]; then + echo "N/A" + return + fi + + if [[ "$value" =~ ^([0-9]+(\.[0-9]+)?)(ms|us|µs|s)$ ]]; then + echo "${BASH_REMATCH[1]} ${BASH_REMATCH[3]}" + return + fi + + echo "$value" | awk '{ if ($1 == "" || $1 == "N/A") print "N/A"; else print $1" "$2 }' +} + extract_metrics() { local log_file="$1" - local throughput reqps latency + local throughput reqps latency req_p90 req_p99 throughput="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")" reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")" latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')" + req_p90="$(rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^90%:[[:space:]]+//')" + req_p99="$(rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^99%:[[:space:]]+//')" if [[ -z "$latency" ]]; then latency="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(ms|us|µs|s)' "$log_file")" @@ -303,16 +322,16 @@ extract_metrics() { throughput="$(trim "${throughput:-N/A}")" reqps="$(trim "${reqps:-N/A}")" latency="$(trim "${latency:-N/A}")" + req_p90="$(trim "${req_p90:-N/A}")" + req_p99="$(trim "${req_p99:-N/A}")" # Normalize to " " shape for downstream conversion helpers. - if [[ "$latency" =~ ^([0-9]+(\.[0-9]+)?)(ms|us|µs|s)$ ]]; then - latency="${BASH_REMATCH[1]} ${BASH_REMATCH[3]}" - else - latency="$(echo "$latency" | awk '{print $1" "$2}')" - fi + latency="$(normalize_duration_metric "$latency")" + req_p90="$(normalize_duration_metric "$req_p90")" + req_p99="$(normalize_duration_metric "$req_p99")" reqps_num="$(echo "$reqps" | awk '{print $1}')" - echo "$throughput,${reqps_num:-N/A},$latency" + echo "$throughput,${reqps_num:-N/A},$latency,$req_p90,$req_p99" } median_from_numbers() { @@ -414,20 +433,28 @@ run_one_attempt() { fi finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - local metrics throughput_human reqps latency_human throughput_bps latency_ms + local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms if [[ "$DRY_RUN" == "true" ]]; then throughput_human="N/A" reqps="N/A" latency_human="N/A" throughput_bps="N/A" latency_ms="N/A" + req_p90_human="N/A" + req_p90_ms="N/A" + req_p99_human="N/A" + req_p99_ms="N/A" else metrics="$(extract_metrics "$log_file")" throughput_human="$(echo "$metrics" | cut -d',' -f1)" reqps="$(echo "$metrics" | cut -d',' -f2)" latency_human="$(echo "$metrics" | cut -d',' -f3)" + req_p90_human="$(echo "$metrics" | cut -d',' -f4)" + req_p99_human="$(echo "$metrics" | cut -d',' -f5)" throughput_bps="$(to_bps "$throughput_human")" latency_ms="$(to_ms "$latency_human")" + req_p90_ms="$(to_ms "$req_p90_human")" + req_p99_ms="$(to_ms "$req_p99_human")" fi if [[ "$DRY_RUN" != "true" && "$status" == "ok" ]]; then @@ -437,7 +464,7 @@ run_one_attempt() { fi fi - echo "$size,$TOOL,$round,$attempt,$CONCURRENCY,$status,$exit_code,$started_at_utc,$finished_at_utc,$throughput_human,$throughput_bps,$reqps,$latency_human,$latency_ms,$log_file" >> "$ROUND_CSV" + echo "$size,$TOOL,$round,$attempt,$CONCURRENCY,$status,$exit_code,$started_at_utc,$finished_at_utc,$throughput_human,$throughput_bps,$reqps,$latency_human,$latency_ms,$log_file,$req_p90_human,$req_p90_ms,$req_p99_human,$req_p99_ms" >> "$ROUND_CSV" echo "$status" } @@ -481,20 +508,24 @@ build_median_summary() { size="$(trim "$raw")" [[ -z "$size" ]] && continue - local ok_rounds fail_rounds t_vals r_vals l_vals + local ok_rounds fail_rounds t_vals r_vals l_vals p90_vals p99_vals ok_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" {c++} END{print c+0}' "$ROUND_CSV")" fail_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6!="ok" {c++} END{print c+0}' "$ROUND_CSV")" t_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $11!="N/A" {print $11}' "$ROUND_CSV")" r_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $12!="N/A" {print $12}' "$ROUND_CSV")" l_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $14!="N/A" {print $14}' "$ROUND_CSV")" + p90_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $17!="N/A" {print $17}' "$ROUND_CSV")" + p99_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $19!="N/A" {print $19}' "$ROUND_CSV")" - local m_t m_r m_l + local m_t m_r m_l m_p90 m_p99 m_t="$(median_from_numbers "$t_vals")" m_r="$(median_from_numbers "$r_vals")" m_l="$(median_from_numbers "$l_vals")" + m_p90="$(median_from_numbers "$p90_vals")" + m_p99="$(median_from_numbers "$p99_vals")" - echo "$size,$TOOL,$CONCURRENCY,$ok_rounds,$fail_rounds,$m_t,$m_r,$m_l" >> "$MEDIAN_CSV" + echo "$size,$TOOL,$CONCURRENCY,$ok_rounds,$fail_rounds,$m_t,$m_r,$m_l,$m_p90,$m_p99" >> "$MEDIAN_CSV" done }