mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 23:47:28 +00:00
feat(get): harden codec streaming rollout (#3981)
* feat(get): consolidate GET performance optimization Consolidated implementation of all GET performance optimizations into a single, well-organized commit replacing the previous patch-on-patch approach. ## Changes ### Configuration (set_disk/mod.rs) - Consolidated all GET optimization flags into a single organized section - Enabled by default: codec streaming, metadata early-stop, page cache reclaim - Added codec streaming multipart flag (default: disabled) - Added version-aware early-stop flag (default: disabled) - Added adaptive duplex buffer sizing based on object size - All flags use OnceLock caching with rollout percentage support ### Metadata Early-Stop (set_disk/read.rs) - Delete marker early-stop when quorum agrees - Version-aware early-stop for versioned GET requests - MetadataQuorumAccumulator enhanced with: - delete_marker_votes tracking - requested_version_id and matching_version_votes tracking - version_early_stop_decision() method - 6 new tests for version early-stop scenarios ### Codec Streaming (erasure/coding/decode_reader.rs) - DualInFlight (2-stripe lookahead) enabled by default ### Decode Pipeline (erasure/coding/decode.rs) - Stripe prefetch count configuration - Bitrot-decode overlap configuration ### Disk Layer (disk/local.rs) - O_DIRECT read configuration constants (preparation) ### Metrics (io-metrics/lib.rs) - BytesPool acquisition/return metrics - Metadata phase duration with early-stop label - Total duration with reader_path label ### Diagnostics (diagnostics/) - Early-stop reason constants - Pool tier/outcome label constants ### Observability (.docker/observability/) - 3 Grafana dashboards for GET optimization monitoring - Prometheus alert rules (6 alerts: 3 critical, 3 warning) - Updated README.md and README_ZH.md with usage docs ### Config (config/src/constants/runtime.rs) - Page cache reclaim read enabled by default ## Environment Variables | Variable | Default | Description | |----------|---------|-------------| | RUSTFS_GET_CODEC_STREAMING_ENABLE | true | Codec streaming base flag | | RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT | 100 | Codec streaming rollout % | | RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE | false | Multipart codec streaming | | RUSTFS_GET_METADATA_EARLY_STOP_ENABLE | true | Early-stop base flag | | RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT | 100 | Early-stop rollout % | | RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE | false | Version-aware early-stop | | RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE | true | Page cache reclaim | | RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE | false | O_DIRECT (preparation) | | RUSTFS_GET_DECODE_STRIPE_PREFETCH_COUNT | 1 | Stripe prefetch | | RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE | false | Bitrot-decode overlap | | RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT | 2 | DualInFlight stripes | ## Rollback All optimizations can be disabled via environment variables: RUSTFS_GET_CODEC_STREAMING_ENABLE=false RUSTFS_GET_METADATA_EARLY_STOP_ENABLE=false RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE=false Co-Authored-By: heihutu <heihutu@gmail.com> * test(get): add stress test scripts for GET optimization validation - quick-validate-get-optimization.sh: Quick 5-minute validation - stress-test-get-optimization.sh: Full 30+ minute stress test - README-stress-test.md: Usage documentation Co-Authored-By: heihutu <heihutu@gmail.com> * test(ecstore): align file cache reclaim defaults * chore(deps): update redis and erasure codec * test(ecstore): align decode fill policy default * fix(get): wire codec streaming rollout gate * perf(get): skip metrics-off codec timers * test(get): capture codec streaming diagnostics * test(get): add multipart fallback probe * test(get): add encrypted fallback probe * test(get): add compressed fallback probe * test(get): add degraded read fallback probe * test(get): cover remote fallback probe * test(get): report warp request p99 * test(get): capture OTLP metric deltas * perf(get): align codec streaming inflight default * perf(get): reuse codec reader output buffers * test(get): count codec reader fill starts * perf(get): reuse codec reader fill worker * perf(get): lazy init rustfs codec reconstruct * test(get): cover rustfs codec source faults * docs(get): record rustfs codec fallback scope * feat(get): add multipart codec reader opt-in * test(get): add multipart codec smoke option * test(get): cover multipart codec degraded fallback * perf(get): bound multipart codec eager setup * test(get): satisfy codec hardening PR gate --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -16,7 +16,8 @@ use crate::diagnostics::get::{
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_SHARD_READ_ERROR_MISSING, GET_SHARD_READ_ERROR_NONE, GET_SHARD_READ_OUTCOME_ERROR,
|
||||
GET_SHARD_READ_OUTCOME_MISSING, GET_SHARD_READ_OUTCOME_SUCCESS, GET_SHARD_ROLE_DATA, GET_SHARD_ROLE_PARITY, GET_STAGE_EMIT,
|
||||
GET_STAGE_RANGE, GET_STAGE_RECONSTRUCT, GET_STAGE_STRIPE_READ, GET_STAGE_STRIPE_READ_FIRST_SHARD,
|
||||
GET_STAGE_STRIPE_READ_QUORUM, GetObjectFailureReason, classify_io_error, record_get_object_pipeline_failure,
|
||||
GET_STAGE_STRIPE_READ_QUORUM, GetObjectFailureReason, classify_io_error, get_stage_timer_if_enabled,
|
||||
record_get_object_pipeline_failure, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
use crate::disk::error::Error;
|
||||
@@ -182,7 +183,7 @@ where
|
||||
Box::pin(async move {
|
||||
let mut buf = recycled_buf.unwrap_or_else(|| vec![0; shard_size]);
|
||||
debug_assert_eq!(buf.len(), shard_size);
|
||||
let read_start = Instant::now();
|
||||
let read_start = metrics_path.map(|_| Instant::now());
|
||||
let read_result = if read_timeout.is_zero() {
|
||||
reader.read(&mut buf).await
|
||||
} else {
|
||||
@@ -200,7 +201,7 @@ where
|
||||
GET_SHARD_READ_OUTCOME_ERROR,
|
||||
error_class,
|
||||
0,
|
||||
read_start.elapsed().as_secs_f64(),
|
||||
read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()),
|
||||
reader.last_verify_duration().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
@@ -221,7 +222,7 @@ where
|
||||
GET_SHARD_READ_OUTCOME_SUCCESS,
|
||||
GET_SHARD_READ_ERROR_NONE,
|
||||
n,
|
||||
read_start.elapsed().as_secs_f64(),
|
||||
read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()),
|
||||
reader.last_verify_duration().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
@@ -240,7 +241,7 @@ where
|
||||
GET_SHARD_READ_OUTCOME_ERROR,
|
||||
error_class,
|
||||
0,
|
||||
read_start.elapsed().as_secs_f64(),
|
||||
read_start.map_or(0.0, |read_start| read_start.elapsed().as_secs_f64()),
|
||||
verify_duration_secs,
|
||||
);
|
||||
}
|
||||
@@ -450,6 +451,7 @@ where
|
||||
read_timeout: Duration,
|
||||
verify_reconstruction: bool,
|
||||
) -> Self {
|
||||
let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
let shard_size = e.shard_size();
|
||||
let shard_file_size = e.shard_file_size(total_length as i64) as usize;
|
||||
|
||||
@@ -615,7 +617,7 @@ where
|
||||
let mut reader_iter = ReaderLaunchIter::new(&mut self.readers, read_costs, locality_preference_enabled);
|
||||
let mut sets = FuturesUnordered::new();
|
||||
let mut active_readers = vec![false; num_readers];
|
||||
let stripe_read_start = Instant::now();
|
||||
let stripe_read_start = self.metrics_path.map(|_| Instant::now());
|
||||
let mut scheduled = 0usize;
|
||||
for _ in 0..self.data_shards {
|
||||
if let Some((i, reader)) = reader_iter.next() {
|
||||
@@ -722,11 +724,7 @@ where
|
||||
completed += 1;
|
||||
if !first_shard_recorded {
|
||||
if let Some(path) = self.metrics_path {
|
||||
rustfs_io_metrics::record_get_object_stage_duration(
|
||||
path,
|
||||
GET_STAGE_STRIPE_READ_FIRST_SHARD,
|
||||
stripe_read_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_FIRST_SHARD, stripe_read_start);
|
||||
}
|
||||
first_shard_recorded = true;
|
||||
}
|
||||
@@ -876,11 +874,7 @@ where
|
||||
}
|
||||
|
||||
if let Some(path) = self.metrics_path {
|
||||
rustfs_io_metrics::record_get_object_stage_duration(
|
||||
path,
|
||||
GET_STAGE_STRIPE_READ_QUORUM,
|
||||
stripe_read_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
record_get_stage_duration_if_enabled(path, GET_STAGE_STRIPE_READ_QUORUM, stripe_read_start);
|
||||
rustfs_io_metrics::record_get_object_shard_read_fanout(path, scheduled, completed, success, failed);
|
||||
if locality_preference_enabled {
|
||||
let remote_avoided = remote_available.saturating_sub(remote_scheduled);
|
||||
@@ -1038,9 +1032,11 @@ where
|
||||
offset = 0;
|
||||
|
||||
let write_len = write_left.min(block_slice.len());
|
||||
let write_stage_start = Instant::now();
|
||||
let write_stage_start = get_stage_timer_if_enabled(rustfs_io_metrics::get_stage_metrics_enabled());
|
||||
if let Err(e) = writer.write_all(&block_slice[..write_len]).await {
|
||||
rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
|
||||
if let Some(write_stage_start) = write_stage_start {
|
||||
rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
|
||||
}
|
||||
let reason = classify_io_error(&e);
|
||||
record_get_object_pipeline_failure(GET_STAGE_EMIT, reason);
|
||||
error!(
|
||||
@@ -1054,7 +1050,9 @@ where
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
|
||||
if let Some(write_stage_start) = write_stage_start {
|
||||
rustfs_io_metrics::record_get_object_duplex_backpressure_duration(write_stage_start.elapsed().as_secs_f64());
|
||||
}
|
||||
|
||||
total_written += write_len;
|
||||
write_left -= write_len;
|
||||
@@ -1180,13 +1178,10 @@ impl Erasure {
|
||||
break;
|
||||
}
|
||||
|
||||
let stripe_read_stage_start = Instant::now();
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let (mut shards, errs) = reader.read().await;
|
||||
rustfs_io_metrics::record_get_object_stage_duration(
|
||||
"legacy_duplex",
|
||||
"stripe_read",
|
||||
stripe_read_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_STRIPE_READ, stripe_read_stage_start);
|
||||
|
||||
if ret_err.is_none()
|
||||
&& let (_, Some(err)) = reduce_errs(&errs, &[])
|
||||
@@ -1216,12 +1211,12 @@ impl Erasure {
|
||||
// Decode the shards. If this stripe needed parity to reconstruct a
|
||||
// missing data shard and an extra source shard was available, verify
|
||||
// the reconstructed data against that source before streaming bytes.
|
||||
let reconstruct_stage_start = Instant::now();
|
||||
let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
if let Err(e) = self.decode_data_with_reconstruction_verification(&mut shards) {
|
||||
rustfs_io_metrics::record_get_object_stage_duration(
|
||||
"legacy_duplex",
|
||||
"reconstruct",
|
||||
reconstruct_stage_start.elapsed().as_secs_f64(),
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_STAGE_RECONSTRUCT,
|
||||
reconstruct_stage_start,
|
||||
);
|
||||
let reason = GetObjectFailureReason::DecodeError;
|
||||
error!(
|
||||
@@ -1238,28 +1233,16 @@ impl Erasure {
|
||||
ret_err = Some(e);
|
||||
break;
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_stage_duration(
|
||||
"legacy_duplex",
|
||||
"reconstruct",
|
||||
reconstruct_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
|
||||
|
||||
let emit_stage_start = Instant::now();
|
||||
let emit_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let n = match write_data_blocks(writer, &shards, self.data_shards, block_offset, block_length).await {
|
||||
Ok(n) => {
|
||||
rustfs_io_metrics::record_get_object_stage_duration(
|
||||
"legacy_duplex",
|
||||
"emit",
|
||||
emit_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
|
||||
n
|
||||
}
|
||||
Err(e) => {
|
||||
rustfs_io_metrics::record_get_object_stage_duration(
|
||||
"legacy_duplex",
|
||||
"emit",
|
||||
emit_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
|
||||
error!(
|
||||
block_offset,
|
||||
block_length,
|
||||
@@ -1624,6 +1607,24 @@ mod tests {
|
||||
assert_eq!(shard_read_launch_order(&read_costs, read_costs.len(), true), vec![1, 3, 2, 0, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn parallel_reader_drops_metrics_path_when_stage_metrics_disabled() {
|
||||
let erasure = Erasure::new(2, 1, 32);
|
||||
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
|
||||
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
|
||||
let reader = ParallelReader::new_with_metrics_path(readers, erasure.clone(), 0, 1, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
|
||||
assert_eq!(reader.metrics_path, None);
|
||||
|
||||
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let reader = ParallelReader::new_with_metrics_path(readers, erasure, 0, 1, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
|
||||
assert_eq!(reader.metrics_path, Some(GET_OBJECT_PATH_LEGACY_DUPLEX));
|
||||
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_parallel_reader_local_first_avoids_remote_when_local_quorum_exists() {
|
||||
|
||||
Reference in New Issue
Block a user