mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-30 16:59:52 +00:00
fix(ecstore): correct codec-streaming byte accounting and partNumber routing (#4535)
Two correctness defects on the opt-in codec-streaming GET path. ECA-02 (#943): ErasureDecodeReader only decremented `remaining` for the main fill buffer. Under the default DualInFlight policy each fill also produces a queued stripe that is delivered to the client via `prefetched_bufs.pop_front()` without touching `remaining`, so any object larger than one erasure block finished with `remaining > 0` and the GET terminated with LessData despite delivering all bytes. The inflated `remaining` was also fed back into the fill worker, which used it to trim the final stripe and to decide whether to read past EOF. Account for the queued-stripe bytes when they enter the prefetch queue; queued buffers come only from `Ok(true)` decodes so they are non-empty and bounded by `remaining - main_buf.len()`, ruling out underflow. ECA-04 (#945): the codec-streaming gate did not inspect `opts.part_number`. A partNumber GET carries `range == None`, so it was not classified as a Range request and reached the full-object codec-streaming reader, which drops the storage offset/length returned by GetObjectReader::new. A partNumber >= 2 request would then stream the whole object. Mirror the direct-memory part_number fallback and route any partNumber request back to the legacy duplex path, which applies the offset/length correctly. Regression tests: DualInFlight read_to_end on a multi-block object and on a non-block-aligned object; SingleInFlight vs DualInFlight byte-identical output; gate fallback on partNumber requests. Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -2719,6 +2719,25 @@ mod tests {
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
range,
|
||||
None,
|
||||
object_info,
|
||||
fi,
|
||||
lock_optimization_enabled,
|
||||
)
|
||||
}
|
||||
|
||||
fn codec_streaming_reader_gate_for_test_with_part_number(
|
||||
range: &Option<HTTPRangeSpec>,
|
||||
part_number: Option<usize>,
|
||||
object_info: &ObjectInfo,
|
||||
fi: &FileInfo,
|
||||
lock_optimization_enabled: bool,
|
||||
) -> GetCodecStreamingGate {
|
||||
get_codec_streaming_reader_gate(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
range,
|
||||
part_number,
|
||||
object_info,
|
||||
fi,
|
||||
lock_optimization_enabled,
|
||||
@@ -4080,6 +4099,48 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_streaming_reader_gate_falls_back_on_part_number_request() {
|
||||
// A partNumber GET has `range == None`, so it is classified as a plain
|
||||
// object and would otherwise reach the codec-streaming path. That path
|
||||
// builds a full-object reader and drops the storage offset/length, so
|
||||
// partNumber >= 2 would stream the whole object. The gate must route any
|
||||
// partNumber request back to the legacy duplex path via Fallback.
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("100")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
// Baseline: without a partNumber the gate uses codec streaming.
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test_with_part_number(&None, None, &object_info, &fi, true).decision,
|
||||
GetCodecStreamingDecision::Use
|
||||
);
|
||||
|
||||
// partNumber >= 2 must fall back to legacy duplex.
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test_with_part_number(&None, Some(2), &object_info, &fi, true).decision,
|
||||
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::PartNumber)
|
||||
);
|
||||
|
||||
// partNumber == 1 also falls back even though its offset is 0, so the
|
||||
// legacy path stays the single owner of part offset application.
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test_with_part_number(&None, Some(1), &object_info, &fi, true).decision,
|
||||
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::PartNumber)
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codec_streaming_reader_build_falls_back_when_read_quorum_is_not_safe() {
|
||||
let setup = setup_inline_bitrot_readers(
|
||||
|
||||
Reference in New Issue
Block a user