gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 17:28:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(get): include small objects in codec streaming (#6004)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-13 00:04:15 +08:00
committed by GitHub
parent d92c563b9e
commit 24ca61eb6e
3 changed files with 50 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/get_codec_streaming_compat_test.rs
+7 -2
View File
@@ -189,8 +189,6 @@ mod tests {
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"), ("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"),
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"), ("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"),
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"), ("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"),
// Lower the min-size floor so every non-inline object below is eligible.
("RUSTFS_GET_CODEC_STREAMING_MIN_SIZE", "4096"),
// Route multipart objects through per-part codec streaming too. // Route multipart objects through per-part codec streaming too.
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"), ("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"),
// Lock optimization is on by default, but pin it so the gate's // Lock optimization is on by default, but pin it so the gate's
@@ -315,6 +313,13 @@ mod tests {
}, },
payload(64 * 1024, 2), payload(64 * 1024, 2),
), ),
(
Shape {
key: "small-non-inline-256kib-plus",
expect_large: true,
},
payload(256 * 1024 + 1, 6),
),
( (
Shape { Shape {
key: "mid-1_5mib", key: "mid-1_5mib",
crates/ecstore/src/set_disk/mod.rs
+4 -2
View File
@@ -639,9 +639,11 @@ const ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = true; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = true;
const ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE"; const ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = MI_B; // Meet the direct-memory path at its default ceiling. Codec streaming remains
// rollout-gated and starts where the eager small-object path ends.
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD;
const ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE"; const ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = MI_B; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE;
const ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = "RUSTFS_GET_CODEC_STREAMING_ENGINE"; const ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = "RUSTFS_GET_CODEC_STREAMING_ENGINE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = GET_CODEC_STREAMING_ENGINE_LEGACY; const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = GET_CODEC_STREAMING_ENGINE_LEGACY;
crates/ecstore/src/set_disk/read.rs
+39 -36
View File
@@ -5479,33 +5479,36 @@ mod tests {
} }
#[test] #[test]
fn rustfs_codec_streaming_uses_conservative_default_min_size() { fn codec_streaming_default_min_size_meets_direct_memory_ceiling() {
temp_env::with_vars( for engine in [None, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)] {
[ temp_env::with_vars(
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), [
(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)), (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")), (ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, engine),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>), (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>), (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
], (ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>),
|| { ],
let below_threshold_fi = codec_streaming_test_fileinfo(512 * 1024, 1); || {
let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi); let below_threshold_fi = codec_streaming_test_fileinfo(128 * 1024 - 1, 1);
assert_eq!( let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi);
codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true).decision, assert_eq!(
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize) codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true)
); .decision,
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize)
);
let threshold_fi = codec_streaming_test_fileinfo(1_048_576, 1); let threshold_fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let threshold_object_info = codec_streaming_test_object_info(&threshold_fi); let threshold_object_info = codec_streaming_test_object_info(&threshold_fi);
assert_eq!( assert_eq!(
codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision, codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision,
GetCodecStreamingDecision::Use GetCodecStreamingDecision::Use
); );
}, },
); );
}
} }
#[test] #[test]
@@ -5803,10 +5806,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, None::<&str>), (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>), (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>), (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
], ],
|| { || {
let fi = codec_streaming_test_fileinfo(1024, 1); let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi); let object_info = codec_streaming_test_object_info(&fi);
assert_eq!( assert_eq!(
@@ -5827,10 +5830,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")), (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>), (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>), (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
], ],
|| { || {
let fi = codec_streaming_test_fileinfo(1024, 1); let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi); let object_info = codec_streaming_test_object_info(&fi);
assert_eq!( assert_eq!(
@@ -5848,10 +5851,10 @@ mod tests {
[ [
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("false")), (ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("false")),
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")), (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
], ],
|| { || {
let fi = codec_streaming_test_fileinfo(1024, 1); let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi); let object_info = codec_streaming_test_object_info(&fi);
assert_eq!( assert_eq!(
@@ -5931,10 +5934,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("0")), (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("0")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
], ],
|| { || {
let fi = codec_streaming_test_fileinfo(1024, 1); let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi); let object_info = codec_streaming_test_object_info(&fi);
assert_eq!( assert_eq!(
@@ -5951,10 +5954,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("100")), (ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("100")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), (ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), (ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), (ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
], ],
|| { || {
let fi = codec_streaming_test_fileinfo(1024, 1); let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi); let object_info = codec_streaming_test_object_info(&fi);
assert_eq!( assert_eq!(