gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(get): tune body stream buffers (#5959)

Browse Source
This commit is contained in:
houseme
2026-08-11 20:48:03 +08:00
committed by GitHub
parent 5e3010c6b5
commit ba6a0f25d9
2 changed files with 36 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+23 -7
View File
@@ -584,10 +584,14 @@ fn capacity_scope_from_disks(disks: &[Option<DiskStore>]) -> CapacityScope {
///
/// **Deprecated**: Use `adaptive_duplex_buffer_size()` for object-size-aware sizing.
pub fn get_duplex_buffer_size() -> usize {
rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
)
static CACHED: OnceLock<usize> = OnceLock::new();
*CACHED.get_or_init(|| {
rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
)
.max(1)
})
}
/// Get adaptive duplex buffer size based on object size.
@@ -597,12 +601,15 @@ pub fn get_duplex_buffer_size() -> usize {
fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
const KB: usize = 1024;
const MB: usize = 1024 * 1024;
match object_size {
0..=1_048_576 => 64 * KB, // <= 1MB: 64KB
let target = match object_size {
0..=131_072 => 64 * KB, // <= 128KB: 64KB
131_073..=1_048_576 => 512 * KB, // <= 1MB: reduce duplex backpressure without a 1MB pipe per request
1_048_577..=16_777_216 => MB, // <= 16MB: 1MB
16_777_217..=268_435_456 => 4 * MB, // <= 256MB: 4MB
_ => 8 * MB, // > 256MB: 8MB
}
};
let object_cap = usize::try_from(object_size).ok().filter(|size| *size > 0).unwrap_or(target);
target.min(object_cap.max(64 * KB)).min(get_duplex_buffer_size())
}
// ============================================================================
@@ -11129,4 +11136,13 @@ mod tests {
);
}
}
#[test]
fn adaptive_duplex_buffer_size_raises_mid_sized_gets_without_penalizing_tiny_objects() {
assert_eq!(adaptive_duplex_buffer_size(64 * 1024), 64 * 1024);
assert_eq!(adaptive_duplex_buffer_size(128 * 1024), 64 * 1024);
assert_eq!(adaptive_duplex_buffer_size(256 * 1024), 256 * 1024);
assert_eq!(adaptive_duplex_buffer_size(1024 * 1024), 512 * 1024);
assert_eq!(adaptive_duplex_buffer_size(2 * 1024 * 1024), 1024 * 1024);
}
}
rustfs/src/app/object_usecase.rs
+13 -1
View File
@@ -708,6 +708,8 @@ const LARGE_SEQUENTIAL_GET_STREAM_BUFFER_CAP_BYTES: usize = 4 * MI_B;
const LARGE_SEQUENTIAL_GET_READAHEAD_MULTIPLIER: usize = 2;
const LARGE_BODY_READER_STREAM_BUFFER_FLOOR_BYTES: usize = MI_B;
const LARGE_BODY_READER_STREAM_BUFFER_THRESHOLD_BYTES: i64 = 4 * MI_B as i64;
const MID_BODY_READER_STREAM_BUFFER_FLOOR_BYTES: usize = 512 * 1024;
const MID_BODY_READER_STREAM_BUFFER_THRESHOLD_BYTES: i64 = MI_B as i64;
const ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE: &str = "RUSTFS_GET_SEEK_BUFFER_ENABLE";
const ENV_RUSTFS_GET_READER_STREAM_BUFFER_SIZE: &str = "RUSTFS_GET_READER_STREAM_BUFFER_SIZE";
const ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE: &str = "RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE";
@@ -763,6 +765,12 @@ fn tune_reader_stream_buffer_size(
return selected_size.max(LARGE_BODY_READER_STREAM_BUFFER_FLOOR_BYTES);
}
if stream_strategy == GetObjectStreamStrategy::Standard
&& response_content_length >= MID_BODY_READER_STREAM_BUFFER_THRESHOLD_BYTES
{
return selected_size.max(MID_BODY_READER_STREAM_BUFFER_FLOOR_BYTES);
}
selected_size
}
@@ -14938,7 +14946,11 @@ mod tests {
);
assert_eq!(
tune_reader_stream_buffer_size(128 * 1024, MI_B as i64, GetObjectStreamStrategy::Standard),
128 * 1024
MID_BODY_READER_STREAM_BUFFER_FLOOR_BYTES
);
assert_eq!(
tune_reader_stream_buffer_size(256 * 1024, 2 * MI_B as i64, GetObjectStreamStrategy::Standard),
MID_BODY_READER_STREAM_BUFFER_FLOOR_BYTES
);
assert_eq!(
tune_reader_stream_buffer_size(128 * 1024, 10 * MI_B as i64, GetObjectStreamStrategy::LargeSequentialReadahead),