gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ecstore): make GET codec-streaming a single rollout switch (backlog#1183) (#4752)

Browse Source 
backlog#1183, staged rollout step. Simplify enabling the zero-duplex GET
codec-streaming fast path to a single switch — RUSTFS_GET_CODEC_STREAMING_ROLLOUT
(default "off") — now that body/header parity is proven (parity e2e net + bench
A/B on backlog#1183).

- Add a clean production rollout token "on" (aliases "full"/"production"); the
  legacy "internal"/"benchmark" tokens remain accepted.
- Flip DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE and the two ..._COMPAT_CONFIRMED
  defaults to true. They are retained as emergency kill-switches (set any to
  false to force the fast path off) but no longer gate enablement — the rollout
  switch does.

No production behavior change: with no env set the rollout switch defaults to
"off", so GET stays on the legacy duplex path exactly as before. Flipping the
hard default to on is deferred to a follow-up after a production soak.

Note (intentional semantics change): with the rollout switch opted in
("on"/"internal"/"benchmark"), codec streaming now activates without also
setting the two ..._COMPAT_CONFIRMED vars — compatibility is confirmed, so those
confirmations are baked in.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-12 01:36:58 +08:00
committed by GitHub
parent 028ba6a675
commit 6886dca7d1
2 changed files with 84 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+34 -3
View File
@@ -429,7 +429,13 @@ const ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: &str = "RUSTFS_GET_OBJEC
// --- Codec Streaming Configuration ---
const ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_ENABLE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = false; // Disabled until rollout gates are ready
// Emergency kill-switch, not the primary enablement knob. The single switch that
// turns codec streaming on/off is `RUSTFS_GET_CODEC_STREAMING_ROLLOUT` (default
// `off`). This flag defaults to `true` and only exists to force-disable the fast
// path regardless of rollout (set to `false`). Body/header compatibility is
// confirmed by the GET codec-streaming parity e2e net + bench A/B (backlog#1183),
// so it no longer gates enablement. See `get_codec_streaming_reader_gate`.
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = true;
const ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = MI_B;
@@ -439,6 +445,10 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = MI_B;
const ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = "RUSTFS_GET_CODEC_STREAMING_ENGINE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = GET_CODEC_STREAMING_ENGINE_LEGACY;
// Primary single switch for the codec-streaming GET fast path. `off` (default)
// keeps GET on the legacy duplex path; `on` (aliases `full`/`production`, plus
// the legacy `internal`/`benchmark`) opts it in. Combine with
// `..._ROLLOUT_PCT` for a partial (percentage-based) rollout.
const ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT: &str = "RUSTFS_GET_CODEC_STREAMING_ROLLOUT";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ROLLOUT: &str = "off";
@@ -918,18 +928,35 @@ fn get_codec_streaming_engine() -> GetCodecStreamingEngine {
fn get_codec_streaming_rollout() -> GetCodecStreamingRollout {
let rollout = rustfs_utils::get_env_str(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, DEFAULT_RUSTFS_GET_CODEC_STREAMING_ROLLOUT);
match rollout.trim() {
// Clean production token. `internal`/`benchmark` remain accepted aliases
// for backward compatibility; all three opt the fast path in.
value
if value.eq_ignore_ascii_case("on")
|| value.eq_ignore_ascii_case("full")
|| value.eq_ignore_ascii_case("production") =>
{
GetCodecStreamingRollout::On
}
value if value.eq_ignore_ascii_case("internal") => GetCodecStreamingRollout::Internal,
value if value.eq_ignore_ascii_case("benchmark") => GetCodecStreamingRollout::Benchmark,
_ => GetCodecStreamingRollout::Off,
}
}
/// Emergency kill-switch (defaults to `true`). Set
/// `RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED=false` to force the fast path
/// off. Body compatibility is confirmed by the parity e2e net + bench (backlog#1183),
/// so this no longer gates enablement — the `..._ROLLOUT` switch does.
fn is_get_codec_streaming_body_compat_confirmed() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, false)
rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, true)
}
/// Emergency kill-switch (defaults to `true`). Set
/// `RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED=false` to force the fast path
/// off. Header compatibility is confirmed by the parity e2e net + bench (backlog#1183),
/// so this no longer gates enablement — the `..._ROLLOUT` switch does.
fn is_get_codec_streaming_header_compat_confirmed() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, false)
rustfs_utils::get_env_bool(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, true)
}
fn build_get_codec_streaming_decode_engine(erasure: coding::Erasure) -> std::io::Result<CodecStreamingDecodeEngine> {
@@ -954,7 +981,11 @@ enum GetCodecStreamingDecision {
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum GetCodecStreamingRollout {
/// Default. Codec streaming disabled; GET uses the legacy duplex path.
Off,
/// Production enablement token (`on`/`full`/`production`).
On,
/// Backward-compatible aliases that also opt the fast path in.
Internal,
Benchmark,
}
crates/ecstore/src/set_disk/read.rs
+50 -1
View File
@@ -4840,7 +4840,11 @@ mod tests {
}
#[test]
fn codec_streaming_reader_gate_defaults_to_disabled() {
fn codec_streaming_reader_gate_defaults_to_off() {
// With no env set the `..._ROLLOUT` switch defaults to `off`, so GET stays
// on the legacy duplex path. The compat kill-switches now default to
// confirmed (backlog#1183), so the fallback reason is the rollout switch,
// not the retired `Disabled`/`*Unconfirmed` reasons.
temp_env::with_vars(
[
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, None::<&str>),
@@ -4853,6 +4857,51 @@ mod tests {
let fi = codec_streaming_test_fileinfo(1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision,
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::RolloutNotOptedIn)
);
},
);
}
#[test]
fn codec_streaming_reader_gate_enabled_by_rollout_switch_alone() {
// The single switch: `..._ROLLOUT=on` opts the fast path in with the
// ENABLE / *_COMPAT_CONFIRMED kill-switches left unset (they default on).
temp_env::with_vars(
[
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
],
|| {
let fi = codec_streaming_test_fileinfo(1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision,
GetCodecStreamingDecision::Use
);
},
);
}
#[test]
fn codec_streaming_reader_gate_force_disabled_by_enable_kill_switch() {
// Even with the rollout switch on, `ENABLE=false` force-disables the fast path.
temp_env::with_vars(
[
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("false")),
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
],
|| {
let fi = codec_streaming_test_fileinfo(1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
codec_streaming_reader_gate_for_test(&None, &object_info, &fi, true).decision,
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::Disabled)