perf(ecstore): keep bounded GET fanout opt-in (#5929)

Keep GET data-read metadata early-stop and bounded fanout behind explicit environment switches so the default path preserves full fanout read-failure tolerance.

Retain the focused opt-in A/B coverage and the invalid parity full-fanout guard for heterogeneous set layouts.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-10 22:15:34 +08:00
committed by GitHub
parent 7ca69eb39c
commit fe2516ee86
3 changed files with 74 additions and 16 deletions
@@ -538,7 +538,7 @@ impl MetadataQuorumAccumulator {
} }
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize { pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 { if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks; return self.total_disks;
} }
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count); let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
@@ -550,7 +550,7 @@ impl MetadataQuorumAccumulator {
} }
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize { pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
if self.default_parity_count == 0 { if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
self.total_disks self.total_disks
} else { } else {
self.total_disks / 2 self.total_disks / 2
@@ -5376,7 +5376,7 @@ mod tests {
temp_env::async_with_vars( temp_env::async_with_vars(
[ [
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None), ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("false")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
], ],
async { async {
@@ -5389,7 +5389,7 @@ mod tests {
assert_eq!( assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION), calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64, DISKS as u64,
"control path should keep the default data-read full fanout" "control path should keep full fanout when data-read early stop is explicitly disabled"
); );
assert_eq!(diagnostics.total_responses(), DISKS); assert_eq!(diagnostics.total_responses(), DISKS);
}, },
@@ -5434,6 +5434,42 @@ mod tests {
drop(dirs); drop(dirs);
} }
#[tokio::test]
async fn bounded_metadata_early_stop_defaults_keep_data_get_full_fanout() {
const DISKS: usize = 4;
let bucket = "bounded-data-get-default-bucket";
let object = "bounded-data-get-default-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
],
async {
let calls = disk_call_counters::observe(object);
let (parts_metadata, errs, diagnostics) =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
.await
.expect("default data-read metadata should resolve");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"default GET data-read metadata must keep full fanout for read-failure tolerance"
);
assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test] #[tokio::test]
async fn bounded_metadata_early_stop_falls_back_to_full_fanout_on_data_read_error() { async fn bounded_metadata_early_stop_falls_back_to_full_fanout_on_data_read_error() {
const DISKS: usize = 4; const DISKS: usize = 4;
@@ -6009,6 +6045,16 @@ mod tests {
assert_eq!(accumulator.candidate_latest_quorum(&impossible_parity), None); assert_eq!(accumulator.candidate_latest_quorum(&impossible_parity), None);
} }
#[test]
fn metadata_quorum_accumulator_treats_invalid_default_parity_as_full_fanout() {
let accumulator = MetadataQuorumAccumulator::new(2, 2, true);
assert_eq!(accumulator.default_write_quorum(), 2);
assert_eq!(accumulator.missing_response_quorum(), 2);
assert!(accumulator.can_still_reach_early_stop_with_pending(2));
assert!(!accumulator.can_still_reach_early_stop_with_pending(1));
}
#[test] #[test]
fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() { fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() {
assert!(!is_confirmed_missing_part_error(None)); assert!(!is_confirmed_missing_part_error(None));
+8 -8
View File
@@ -672,10 +672,10 @@ const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 102
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE"; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE";
// Enabled by default (backlog#872): the early-stop path only engages for // Enabled by default (backlog#872): the early-stop path only engages for
// requests `should_allow_metadata_early_stop` classifies as safe (metadata-only // requests `should_allow_metadata_early_stop` classifies as safe (latest-version
// reads by default, without version_id / healing / free-version needs) and // metadata-only reads by default, without version_id / healing / free-version
// still requires a full read-quorum agreement before stopping. Set the env var // needs) and still requires a full read-quorum agreement before stopping. Set
// to `false` to fall back to full-wait metadata fanout. // the env var to `false` to fall back to full-wait metadata fanout.
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true; const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT"; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
@@ -836,10 +836,10 @@ mod prepared_get_object_metadata_tests {
.prepare_get_object_metadata(bucket, object, &opts) .prepare_get_object_metadata(bucket, object, &opts)
.await .await
.expect("prepared metadata should resolve"); .expect("prepared metadata should resolve");
let prepared_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
assert_eq!( assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION), prepared_calls, 3,
4, "default prepared GET metadata should stop after the 2+2 read/write quorum"
"preparation should fan out to each online disk exactly once"
); );
let mut reader = set_disks let mut reader = set_disks
@@ -864,7 +864,7 @@ mod prepared_get_object_metadata_tests {
); );
assert_eq!( assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION), calls.total(disk_call_counters::KIND_READ_VERSION),
4, 3,
"reader construction must consume prepared metadata instead of repeating the fanout" "reader construction must consume prepared metadata instead of repeating the fanout"
); );
} }
+16 -4
View File
@@ -3834,25 +3834,24 @@ mod tests {
assert!(metadata_early_stop_permitted(true, true, false, "", false, false)); assert!(metadata_early_stop_permitted(true, true, false, "", false, false));
// observe=false (non-observed fanout) also disables early-stop. // observe=false (non-observed fanout) also disables early-stop.
assert!(!metadata_early_stop_permitted(true, false, false, "", false, false)); assert!(!metadata_early_stop_permitted(true, false, false, "", false, false));
// Data reads require their own explicit rollout gate.
assert!(!metadata_early_stop_permitted(true, true, true, "", false, false)); assert!(!metadata_early_stop_permitted(true, true, true, "", false, false));
}, },
); );
} }
#[test] #[test]
fn metadata_early_stop_requires_explicit_data_read_opt_in() { fn metadata_early_stop_keeps_data_reads_opt_in_by_default() {
temp_env::with_vars( temp_env::with_vars(
[ [
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, Some("true")), (ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, None),
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None), (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
], ],
|| { || {
assert!(!should_allow_metadata_early_stop(true, "", false, false)); assert!(!should_allow_metadata_early_stop(true, "", false, false));
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false)); assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
assert!(should_allow_metadata_early_stop(false, "", false, false)); assert!(should_allow_metadata_early_stop(false, "", false, false));
assert!(should_allow_metadata_early_stop(false, "version-id", false, false)); assert!(!should_allow_metadata_early_stop(false, "version-id", false, false));
}, },
); );
temp_env::with_vars( temp_env::with_vars(
@@ -3866,6 +3865,19 @@ mod tests {
assert!(should_allow_metadata_early_stop(true, "version-id", false, false)); assert!(should_allow_metadata_early_stop(true, "version-id", false, false));
}, },
); );
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")),
],
|| {
assert!(!should_allow_metadata_early_stop(true, "", false, false));
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
assert!(should_allow_metadata_early_stop(false, "", false, false));
assert!(should_allow_metadata_early_stop(false, "version-id", false, false));
},
);
} }
#[test] #[test]