gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 11:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): retry idempotent reads to survive read-after-write races (#4597)

Browse Source
This commit is contained in:
Zhengchao An
2026-07-09 12:16:43 +08:00
committed by GitHub
parent 0c040999b1
commit d8c32d3754
4 changed files with 211 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/config/src/constants/internode.rs
+12 -7
View File
@@ -148,14 +148,18 @@ pub const DEFAULT_INTERNODE_OFFLINE_REPROBE_SECS: u64 = 5;
const _: () = assert!(!DEFAULT_INTERNODE_PREWARM);
const _: () = assert!(!DEFAULT_INTERNODE_OFFLINE_BYPASS);
/// Extra attempts for idempotent, read-only/reentrant control-plane RPCs (e.g. `DiskInfo`) on
/// transient network failures, with exponential backoff (grpc-optimization P3-3).
/// Extra attempts for idempotent, read-only/reentrant control-plane and object-read RPCs
/// (`DiskInfo`, `ReadAll`, `ReadMetadata`, `ReadVersion`) on transient network failures, with
/// exponential backoff (grpc-optimization P3-3).
///
/// Defaults to `0` (disabled) — retries change latency-on-failure behavior and are opt-in. **Only**
/// idempotent reads use this; write/lock RPCs (`WriteAll`/`RenameData`/`Delete*`/`Lock`/`UnLock`)
/// must never auto-retry, to preserve quorum and idempotency semantics (see `CLAUDE.md`).
/// Defaults to `1`: a freshly-committed object lives on only `write_quorum` disks until heal
/// reconstructs the rest, so a single reset-by-peer on a metadata read during that
/// read-after-write window can erode the read quorum and surface as a spurious
/// `InsufficientReadQuorum` (issue #2761). One bounded retry absorbs that transient failure.
/// **Only** idempotent reads use this; write/lock RPCs (`WriteAll`/`RenameData`/`Delete*`/`Lock`
/// /`UnLock`) must never auto-retry, to preserve quorum and idempotency semantics (see `CLAUDE.md`).
pub const ENV_INTERNODE_IDEMPOTENT_READ_RETRIES: &str = "RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES";
pub const DEFAULT_INTERNODE_IDEMPOTENT_READ_RETRIES: usize = 0;
pub const DEFAULT_INTERNODE_IDEMPOTENT_READ_RETRIES: usize = 1;
// ── Control/bulk channel isolation (P1) ──
// Large `bytes`-carrying unary RPCs (ReadAll/WriteAll/ReadMultiple/BatchReadVersion) otherwise
@@ -283,7 +287,8 @@ mod tests {
fn internode_p3_lifecycle_defaults_are_opt_in() {
// The opt-in (default-off) invariants are asserted at compile time next to the definitions.
assert_eq!(DEFAULT_INTERNODE_OFFLINE_REPROBE_SECS, 5);
assert_eq!(DEFAULT_INTERNODE_IDEMPOTENT_READ_RETRIES, 0);
// One bounded retry for idempotent reads to absorb read-after-write transient failures (#2761).
assert_eq!(DEFAULT_INTERNODE_IDEMPOTENT_READ_RETRIES, 1);
assert_eq!(ENV_INTERNODE_PREWARM, "RUSTFS_INTERNODE_PREWARM");
assert_eq!(ENV_INTERNODE_OFFLINE_BYPASS, "RUSTFS_INTERNODE_OFFLINE_BYPASS");
assert_eq!(ENV_INTERNODE_OFFLINE_REPROBE_SECS, "RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS");