gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 21:26:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(heal,scanner): single-flight MRF producers per detection event (#6282)

Browse Source 
Two producer paths double-booked the same damage across repair records
(backlog#1894 axis A):

- The scanner's corrupt-metadata branch fired a durable MRF journal
  intent, an immediate High heal request, and a pending-ledger entry for
  the same object. When the MRF intent is accepted into the channel it
  already covers the repair durably (the consumer files a High Metadata
  heal and the journal replays it across restarts), so the immediate
  request and ledger entry are dropped in that case; on delivery failure
  (feature disabled, channel uninitialized, or full) the old immediate
  request + ledger path runs unchanged, keeping the repair safety net.
- The read path filed a journal intent before the read-repair
  reservation check, so a burst of reads failing on one object booked a
  journal record per retry. The intent now rides the submission: it is
  filed only when the sighting wins the dedup TTL, next to the Low
  request, via a new optional mrf_intent field on
  ReadRepairHealSubmission (None keeps the historical no-intent
  behavior for the other read-repair call sites).

Manager dedup-key semantics are untouched; the fix is that competing
producers stop double-booking. With RUSTFS_HEAL_MRF_ENABLE off both
paths behave exactly as before.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-20 01:32:16 +08:00
committed by GitHub
parent d6efb65588
commit b1b4e443b2
4 changed files with 153 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/core/io_primitives.rs
+55
View File
@@ -1095,6 +1095,14 @@ pub(in crate::set_disk) struct ReadRepairHealSubmission<'a> {
pub(in crate::set_disk) set_index: usize,
pub(in crate::set_disk) part_number: Option<usize>,
pub(in crate::set_disk) reason: &'static str,
/// Durable MRF journal intent to file alongside the read-repair request
/// (backlog#1894 axis A): the intent kind plus its native `Uuid`
/// version id (the submission's string form stays display-only). Bound
/// to the reservation — the intent is only delivered when this sighting
/// wins the dedup TTL, so a burst of reads failing on the same object
/// books exactly one journal record instead of one per retry. `None`
/// keeps the historical no-intent behavior.
pub(in crate::set_disk) mrf_intent: Option<(rustfs_common::mrf_channel::MrfKind, Option<uuid::Uuid>)>,
}
pub(in crate::set_disk) fn send_read_repair_heal_request(
@@ -1126,6 +1134,7 @@ pub(in crate::set_disk) async fn submit_read_repair_heal(
set_index,
part_number,
reason,
mrf_intent: None,
},
send_read_repair_heal_request,
)
@@ -1144,6 +1153,7 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
set_index,
part_number,
reason,
mrf_intent,
} = submission;
let Some(dedup_key) = reserve_read_repair_heal(bucket, object, version_id, pool_index, set_index).await else {
@@ -1155,6 +1165,12 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
return;
};
// Reservation won: this sighting owns the repair records for the object,
// including the durable journal intent when the caller asked for one.
if let Some((kind, version_uuid)) = mrf_intent {
rustfs_common::mrf_channel::try_send_mrf_intent(kind, bucket, object, version_uuid);
}
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
@@ -8710,6 +8726,42 @@ mod tests {
assert_eq!(responses[0].error, Error::ErasureReadQuorum.to_string());
}
#[tokio::test]
#[serial_test::serial]
async fn mrf_intent_is_filed_once_per_read_repair_reservation() {
// Serial: owns the process-global MRF channel for this test binary
// (same key as the other channel-owning tests above).
let bucket = format!("mrf-intent-bucket-{}", Uuid::new_v4());
let object = format!("object-{}", Uuid::new_v4());
let mut receiver = rustfs_common::mrf_channel::init_mrf_channel().expect("first channel init in this binary");
rustfs_common::mrf_channel::set_mrf_delivery_enabled(true);
fn intent_submission<'a>(bucket: &'a str, object: &'a str) -> ReadRepairHealSubmission<'a> {
ReadRepairHealSubmission {
bucket,
object,
version_id: None,
pool_index: 9,
set_index: 9,
part_number: Some(1),
reason: "decode_error",
mrf_intent: Some((rustfs_common::mrf_channel::MrfKind::DecodeFailure, None)),
}
}
// First sighting wins the reservation: the journal intent is filed
// synchronously before the admission task is spawned.
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
let first = receiver.try_recv().expect("first sighting must file exactly one MRF intent");
assert_eq!(*first.bucket, bucket);
assert_eq!(*first.object, object);
// Second sighting within the dedup TTL is a duplicate: no request, no
// second journal record.
submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
assert!(receiver.try_recv().is_err(), "duplicate sighting must not file another MRF intent");
}
#[tokio::test]
async fn reserve_read_repair_heal_dedupes_by_object_version_and_set() {
let object = format!("object-{}", Uuid::new_v4());
@@ -8818,6 +8870,7 @@ mod tests {
set_index: 2,
part_number: Some(1),
reason: "test",
mrf_intent: None,
},
failed_read_repair_submitter,
)
@@ -8846,6 +8899,7 @@ mod tests {
set_index: 3,
part_number: Some(2),
reason: "test",
mrf_intent: None,
},
dropped_read_repair_submitter,
)
@@ -8874,6 +8928,7 @@ mod tests {
set_index: 4,
part_number: None,
reason: "test",
mrf_intent: None,
},
accepted_read_repair_submitter,
)
crates/ecstore/src/set_disk/read.rs
+20 -17
View File
@@ -1077,23 +1077,23 @@ impl SetDisks {
"Recoverable decode error triggered read repair"
);
let version_id = fi.version_id.as_ref().map(ToString::to_string);
// MRF journal intent: keeps a durable Urgent ECDecode
// request alive across restarts even when the in-memory
// read-repair request is dropped or lost (HS-01).
rustfs_common::mrf_channel::try_send_mrf_intent(
rustfs_common::mrf_channel::MrfKind::DecodeFailure,
bucket,
object,
fi.version_id,
);
submit_read_repair_heal(
bucket,
object,
version_id.as_deref(),
pool_index,
set_index,
Some(part_number),
"decode_error",
// Single-flight (backlog#1894 axis A): the durable
// MRF intent (Urgent ECDecode across restarts, HS-01)
// is bound to the read-repair reservation, so only the
// first sighting within the dedup TTL books a journal
// record instead of one per retried read.
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket,
object,
version_id: version_id.as_deref(),
pool_index,
set_index,
part_number: Some(part_number),
reason: "decode_error",
mrf_intent: Some((rustfs_common::mrf_channel::MrfKind::DecodeFailure, fi.version_id)),
},
send_read_repair_heal_request,
)
.await;
has_err = false;
@@ -2577,6 +2577,7 @@ mod metadata_cache_tests {
set_index: 0,
part_number: Some(1),
reason: "missing_shards",
mrf_intent: None,
},
slow_read_repair_submitter,
)
@@ -2611,6 +2612,7 @@ mod metadata_cache_tests {
set_index: 0,
part_number: Some(1),
reason: "missing_shards",
mrf_intent: None,
},
dropped_read_repair_submitter,
)
@@ -2647,6 +2649,7 @@ mod metadata_cache_tests {
set_index: 0,
part_number: Some(1),
reason: "missing_shards",
mrf_intent: None,
},
capture_read_repair_submitter,
)