perf(heal,scanner): single-flight MRF producers per detection event (#6282)

Two producer paths double-booked the same damage across repair records
(backlog#1894 axis A):

- The scanner's corrupt-metadata branch fired a durable MRF journal
  intent, an immediate High heal request, and a pending-ledger entry for
  the same object. When the MRF intent is accepted into the channel it
  already covers the repair durably (the consumer files a High Metadata
  heal and the journal replays it across restarts), so the immediate
  request and ledger entry are dropped in that case; on delivery failure
  (feature disabled, channel uninitialized, or full) the old immediate
  request + ledger path runs unchanged, keeping the repair safety net.
- The read path filed a journal intent before the read-repair
  reservation check, so a burst of reads failing on one object booked a
  journal record per retry. The intent now rides the submission: it is
  filed only when the sighting wins the dedup TTL, next to the Low
  request, via a new optional mrf_intent field on
  ReadRepairHealSubmission (None keeps the historical no-intent
  behavior for the other read-repair call sites).

Manager dedup-key semantics are untouched; the fix is that competing
producers stop double-booking. With RUSTFS_HEAL_MRF_ENABLE off both
paths behave exactly as before.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-20 01:32:16 +08:00
committed by GitHub
parent d6efb65588
commit b1b4e443b2
4 changed files with 153 additions and 38 deletions
+72 -18
View File
@@ -645,6 +645,32 @@ enum GetSizeFailureAction {
HealMetadata { object: String },
}
/// How the corrupt-metadata branch records the repair after attempting an
/// MRF intent (backlog#1894 axis A).
#[derive(Debug, PartialEq, Eq)]
enum CorruptMetadataRecording {
/// Intent accepted: the MRF consumer owns the repair (High Metadata
/// heal, durable after the journal's group-commit flush), so the
/// immediate heal request is skipped — the manager would otherwise book
/// two tasks for one target. A pending-ledger entry stays behind as the
/// backstop for what the journal cannot cover on its own (a crash inside
/// the flush window, or the consumer exhausting its admission attempts);
/// the repaired-notice fanout (axis B) drops the entry once the repair
/// lands.
LedgerOnly,
/// Intent rejected (feature disabled, channel uninitialized, or full):
/// the historical immediate heal request plus the ledger entry.
ImmediateAndLedger,
}
fn corrupt_metadata_recording(mrf_accepted: bool) -> CorruptMetadataRecording {
if mrf_accepted {
CorruptMetadataRecording::LedgerOnly
} else {
CorruptMetadataRecording::ImmediateAndLedger
}
}
fn build_bucket_heal_request(bucket: String, priority: HealChannelPriority) -> HealChannelRequest {
HealChannelRequest {
bucket,
@@ -2478,29 +2504,46 @@ impl FolderScanner {
}
if let GetSizeFailureAction::HealMetadata { object } = failure_action {
// MRF journal intent: durable High-priority Metadata
// heal across restarts (HS-01); the scanner heal
// request below stays as the immediate path.
rustfs_common::mrf_channel::try_send_mrf_intent(
// Single-flight (backlog#1894 axis A) — the
// recording mode and its guarantees are pinned by
// corrupt_metadata_recording below.
let mrf_accepted = rustfs_common::mrf_channel::try_send_mrf_intent(
rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
&item.bucket,
&object,
None,
);
self.send_required_scanner_heal_request(
PendingScannerHealKind::Object,
item.bucket.clone(),
Some(object.clone()),
None,
build_object_heal_request(
item.bucket.clone(),
object.clone(),
None,
self.scan_mode,
HealChannelPriority::High,
),
)
.await?;
match corrupt_metadata_recording(mrf_accepted) {
CorruptMetadataRecording::LedgerOnly => {
// Recorded as Full (retry-later): admission
// for this target happens in the MRF
// consumer, not in the manager's queue here.
self.update_pending_scanner_heal_after_admission(
PendingScannerHealKind::Object,
&item.bucket,
Some(&object),
None,
self.scan_mode,
HealAdmissionResult::Full,
);
}
CorruptMetadataRecording::ImmediateAndLedger => {
self.send_required_scanner_heal_request(
PendingScannerHealKind::Object,
item.bucket.clone(),
Some(object.clone()),
None,
build_object_heal_request(
item.bucket.clone(),
object.clone(),
None,
self.scan_mode,
HealChannelPriority::High,
),
)
.await?;
}
}
}
timer.sleep().await;
@@ -3395,6 +3438,17 @@ mod tests {
assert_eq!(EVENT_SCANNER_BIG_PREFIX, EventName::ScannerBigPrefix.to_string());
}
/// Single-flight decision for the corrupt-metadata branch (backlog#1894
/// axis A): an accepted MRF intent must drop the immediate heal request
/// (the consumer files one; the manager would double-book) while a
/// rejected one must keep it — in both cases a ledger entry remains, so
/// the backstop survives regardless of delivery.
#[test]
fn corrupt_metadata_recording_maps_delivery_to_backstop() {
assert_eq!(corrupt_metadata_recording(true), CorruptMetadataRecording::LedgerOnly);
assert_eq!(corrupt_metadata_recording(false), CorruptMetadataRecording::ImmediateAndLedger);
}
fn cooldown_map_len() -> usize {
SCANNER_ALERT_EMISSION_COOLDOWN
.lock()