mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 12:26:37 +00:00
perf(heal,scanner): single-flight MRF producers per detection event (#6282)
Two producer paths double-booked the same damage across repair records (backlog#1894 axis A): - The scanner's corrupt-metadata branch fired a durable MRF journal intent, an immediate High heal request, and a pending-ledger entry for the same object. When the MRF intent is accepted into the channel it already covers the repair durably (the consumer files a High Metadata heal and the journal replays it across restarts), so the immediate request and ledger entry are dropped in that case; on delivery failure (feature disabled, channel uninitialized, or full) the old immediate request + ledger path runs unchanged, keeping the repair safety net. - The read path filed a journal intent before the read-repair reservation check, so a burst of reads failing on one object booked a journal record per retry. The intent now rides the submission: it is filed only when the sighting wins the dedup TTL, next to the Low request, via a new optional mrf_intent field on ReadRepairHealSubmission (None keeps the historical no-intent behavior for the other read-repair call sites). Manager dedup-key semantics are untouched; the fix is that competing producers stop double-booking. With RUSTFS_HEAL_MRF_ENABLE off both paths behave exactly as before. Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -645,6 +645,32 @@ enum GetSizeFailureAction {
|
||||
HealMetadata { object: String },
|
||||
}
|
||||
|
||||
/// How the corrupt-metadata branch records the repair after attempting an
|
||||
/// MRF intent (backlog#1894 axis A).
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum CorruptMetadataRecording {
|
||||
/// Intent accepted: the MRF consumer owns the repair (High Metadata
|
||||
/// heal, durable after the journal's group-commit flush), so the
|
||||
/// immediate heal request is skipped — the manager would otherwise book
|
||||
/// two tasks for one target. A pending-ledger entry stays behind as the
|
||||
/// backstop for what the journal cannot cover on its own (a crash inside
|
||||
/// the flush window, or the consumer exhausting its admission attempts);
|
||||
/// the repaired-notice fanout (axis B) drops the entry once the repair
|
||||
/// lands.
|
||||
LedgerOnly,
|
||||
/// Intent rejected (feature disabled, channel uninitialized, or full):
|
||||
/// the historical immediate heal request plus the ledger entry.
|
||||
ImmediateAndLedger,
|
||||
}
|
||||
|
||||
fn corrupt_metadata_recording(mrf_accepted: bool) -> CorruptMetadataRecording {
|
||||
if mrf_accepted {
|
||||
CorruptMetadataRecording::LedgerOnly
|
||||
} else {
|
||||
CorruptMetadataRecording::ImmediateAndLedger
|
||||
}
|
||||
}
|
||||
|
||||
fn build_bucket_heal_request(bucket: String, priority: HealChannelPriority) -> HealChannelRequest {
|
||||
HealChannelRequest {
|
||||
bucket,
|
||||
@@ -2478,29 +2504,46 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
if let GetSizeFailureAction::HealMetadata { object } = failure_action {
|
||||
// MRF journal intent: durable High-priority Metadata
|
||||
// heal across restarts (HS-01); the scanner heal
|
||||
// request below stays as the immediate path.
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
// Single-flight (backlog#1894 axis A) — the
|
||||
// recording mode and its guarantees are pinned by
|
||||
// corrupt_metadata_recording below.
|
||||
let mrf_accepted = rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
|
||||
&item.bucket,
|
||||
&object,
|
||||
None,
|
||||
);
|
||||
self.send_required_scanner_heal_request(
|
||||
PendingScannerHealKind::Object,
|
||||
item.bucket.clone(),
|
||||
Some(object.clone()),
|
||||
None,
|
||||
build_object_heal_request(
|
||||
item.bucket.clone(),
|
||||
object.clone(),
|
||||
None,
|
||||
self.scan_mode,
|
||||
HealChannelPriority::High,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
match corrupt_metadata_recording(mrf_accepted) {
|
||||
CorruptMetadataRecording::LedgerOnly => {
|
||||
// Recorded as Full (retry-later): admission
|
||||
// for this target happens in the MRF
|
||||
// consumer, not in the manager's queue here.
|
||||
self.update_pending_scanner_heal_after_admission(
|
||||
PendingScannerHealKind::Object,
|
||||
&item.bucket,
|
||||
Some(&object),
|
||||
None,
|
||||
self.scan_mode,
|
||||
HealAdmissionResult::Full,
|
||||
);
|
||||
}
|
||||
CorruptMetadataRecording::ImmediateAndLedger => {
|
||||
self.send_required_scanner_heal_request(
|
||||
PendingScannerHealKind::Object,
|
||||
item.bucket.clone(),
|
||||
Some(object.clone()),
|
||||
None,
|
||||
build_object_heal_request(
|
||||
item.bucket.clone(),
|
||||
object.clone(),
|
||||
None,
|
||||
self.scan_mode,
|
||||
HealChannelPriority::High,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timer.sleep().await;
|
||||
@@ -3395,6 +3438,17 @@ mod tests {
|
||||
assert_eq!(EVENT_SCANNER_BIG_PREFIX, EventName::ScannerBigPrefix.to_string());
|
||||
}
|
||||
|
||||
/// Single-flight decision for the corrupt-metadata branch (backlog#1894
|
||||
/// axis A): an accepted MRF intent must drop the immediate heal request
|
||||
/// (the consumer files one; the manager would double-book) while a
|
||||
/// rejected one must keep it — in both cases a ledger entry remains, so
|
||||
/// the backstop survives regardless of delivery.
|
||||
#[test]
|
||||
fn corrupt_metadata_recording_maps_delivery_to_backstop() {
|
||||
assert_eq!(corrupt_metadata_recording(true), CorruptMetadataRecording::LedgerOnly);
|
||||
assert_eq!(corrupt_metadata_recording(false), CorruptMetadataRecording::ImmediateAndLedger);
|
||||
}
|
||||
|
||||
fn cooldown_map_len() -> usize {
|
||||
SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
.lock()
|
||||
|
||||
Reference in New Issue
Block a user