feat(heal,scanner): best-effort repaired notices from the MRF consumer (#6283)

The scanner's pending-heal ledger and the MRF journal tracked the same
damaged objects with no cross-talk: once the consumer landed an intent
with the heal manager, the ledger's retry entry for that target kept
re-submitting a heal the manager already owned (backlog#1894 axis B).

Fan the acceptance out: both dispatch sites in the MRF queue (the live
consumer and the startup replay) record a compact MrfRepairedEvent
(bucket, object, version bytes) in a bounded process-wide ring owned by
rustfs-common. The scanner drains its own bucket's notices at the top
of retry_pending_scanner_heals and clears the matching Object-kind
ledger entries in one batched retain + sync (a mass-recovery first
sweep must not turn into thousands of full-table ledger clones on the
scan task), with nil notice UUIDs mapping to None per the repo-wide
defensive-UUID invariant so unversioned entries match unversioned
notices only. Notices are best-effort by design — a lost or capped-out
notice leaves the entry to expire through its own attempts/age limits,
because the ledger is a retry oracle, not a source of truth; other
buckets' notices stay queued for their own scanners. Neither persistent
format changes; old nodes that keep double-booking remain harmless.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-20 01:31:26 +08:00
committed by GitHub
parent 99c3811d93
commit d6efb65588
4 changed files with 232 additions and 3 deletions
+84
View File
@@ -148,6 +148,62 @@ fn unix_now_ms() -> u64 {
.unwrap_or(0)
}
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
/// mirrors the intent identity so consumers match without re-parsing.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MrfRepairedEvent {
pub bucket: Arc<str>,
pub object: Arc<str>,
pub version_id: Option<[u8; 16]>,
}
/// Bound on the repaired-event backlog. Notices are best-effort hints; when
/// the ring is full the oldest are dropped and the affected ledger entries
/// simply expire through their own attempts/age limits.
const MRF_REPAIRED_EVENT_CAP: usize = 4096;
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
/// Record that the MRF consumer landed a repair. Never blocks: the critical
/// section is a deque push under a std mutex.
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
let Ok(mut events) = registry.lock() else {
return;
};
if events.len() >= MRF_REPAIRED_EVENT_CAP {
events.pop_front();
}
events.push_back(MrfRepairedEvent {
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
});
}
/// Take the repair notices recorded for `bucket`, leaving other buckets'
/// notices in place for their own scanners.
pub fn take_mrf_repaired_events_for(bucket: &str) -> Vec<MrfRepairedEvent> {
let Some(registry) = MRF_REPAIRED_EVENTS.get() else {
return Vec::new();
};
let Ok(mut events) = registry.lock() else {
return Vec::new();
};
let mut taken = Vec::new();
let mut retained = std::collections::VecDeque::with_capacity(events.len());
while let Some(event) = events.pop_front() {
if event.bucket.as_ref() == bucket {
taken.push(event);
} else {
retained.push_back(event);
}
}
*events = retained;
taken
}
#[cfg(test)]
mod tests {
use super::*;
@@ -200,4 +256,32 @@ mod tests {
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
set_mrf_delivery_enabled(true);
}
#[test]
fn repaired_events_take_is_bucket_scoped_and_cap_bounded() {
// Distinct buckets keep their notices until their own scanner takes
// them; a take for one bucket leaves the others' notices in place.
note_mrf_repaired("bucket-a", "object-1", None);
note_mrf_repaired("bucket-b", "object-2", None);
note_mrf_repaired("bucket-a", "object-3", None);
let taken_a = take_mrf_repaired_events_for("bucket-a");
assert_eq!(taken_a.len(), 2);
assert_eq!(taken_a[0].object.as_ref(), "object-1");
assert_eq!(taken_a[1].object.as_ref(), "object-3");
assert!(take_mrf_repaired_events_for("bucket-a").is_empty(), "take is destructive per bucket");
let taken_b = take_mrf_repaired_events_for("bucket-b");
assert_eq!(taken_b.len(), 1);
assert_eq!(taken_b[0].object.as_ref(), "object-2");
// Cap bound: flooding the ring drops the oldest notices rather than
// growing unbounded.
for i in 0..=(MRF_REPAIRED_EVENT_CAP + 8) {
note_mrf_repaired("flood-bucket", &format!("object-{i}"), None);
}
let flooded = take_mrf_repaired_events_for("flood-bucket");
assert_eq!(flooded.len(), MRF_REPAIRED_EVENT_CAP);
assert_eq!(flooded[0].object.as_ref(), "object-9", "the oldest notices past the cap are dropped");
}
}