mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 11:56:38 +00:00
feat(heal,scanner): best-effort repaired notices from the MRF consumer (#6283)
The scanner's pending-heal ledger and the MRF journal tracked the same damaged objects with no cross-talk: once the consumer landed an intent with the heal manager, the ledger's retry entry for that target kept re-submitting a heal the manager already owned (backlog#1894 axis B). Fan the acceptance out: both dispatch sites in the MRF queue (the live consumer and the startup replay) record a compact MrfRepairedEvent (bucket, object, version bytes) in a bounded process-wide ring owned by rustfs-common. The scanner drains its own bucket's notices at the top of retry_pending_scanner_heals and clears the matching Object-kind ledger entries in one batched retain + sync (a mass-recovery first sweep must not turn into thousands of full-table ledger clones on the scan task), with nil notice UUIDs mapping to None per the repo-wide defensive-UUID invariant so unversioned entries match unversioned notices only. Notices are best-effort by design — a lost or capped-out notice leaves the entry to expire through its own attempts/age limits, because the ledger is a retry oracle, not a source of truth; other buckets' notices stay queued for their own scanners. Neither persistent format changes; old nodes that keep double-booking remain harmless. Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -148,6 +148,62 @@ fn unix_now_ms() -> u64 {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
|
||||
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
|
||||
/// mirrors the intent identity so consumers match without re-parsing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MrfRepairedEvent {
|
||||
pub bucket: Arc<str>,
|
||||
pub object: Arc<str>,
|
||||
pub version_id: Option<[u8; 16]>,
|
||||
}
|
||||
|
||||
/// Bound on the repaired-event backlog. Notices are best-effort hints; when
|
||||
/// the ring is full the oldest are dropped and the affected ledger entries
|
||||
/// simply expire through their own attempts/age limits.
|
||||
const MRF_REPAIRED_EVENT_CAP: usize = 4096;
|
||||
|
||||
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
|
||||
|
||||
/// Record that the MRF consumer landed a repair. Never blocks: the critical
|
||||
/// section is a deque push under a std mutex.
|
||||
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
|
||||
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return;
|
||||
};
|
||||
if events.len() >= MRF_REPAIRED_EVENT_CAP {
|
||||
events.pop_front();
|
||||
}
|
||||
events.push_back(MrfRepairedEvent {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id,
|
||||
});
|
||||
}
|
||||
|
||||
/// Take the repair notices recorded for `bucket`, leaving other buckets'
|
||||
/// notices in place for their own scanners.
|
||||
pub fn take_mrf_repaired_events_for(bucket: &str) -> Vec<MrfRepairedEvent> {
|
||||
let Some(registry) = MRF_REPAIRED_EVENTS.get() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut taken = Vec::new();
|
||||
let mut retained = std::collections::VecDeque::with_capacity(events.len());
|
||||
while let Some(event) = events.pop_front() {
|
||||
if event.bucket.as_ref() == bucket {
|
||||
taken.push(event);
|
||||
} else {
|
||||
retained.push_back(event);
|
||||
}
|
||||
}
|
||||
*events = retained;
|
||||
taken
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -200,4 +256,32 @@ mod tests {
|
||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repaired_events_take_is_bucket_scoped_and_cap_bounded() {
|
||||
// Distinct buckets keep their notices until their own scanner takes
|
||||
// them; a take for one bucket leaves the others' notices in place.
|
||||
note_mrf_repaired("bucket-a", "object-1", None);
|
||||
note_mrf_repaired("bucket-b", "object-2", None);
|
||||
note_mrf_repaired("bucket-a", "object-3", None);
|
||||
|
||||
let taken_a = take_mrf_repaired_events_for("bucket-a");
|
||||
assert_eq!(taken_a.len(), 2);
|
||||
assert_eq!(taken_a[0].object.as_ref(), "object-1");
|
||||
assert_eq!(taken_a[1].object.as_ref(), "object-3");
|
||||
assert!(take_mrf_repaired_events_for("bucket-a").is_empty(), "take is destructive per bucket");
|
||||
|
||||
let taken_b = take_mrf_repaired_events_for("bucket-b");
|
||||
assert_eq!(taken_b.len(), 1);
|
||||
assert_eq!(taken_b[0].object.as_ref(), "object-2");
|
||||
|
||||
// Cap bound: flooding the ring drops the oldest notices rather than
|
||||
// growing unbounded.
|
||||
for i in 0..=(MRF_REPAIRED_EVENT_CAP + 8) {
|
||||
note_mrf_repaired("flood-bucket", &format!("object-{i}"), None);
|
||||
}
|
||||
let flooded = take_mrf_repaired_events_for("flood-bucket");
|
||||
assert_eq!(flooded.len(), MRF_REPAIRED_EVENT_CAP);
|
||||
assert_eq!(flooded[0].object.as_ref(), "object-9", "the oldest notices past the cap are dropped");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user