perf(heal): trim scanner and heal queue hot paths (#6307)

Cache heal queue dedup keys, avoid retry request double construction, clear task aliases after terminal completion, and age out stale scanner pending-heal ledger entries during retry sweeps.

Co-authored-by: heihutu <heihutu@gmail.com>

crates/scanner/src/scanner_folder.rs

@@ -193,6 +193,7 @@ fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size:
     });
 }
 const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000;
+const MAX_PENDING_SCANNER_HEAL_AGE_SECS: u64 = 24 * 60 * 60;
 
 static SCANNER_ALERT_METRICS_ONCE: Once = Once::new();
 

crates/scanner/src/scanner_folder/ledger.rs

@@ -100,19 +100,52 @@ impl FolderScanner {
             last_admission_result: result.result_label().to_string(),
             last_admission_reason: result.reason_label().to_string(),
         });
-        self.prune_pending_scanner_heals();
-        self.sync_pending_heals();
+        if self.prune_pending_scanner_heal_capacity() == 0 {
+            self.sync_pending_heals();
+        }
     }
 
     pub(super) fn prune_pending_scanner_heals(&mut self) {
+        let now = Self::now_secs();
+        let before_expiry = self.new_cache.info.pending_heals.len();
+        self.new_cache
+            .info
+            .pending_heals
+            .retain(|entry| now.saturating_sub(entry.first_seen) <= MAX_PENDING_SCANNER_HEAL_AGE_SECS);
+        let expired = before_expiry.saturating_sub(self.new_cache.info.pending_heals.len());
+        if expired > 0 {
+            counter!(
+                METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL,
+                "bucket" => self.new_cache.info.name.clone()
+            )
+            .increment(u64::try_from(expired).unwrap_or(u64::MAX));
+            warn!(
+                target: "rustfs::scanner::folder",
+                event = EVENT_SCANNER_HEAL_ADMISSION,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_HEAL,
+                bucket = %self.new_cache.info.name,
+                pruned = expired,
+                remaining = self.new_cache.info.pending_heals.len(),
+                state = "pending_heal_expired",
+                "Scanner pending heal ledger expired old entries"
+            );
+            self.sync_pending_heals();
+        }
+
+        self.prune_pending_scanner_heal_capacity();
+    }
+
+    fn prune_pending_scanner_heal_capacity(&mut self) -> usize {
         let len = self.new_cache.info.pending_heals.len();
         if len <= MAX_PENDING_SCANNER_HEALS_PER_BUCKET {
-            return;
+            return 0;
         }
 
         sort_pending_scanner_heals_for_retry(&mut self.new_cache.info.pending_heals);
         let remove_count = len.saturating_sub(MAX_PENDING_SCANNER_HEALS_PER_BUCKET);
         self.new_cache.info.pending_heals.drain(..remove_count);
+        self.sync_pending_heals();
         counter!(
             METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL,
             "bucket" => self.new_cache.info.name.clone()
@@ -129,6 +162,7 @@ impl FolderScanner {
             state = "pending_heal_pruned",
             "Scanner pending heal ledger pruned oldest entries"
         );
+        remove_count
     }
 
     pub(super) fn update_pending_scanner_heal_after_admission(
@@ -174,6 +208,7 @@ impl FolderScanner {
         if !repaired.is_empty() {
             self.clear_pending_scanner_heals_for_repaired(&repaired);
         }
+        self.prune_pending_scanner_heals();
         for pending in pending_scanner_heal_retry_candidates(&self.new_cache.info.pending_heals, &bucket) {
             if !self.should_heal().await {
                 break;

crates/scanner/src/scanner_folder/tests.rs

@@ -1018,7 +1018,7 @@ fn pending_heal(
         object: object.map(ToOwned::to_owned),
         version_id: version_id.map(ToOwned::to_owned),
         scan_mode: HealScanMode::Deep,
-        first_seen: 1,
+        first_seen: FolderScanner::now_secs(),
         last_attempt,
         attempts,
         last_admission_result: "full".to_string(),
@@ -1155,6 +1155,52 @@ fn test_pending_heal_retry_candidates_respect_cap_and_order() {
     assert_eq!(candidates.last().and_then(|entry| entry.object.as_deref()), Some("object-127"));
 }
 
+#[tokio::test]
+async fn test_pending_heal_prune_expires_stale_entries() {
+    let (mut scanner, temp_dir) = build_test_scanner().await;
+    let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
+    scanner.new_cache.info.name = "bucket".to_string();
+    scanner.update_cache.info.name = "bucket".to_string();
+
+    let mut stale = pending_heal(PendingScannerHealKind::Object, "bucket", Some("stale"), None, 1, 1);
+    stale.first_seen = FolderScanner::now_secs().saturating_sub(MAX_PENDING_SCANNER_HEAL_AGE_SECS + 1);
+    let fresh = pending_heal(PendingScannerHealKind::Object, "bucket", Some("fresh"), None, 1, 1);
+    scanner.new_cache.info.pending_heals = vec![stale, fresh];
+
+    scanner.prune_pending_scanner_heals();
+
+    assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
+    assert_eq!(scanner.new_cache.info.pending_heals[0].object.as_deref(), Some("fresh"));
+    assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
+    assert!(scanner.pending_heals_changed);
+}
+
+#[tokio::test]
+async fn test_pending_heal_update_keeps_stale_entry_until_retry_prune() {
+    let (mut scanner, temp_dir) = build_test_scanner().await;
+    let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
+    scanner.new_cache.info.name = "bucket".to_string();
+    scanner.update_cache.info.name = "bucket".to_string();
+
+    let mut stale = pending_heal(PendingScannerHealKind::Object, "bucket", Some("object"), None, 1, 1);
+    stale.first_seen = FolderScanner::now_secs().saturating_sub(MAX_PENDING_SCANNER_HEAL_AGE_SECS + 1);
+    scanner.new_cache.info.pending_heals = vec![stale];
+
+    scanner.update_pending_scanner_heal_after_admission(
+        PendingScannerHealKind::Object,
+        "bucket",
+        Some("object"),
+        None,
+        HealScanMode::Deep,
+        HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull),
+    );
+
+    assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
+    assert_eq!(scanner.new_cache.info.pending_heals[0].attempts, 2);
+    assert_eq!(scanner.new_cache.info.pending_heals[0].object.as_deref(), Some("object"));
+    assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
+}
+
 #[tokio::test]
 async fn test_pending_heal_queue_full_deduplicates_object_entry() {
     let (mut scanner, temp_dir) = build_test_scanner().await;