From 8c8d1574181ebe6adc05169e1d44375b0041aae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Tue, 24 Mar 2026 19:09:45 +0800 Subject: [PATCH] fix(object): always unregister deadlock-tracked get requests (#2275) Signed-off-by: heihutu Co-authored-by: heihutu Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: cxymds --- rustfs/src/app/object_usecase.rs | 52 ++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index e22e1e3ee..d26c10629 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -121,6 +121,26 @@ use tokio_util::io::{ReaderStream, StreamReader}; use tracing::{debug, error, info, instrument, warn}; use uuid::Uuid; +struct DeadlockRequestGuard { + deadlock_detector: Arc, + request_id: String, +} + +impl DeadlockRequestGuard { + fn new(deadlock_detector: Arc, request_id: String) -> Self { + Self { + deadlock_detector, + request_id, + } + } +} + +impl Drop for DeadlockRequestGuard { + fn drop(&mut self) { + self.deadlock_detector.unregister_request(&self.request_id); + } +} + pin_project! { struct ExtractArchiveEtagReader { #[pin] @@ -169,6 +189,31 @@ impl AsyncRead for ExtractArchiveEtagReader { } } +#[cfg(test)] +mod deadlock_request_guard_tests { + use super::DeadlockRequestGuard; + use crate::storage::deadlock_detector::{DeadlockDetector, DeadlockDetectorConfig}; + use std::sync::Arc; + + #[test] + fn deadlock_request_guard_unregisters_on_drop() { + let detector = Arc::new(DeadlockDetector::new(DeadlockDetectorConfig { + enabled: true, + ..DeadlockDetectorConfig::default() + })); + let request_id = "test-request-id".to_string(); + + detector.register_request(&request_id, "test request"); + assert_eq!(detector.tracked_count(), 1); + + { + let _guard = DeadlockRequestGuard::new(Arc::clone(&detector), request_id.clone()); + // `_guard` is dropped at the end of this scope, which should unregister the request. + } + + assert_eq!(detector.tracked_count(), 0); + } +} async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventSrc) { enqueue_transition_immediate(obj_info, src).await; } @@ -1268,6 +1313,7 @@ impl DefaultObjectUsecase { let deadlock_detector = crate::storage::deadlock_detector::get_deadlock_detector(); let request_id = wrapper.request_id().to_string(); deadlock_detector.register_request(&request_id, format!("GetObject {}/{}", req.input.bucket, req.input.key)); + let _deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector.clone(), request_id); // Check for request timeout before proceeding if wrapper.is_timeout() { @@ -1278,7 +1324,6 @@ impl DefaultObjectUsecase { elapsed_ms = wrapper.elapsed().as_millis(), "GetObject request timed out before processing" ); - deadlock_detector.unregister_request(&request_id); return Err(s3_error!(InternalError, "Request timeout before processing")); } @@ -1470,7 +1515,6 @@ impl DefaultObjectUsecase { elapsed_ms = wrapper.elapsed().as_millis(), "GetObject request timed out while waiting for disk permit" ); - deadlock_detector.unregister_request(&request_id); #[cfg(feature = "metrics")] metrics::counter!("rustfs.get.object.timeout.total", "stage" => "disk_permit").increment(1); return Err(s3_error!(InternalError, "Request timeout while waiting for disk permit")); @@ -1576,7 +1620,6 @@ impl DefaultObjectUsecase { elapsed_ms = wrapper.elapsed().as_millis(), "GetObject request timed out before reading object" ); - deadlock_detector.unregister_request(&request_id); #[cfg(feature = "metrics")] metrics::counter!("rustfs.get.object.timeout.total", "stage" => "before_read").increment(1); return Err(s3_error!(InternalError, "Request timeout before reading object")); @@ -1964,9 +2007,6 @@ impl DefaultObjectUsecase { cache_key, response_content_length, total_duration, optimal_buffer_size ); - // Unregister from deadlock detector - deadlock_detector.unregister_request(&request_id); - let response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; let result = Ok(response); let _ = helper.complete(&result);