fix(put): reap cancelled eager commit owners (#6569)

rustfs/src/app/object_usecase.rs

@@ -396,6 +396,7 @@ const LOG_COMPONENT_APP: &str = "app";
 const LOG_SUBSYSTEM_OBJECT: &str = "object";
 const EVENT_PUT_OBJECT_STORE_INFLIGHT_SLOW: &str = "put_object_store_inflight_slow";
 const EVENT_PUT_OBJECT_STORE_RETURNED: &str = "put_object_store_returned";
+const EVENT_PUT_OBJECT_COMMIT_OWNER_DEADLINE: &str = "put_object_commit_owner_deadline";
 const EVENT_GET_OBJECT_STREAM_BODY: &str = "get_object_stream_body";
 const EVENT_PUT_OBJECT_BODY_READ_STALLED: &str = "put_object_body_read_stalled";
 const GET_OBJECT_STAGE_PATH_S3_HANDLER: &str = "s3_handler";
@@ -411,6 +412,12 @@ const GET_OBJECT_STAGE_CHECKSUM_HEADERS: &str = "checksum_headers";
 const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration";
 const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter";
 const PUT_OBJECT_STORE_WARN_THRESHOLD: Duration = Duration::from_secs(5);
+// Eager PUT bodies are fully materialized before the storage owner starts. On
+// request cancellation, keep the commit/publication tail alive briefly, then
+// request pre-commit rollback and await cleanup so its write-health guard is
+// reaped without abandoning staged shards.
+const EAGER_PUT_COMMIT_CANCELLATION_GRACE: Duration =
+    Duration::from_secs(rustfs_config::DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS * 4);
 const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5);
 static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false);
 
@@ -3203,6 +3210,62 @@ struct PutObjectCommitResult {
     put_versioned: bool,
 }
 
+struct EagerPutCommitOwner<T: Send + 'static> {
+    task: Option<tokio::task::JoinHandle<T>>,
+    cancellation: tokio_util::sync::CancellationToken,
+    cancellation_grace: Duration,
+}
+
+impl<T: Send + 'static> EagerPutCommitOwner<T> {
+    fn new(
+        task: tokio::task::JoinHandle<T>,
+        cancellation: tokio_util::sync::CancellationToken,
+        cancellation_grace: Duration,
+    ) -> Self {
+        Self {
+            task: Some(task),
+            cancellation,
+            cancellation_grace,
+        }
+    }
+
+    async fn join(mut self) -> Result<T, tokio::task::JoinError> {
+        let result = self.task.as_mut().expect("eager PUT commit owner task must be present").await;
+        self.task = None;
+        result
+    }
+}
+
+impl<T: Send + 'static> Drop for EagerPutCommitOwner<T> {
+    fn drop(&mut self) {
+        let Some(mut task) = self.task.take() else {
+            return;
+        };
+        if tokio::runtime::Handle::try_current().is_err() {
+            task.abort();
+            return;
+        }
+        let cancellation = self.cancellation.clone();
+        let cancellation_grace = self.cancellation_grace;
+        spawn_traced(async move {
+            if tokio::time::timeout(cancellation_grace, &mut task).await.is_err() {
+                cancellation.cancel();
+                metrics::counter!("rustfs_put_commit_owner_deadline_total", "put_path" => "eager").increment(1);
+                warn!(
+                    target: "rustfs::app::object_usecase",
+                    event = EVENT_PUT_OBJECT_COMMIT_OWNER_DEADLINE,
+                    component = LOG_COMPONENT_APP,
+                    subsystem = LOG_SUBSYSTEM_OBJECT,
+                    state = "cancellation_requested",
+                    cancellation_grace_ms = cancellation_grace.as_millis() as u64,
+                    "cancelled eager PutObject commit owner exceeded its grace period and requested storage cleanup"
+                );
+                let _ = task.await;
+            }
+        });
+    }
+}
+
 fn normalize_delete_objects_version_id(
     version_id: Option<String>,
 ) -> std::result::Result<(Option<String>, Option<Uuid>), String> {
@@ -6106,6 +6169,9 @@ impl DefaultObjectUsecase {
             object_lock_retain_until_date,
             &mut opts,
         )?;
+        let eager_put_commit_cancellation =
+            (use_zero_copy_eager_put_path || use_empty_or_small_eager_put_path).then(tokio_util::sync::CancellationToken::new);
+        opts.put_object_cancellation = eager_put_commit_cancellation.clone();
 
         // rustfs/backlog#1009: the pre-PUT lookup has exactly two consumers —
         // the existing-object WORM validation and usage accounting's
@@ -6488,7 +6554,14 @@ impl DefaultObjectUsecase {
                 Ok::<_, S3Error>(PutObjectCommitResult { obj_info, put_versioned })
             }
         });
-        let PutObjectCommitResult { obj_info, put_versioned } = match put_commit.await {
+        let put_commit_result = if let Some(cancellation) = eager_put_commit_cancellation {
+            EagerPutCommitOwner::new(put_commit, cancellation, EAGER_PUT_COMMIT_CANCELLATION_GRACE)
+                .join()
+                .await
+        } else {
+            put_commit.await
+        };
+        let PutObjectCommitResult { obj_info, put_versioned } = match put_commit_result {
             Ok(Ok(result)) => result,
             Ok(Err(err)) => {
                 let result: S3Result<S3Response<PutObjectOutput>> = Err(err);
@@ -10274,6 +10347,38 @@ mod tests {
     use tokio::io::{AsyncRead, ReadBuf};
     use tokio_tar::{Builder, EntryType, Header};
 
+    #[tokio::test]
+    async fn cancelled_eager_put_commit_owner_reaps_stalled_storage_task() {
+        let health = Arc::new(ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+        let task_health = Arc::clone(&health);
+        let cancellation = tokio_util::sync::CancellationToken::new();
+        let task_cancellation = cancellation.clone();
+        let task = spawn_traced_join(async move {
+            let _progress = task_health.track_write_storage().expect("write tracking must be enabled");
+            task_cancellation.cancelled().await;
+        });
+        let owner = EagerPutCommitOwner::new(task, cancellation, Duration::from_millis(10));
+        let request = spawn_traced_join(owner.join());
+
+        tokio::time::timeout(Duration::from_secs(2), async {
+            while !health.snapshot().write_stalled {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("stalled owner must publish write-storage progress");
+
+        request.abort();
+        let _ = request.await;
+        tokio::time::timeout(Duration::from_secs(2), async {
+            while health.snapshot().write_stalled {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("cancelled owner must abort and reap the stalled storage task");
+    }
+
     #[test]
     fn delete_response_version_id_preserves_null_and_synthetic_semantics() {
         let version_id = Uuid::new_v4();