fix(multipart): clean temp part data on failure (#4412)

fix(multipart): clean failed part temp data

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -298,6 +298,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp());
         let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
 
+        let result: Result<PartInfo> = async {
             let erasure = coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
             let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
 
@@ -363,10 +364,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                         write_quorum,
                         &write_err,
                     );
-                return Err(to_object_err(write_err.into(), vec![bucket, object]));
+                    Err(to_object_err(write_err.into(), vec![bucket, object]))?;
                 }
 
-            return Err(Error::other(format!("not enough disks to write: {errors:?}")));
+                Err(Error::other(format!("not enough disks to write: {errors:?}")))?;
             }
 
             // Capture the original part size before swapping the stream out for encoding.
@@ -386,11 +387,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                         .encode_single_block_non_inline(stream, &mut writers, write_quorum)
                         .await?
                 }
-            SmallWritePath::PipelineBatchedLarge => Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await?,
+                SmallWritePath::PipelineBatchedLarge => {
+                    Arc::new(erasure).encode_batched(stream, &mut writers, write_quorum).await?
+                }
                 SmallWritePath::Inline | SmallWritePath::Pipeline => {
                     Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?
                 }
-        }; // TODO: delete temporary directory on error
+            };
 
             if let Some(stage_start) = encode_stage_start {
                 rustfs_io_metrics::record_put_object_stage_duration(
@@ -414,11 +417,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                     state = "short_write",
                     "Set disk multipart write produced fewer bytes than expected"
                 );
-            return Err(Error::other(format!(
+                Err(Error::other(format!(
                     "put_object_part write size < data.size(), w_size={}, data.size={}",
                     w_size,
                     data.size()
-            )));
+                )))?;
             }
 
             let index_op = data
@@ -518,6 +521,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 
             Ok(ret)
         }
+        .await;
+
+        if result.is_err()
+            && let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_part).await
+        {
+            warn!(tmp_part = %tmp_part, error = ?err, "failed to cleanup multipart temporary data");
+        }
+
+        result
+    }
 
     #[tracing::instrument(skip(self))]
     async fn list_object_parts(
@@ -1434,6 +1447,28 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::disk::DiskAPI as _;
+    use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks;
+    use tempfile::TempDir;
+
+    async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec<String> {
+        let mut leftovers = Vec::new();
+        for temp_dir in temp_dirs {
+            let tmp_path = temp_dir.path().join(RUSTFS_META_TMP_BUCKET);
+            let mut read_dir = match tokio::fs::read_dir(&tmp_path).await {
+                Ok(read_dir) => read_dir,
+                Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
+                Err(err) => panic!("tmp dir {tmp_path:?} should be listable: {err}"),
+            };
+            while let Some(entry) = read_dir.next_entry().await.expect("tmp dir entry should be readable") {
+                let name = entry.file_name().to_string_lossy().to_string();
+                if name != ".trash" {
+                    leftovers.push(format!("{}/{name}", tmp_path.display()));
+                }
+            }
+        }
+        leftovers
+    }
 
     #[tokio::test]
     async fn collect_list_parts_results_fails_early_when_quorum_is_impossible() {
@@ -1522,6 +1557,38 @@ mod tests {
         assert!(started.elapsed() < Duration::from_millis(120));
     }
 
+    #[tokio::test]
+    async fn put_object_part_failure_cleans_tmp_workspace_inline() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-tmp-clean-bucket";
+        let object = "object";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let upload = set_disks
+            .new_multipart_upload(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("multipart upload should be created");
+        let declared_size = 1024 * 1024;
+        let short_stream = Cursor::new(vec![7u8; 512]);
+        let mut reader = PutObjReader::new(
+            HashReader::from_stream(short_stream, declared_size, declared_size, None, None, false)
+                .expect("hash reader should be constructed"),
+        );
+
+        let err = set_disks
+            .put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
+            .await
+            .expect_err("short multipart stream should fail");
+
+        let leftovers = non_trash_tmp_entries(&temp_dirs).await;
+        assert!(
+            leftovers.is_empty(),
+            "failed multipart upload part must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
+        );
+    }
+
     #[test]
     fn reduce_quorum_part_numbers_only_keeps_parts_present_on_quorum_of_drives() {
         let object_parts = vec![

crates/ecstore/src/set_disk/ops/object.rs

@@ -2322,7 +2322,7 @@ mod b3_write_quorum_tests {
 }
 
 #[cfg(test)]
-mod hermetic_set_disks_support {
+pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
     //! Shared hermetic `SetDisks` construction for the ops tests below: the
     //! `SetDisks` under test is built directly on formatted local disks (same
     //! pattern as the `ops/locking.rs` tests) so the tests stay hermetic — no
@@ -2334,7 +2334,10 @@ mod hermetic_set_disks_support {
     use tempfile::TempDir;
     use tokio::sync::RwLock;
 
-    pub(super) async fn make_formatted_local_disk(disk_idx: usize, format: &FormatV3) -> (TempDir, Endpoint, DiskStore) {
+    pub(in crate::set_disk::ops) async fn make_formatted_local_disk(
+        disk_idx: usize,
+        format: &FormatV3,
+    ) -> (TempDir, Endpoint, DiskStore) {
         let dir = tempfile::tempdir().expect("tempdir should be created");
         let mut endpoint =
             Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
@@ -2361,7 +2364,7 @@ mod hermetic_set_disks_support {
         (dir, endpoint, disk)
     }
 
-    pub(super) async fn hermetic_set_disks(disk_count: usize) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
+    pub(in crate::set_disk::ops) async fn hermetic_set_disks(disk_count: usize) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
         let format = FormatV3::new(1, disk_count);
 
         let mut temp_dirs = Vec::with_capacity(disk_count);