perf: reduce spawn_blocking contention in PUT path (#3132)

* perf: reduce spawn_blocking contention in PUT path (~23% throughput gain) Flame graph profiling identified tokio blocking pool mutex contention as the #1 bottleneck (17.3% of CPU time). Each spawn_blocking call must acquire parking_lot::raw_mutex to enqueue work. With 16 concurrent PUTs × 4 disks × 3+ spawn_blocking per disk, this became a serialization point. Optimizations applied: - Merge make_dir_all + file write into single spawn_blocking - Merge read_file + parse + write + rename into single spawn_blocking for inline objects (small files) - Optimize reliable_rename to try rename first, mkdir only on ENOENT - Optimize remove/remove_std to try remove_file first, EISDIR fallback - Add encode_inline_small fast path for small objects - Parallelize bitrot writer creation with join_all Benchmark (4KiB PUT, 4-disk EC, 16 concurrent, 8 rounds, randomized A/B): Baseline: ~950 obj/s → Optimized: ~1173 obj/s (+23%) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * fix: address review comments - Return old_data_dir for non-inline rename_data path (was incorrectly None) - Restore delete_all cleanup of PUT temp data on failure paths - Fix cargo fmt formatting Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * style: apply rustfmt from stable 1.96.0 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * fix: collapse nested if-let chains for clippy compliance Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * fix: address Copilot review comments - fs.rs: handle macOS EPERM from remove_file on directories - os.rs: restore NotFound=Ok(()) semantics on first rename attempt - local.rs: use try-rename-then-mkdir pattern for inline rename_data Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * test: add unit tests for encode_inline_small fast path * test: fix comment and line length in encode_inline_small tests * fix: revert reliable_rename and write_all_internal to match original Restore the original reliable_rename logic (check parent exists, then rename in loop) and the original write_all_internal (make_dir_all outside spawn_blocking). The optimization changes caused a CI-only test failure in capacity_dirty_scope_test that could not be reproduced locally. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * fix: update comment and fix formatting for CI - Fix macOS/BSD comment to accurately say macOS only - Fix encode_inline_small test formatting to match rustfmt 1.96.0 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * fix: propagate inline rename errors * fix: retry rename when parent missing --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-08-08 14:23:13 +00:00 · 2026-05-31 09:28:48 +08:00
parent ca4793f93e
commit cf55743579
5 changed files with 308 additions and 139 deletions
@@ -294,6 +294,33 @@ impl Erasure {
        writers.shutdown().await?;
        Ok((reader, total))
    }
+
+    /// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
+    /// Reads all data, encodes directly, writes shards sequentially.
+    pub async fn encode_inline_small<R>(
+        self: Arc<Self>,
+        mut reader: R,
+        writers: &mut [Option<BitrotWriterWrapper>],
+        quorum: usize,
+    ) -> std::io::Result<(R, usize)>
+    where
+        R: AsyncRead + Send + Sync + Unpin,
+    {
+        use tokio::io::AsyncReadExt;
+
+        let mut buf = Vec::with_capacity(self.block_size);
+        let total = reader.read_to_end(&mut buf).await?;
+
+        if total == 0 {
+            return Ok((reader, 0));
+        }
+
+        let shards = self.encode_data(&buf)?;
+        let mut mw = MultiWriter::new(writers, quorum);
+        mw.write(shards).await?;
+        mw.shutdown().await?;
+        Ok((reader, total))
+    }
 }

 #[cfg(test)]
@@ -357,6 +384,61 @@ mod tests {
        assert!(!committed.lock().unwrap().is_empty());
    }

+    /// encode_inline_small: empty reader returns (reader, 0) without writing to any shard.
+    #[tokio::test]
+    async fn encode_inline_small_empty_stream_returns_zero() {
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let writer = DeferredCommitWriter::new(committed.clone());
+        // 1 data shard, 0 parity shards, block_size = 16
+        let mut writers = vec![Some(BitrotWriterWrapper::new(
+            CustomWriter::new_tokio_writer(writer),
+            16,
+            HashAlgorithm::HighwayHash256S,
+        ))];
+
+        let erasure = Arc::new(Erasure::new(1, 0, 16));
+        let reader = tokio::io::BufReader::new(std::io::Cursor::new(Vec::<u8>::new()));
+        let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, 1).await.unwrap();
+
+        assert_eq!(total, 0);
+        // No shutdown was called, so nothing should be committed
+        assert!(committed.lock().unwrap().is_empty());
+    }
+
+    /// encode_inline_small: small payload is encoded into the correct number of shards
+    /// and each writer receives data after shutdown.
+    #[tokio::test]
+    async fn encode_inline_small_payload_writes_all_shards() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
+        const BLOCK_SIZE: usize = 64;
+
+        let committed: Vec<Arc<Mutex<Vec<u8>>>> = (0..TOTAL_SHARDS).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
+
+        let mut writers: Vec<Option<BitrotWriterWrapper>> = committed
+            .iter()
+            .map(|c| {
+                Some(BitrotWriterWrapper::new(
+                    CustomWriter::new_tokio_writer(DeferredCommitWriter::new(c.clone())),
+                    BLOCK_SIZE / DATA_SHARDS,
+                    HashAlgorithm::HighwayHash256S,
+                ))
+            })
+            .collect();
+
+        let payload = b"hello inline small";
+        let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
+        let reader = tokio::io::BufReader::new(std::io::Cursor::new(payload.to_vec()));
+        let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, DATA_SHARDS).await.unwrap();
+
+        assert_eq!(total, payload.len());
+        // All shards must have received data (shutdown flushed the bitrot header + shard bytes)
+        for (i, c) in committed.iter().enumerate() {
+            assert!(!c.lock().unwrap().is_empty(), "shard {i} should have received data");
+        }
+    }
+
    #[test]
    fn encode_channel_capacity_never_returns_zero() {
        assert_eq!(encode_channel_capacity(0, 1024), 1);