feat(observability): feature-gated hotpath instrumentation for the data path (#4394)

Merge the hotpath-rs wall-time instrumentation from the backlog#936 analysis worktree behind an opt-in 'hotpath' cargo feature, keeping the default build at zero overhead and zero dependency.

- hotpath is an optional dependency everywhere (dep:hotpath feature syntax); the default dependency tree contains no hotpath crate at all
- 40+ measurement points across S3 handlers, ECStore/SetDisks object and multipart ops, erasure encode/decode, bitrot, LocalDisk I/O, FileMeta codec, and HashReader
- attribute sites use #[cfg_attr(feature = "hotpath", hotpath::measure)]; async_trait bodies use per-crate hp_guard! macros (ecstore + rustfs bin); rio gates measure_block! behind hp_measure_block!
- feature chain: rustfs -> rustfs-ecstore -> rustfs-rio / rustfs-filemeta, each crate owning its own gate
- hotpath-alloc is intentionally not wired up (hotpath 0.21.x TLS panic on cross-thread guard drop under tokio, see backlog#935); mimalloc stays the unconditional global allocator
- docs/development/hotpath-profiling.md documents building, HOTPATH_* env vars, SIGTERM report flow, and how to reproduce the backlog#936 timing reports

Refs: https://github.com/rustfs/backlog/issues/935 (HP-14, item 2), https://github.com/rustfs/backlog/issues/936

Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/set_disk/read.rs

@@ -120,6 +120,7 @@ impl SetDisks {
             .await;
     }
 
+    #[cfg_attr(feature = "hotpath", hotpath::measure)]
     pub async fn read_version_optimized(
         &self,
         bucket: &str,
@@ -161,6 +162,7 @@ impl SetDisks {
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
+    #[cfg_attr(feature = "hotpath", hotpath::measure)]
     pub(super) async fn get_object_fileinfo(
         &self,
         bucket: &str,
@@ -319,6 +321,7 @@ impl SetDisks {
         Ok((fi, parts_metadata, op_online_disks))
     }
 
+    #[cfg_attr(feature = "hotpath", hotpath::measure)]
     pub(super) async fn get_object_info_and_quorum(
         &self,
         bucket: &str,
@@ -515,6 +518,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
+    #[cfg_attr(feature = "hotpath", hotpath::measure)]
     pub(super) async fn get_object_with_fileinfo<W>(
         // &self,
         bucket: &str,
@@ -1042,6 +1046,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
+    #[cfg_attr(feature = "hotpath", hotpath::measure)]
     pub(super) async fn get_object_decode_reader_with_fileinfo(
         bucket: &str,
         object: &str,
@@ -1193,6 +1198,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
+    #[cfg_attr(feature = "hotpath", hotpath::measure)]
     async fn build_codec_streaming_part_reader(
         bucket: &str,
         object: &str,
@@ -1330,6 +1336,7 @@ fn multipart_part_checksum_algo(fi: &FileInfo, part_number: usize) -> HashAlgori
 /// `get_object_with_fileinfo` (backlog#870) so both report the same
 /// stage-duration semantics.
 #[allow(clippy::too_many_arguments)]
+#[cfg_attr(feature = "hotpath", hotpath::measure)]
 async fn setup_multipart_part_readers(
     files: &[FileInfo],
     disks: &[Option<DiskStore>],