From 707c773dd9be8f55c5f1bdeae1a8d81c763af3d6 Mon Sep 17 00:00:00 2001
From: weisd <weishidavip@163.com>
Date: Wed, 23 Apr 2025 17:20:14 +0800
Subject: [PATCH 01/13] rm unuse log

---
 ecstore/src/set_disk.rs | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs
index c64bc189b..3a192898d 100644
--- a/ecstore/src/set_disk.rs
+++ b/ecstore/src/set_disk.rs
@@ -861,7 +861,6 @@ impl SetDisks {
         };
 
         if let Some(err) = reduce_read_quorum_errs(errs, object_op_ignored_errs().as_ref(), expected_rquorum) {
-            warn!("object_quorum_from_meta err {:?}", &err);
             return Err(err);
         }
 
@@ -874,7 +873,6 @@ impl SetDisks {
         let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
 
         if parity_blocks < 0 {
-            warn!("QuorumError::Read, common_parity < 0 ");
             return Err(Error::new(QuorumError::Read));
         }
 
@@ -1746,7 +1744,7 @@ impl SetDisks {
         // TODO: 优化并发 可用数量中断
         let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await;
         // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
-        warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
+        // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
 
         let _min_disks = self.set_drive_count - self.default_parity_count;
 
@@ -1754,12 +1752,6 @@ impl SetDisks {
             .map_err(|err| to_object_err(err, vec![bucket, object]))?;
 
         if let Some(err) = reduce_read_quorum_errs(&errs, object_op_ignored_errs().as_ref(), read_quorum as usize) {
-            error!(
-                "reduce_read_quorum_errs disks: {} read_quorum {} \n {:?}",
-                disks.len(),
-                read_quorum,
-                &errs
-            );
             return Err(to_object_err(err, vec![bucket, object]));
         }
 
@@ -2587,7 +2579,6 @@ impl SetDisks {
                 }
             }
             Err(err) => {
-                warn!("object_quorum_from_meta failed, err: {}", err.to_string());
                 let data_errs_by_part = HashMap::new();
                 match self
                     .delete_if_dang_ling(

From 08d0021cd6c1806a1f5fae1f9e48bdd78d0062c9 Mon Sep 17 00:00:00 2001
From: junxiang Mu <1948535941@qq.com>
Date: Thu, 24 Apr 2025 03:04:05 +0000
Subject: [PATCH 02/13] support async calculate etag

Signed-off-by: junxiang Mu <1948535941@qq.com>
---
 ecstore/src/io.rs       | 50 ++++++++++++++++++++++++++++-------------
 ecstore/src/set_disk.rs |  4 ++--
 2 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/ecstore/src/io.rs b/ecstore/src/io.rs
index 3ca27fe0d..7ecf8f84c 100644
--- a/ecstore/src/io.rs
+++ b/ecstore/src/io.rs
@@ -1,6 +1,8 @@
+use bytes::Bytes;
 use futures::TryStreamExt;
 use md5::Digest;
 use md5::Md5;
+use tokio::sync::mpsc;
 use std::pin::Pin;
 use std::task::Context;
 use std::task::Poll;
@@ -125,33 +127,51 @@ impl AsyncRead for HttpFileReader {
 
 pub struct EtagReader<R> {
     inner: R,
-    md5: Md5,
+    bytes_tx: mpsc::Sender<Bytes>,
+    md5_rx: oneshot::Receiver<String>,
 }
 
 impl<R> EtagReader<R> {
     pub fn new(inner: R) -> Self {
-        EtagReader { inner, md5: Md5::new() }
+        let (bytes_tx, mut bytes_rx) = mpsc::channel::<Bytes>(8);
+        let (md5_tx, md5_rx) = oneshot::channel::<String>();
+
+        tokio::task::spawn_blocking(move || {
+            let mut md5 = Md5::new();
+            while let Some(bytes) = bytes_rx.blocking_recv() {
+                md5.update(&bytes);
+            }
+            let digest = md5.finalize();
+            let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
+            let _ = md5_tx.send(etag);
+        });
+
+        EtagReader { inner, bytes_tx, md5_rx }
     }
 
-    pub fn etag(self) -> String {
-        hex_simd::encode_to_string(self.md5.finalize(), hex_simd::AsciiCase::Lower)
+    pub async fn etag(self) -> String {
+        drop(self.inner);
+        drop(self.bytes_tx);
+        self.md5_rx.await.unwrap()
     }
 }
 
 impl<R: AsyncRead + Unpin> AsyncRead for EtagReader<R> {
+    #[tracing::instrument(level = "debug", skip_all)]
     fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<tokio::io::Result<()>> {
-        let befor_size = buf.filled().len();
-
-        match Pin::new(&mut self.inner).poll_read(cx, buf) {
-            Poll::Ready(Ok(())) => {
-                if buf.filled().len() > befor_size {
-                    let bytes = &buf.filled()[befor_size..];
-                    self.md5.update(bytes);
-                }
-
-                Poll::Ready(Ok(()))
+        let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
+        if let Poll::Ready(Ok(())) = &poll {
+            if buf.remaining() == 0 {
+                let bytes = buf.filled();
+                let bytes = Bytes::copy_from_slice(bytes);
+                let tx = self.bytes_tx.clone();
+                tokio::spawn(async move {
+                    if let Err(e) = tx.send(bytes).await {
+                        warn!("EtagReader send error: {:?}", e);
+                    }
+                });
             }
-            other => other,
         }
+        poll
     }
 }
diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs
index c64bc189b..b35c6ad89 100644
--- a/ecstore/src/set_disk.rs
+++ b/ecstore/src/set_disk.rs
@@ -3802,7 +3802,7 @@ impl ObjectIO for SetDisks {
             error!("close_bitrot_writers err {:?}", err);
         }
 
-        let etag = etag_stream.etag();
+        let etag = etag_stream.etag().await;
         //TODO: userDefined
 
         user_defined.insert("etag".to_owned(), etag.clone());
@@ -4393,7 +4393,7 @@ impl StorageAPI for SetDisks {
             error!("close_bitrot_writers err {:?}", err);
         }
 
-        let mut etag = etag_stream.etag();
+        let mut etag = etag_stream.etag().await;
 
         if let Some(ref tag) = opts.preserve_etag {
             etag = tag.clone();

From a745b91ded2727a943c17ead2c5f62db53a607f9 Mon Sep 17 00:00:00 2001
From: weisd <weishidavip@163.com>
Date: Thu, 24 Apr 2025 13:22:34 +0800
Subject: [PATCH 03/13] fix:#351 delete object err

---
 ecstore/src/disk/local.rs | 6 +++++-
 scripts/dev.sh            | 7 -------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs
index ddefb55fd..ec2a60a60 100644
--- a/ecstore/src/disk/local.rs
+++ b/ecstore/src/disk/local.rs
@@ -629,7 +629,11 @@ impl LocalDisk {
                 let _ = fm.data.remove(vec![vid, dir]);
 
                 let dir_path = self.get_object_path(volume, format!("{}/{}", path, dir).as_str())?;
-                self.move_to_trash(&dir_path, true, false).await?;
+                if let Err(err) = self.move_to_trash(&dir_path, true, false).await {
+                    if !(is_err_file_not_found(&err) || is_err_os_not_exist(&err)) {
+                        return Err(err);
+                    }
+                };
             }
         }
 
diff --git a/scripts/dev.sh b/scripts/dev.sh
index 8dc19c88b..0df907a84 100755
--- a/scripts/dev.sh
+++ b/scripts/dev.sh
@@ -15,13 +15,6 @@ REMOTE_PATH="~"
 # 格式：服务器IP 用户名 目标路径
 SERVER_LIST=(
     "root@121.89.80.13"
-    "root@121.89.80.198"
-    "root@8.130.78.237"
-    "root@8.130.189.236"
-    "root@121.89.80.230"
-    "root@121.89.80.45"
-    "root@8.130.191.95"
-    "root@121.89.80.91"
 )
 
 # 遍历服务器列表

From 56575d58f490ae56c08ad342cf9fcb8f4cf8189c Mon Sep 17 00:00:00 2001
From: weisd <weishidavip@163.com>
Date: Thu, 24 Apr 2025 15:14:57 +0800
Subject: [PATCH 04/13] fix:#352, #353

fix: verify_file bug
---
 ecstore/src/disk/local.rs              |  2 +-
 ecstore/src/store.rs                   |  4 ++--
 rustfs/src/admin/handlers/rebalance.rs | 11 ++++++++---
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs
index ec2a60a60..4ab9bb183 100644
--- a/ecstore/src/disk/local.rs
+++ b/ecstore/src/disk/local.rs
@@ -1232,7 +1232,7 @@ impl DiskAPI for LocalDisk {
         }
 
         let mut resp = CheckPartsResp {
-            results: Vec::with_capacity(fi.parts.len()),
+            results: vec![0; fi.parts.len()],
         };
 
         let erasure = &fi.erasure;
diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs
index 4a7e06ec1..23a17596b 100644
--- a/ecstore/src/store.rs
+++ b/ecstore/src/store.rs
@@ -223,7 +223,7 @@ impl ECStore {
             disk_map,
             pools,
             peer_sys,
-            pool_meta: pool_meta.into(),
+            pool_meta: RwLock::new(pool_meta),
             rebalance_meta: RwLock::new(None),
             decommission_cancelers,
         });
@@ -268,7 +268,7 @@ impl ECStore {
         meta.load(self.pools[0].clone(), self.pools.clone()).await?;
         let update = meta.validate(self.pools.clone())?;
 
-        if update {
+        if !update {
             {
                 let mut pool_meta = self.pool_meta.write().await;
                 *pool_meta = meta.clone();
diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs
index 65b5277fc..fefd7df42 100644
--- a/rustfs/src/admin/handlers/rebalance.rs
+++ b/rustfs/src/admin/handlers/rebalance.rs
@@ -1,4 +1,5 @@
 use ecstore::{
+    config::error::is_err_config_not_found,
     new_object_layer_fn,
     notification_sys::get_global_notification_sys,
     rebalance::{DiskStat, RebalSaveOpt},
@@ -128,9 +129,13 @@ impl Operation for RebalanceStatus {
         };
 
         let mut meta = RebalanceMeta::new();
-        meta.load(store.pools[0].clone())
-            .await
-            .map_err(|e| s3_error!(InternalError, "Failed to load rebalance meta: {}", e))?;
+        if let Err(err) = meta.load(store.pools[0].clone()).await {
+            if is_err_config_not_found(&err) {
+                return Err(s3_error!(NoSuchResource, "Pool rebalance is not started"));
+            }
+
+            return Err(s3_error!(InternalError, "Failed to load rebalance meta: {}", err));
+        }
 
         // Compute disk usage percentage
         let si = store.storage_info().await;

From b2da8148bd4d19f12722eef2403c96c29d1024d9 Mon Sep 17 00:00:00 2001
From: junxiang Mu <1948535941@qq.com>
Date: Thu, 24 Apr 2025 08:12:57 +0000
Subject: [PATCH 05/13] support concurrency write

Signed-off-by: junxiang Mu <1948535941@qq.com>
---
 ecstore/src/bitrot.rs   | 19 +++++++++++++------
 ecstore/src/erasure.rs  | 31 ++++++++++++++++---------------
 ecstore/src/io.rs       |  2 +-
 ecstore/src/set_disk.rs | 39 ++++++++++++++++++++++-----------------
 4 files changed, 52 insertions(+), 39 deletions(-)

diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs
index 757401557..68ccff71e 100644
--- a/ecstore/src/bitrot.rs
+++ b/ecstore/src/bitrot.rs
@@ -6,6 +6,7 @@ use crate::{
 };
 use blake2::Blake2b512;
 use blake2::Digest as _;
+use bytes::Bytes;
 use common::error::{Error, Result};
 use highway::{HighwayHash, HighwayHasher, Key};
 use lazy_static::lazy_static;
@@ -533,20 +534,26 @@ impl Writer for BitrotFileWriter {
         self
     }
 
-    async fn write(&mut self, buf: &[u8]) -> Result<()> {
+    async fn write(&mut self, buf: Bytes) -> Result<()> {
         if buf.is_empty() {
             return Ok(());
         }
-        self.hasher.reset();
-        self.hasher.update(buf);
-        let hash_bytes = self.hasher.clone().finalize();
+        let mut hasher = self.hasher.clone();
+        let h_buf = buf.clone();
+        let hash_bytes = tokio::spawn(async move {
+            hasher.reset();
+            hasher.update(h_buf);
+            hasher.finalize()
+        })
+        .await
+        .unwrap();
 
         if let Some(f) = self.inner.as_mut() {
             f.write_all(&hash_bytes).await?;
-            f.write_all(buf).await?;
+            f.write_all(&buf).await?;
         } else {
             self.inline_data.extend_from_slice(&hash_bytes);
-            self.inline_data.extend_from_slice(buf);
+            self.inline_data.extend_from_slice(&buf);
         }
 
         Ok(())
diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs
index d99c7eea5..b4c45ff11 100644
--- a/ecstore/src/erasure.rs
+++ b/ecstore/src/erasure.rs
@@ -86,7 +86,6 @@ impl Erasure {
                 }
 
                 self.buf.resize(new_len, 0u8);
-
                 match reader.read_exact(&mut self.buf).await {
                     Ok(res) => res,
                     Err(e) => {
@@ -101,20 +100,22 @@ impl Erasure {
             }
 
             self.encode_data(&self.buf, &mut blocks)?;
-            let mut errs = Vec::new();
 
-            // TODO: 并发写入
-            for (i, w_op) in writers.iter_mut().enumerate() {
-                if let Some(w) = w_op {
-                    match w.write(blocks[i].as_ref()).await {
-                        Ok(_) => errs.push(None),
-                        Err(e) => errs.push(Some(e)),
+            let write_futures = writers.iter_mut().enumerate().map(|(i, w_op)| {
+                let i_inner = i.clone();
+                let blocks_inner = blocks.clone();
+                async move {
+                    if let Some(w) = w_op {
+                        match w.write(blocks_inner[i_inner].clone()).await {
+                            Ok(_) => None,
+                            Err(e) => Some(e),
+                        }
+                    } else {
+                        Some(Error::new(DiskError::DiskNotFound))
                     }
-                } else {
-                    errs.push(Some(Error::new(DiskError::DiskNotFound)));
                 }
-            }
-
+            });
+            let errs = join_all(write_futures).await;
             let none_count = errs.iter().filter(|&x| x.is_none()).count();
             if none_count >= write_quorum {
                 if total_size == 0 {
@@ -472,7 +473,7 @@ impl Erasure {
                 self.encoder.as_ref().unwrap().reconstruct(&mut bufs)?;
             }
 
-            let shards: Vec<Vec<u8>> = bufs.into_iter().flatten().collect::<Vec<_>>();
+            let shards = bufs.into_iter().flatten().collect::<Vec<_>>();
             if shards.len() != self.parity_shards + self.data_shards {
                 return Err(Error::from_string("can not reconstruct data"));
             }
@@ -481,7 +482,7 @@ impl Erasure {
                 if w.is_none() {
                     continue;
                 }
-                match w.as_mut().unwrap().write(shards[i].as_ref()).await {
+                match w.as_mut().unwrap().write(shards[i].clone().into()).await {
                     Ok(_) => {}
                     Err(e) => {
                         info!("write failed, err: {:?}", e);
@@ -501,7 +502,7 @@ impl Erasure {
 #[async_trait::async_trait]
 pub trait Writer {
     fn as_any(&self) -> &dyn Any;
-    async fn write(&mut self, buf: &[u8]) -> Result<()>;
+    async fn write(&mut self, buf: Bytes) -> Result<()>;
     async fn close(&mut self) -> Result<()> {
         Ok(())
     }
diff --git a/ecstore/src/io.rs b/ecstore/src/io.rs
index 7ecf8f84c..f2affe8cb 100644
--- a/ecstore/src/io.rs
+++ b/ecstore/src/io.rs
@@ -2,13 +2,13 @@ use bytes::Bytes;
 use futures::TryStreamExt;
 use md5::Digest;
 use md5::Md5;
-use tokio::sync::mpsc;
 use std::pin::Pin;
 use std::task::Context;
 use std::task::Poll;
 use tokio::io::AsyncRead;
 use tokio::io::AsyncWrite;
 use tokio::io::ReadBuf;
+use tokio::sync::mpsc;
 use tokio::sync::oneshot;
 use tokio_util::io::ReaderStream;
 use tokio_util::io::StreamReader;
diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs
index b35c6ad89..77eafe01a 100644
--- a/ecstore/src/set_disk.rs
+++ b/ecstore/src/set_disk.rs
@@ -325,6 +325,7 @@ impl SetDisks {
             }
         }
 
+        let mut futures = Vec::with_capacity(disks.len());
         if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) {
             // TODO: 并发
             for (i, err) in errs.iter().enumerate() {
@@ -334,26 +335,30 @@ impl SetDisks {
 
                 if let Some(disk) = disks[i].as_ref() {
                     let fi = file_infos[i].clone();
-                    let _ = disk
-                        .delete_version(
-                            src_bucket,
-                            src_object,
-                            fi,
-                            false,
-                            DeleteOptions {
-                                undo_write: true,
-                                old_data_dir: data_dirs[i],
-                                ..Default::default()
-                            },
-                        )
-                        .await
-                        .map_err(|e| {
-                            debug!("rename_data delete_version err {:?}", e);
-                            e
-                        });
+                    let old_data_dir = data_dirs[i];
+                    futures.push(async move {
+                        let _ = disk
+                            .delete_version(
+                                src_bucket,
+                                src_object,
+                                fi,
+                                false,
+                                DeleteOptions {
+                                    undo_write: true,
+                                    old_data_dir: old_data_dir,
+                                    ..Default::default()
+                                },
+                            )
+                            .await
+                            .map_err(|e| {
+                                debug!("rename_data delete_version err {:?}", e);
+                                e
+                            });
+                    });
                 }
             }
 
+            let _ = join_all(futures).await;
             return Err(err);
         }
 

From e38dd25bf679d09a09dc4231b03901d2b688c3a8 Mon Sep 17 00:00:00 2001
From: junxiang Mu <1948535941@qq.com>
Date: Thu, 24 Apr 2025 08:21:09 +0000
Subject: [PATCH 06/13] fix admin info

Signed-off-by: junxiang Mu <1948535941@qq.com>
---
 madmin/src/info_commands.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/madmin/src/info_commands.rs b/madmin/src/info_commands.rs
index 72c5972f2..53afc8c13 100644
--- a/madmin/src/info_commands.rs
+++ b/madmin/src/info_commands.rs
@@ -45,7 +45,9 @@ pub struct DiskMetrics {
 #[derive(Serialize, Deserialize, Debug, Default, Clone)]
 pub struct Disk {
     pub endpoint: String,
+    #[serde(rename = "rootDisk")]
     pub root_disk: bool,
+    #[serde(rename = "path")]
     pub drive_path: String,
     pub healing: bool,
     pub scanning: bool,
@@ -54,12 +56,19 @@ pub struct Disk {
     pub major: u32,
     pub minor: u32,
     pub model: Option<String>,
+    #[serde(rename = "totalspace")]
     pub total_space: u64,
+    #[serde(rename = "usedspace")]
     pub used_space: u64,
+    #[serde(rename = "availspace")]
     pub available_space: u64,
+    #[serde(rename = "readthroughput")]
     pub read_throughput: f64,
+    #[serde(rename = "writethroughput")]
     pub write_throughput: f64,
+    #[serde(rename = "readlatency")]
     pub read_latency: f64,
+    #[serde(rename = "writelatency")]
     pub write_latency: f64,
     pub utilization: f64,
     pub metrics: Option<DiskMetrics>,

From 4613b366979382cd5baed434f855d2c8b12584f0 Mon Sep 17 00:00:00 2001
From: junxiang Mu <1948535941@qq.com>
Date: Thu, 24 Apr 2025 08:41:40 +0000
Subject: [PATCH 07/13] fix

Signed-off-by: junxiang Mu <1948535941@qq.com>
---
 ecstore/src/heal/data_scanner.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs
index 71a8d8a25..ff4fe850b 100644
--- a/ecstore/src/heal/data_scanner.rs
+++ b/ecstore/src/heal/data_scanner.rs
@@ -1029,7 +1029,7 @@ impl FolderScanner {
 
 #[tracing::instrument(level = "info", skip(into, folder_scanner))]
 async fn scan(folder: &CachedFolder, into: &mut DataUsageEntry, folder_scanner: &mut FolderScanner) {
-    if !into.compacted {
+    if into.compacted {
         *into = DataUsageEntry::default();
     }
 

From 86353d98d5a1fc60c7a91aecd6dd93f0fbf47295 Mon Sep 17 00:00:00 2001
From: houseme <housemecn@gmail.com>
Date: Thu, 24 Apr 2025 19:04:57 +0800
Subject: [PATCH 08/13] feat: add TraceLayer for HTTP service and improve
 metrics (#361)

* improve code for opentelemetry and add system metrics

* feat: add TraceLayer for HTTP service and improve metrics

- Add TraceLayer to HTTP server for request tracing
- Implement system metrics for process monitoring
- Optimize init_telemetry method for better resource management
- Add graceful shutdown handling for telemetry components
- Fix GracefulShutdown ownership issues with Arc wrapper

* improve code for init_process_observer

* remove tomlfmt.toml

* Translation comment

* improve code for console CompressionLayer params
---
 Cargo.lock                               | 153 +++++++---
 Cargo.toml                               |  46 +--
 crates/event-notifier/src/config.rs      |   1 -
 crates/obs/Cargo.toml                    |  11 +-
 crates/obs/examples/config.toml          |   2 +-
 crates/obs/examples/server.rs            |   9 +-
 crates/obs/src/config.rs                 |   2 +-
 crates/obs/src/global.rs                 |  28 +-
 crates/obs/src/lib.rs                    |  80 +++--
 crates/obs/src/logger.rs                 | 102 ++-----
 crates/obs/src/system/attributes.rs      |  44 +++
 crates/obs/src/system/collector.rs       | 156 ++++++++++
 crates/obs/src/system/gpu.rs             |  70 +++++
 crates/obs/src/system/metrics.rs         | 100 +++++++
 crates/obs/src/system/mod.rs             |  24 ++
 crates/obs/src/telemetry.rs              | 361 ++++++++++-------------
 rustfs/Cargo.toml                        |   3 +-
 rustfs/src/console.rs                    |   2 +-
 rustfs/src/main.rs                       | 106 +++++--
 s3select/query/src/dispatcher/manager.rs |   3 +-
 scripts/run.sh                           |   4 +-
 21 files changed, 900 insertions(+), 407 deletions(-)
 create mode 100644 crates/obs/src/system/attributes.rs
 create mode 100644 crates/obs/src/system/collector.rs
 create mode 100644 crates/obs/src/system/gpu.rs
 create mode 100644 crates/obs/src/system/metrics.rs
 create mode 100644 crates/obs/src/system/mod.rs

diff --git a/Cargo.lock b/Cargo.lock
index 53ba1e2b0..81d6b4c3e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -525,7 +525,6 @@ version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c"
 dependencies = [
- "brotli",
  "bzip2",
  "flate2",
  "futures-core",
@@ -5251,6 +5250,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "ntapi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -5413,6 +5421,29 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093"
 
+[[package]]
+name = "nvml-wrapper"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c9bff0aa1d48904a1385ea2a8b97576fbdcbc9a3cfccd0d31fe978e1c4038c5"
+dependencies = [
+ "bitflags 2.9.0",
+ "libloading 0.8.6",
+ "nvml-wrapper-sys",
+ "static_assertions",
+ "thiserror 1.0.69",
+ "wrapcenum-derive",
+]
+
+[[package]]
+name = "nvml-wrapper-sys"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "698d45156f28781a4e79652b6ebe2eaa0589057d588d3aec1333f6466f13fcb5"
+dependencies = [
+ "libloading 0.8.6",
+]
+
 [[package]]
 name = "objc"
 version = "0.2.7"
@@ -5716,19 +5747,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "opentelemetry-prometheus"
-version = "0.29.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "098a71a4430bb712be6130ed777335d2e5b19bc8566de5f2edddfce906def6ab"
-dependencies = [
- "once_cell",
- "opentelemetry",
- "opentelemetry_sdk",
- "prometheus",
- "tracing",
-]
-
 [[package]]
 name = "opentelemetry-proto"
 version = "0.29.0"
@@ -6487,21 +6505,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "prometheus"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
-dependencies = [
- "cfg-if",
- "fnv",
- "lazy_static",
- "memchr",
- "parking_lot 0.12.3",
- "protobuf",
- "thiserror 2.0.12",
-]
-
 [[package]]
 name = "prost"
 version = "0.13.5"
@@ -7248,6 +7251,7 @@ dependencies = [
  "mime",
  "mime_guess",
  "netif",
+ "opentelemetry",
  "pin-project-lite",
  "policy",
  "prost-build",
@@ -7332,18 +7336,19 @@ dependencies = [
  "chrono",
  "config",
  "local-ip-address",
+ "nvml-wrapper",
  "opentelemetry",
  "opentelemetry-appender-tracing",
  "opentelemetry-otlp",
- "opentelemetry-prometheus",
  "opentelemetry-semantic-conventions",
  "opentelemetry-stdout",
  "opentelemetry_sdk",
- "prometheus",
  "rdkafka",
  "reqwest",
  "serde",
  "serde_json",
+ "smallvec",
+ "sysinfo",
  "thiserror 2.0.12",
  "tokio",
  "tracing",
@@ -8274,6 +8279,19 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.34.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4b93974b3d3aeaa036504b8eefd4c039dced109171c1ae973f1dc63b2c7e4b2"
+dependencies = [
+ "libc",
+ "memchr",
+ "ntapi",
+ "objc2-core-foundation",
+ "windows 0.57.0",
+]
+
 [[package]]
 name = "system-configuration"
 version = "0.6.1"
@@ -8342,7 +8360,7 @@ dependencies = [
  "tao-macros",
  "unicode-segmentation",
  "url",
- "windows",
+ "windows 0.58.0",
  "windows-core 0.58.0",
  "windows-version",
  "x11-dl",
@@ -9483,7 +9501,7 @@ checksum = "6f61ff3d9d0ee4efcb461b14eb3acfda2702d10dc329f339303fc3e57215ae2c"
 dependencies = [
  "webview2-com-macros",
  "webview2-com-sys",
- "windows",
+ "windows 0.58.0",
  "windows-core 0.58.0",
  "windows-implement 0.58.0",
  "windows-interface 0.58.0",
@@ -9507,7 +9525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3a3e2eeb58f82361c93f9777014668eb3d07e7d174ee4c819575a9208011886"
 dependencies = [
  "thiserror 1.0.69",
- "windows",
+ "windows 0.58.0",
  "windows-core 0.58.0",
 ]
 
@@ -9554,6 +9572,16 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
+dependencies = [
+ "windows-core 0.57.0",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows"
 version = "0.58.0"
@@ -9564,6 +9592,18 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-core"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
+dependencies = [
+ "windows-implement 0.57.0",
+ "windows-interface 0.57.0",
+ "windows-result 0.1.2",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-core"
 version = "0.58.0"
@@ -9590,6 +9630,17 @@ dependencies = [
  "windows-strings 0.4.0",
 ]
 
+[[package]]
+name = "windows-implement"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "windows-implement"
 version = "0.58.0"
@@ -9612,6 +9663,17 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "windows-interface"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "windows-interface"
 version = "0.58.0"
@@ -9651,6 +9713,15 @@ dependencies = [
  "windows-targets 0.53.0",
 ]
 
+[[package]]
+name = "windows-result"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-result"
 version = "0.2.0"
@@ -10020,6 +10091,18 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "wrapcenum-derive"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a76ff259533532054cfbaefb115c613203c73707017459206380f03b3b3f266e"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "write16"
 version = "1.0.0"
@@ -10066,7 +10149,7 @@ dependencies = [
  "webkit2gtk",
  "webkit2gtk-sys",
  "webview2-com",
- "windows",
+ "windows 0.58.0",
  "windows-core 0.58.0",
  "windows-version",
  "x11-dl",
diff --git a/Cargo.toml b/Cargo.toml
index d0eb484d1..b9eb8dfd1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,21 +1,21 @@
 [workspace]
 members = [
-    "madmin",                # Management dashboard and admin API interface
-    "rustfs",                # Core file system implementation
-    "ecstore",               # Erasure coding storage implementation
-    "e2e_test",              # End-to-end test suite
-    "common/common",         # Shared utilities and data structures
-    "common/lock",           # Distributed locking implementation
-    "common/protos",         # Protocol buffer definitions
-    "common/workers",        # Worker thread pools and task scheduling
-    "iam",                   # Identity and Access Management
-    "crypto",                # Cryptography and security features
-    "cli/rustfs-gui",        # Graphical user interface client
-    "crates/obs",            # Observability utilities
+    "madmin", # Management dashboard and admin API interface
+    "rustfs", # Core file system implementation
+    "ecstore", # Erasure coding storage implementation
+    "e2e_test", # End-to-end test suite
+    "common/common", # Shared utilities and data structures
+    "common/lock", # Distributed locking implementation
+    "common/protos", # Protocol buffer definitions
+    "common/workers", # Worker thread pools and task scheduling
+    "iam", # Identity and Access Management
+    "crypto", # Cryptography and security features
+    "cli/rustfs-gui", # Graphical user interface client
+    "crates/obs", # Observability utilities
     "crates/event-notifier", # Event notification system
-    "s3select/api",          # S3 Select API interface
-    "s3select/query",        # S3 Select query engine
-    "appauth",               # Application authentication and authorization
+    "s3select/api", # S3 Select API interface
+    "s3select/query", # S3 Select query engine
+    "appauth", # Application authentication and authorization
 ]
 resolver = "2"
 
@@ -93,6 +93,7 @@ md-5 = "0.10.6"
 mime = "0.3.17"
 mime_guess = "2.0.5"
 netif = "0.1.6"
+nvml-wrapper = "0.10.0"
 object_store = "0.11.2"
 opentelemetry = { version = "0.29.1" }
 opentelemetry-appender-tracing = { version = "0.29.1", features = [
@@ -102,10 +103,7 @@ opentelemetry-appender-tracing = { version = "0.29.1", features = [
 opentelemetry_sdk = { version = "0.29.0" }
 opentelemetry-stdout = { version = "0.29.0" }
 opentelemetry-otlp = { version = "0.29.0" }
-opentelemetry-prometheus = { version = "0.29.1" }
-opentelemetry-semantic-conventions = { version = "0.29.0", features = [
-    "semconv_experimental",
-] }
+opentelemetry-semantic-conventions = { version = "0.29.0", features = ["semconv_experimental"] }
 parking_lot = "0.12.3"
 pin-project-lite = "0.2.16"
 prometheus = "0.14.0"
@@ -143,10 +141,11 @@ serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.140"
 serde_urlencoded = "0.7.1"
 serde_with = "3.12.0"
-smallvec = { version = "1.15.0", features = ["serde"] }
-strum = { version = "0.27.1", features = ["derive"] }
 sha2 = "0.10.8"
+smallvec = { version = "1.15.0", features = ["serde"] }
 snafu = "0.8.5"
+strum = { version = "0.27.1", features = ["derive"] }
+sysinfo = "0.34.2"
 tempfile = "3.19.1"
 test-case = "3.3.1"
 thiserror = "2.0.12"
@@ -192,7 +191,10 @@ inherits = "dev"
 
 [profile.release]
 opt-level = 3
-lto = "thin"
+lto = "fat"
+codegen-units = 1
+panic = "abort"  # Optional, remove the panic expansion code
+strip = true     # strip symbol information to reduce binary size
 
 [profile.production]
 inherits = "release"
diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs
index ae718abc2..ab46d8e8f 100644
--- a/crates/event-notifier/src/config.rs
+++ b/crates/event-notifier/src/config.rs
@@ -149,7 +149,6 @@ impl NotifierConfig {
             )
             .build()
             .unwrap_or_default();
-        println!("Loaded config: {:?}", app_config);
         match app_config.try_deserialize::<NotifierConfig>() {
             Ok(app_config) => {
                 println!("Parsed AppConfig: {:?} \n", app_config);
diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml
index 4290b76b3..f6d9c4787 100644
--- a/crates/obs/Cargo.toml
+++ b/crates/obs/Cargo.toml
@@ -11,32 +11,35 @@ workspace = true
 
 [features]
 default = ["file"]
+file = []
+gpu = ["dep:nvml-wrapper"]
 kafka = ["dep:rdkafka"]
 webhook = ["dep:reqwest"]
-file = []
+full = ["file", "gpu", "kafka", "webhook"]
 
 [dependencies]
 async-trait = { workspace = true }
 chrono = { workspace = true }
 config = { workspace = true }
+nvml-wrapper = { workspace = true, optional = true }
 opentelemetry = { workspace = true }
 opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] }
 opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 opentelemetry-stdout = { workspace = true }
 opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] }
-opentelemetry-prometheus = { workspace = true }
 opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
-prometheus = { workspace = true }
 serde = { workspace = true }
+smallvec = { workspace = true, features = ["serde"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
 tracing-core = { workspace = true }
 tracing-error = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
-tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 rdkafka = { workspace = true, features = ["tokio"], optional = true }
 reqwest = { workspace = true, optional = true, default-features = false }
 serde_json = { workspace = true }
+sysinfo = { workspace = true }
 thiserror = { workspace = true }
 local-ip-address = { workspace = true }
 
diff --git a/crates/obs/examples/config.toml b/crates/obs/examples/config.toml
index 929867f03..c1b3df148 100644
--- a/crates/obs/examples/config.toml
+++ b/crates/obs/examples/config.toml
@@ -25,7 +25,7 @@ batch_timeout_ms = 1000 # Default is 100ms if not specified
 
 [sinks.file]
 enabled = true
-path = "logs/app.log"
+path = "deploy/logs/app.log"
 batch_size = 100
 batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
 
diff --git a/crates/obs/examples/server.rs b/crates/obs/examples/server.rs
index 55977b10c..0310c158f 100644
--- a/crates/obs/examples/server.rs
+++ b/crates/obs/examples/server.rs
@@ -1,8 +1,8 @@
 use opentelemetry::global;
-use rustfs_obs::{get_logger, init_obs, load_config, log_info, BaseLogEntry, ServerLogEntry};
+use rustfs_obs::{get_logger, init_obs, init_process_observer, load_config, log_info, BaseLogEntry, ServerLogEntry};
 use std::collections::HashMap;
 use std::time::{Duration, SystemTime};
-use tracing::{info, instrument};
+use tracing::{error, info, instrument};
 use tracing_core::Level;
 
 #[tokio::main]
@@ -38,6 +38,11 @@ async fn run(bucket: String, object: String, user: String, service_name: String)
         &[opentelemetry::KeyValue::new("operation", "run")],
     );
 
+    match init_process_observer(meter).await {
+        Ok(_) => info!("Process observer initialized successfully"),
+        Err(e) => error!("Failed to initialize process observer: {:?}", e),
+    }
+
     let base_entry = BaseLogEntry::new()
         .message(Some("run logger api_handler info".to_string()))
         .request_id(Some("request_id".to_string()))
diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs
index a9e620161..33f974f97 100644
--- a/crates/obs/src/config.rs
+++ b/crates/obs/src/config.rs
@@ -178,7 +178,6 @@ pub struct AppConfig {
     pub logger: Option<LoggerConfig>,
 }
 
-// 为 AppConfig 实现 Default
 impl AppConfig {
     pub fn new() -> Self {
         Self {
@@ -189,6 +188,7 @@ impl AppConfig {
     }
 }
 
+// implement default for AppConfig
 impl Default for AppConfig {
     fn default() -> Self {
         Self::new()
diff --git a/crates/obs/src/global.rs b/crates/obs/src/global.rs
index 41c5e555e..8d392ad94 100644
--- a/crates/obs/src/global.rs
+++ b/crates/obs/src/global.rs
@@ -16,11 +16,27 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
 
 /// Error type for global guard operations
 #[derive(Debug, thiserror::Error)]
-pub enum GuardError {
+pub enum GlobalError {
     #[error("Failed to set global guard: {0}")]
     SetError(#[from] SetError<Arc<Mutex<OtelGuard>>>),
     #[error("Global guard not initialized")]
     NotInitialized,
+    #[error("Global system metrics err: {0}")]
+    MetricsError(String),
+    #[error("Failed to get current PID: {0}")]
+    PidError(String),
+    #[error("Process with PID {0} not found")]
+    ProcessNotFound(u32),
+    #[error("Failed to get physical core count")]
+    CoreCountError,
+    #[error("GPU initialization failed: {0}")]
+    GpuInitError(String),
+    #[error("GPU device not found: {0}")]
+    GpuDeviceError(String),
+    #[error("Failed to send log: {0}")]
+    SendFailed(&'static str),
+    #[error("Operation timed out: {0}")]
+    Timeout(&'static str),
 }
 
 /// Set the global guard for OpenTelemetry
@@ -43,9 +59,9 @@ pub enum GuardError {
 ///     Ok(())
 /// }
 /// ```
-pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
+pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
     info!("Initializing global OpenTelemetry guard");
-    GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GuardError::SetError)
+    GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GlobalError::SetError)
 }
 
 /// Get the global guard for OpenTelemetry
@@ -65,8 +81,8 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
 ///     Ok(())
 /// }
 /// ```
-pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GuardError> {
-    GLOBAL_GUARD.get().cloned().ok_or(GuardError::NotInitialized)
+pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GlobalError> {
+    GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized)
 }
 
 /// Try to get the global guard for OpenTelemetry
@@ -84,6 +100,6 @@ mod tests {
     #[tokio::test]
     async fn test_get_uninitialized_guard() {
         let result = get_global_guard();
-        assert!(matches!(result, Err(GuardError::NotInitialized)));
+        assert!(matches!(result, Err(GlobalError::NotInitialized)));
     }
 }
diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs
index 843c52367..d41d0e66d 100644
--- a/crates/obs/src/lib.rs
+++ b/crates/obs/src/lib.rs
@@ -1,23 +1,24 @@
-/// # obs
-///
-/// `obs` is a logging and observability library for Rust.
-/// It provides a simple and easy-to-use interface for logging and observability.
-/// It is built on top of the `log` crate and `opentelemetry` crate.
-///
-/// ## Features
-/// - Structured logging
-/// - Distributed tracing
-/// - Metrics collection
-/// - Log processing worker
-/// - Multiple sinks
-/// - Configuration-based setup
-/// - Telemetry guard
-/// - Global logger
-/// - Log levels
-/// - Log entry types
-/// - Log record
-/// - Object version
-/// - Local IP address
+//! # RustFS Observability
+//!
+//! provides tools for system and service monitoring
+//!
+//! ## feature mark
+//!
+//! - `file`: enable file logging enabled by default
+//! - `gpu`: gpu monitoring function
+//! - `kafka`: enable kafka metric output
+//! - `webhook`: enable webhook notifications
+//! - `full`: includes all functions
+//!
+//! to enable gpu monitoring add in cargo toml
+//!
+//! ```toml
+//! # using gpu monitoring
+//! rustfs-obs = { version = "0.1.0", features = ["gpu"] }
+//!
+//! # use all functions
+//! rustfs-obs = { version = "0.1.0", features = ["full"] }
+//! ```
 ///
 /// ## Usage
 ///
@@ -32,28 +33,34 @@ mod entry;
 mod global;
 mod logger;
 mod sink;
+mod system;
 mod telemetry;
 mod utils;
 mod worker;
+
+use crate::logger::InitLogStatus;
 pub use config::load_config;
-pub use config::{AppConfig, OtelConfig};
+#[cfg(feature = "file")]
+pub use config::FileSinkConfig;
+#[cfg(feature = "kafka")]
+pub use config::KafkaSinkConfig;
+#[cfg(feature = "webhook")]
+pub use config::WebhookSinkConfig;
+pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
 pub use entry::args::Args;
 pub use entry::audit::{ApiDetails, AuditLogEntry};
 pub use entry::base::BaseLogEntry;
 pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
 pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
-pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GuardError};
-pub use logger::{ensure_logger_initialized, log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
-pub use logger::{get_global_logger, init_global_logger, locked_logger, start_logger};
-pub use logger::{log_init_state, InitLogStatus};
-pub use logger::{LogError, Logger};
-pub use sink::Sink;
+pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GlobalError};
+pub use logger::Logger;
+pub use logger::{get_global_logger, init_global_logger, start_logger};
+pub use logger::{log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
 use std::sync::Arc;
+pub use system::{init_process_observer, init_process_observer_for_pid};
 pub use telemetry::init_telemetry;
-pub use telemetry::{get_global_registry, metrics};
 use tokio::sync::Mutex;
-pub use utils::{get_local_ip, get_local_ip_with_default};
-pub use worker::start_worker;
+use tracing::{error, info};
 
 /// Initialize the observability module
 ///
@@ -74,6 +81,19 @@ pub async fn init_obs(config: AppConfig) -> (Arc<Mutex<Logger>>, telemetry::Otel
     let guard = init_telemetry(&config.observability);
     let sinks = sink::create_sinks(&config).await;
     let logger = init_global_logger(&config, sinks).await;
+    let obs_config = config.observability.clone();
+    tokio::spawn(async move {
+        let result = InitLogStatus::init_start_log(&obs_config).await;
+        match result {
+            Ok(_) => {
+                info!("Logger initialized successfully");
+            }
+            Err(e) => {
+                error!("Failed to initialize logger: {}", e);
+            }
+        }
+    });
+
     (logger, guard)
 }
 
diff --git a/crates/obs/src/logger.rs b/crates/obs/src/logger.rs
index 1e62712cb..6329ab8fc 100644
--- a/crates/obs/src/logger.rs
+++ b/crates/obs/src/logger.rs
@@ -1,5 +1,6 @@
 use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
-use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, OtelConfig, ServerLogEntry, Sink, UnifiedLogEntry};
+use crate::sink::Sink;
+use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry};
 use std::sync::Arc;
 use std::time::SystemTime;
 use tokio::sync::mpsc::{self, Receiver, Sender};
@@ -43,26 +44,26 @@ impl Logger {
 
     /// Log a server entry
     #[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
-    pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), LogError> {
+    pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), GlobalError> {
         self.log_entry(UnifiedLogEntry::Server(entry)).await
     }
 
     /// Log an audit entry
     #[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
-    pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), LogError> {
+    pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), GlobalError> {
         self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
     }
 
     /// Log a console entry
     #[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
-    pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), LogError> {
+    pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), GlobalError> {
         self.log_entry(UnifiedLogEntry::Console(entry)).await
     }
 
     /// Asynchronous logging of unified log entries
     #[tracing::instrument(skip(self), fields(log_source = "logger"))]
     #[tracing::instrument(level = "error", skip_all)]
-    pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), LogError> {
+    pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), GlobalError> {
         // Extract information for tracing based on entry type
         match &entry {
             UnifiedLogEntry::Server(server) => {
@@ -123,11 +124,11 @@ impl Logger {
                 tracing::warn!("Log queue full, applying backpressure");
                 match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
                     Ok(Ok(_)) => Ok(()),
-                    Ok(Err(_)) => Err(LogError::SendFailed("Channel closed")),
-                    Err(_) => Err(LogError::Timeout("Queue backpressure timeout")),
+                    Ok(Err(_)) => Err(GlobalError::SendFailed("Channel closed")),
+                    Err(_) => Err(GlobalError::Timeout("Queue backpressure timeout")),
                 }
             }
-            Err(mpsc::error::TrySendError::Closed(_)) => Err(LogError::SendFailed("Logger channel closed")),
+            Err(mpsc::error::TrySendError::Closed(_)) => Err(GlobalError::SendFailed("Logger channel closed")),
         }
     }
 
@@ -160,7 +161,7 @@ impl Logger {
         request_id: Option<String>,
         user_id: Option<String>,
         fields: Vec<(String, String)>,
-    ) -> Result<(), LogError> {
+    ) -> Result<(), GlobalError> {
         let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
 
         let server_entry = ServerLogEntry::new(level, source.to_string())
@@ -190,7 +191,7 @@ impl Logger {
     ///   let _ = logger.write("This is an information message", "example", Level::INFO).await;
     /// }
     /// ```
-    pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), LogError> {
+    pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), GlobalError> {
         self.write_with_context(message, source, level, None, None, Vec::new()).await
     }
 
@@ -208,31 +209,12 @@ impl Logger {
     ///  let _ = logger.shutdown().await;
     /// }
     /// ```
-    pub async fn shutdown(self) -> Result<(), LogError> {
+    pub async fn shutdown(self) -> Result<(), GlobalError> {
         drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
         Ok(())
     }
 }
 
-/// Log error type
-/// This enum defines the error types that can occur when logging.
-/// It is used to provide more detailed error information.
-/// # Example
-/// ```
-/// use rustfs_obs::LogError;
-/// use thiserror::Error;
-///
-/// LogError::SendFailed("Failed to send log");
-/// LogError::Timeout("Operation timed out");
-/// ```
-#[derive(Debug, thiserror::Error)]
-pub enum LogError {
-    #[error("Failed to send log: {0}")]
-    SendFailed(&'static str),
-    #[error("Operation timed out: {0}")]
-    Timeout(&'static str),
-}
-
 /// Start the log module
 /// This function starts the log module.
 /// It initializes the logger and starts the worker to process logs.
@@ -297,48 +279,6 @@ pub fn get_global_logger() -> &'static Arc<Mutex<Logger>> {
     GLOBAL_LOGGER.get().expect("Logger not initialized")
 }
 
-/// Get the global logger instance with a lock
-/// This function returns a reference to the global logger instance with a lock.
-/// It is used to ensure that the logger is thread-safe.
-///
-/// # Returns
-/// A reference to the global logger instance with a lock
-///
-/// # Example
-/// ```
-/// use rustfs_obs::locked_logger;
-///
-/// async fn example() {
-///     let logger = locked_logger().await;
-/// }
-/// ```
-pub async fn locked_logger() -> tokio::sync::MutexGuard<'static, Logger> {
-    get_global_logger().lock().await
-}
-
-/// Initialize with default empty logger if needed (optional)
-/// This function initializes the logger with a default empty logger if needed.
-/// It is used to ensure that the logger is initialized before logging.
-///
-/// # Returns
-/// A reference to the global logger instance
-///
-/// # Example
-/// ```
-/// use rustfs_obs::ensure_logger_initialized;
-///
-/// let logger = ensure_logger_initialized();
-/// ```
-pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
-    if GLOBAL_LOGGER.get().is_none() {
-        let config = AppConfig::default();
-        let sinks = vec![];
-        let logger = Arc::new(Mutex::new(start_logger(&config, sinks)));
-        let _ = GLOBAL_LOGGER.set(logger);
-    }
-    GLOBAL_LOGGER.get().unwrap()
-}
-
 /// Log information
 /// This function logs information messages.
 ///
@@ -357,7 +297,7 @@ pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
 ///    let _ = log_info("This is an information message", "example").await;
 /// }
 /// ```
-pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_info(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::INFO).await
 }
 
@@ -375,7 +315,7 @@ pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
 /// async fn example() {
 ///     let _ = log_error("This is an error message", "example").await;
 /// }
-pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_error(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::ERROR).await
 }
 
@@ -395,7 +335,7 @@ pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
 ///     let _ = log_warn("This is a warning message", "example").await;
 /// }
 /// ```
-pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_warn(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::WARN).await
 }
 
@@ -415,7 +355,7 @@ pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
 ///     let _ = log_debug("This is a debug message", "example").await;
 /// }
 /// ```
-pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_debug(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::DEBUG).await
 }
 
@@ -436,7 +376,7 @@ pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
 ///    let _ = log_trace("This is a trace message", "example").await;
 /// }
 /// ```
-pub async fn log_trace(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_trace(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::TRACE).await
 }
 
@@ -467,7 +407,7 @@ pub async fn log_with_context(
     request_id: Option<String>,
     user_id: Option<String>,
     fields: Vec<(String, String)>,
-) -> Result<(), LogError> {
+) -> Result<(), GlobalError> {
     get_global_logger()
         .lock()
         .await
@@ -477,7 +417,7 @@ pub async fn log_with_context(
 
 /// Log initialization status
 #[derive(Debug)]
-pub struct InitLogStatus {
+pub(crate) struct InitLogStatus {
     pub timestamp: SystemTime,
     pub service_name: String,
     pub version: String,
@@ -508,14 +448,14 @@ impl InitLogStatus {
         }
     }
 
-    pub async fn init_start_log(config: &OtelConfig) -> Result<(), LogError> {
+    pub async fn init_start_log(config: &OtelConfig) -> Result<(), GlobalError> {
         let status = Self::new_config(config);
         log_init_state(Some(status)).await
     }
 }
 
 /// Log initialization details during system startup
-pub async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), LogError> {
+async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), GlobalError> {
     let status = status.unwrap_or_default();
 
     let base_entry = BaseLogEntry::new()
diff --git a/crates/obs/src/system/attributes.rs b/crates/obs/src/system/attributes.rs
new file mode 100644
index 000000000..289b4b66a
--- /dev/null
+++ b/crates/obs/src/system/attributes.rs
@@ -0,0 +1,44 @@
+use crate::GlobalError;
+use opentelemetry::KeyValue;
+use sysinfo::{Pid, System};
+
+pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
+pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
+pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
+pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
+
+/// Struct to hold process attributes
+pub struct ProcessAttributes {
+    pub attributes: Vec<KeyValue>,
+}
+
+impl ProcessAttributes {
+    /// Creates a new instance of `ProcessAttributes` for the given PID.
+    pub fn new(pid: Pid, system: &mut System) -> Result<Self, GlobalError> {
+        system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+        let process = system
+            .process(pid)
+            .ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?;
+
+        let attributes = vec![
+            KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
+            KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()),
+            KeyValue::new(
+                PROCESS_EXECUTABLE_PATH,
+                process
+                    .exe()
+                    .map(|path| path.to_string_lossy().into_owned())
+                    .unwrap_or_default(),
+            ),
+            KeyValue::new(
+                PROCESS_COMMAND,
+                process
+                    .cmd()
+                    .iter()
+                    .fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()),
+            ),
+        ];
+
+        Ok(ProcessAttributes { attributes })
+    }
+}
diff --git a/crates/obs/src/system/collector.rs b/crates/obs/src/system/collector.rs
new file mode 100644
index 000000000..c41840515
--- /dev/null
+++ b/crates/obs/src/system/collector.rs
@@ -0,0 +1,156 @@
+use crate::system::attributes::ProcessAttributes;
+use crate::system::gpu::GpuCollector;
+use crate::system::metrics::{Metrics, DIRECTION, INTERFACE, STATUS};
+use crate::GlobalError;
+use opentelemetry::KeyValue;
+use std::time::SystemTime;
+use sysinfo::{Networks, Pid, ProcessStatus, System};
+use tokio::time::{sleep, Duration};
+
+/// Collector is responsible for collecting system metrics and attributes.
+/// It uses the sysinfo crate to gather information about the system and processes.
+/// It also uses OpenTelemetry to record metrics.
+pub struct Collector {
+    metrics: Metrics,
+    attributes: ProcessAttributes,
+    gpu_collector: GpuCollector,
+    pid: Pid,
+    system: System,
+    networks: Networks,
+    core_count: usize,
+    interval_ms: u64,
+}
+
+impl Collector {
+    pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result<Self, GlobalError> {
+        let mut system = System::new_all();
+        let attributes = ProcessAttributes::new(pid, &mut system)?;
+        let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
+        let metrics = Metrics::new(&meter);
+        let gpu_collector = GpuCollector::new(pid)?;
+        let networks = Networks::new_with_refreshed_list();
+
+        Ok(Collector {
+            metrics,
+            attributes,
+            gpu_collector,
+            pid,
+            system,
+            networks,
+            core_count,
+            interval_ms,
+        })
+    }
+
+    pub async fn run(&mut self) -> Result<(), GlobalError> {
+        loop {
+            self.collect()?;
+            tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now());
+            sleep(Duration::from_millis(self.interval_ms)).await;
+        }
+    }
+
+    fn collect(&mut self) -> Result<(), GlobalError> {
+        self.system
+            .refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), true);
+
+        // refresh the network interface list and statistics
+        self.networks.refresh(false);
+
+        let process = self
+            .system
+            .process(self.pid)
+            .ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?;
+
+        // CPU metrics
+        let cpu_usage = process.cpu_usage();
+        self.metrics.cpu_usage.record(cpu_usage as f64, &[]);
+        self.metrics
+            .cpu_utilization
+            .record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes);
+
+        // Memory metrics
+        self.metrics
+            .memory_usage
+            .record(process.memory() as i64, &self.attributes.attributes);
+        self.metrics
+            .memory_virtual
+            .record(process.virtual_memory() as i64, &self.attributes.attributes);
+
+        // Disk I/O metrics
+        let disk_io = process.disk_usage();
+        self.metrics.disk_io.record(
+            disk_io.read_bytes as i64,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(),
+        );
+        self.metrics.disk_io.record(
+            disk_io.written_bytes as i64,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(),
+        );
+
+        // Network I/O indicators (corresponding to /system/network/internode)
+        let mut total_received: i64 = 0;
+        let mut total_transmitted: i64 = 0;
+
+        // statistics by interface
+        for (interface_name, data) in self.networks.iter() {
+            total_received += data.total_received() as i64;
+            total_transmitted += data.total_transmitted() as i64;
+
+            let received = data.received() as i64;
+            let transmitted = data.transmitted() as i64;
+            self.metrics.network_io_per_interface.record(
+                received,
+                &[
+                    &self.attributes.attributes[..],
+                    &[
+                        KeyValue::new(INTERFACE, interface_name.to_string()),
+                        KeyValue::new(DIRECTION, "received"),
+                    ],
+                ]
+                .concat(),
+            );
+            self.metrics.network_io_per_interface.record(
+                transmitted,
+                &[
+                    &self.attributes.attributes[..],
+                    &[
+                        KeyValue::new(INTERFACE, interface_name.to_string()),
+                        KeyValue::new(DIRECTION, "transmitted"),
+                    ],
+                ]
+                .concat(),
+            );
+        }
+        // global statistics
+        self.metrics.network_io.record(
+            total_received,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(),
+        );
+        self.metrics.network_io.record(
+            total_transmitted,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(),
+        );
+
+        // Process status indicator (corresponding to /system/process)
+        let status_value = match process.status() {
+            ProcessStatus::Run => 0,
+            ProcessStatus::Sleep => 1,
+            ProcessStatus::Zombie => 2,
+            _ => 3, // other status
+        };
+        self.metrics.process_status.record(
+            status_value,
+            &[
+                &self.attributes.attributes[..],
+                &[KeyValue::new(STATUS, format!("{:?}", process.status()))],
+            ]
+            .concat(),
+        );
+
+        // GPU Metrics (Optional) Non-MacOS
+        self.gpu_collector.collect(&self.metrics, &self.attributes)?;
+
+        Ok(())
+    }
+}
diff --git a/crates/obs/src/system/gpu.rs b/crates/obs/src/system/gpu.rs
new file mode 100644
index 000000000..ce47f2c51
--- /dev/null
+++ b/crates/obs/src/system/gpu.rs
@@ -0,0 +1,70 @@
+#[cfg(feature = "gpu")]
+use crate::system::attributes::ProcessAttributes;
+#[cfg(feature = "gpu")]
+use crate::system::metrics::Metrics;
+#[cfg(feature = "gpu")]
+use crate::GlobalError;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::enums::device::UsedGpuMemory;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::Nvml;
+#[cfg(feature = "gpu")]
+use sysinfo::Pid;
+#[cfg(feature = "gpu")]
+use tracing::warn;
+
+/// `GpuCollector` is responsible for collecting GPU memory usage metrics.
+#[cfg(feature = "gpu")]
+pub struct GpuCollector {
+    nvml: Nvml,
+    pid: Pid,
+}
+
+#[cfg(feature = "gpu")]
+impl GpuCollector {
+    pub fn new(pid: Pid) -> Result<Self, GlobalError> {
+        let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
+        Ok(GpuCollector { nvml, pid })
+    }
+
+    pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> {
+        if let Ok(device) = self.nvml.device_by_index(0) {
+            if let Ok(gpu_stats) = device.running_compute_processes() {
+                for stat in gpu_stats.iter() {
+                    if stat.pid == self.pid.as_u32() {
+                        let memory_used = match stat.used_gpu_memory {
+                            UsedGpuMemory::Used(bytes) => bytes,
+                            UsedGpuMemory::Unavailable => 0,
+                        };
+                        metrics.gpu_memory_usage.record(memory_used, &attributes.attributes);
+                        return Ok(());
+                    }
+                }
+            } else {
+                warn!("Could not get GPU stats, recording 0 for GPU memory usage");
+            }
+        } else {
+            return Err(GlobalError::GpuDeviceError("No GPU device found".to_string()));
+        }
+        metrics.gpu_memory_usage.record(0, &attributes.attributes);
+        Ok(())
+    }
+}
+
+#[cfg(not(feature = "gpu"))]
+pub struct GpuCollector;
+
+#[cfg(not(feature = "gpu"))]
+impl GpuCollector {
+    pub fn new(_pid: sysinfo::Pid) -> Result<Self, crate::GlobalError> {
+        Ok(GpuCollector)
+    }
+
+    pub fn collect(
+        &self,
+        _metrics: &crate::system::metrics::Metrics,
+        _attributes: &crate::system::attributes::ProcessAttributes,
+    ) -> Result<(), crate::GlobalError> {
+        Ok(())
+    }
+}
diff --git a/crates/obs/src/system/metrics.rs b/crates/obs/src/system/metrics.rs
new file mode 100644
index 000000000..114c285ea
--- /dev/null
+++ b/crates/obs/src/system/metrics.rs
@@ -0,0 +1,100 @@
+pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
+pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
+pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
+pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
+pub const PROCESS_DISK_IO: &str = "process.disk.io";
+pub const PROCESS_NETWORK_IO: &str = "process.network.io";
+pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
+pub const PROCESS_STATUS: &str = "process.status";
+#[cfg(feature = "gpu")]
+pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
+pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
+pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
+pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
+
+/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
+/// It contains various metrics such as CPU usage, memory usage,
+/// disk I/O, network I/O, and process status.
+///
+/// The `Metrics` struct is designed to be used with OpenTelemetry's
+/// metrics API to record and export these metrics.
+///
+/// The `new` method initializes the metrics using the provided
+/// `opentelemetry::metrics::Meter`.
+pub struct Metrics {
+    pub cpu_usage: opentelemetry::metrics::Gauge<f64>,
+    pub cpu_utilization: opentelemetry::metrics::Gauge<f64>,
+    pub memory_usage: opentelemetry::metrics::Gauge<i64>,
+    pub memory_virtual: opentelemetry::metrics::Gauge<i64>,
+    pub disk_io: opentelemetry::metrics::Gauge<i64>,
+    pub network_io: opentelemetry::metrics::Gauge<i64>,
+    pub network_io_per_interface: opentelemetry::metrics::Gauge<i64>,
+    pub process_status: opentelemetry::metrics::Gauge<i64>,
+    #[cfg(feature = "gpu")]
+    pub gpu_memory_usage: opentelemetry::metrics::Gauge<u64>,
+}
+
+impl Metrics {
+    pub fn new(meter: &opentelemetry::metrics::Meter) -> Self {
+        let cpu_usage = meter
+            .f64_gauge(PROCESS_CPU_USAGE)
+            .with_description("The percentage of CPU in use.")
+            .with_unit("percent")
+            .build();
+        let cpu_utilization = meter
+            .f64_gauge(PROCESS_CPU_UTILIZATION)
+            .with_description("The amount of CPU in use.")
+            .with_unit("percent")
+            .build();
+        let memory_usage = meter
+            .i64_gauge(PROCESS_MEMORY_USAGE)
+            .with_description("The amount of physical memory in use.")
+            .with_unit("byte")
+            .build();
+        let memory_virtual = meter
+            .i64_gauge(PROCESS_MEMORY_VIRTUAL)
+            .with_description("The amount of committed virtual memory.")
+            .with_unit("byte")
+            .build();
+        let disk_io = meter
+            .i64_gauge(PROCESS_DISK_IO)
+            .with_description("Disk bytes transferred.")
+            .with_unit("byte")
+            .build();
+        let network_io = meter
+            .i64_gauge(PROCESS_NETWORK_IO)
+            .with_description("Network bytes transferred.")
+            .with_unit("byte")
+            .build();
+        let network_io_per_interface = meter
+            .i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE)
+            .with_description("Network bytes transferred (per interface).")
+            .with_unit("byte")
+            .build();
+
+        let process_status = meter
+            .i64_gauge(PROCESS_STATUS)
+            .with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)")
+            .build();
+
+        #[cfg(feature = "gpu")]
+        let gpu_memory_usage = meter
+            .u64_gauge(PROCESS_GPU_MEMORY_USAGE)
+            .with_description("The amount of physical GPU memory in use.")
+            .with_unit("byte")
+            .build();
+
+        Metrics {
+            cpu_usage,
+            cpu_utilization,
+            memory_usage,
+            memory_virtual,
+            disk_io,
+            network_io,
+            network_io_per_interface,
+            process_status,
+            #[cfg(feature = "gpu")]
+            gpu_memory_usage,
+        }
+    }
+}
diff --git a/crates/obs/src/system/mod.rs b/crates/obs/src/system/mod.rs
new file mode 100644
index 000000000..a69bc3430
--- /dev/null
+++ b/crates/obs/src/system/mod.rs
@@ -0,0 +1,24 @@
+use crate::GlobalError;
+
+pub(crate) mod attributes;
+mod collector;
+pub(crate) mod gpu;
+pub(crate) mod metrics;
+
+/// Initialize the indicator collector for the current process
+/// This function will create a new `Collector` instance and start collecting metrics.
+/// It will run indefinitely until the process is terminated.
+pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> {
+    let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?;
+    let mut collector = collector::Collector::new(pid, meter, 30000)?;
+    collector.run().await
+}
+
+/// Initialize the metric collector for the specified PID process
+/// This function will create a new `Collector` instance and start collecting metrics.
+/// It will run indefinitely until the process is terminated.
+pub async fn init_process_observer_for_pid(meter: opentelemetry::metrics::Meter, pid: u32) -> Result<(), GlobalError> {
+    let pid = sysinfo::Pid::from_u32(pid);
+    let mut collector = collector::Collector::new(pid, meter, 30000)?;
+    collector.run().await
+}
diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs
index 46fdbc8d7..9ffc265dd 100644
--- a/crates/obs/src/telemetry.rs
+++ b/crates/obs/src/telemetry.rs
@@ -1,5 +1,6 @@
 use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
-use crate::{get_local_ip_with_default, OtelConfig};
+use crate::utils::get_local_ip_with_default;
+use crate::OtelConfig;
 use opentelemetry::trace::TracerProvider;
 use opentelemetry::{global, KeyValue};
 use opentelemetry_appender_tracing::layer;
@@ -14,11 +15,10 @@ use opentelemetry_semantic_conventions::{
     attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
     SCHEMA_URL,
 };
-use prometheus::{Encoder, Registry, TextEncoder};
+use smallvec::SmallVec;
+use std::borrow::Cow;
 use std::io::IsTerminal;
-use std::sync::Arc;
-use tokio::sync::{Mutex, OnceCell};
-use tracing::{info, warn};
+use tracing::info;
 use tracing_error::ErrorLayer;
 use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
@@ -67,66 +67,20 @@ impl Drop for OtelGuard {
     }
 }
 
-/// Global registry for Prometheus metrics
-static GLOBAL_REGISTRY: OnceCell<Arc<Mutex<Registry>>> = OnceCell::const_new();
-
-/// Get the global registry instance
-/// This function returns a reference to the global registry instance.
-///
-/// # Returns
-/// A reference to the global registry instance
-///
-/// # Example
-/// ```
-/// use rustfs_obs::get_global_registry;
-///
-/// let registry = get_global_registry();
-/// ```
-pub fn get_global_registry() -> Arc<Mutex<Registry>> {
-    GLOBAL_REGISTRY.get().unwrap().clone()
-}
-
-/// Prometheus metric endpoints
-/// This function returns a string containing the Prometheus metrics.
-/// The metrics are collected from the global registry.
-/// The function is used to expose the metrics via an HTTP endpoint.
-///
-/// # Returns
-/// A string containing the Prometheus metrics
-///
-/// # Example
-/// ```
-/// use rustfs_obs::metrics;
-///
-/// async fn main() {
-///    let metrics = metrics().await;
-///   println!("{}", metrics);
-/// }
-/// ```
-pub async fn metrics() -> String {
-    let encoder = TextEncoder::new();
-    // Get a reference to the registry for reading metrics
-    let registry = get_global_registry().lock().await.to_owned();
-    let metric_families = registry.gather();
-    if metric_families.is_empty() {
-        warn!("No metrics available in Prometheus registry");
-    } else {
-        info!("Metrics collected: {} families", metric_families.len());
-    }
-    let mut buffer = Vec::new();
-    encoder.encode(&metric_families, &mut buffer).unwrap();
-    String::from_utf8(buffer).unwrap_or_else(|_| "Error encoding metrics".to_string())
-}
-
 /// create OpenTelemetry Resource
 fn resource(config: &OtelConfig) -> Resource {
-    let config = config.clone();
     Resource::builder()
-        .with_service_name(config.service_name.unwrap_or(SERVICE_NAME.to_string()))
+        .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string())
         .with_schema_url(
             [
-                KeyValue::new(OTEL_SERVICE_VERSION, config.service_version.unwrap_or(SERVICE_VERSION.to_string())),
-                KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, config.environment.unwrap_or(ENVIRONMENT.to_string())),
+                KeyValue::new(
+                    OTEL_SERVICE_VERSION,
+                    Cow::Borrowed(config.service_version.as_deref().unwrap_or(SERVICE_VERSION)).to_string(),
+                ),
+                KeyValue::new(
+                    DEPLOYMENT_ENVIRONMENT_NAME,
+                    Cow::Borrowed(config.environment.as_deref().unwrap_or(ENVIRONMENT)).to_string(),
+                ),
                 KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()),
             ],
             SCHEMA_URL,
@@ -141,167 +95,169 @@ fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout:
         .build()
 }
 
-/// Initialize Meter Provider
-fn init_meter_provider(config: &OtelConfig) -> SdkMeterProvider {
-    let mut builder = MeterProviderBuilder::default().with_resource(resource(config));
-    // If endpoint is empty, use stdout output
-    if config.endpoint.is_empty() {
-        builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
-    } else {
-        // If endpoint is not empty, use otlp output
-        let exporter = opentelemetry_otlp::MetricExporter::builder()
-            .with_tonic()
-            .with_endpoint(&config.endpoint)
-            .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
-            .build()
-            .unwrap();
-        builder = builder.with_reader(
-            PeriodicReader::builder(exporter)
-                .with_interval(std::time::Duration::from_secs(config.meter_interval.unwrap_or(METER_INTERVAL)))
-                .build(),
-        );
-        // If use_stdout is true, output to stdout at the same time
-        if config.use_stdout.unwrap_or(USE_STDOUT) {
-            builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
-        }
-    }
-    let registry = Registry::new();
-    // Set global registry
-    GLOBAL_REGISTRY.set(Arc::new(Mutex::new(registry.clone()))).unwrap();
-    // Create Prometheus exporter
-    let prometheus_exporter = opentelemetry_prometheus::exporter().with_registry(registry).build().unwrap();
-    // Build meter provider
-    let meter_provider = builder.with_reader(prometheus_exporter).build();
-    global::set_meter_provider(meter_provider.clone());
-    meter_provider
-}
-
-/// Initialize Tracer Provider
-fn init_tracer_provider(config: &OtelConfig) -> SdkTracerProvider {
-    let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
-    let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
-        Sampler::TraceIdRatioBased(sample_ratio)
-    } else {
-        Sampler::AlwaysOn
-    };
-    let builder = SdkTracerProvider::builder()
-        .with_sampler(sampler)
-        .with_id_generator(RandomIdGenerator::default())
-        .with_resource(resource(config));
-
-    let tracer_provider = if config.endpoint.is_empty() {
-        builder
-            .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
-            .build()
-    } else {
-        let exporter = opentelemetry_otlp::SpanExporter::builder()
-            .with_tonic()
-            .with_endpoint(&config.endpoint)
-            .build()
-            .unwrap();
-        if config.use_stdout.unwrap_or(USE_STDOUT) {
-            builder
-                .with_batch_exporter(exporter)
-                .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
-        } else {
-            builder.with_batch_exporter(exporter)
-        }
-        .build()
-    };
-
-    global::set_tracer_provider(tracer_provider.clone());
-    tracer_provider
-}
-
 /// Initialize Telemetry
 pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
-    let tracer_provider = init_tracer_provider(config);
-    let meter_provider = init_meter_provider(config);
+    // avoid repeated access to configuration fields
+    let endpoint = &config.endpoint;
+    let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT);
+    let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
+    let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL);
+    let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME);
 
-    // Initialize logger provider based on configuration
-    let logger_provider = {
-        let mut builder = SdkLoggerProvider::builder().with_resource(resource(config));
+    // Pre-create resource objects to avoid repeated construction
+    let res = resource(config);
 
-        if config.endpoint.is_empty() {
-            // Use stdout exporter when no endpoint is configured
-            builder = builder.with_simple_exporter(opentelemetry_stdout::LogExporter::default());
+    // initialize tracer provider
+    let tracer_provider = {
+        let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
+        let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
+            Sampler::TraceIdRatioBased(sample_ratio)
+        } else {
+            Sampler::AlwaysOn
+        };
+
+        let builder = SdkTracerProvider::builder()
+            .with_sampler(sampler)
+            .with_id_generator(RandomIdGenerator::default())
+            .with_resource(res.clone());
+
+        let tracer_provider = if endpoint.is_empty() {
+            builder
+                .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
+                .build()
+        } else {
+            let exporter = opentelemetry_otlp::SpanExporter::builder()
+                .with_tonic()
+                .with_endpoint(endpoint)
+                .build()
+                .unwrap();
+
+            let builder = if use_stdout {
+                builder
+                    .with_batch_exporter(exporter)
+                    .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
+            } else {
+                builder.with_batch_exporter(exporter)
+            };
+
+            builder.build()
+        };
+
+        global::set_tracer_provider(tracer_provider.clone());
+        tracer_provider
+    };
+
+    // initialize meter provider
+    let meter_provider = {
+        let mut builder = MeterProviderBuilder::default().with_resource(res.clone());
+
+        if endpoint.is_empty() {
+            builder = builder.with_reader(create_periodic_reader(meter_interval));
+        } else {
+            let exporter = opentelemetry_otlp::MetricExporter::builder()
+                .with_tonic()
+                .with_endpoint(endpoint)
+                .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
+                .build()
+                .unwrap();
+
+            builder = builder.with_reader(
+                PeriodicReader::builder(exporter)
+                    .with_interval(std::time::Duration::from_secs(meter_interval))
+                    .build(),
+            );
+
+            if use_stdout {
+                builder = builder.with_reader(create_periodic_reader(meter_interval));
+            }
+        }
+
+        let meter_provider = builder.build();
+        global::set_meter_provider(meter_provider.clone());
+        meter_provider
+    };
+
+    // initialize logger provider
+    let logger_provider = {
+        let mut builder = SdkLoggerProvider::builder().with_resource(res);
+
+        if endpoint.is_empty() {
+            builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
         } else {
-            // Configure OTLP exporter when endpoint is provided
             let exporter = opentelemetry_otlp::LogExporter::builder()
                 .with_tonic()
-                .with_endpoint(&config.endpoint)
+                .with_endpoint(endpoint)
                 .build()
                 .unwrap();
 
             builder = builder.with_batch_exporter(exporter);
 
-            // Add stdout exporter if requested
-            if config.use_stdout.unwrap_or(USE_STDOUT) {
+            if use_stdout {
                 builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
             }
         }
 
         builder.build()
     };
-    let config = config.clone();
-    let logger_level = config.logger_level.unwrap_or(LOGGER_LEVEL.to_string());
-    let logger_level = logger_level.as_str();
-    // Setup OpenTelemetryTracingBridge layer
-    let otel_layer = {
-        // Filter to prevent infinite telemetry loops
-        // This blocks events from OpenTelemetry and its dependent libraries (tonic, reqwest, etc.)
-        // from being sent back to OpenTelemetry itself
-        let filter_otel = match logger_level {
-            "trace" | "debug" => {
-                info!("OpenTelemetry tracing initialized with level: {}", logger_level);
-                let mut filter = EnvFilter::new(logger_level);
-                for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
-                    filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+
+    // configuring tracing
+    {
+        // optimize filter configuration
+        let otel_layer = {
+            let filter_otel = match logger_level {
+                "trace" | "debug" => {
+                    info!("OpenTelemetry tracing initialized with level: {}", logger_level);
+                    EnvFilter::new(logger_level)
                 }
-                filter
-            }
-            _ => {
-                let mut filter = EnvFilter::new(logger_level);
-                for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
-                    filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+                _ => {
+                    let mut filter = EnvFilter::new(logger_level);
+
+                    // use smallvec to avoid heap allocation
+                    let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
+
+                    for directive in directives {
+                        filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+                    }
+                    filter
                 }
-                filter
-            }
+            };
+            layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
         };
-        layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
-    };
 
-    let tracer = tracer_provider.tracer(config.service_name.unwrap_or(SERVICE_NAME.to_string()));
-    let registry = tracing_subscriber::registry()
-        .with(switch_level(logger_level))
-        .with(OpenTelemetryLayer::new(tracer))
-        .with(MetricsLayer::new(meter_provider.clone()))
-        .with(otel_layer)
-        .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
+        let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
 
-    // Configure formatting layer
-    let enable_color = std::io::stdout().is_terminal();
-    let fmt_layer = tracing_subscriber::fmt::layer()
-        .with_ansi(enable_color)
-        .with_thread_names(true)
-        .with_file(true)
-        .with_line_number(true);
+        // Configure registry to avoid repeated calls to filter methods
+        let level_filter = switch_level(logger_level);
+        let registry = tracing_subscriber::registry()
+            .with(level_filter)
+            .with(OpenTelemetryLayer::new(tracer))
+            .with(MetricsLayer::new(meter_provider.clone()))
+            .with(otel_layer)
+            .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
 
-    // Creating a formatting layer with explicit type to avoid type mismatches
-    let fmt_layer = fmt_layer.with_filter(
-        EnvFilter::new(logger_level).add_directive(
-            format!("opentelemetry={}", if config.endpoint.is_empty() { logger_level } else { "off" })
-                .parse()
-                .unwrap(),
-        ),
-    );
+        // configure the formatting layer
+        let enable_color = std::io::stdout().is_terminal();
+        let fmt_layer = tracing_subscriber::fmt::layer()
+            .with_ansi(enable_color)
+            .with_thread_names(true)
+            .with_file(true)
+            .with_line_number(true)
+            .with_filter(
+                EnvFilter::new(logger_level).add_directive(
+                    format!("opentelemetry={}", if endpoint.is_empty() { logger_level } else { "off" })
+                        .parse()
+                        .unwrap(),
+                ),
+            );
 
-    registry.with(ErrorLayer::default()).with(fmt_layer).init();
-    if !config.endpoint.is_empty() {
-        info!(
-            "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
-            config.endpoint, logger_level
-        );
+        registry.with(ErrorLayer::default()).with(fmt_layer).init();
+
+        if !endpoint.is_empty() {
+            info!(
+                "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
+                endpoint, logger_level
+            );
+        }
     }
 
     OtelGuard {
@@ -313,12 +269,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
 
 /// Switch log level
 fn switch_level(logger_level: &str) -> tracing_subscriber::filter::LevelFilter {
+    use tracing_subscriber::filter::LevelFilter;
     match logger_level {
-        "error" => tracing_subscriber::filter::LevelFilter::ERROR,
-        "warn" => tracing_subscriber::filter::LevelFilter::WARN,
-        "info" => tracing_subscriber::filter::LevelFilter::INFO,
-        "debug" => tracing_subscriber::filter::LevelFilter::DEBUG,
-        "trace" => tracing_subscriber::filter::LevelFilter::TRACE,
-        _ => tracing_subscriber::filter::LevelFilter::OFF,
+        "error" => LevelFilter::ERROR,
+        "warn" => LevelFilter::WARN,
+        "info" => LevelFilter::INFO,
+        "debug" => LevelFilter::DEBUG,
+        "trace" => LevelFilter::TRACE,
+        _ => LevelFilter::OFF,
     }
 }
diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml
index 49ac85414..3019c060e 100644
--- a/rustfs/Cargo.toml
+++ b/rustfs/Cargo.toml
@@ -46,6 +46,7 @@ local-ip-address = { workspace = true }
 matchit = { workspace = true }
 mime.workspace = true
 mime_guess = { workspace = true }
+opentelemetry = { workspace = true }
 pin-project-lite.workspace = true
 protos.workspace = true
 query = { workspace = true }
@@ -76,7 +77,7 @@ tokio-stream.workspace = true
 tonic = { workspace = true }
 tower.workspace = true
 transform-stream.workspace = true
-tower-http = { workspace = true, features = ["trace", "compression-full", "cors"] }
+tower-http = { workspace = true, features = ["trace", "compression-deflate", "compression-gzip", "cors"] }
 uuid = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
diff --git a/rustfs/src/console.rs b/rustfs/src/console.rs
index 127d2144e..1125b24d0 100644
--- a/rustfs/src/console.rs
+++ b/rustfs/src/console.rs
@@ -271,7 +271,7 @@ pub async fn start_static_file_server(
         .route("/config.json", get(config_handler))
         .fallback_service(get(static_handler))
         .layer(cors)
-        .layer(tower_http::compression::CompressionLayer::new())
+        .layer(tower_http::compression::CompressionLayer::new().gzip(true).deflate(true))
         .layer(TraceLayer::new_for_http());
     let local_addr: SocketAddr = addrs.parse().expect("Failed to parse socket address");
     info!("WebUI: http://{}:{} http://127.0.0.1:{}", local_ip, local_addr.port(), local_addr.port());
diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs
index f86f951fc..ba1579abd 100644
--- a/rustfs/src/main.rs
+++ b/rustfs/src/main.rs
@@ -14,6 +14,7 @@ use crate::console::{init_console_cfg, CONSOLE_CONFIG};
 // Ensure the correct path for parse_license is imported
 use crate::server::{wait_for_shutdown, ServiceState, ServiceStateManager, ShutdownSignal, SHUTDOWN_TIMEOUT};
 use crate::utils::error;
+use bytes::Bytes;
 use chrono::Datelike;
 use clap::Parser;
 use common::{
@@ -37,6 +38,7 @@ use ecstore::{
 };
 use ecstore::{global::set_global_rustfs_port, notification_sys::new_global_notification_sys};
 use grpc::make_server;
+use http::{HeaderMap, Request as HttpRequest, Response};
 use hyper_util::server::graceful::GracefulShutdown;
 use hyper_util::{
     rt::{TokioExecutor, TokioIo},
@@ -47,17 +49,20 @@ use iam::init_iam_sys;
 use license::init_license;
 use protos::proto_gen::node_service::node_service_server::NodeServiceServer;
 use rustfs_event_notifier::NotifierConfig;
-use rustfs_obs::{init_obs, load_config, set_global_guard, InitLogStatus};
+use rustfs_obs::{init_obs, init_process_observer, load_config, set_global_guard};
 use rustls::ServerConfig;
 use s3s::{host::MultiDomain, service::S3ServiceBuilder};
 use service::hybrid;
 use std::net::SocketAddr;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::net::TcpListener;
 use tokio::signal::unix::{signal, SignalKind};
 use tokio_rustls::TlsAcceptor;
 use tonic::{metadata::MetadataValue, Request, Status};
 use tower_http::cors::CorsLayer;
+use tower_http::trace::TraceLayer;
+use tracing::Span;
 use tracing::{debug, error, info, info_span, warn};
 
 #[cfg(all(target_os = "linux", target_env = "gnu"))]
@@ -103,9 +108,6 @@ async fn main() -> Result<()> {
     // Store in global storage
     set_global_guard(guard)?;
 
-    // Log initialization status
-    InitLogStatus::init_start_log(&config.observability).await?;
-
     // Initialize event notifier
     let notifier_config = opt.clone().event_config;
     if notifier_config.is_some() {
@@ -247,6 +249,20 @@ async fn run(opt: config::Opt) -> Result<()> {
         b.build()
     };
 
+    tokio::spawn(async move {
+        // Record the PID-related metrics of the current process
+        let meter = opentelemetry::global::meter("system");
+        let obs_result = init_process_observer(meter).await;
+        match obs_result {
+            Ok(_) => {
+                info!("Process observer initialized successfully");
+            }
+            Err(e) => {
+                error!("Failed to initialize process observer: {}", e);
+            }
+        }
+    });
+
     let rpc_service = NodeServiceServer::with_interceptor(make_server(), check_auth);
 
     let tls_path = opt.tls_path.clone().unwrap_or_default();
@@ -300,18 +316,53 @@ async fn run(opt: config::Opt) -> Result<()> {
         let mut sigint_inner = sigint_inner;
         let hybrid_service = TowerToHyperService::new(
             tower::ServiceBuilder::new()
+                .layer(
+                    TraceLayer::new_for_http()
+                        .make_span_with(|request: &HttpRequest<_>| {
+                            let span = tracing::debug_span!("http-request",
+                                status_code = tracing::field::Empty,
+                                method = %request.method(),
+                                uri = %request.uri(),
+                                version = ?request.version(),
+                            );
+                            for (header_name, header_value) in request.headers() {
+                                if header_name == "user-agent" || header_name == "content-type" || header_name == "content-length"
+                                {
+                                    span.record(header_name.as_str(), header_value.to_str().unwrap_or("invalid"));
+                                }
+                            }
+
+                            span
+                        })
+                        .on_request(|request: &HttpRequest<_>, _span: &Span| {
+                            debug!("started method: {}, url path: {}", request.method(), request.uri().path())
+                        })
+                        .on_response(|response: &Response<_>, latency: Duration, _span: &Span| {
+                            _span.record("status_code", tracing::field::display(response.status()));
+                            debug!("response generated in {:?}", latency)
+                        })
+                        .on_body_chunk(|chunk: &Bytes, latency: Duration, _span: &Span| {
+                            debug!("sending {} bytes in {:?}", chunk.len(), latency)
+                        })
+                        .on_eos(|_trailers: Option<&HeaderMap>, stream_duration: Duration, _span: &Span| {
+                            debug!("stream closed after {:?}", stream_duration)
+                        })
+                        .on_failure(|_error, latency: Duration, _span: &Span| {
+                            debug!("request error: {:?} in {:?}", _error, latency)
+                        }),
+                )
                 .layer(CorsLayer::permissive())
                 .service(hybrid(s3_service, rpc_service)),
         );
 
-        let http_server = ConnBuilder::new(TokioExecutor::new());
+        let http_server = Arc::new(ConnBuilder::new(TokioExecutor::new()));
         let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
-        let graceful = GracefulShutdown::new();
+        let graceful = Arc::new(GracefulShutdown::new());
         debug!("graceful initiated");
 
         // 服务准备就绪
         worker_state_manager.update(ServiceState::Ready);
-
+        let value = hybrid_service.clone();
         loop {
             debug!("waiting for SIGINT or SIGTERM has_tls_certs: {}", has_tls_certs);
             // Wait for a connection
@@ -366,12 +417,17 @@ async fn run(opt: config::Opt) -> Result<()> {
                         continue;
                     }
                 };
-                let conn = http_server.serve_connection(TokioIo::new(tls_socket), hybrid_service.clone());
-                let conn = graceful.watch(conn.into_owned());
+
+                let http_server_clone = http_server.clone();
+                let value_clone = value.clone();
+                let graceful_clone = graceful.clone();
+
                 tokio::task::spawn_blocking(move || {
                     tokio::runtime::Runtime::new()
                         .expect("Failed to create runtime")
                         .block_on(async move {
+                            let conn = http_server_clone.serve_connection(TokioIo::new(tls_socket), value_clone);
+                            let conn = graceful_clone.watch(conn);
                             if let Err(err) = conn.await {
                                 error!("Https Connection error: {}", err);
                             }
@@ -380,9 +436,13 @@ async fn run(opt: config::Opt) -> Result<()> {
                 debug!("TLS handshake success");
             } else {
                 debug!("Http handshake start");
-                let conn = http_server.serve_connection(TokioIo::new(socket), hybrid_service.clone());
-                let conn = graceful.watch(conn.into_owned());
+
+                let http_server_clone = http_server.clone();
+                let value_clone = value.clone();
+                let graceful_clone = graceful.clone();
                 tokio::spawn(async move {
+                    let conn = http_server_clone.serve_connection(TokioIo::new(socket), value_clone);
+                    let conn = graceful_clone.watch(conn);
                     if let Err(err) = conn.await {
                         error!("Http Connection error: {}", err);
                     }
@@ -391,12 +451,24 @@ async fn run(opt: config::Opt) -> Result<()> {
             }
         }
         worker_state_manager.update(ServiceState::Stopping);
-        tokio::select! {
-            () = graceful.shutdown() => {
-                 debug!("Gracefully shutdown!");
-            },
-            () = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
-                 debug!("Waited 10 seconds for graceful shutdown, aborting...");
+        match Arc::try_unwrap(graceful) {
+            Ok(g) => {
+                // Successfully obtaining unique ownership, you can call shutdown
+                tokio::select! {
+                    () = g.shutdown() => {
+                        debug!("Gracefully shutdown!");
+                    },
+                    () = tokio::time::sleep(Duration::from_secs(10)) => {
+                        debug!("Waited 10 seconds for graceful shutdown, aborting...");
+                    }
+                }
+            }
+            Err(arc_graceful) => {
+                // There are other references that cannot be obtained for unique ownership
+                error!("Cannot perform graceful shutdown, other references exist err: {:?}", arc_graceful);
+                // In this case, we can only wait for the timeout
+                tokio::time::sleep(Duration::from_secs(10)).await;
+                debug!("Timeout reached, forcing shutdown");
             }
         }
         worker_state_manager.update(ServiceState::Stopped);
diff --git a/s3select/query/src/dispatcher/manager.rs b/s3select/query/src/dispatcher/manager.rs
index 05f763435..80543ed69 100644
--- a/s3select/query/src/dispatcher/manager.rs
+++ b/s3select/query/src/dispatcher/manager.rs
@@ -166,7 +166,8 @@ impl SimpleQueryDispatcher {
                 if let Some(delimiter) = csv.field_delimiter.as_ref() {
                     file_format = file_format.with_delimiter(delimiter.as_bytes().first().copied().unwrap_or_default());
                 }
-                if csv.file_header_info.is_some() {}
+                // TODO waiting for processing @junxiang Mu
+                // if csv.file_header_info.is_some() {}
                 match csv.file_header_info.as_ref() {
                     Some(info) => {
                         if *info == *NONE {
diff --git a/scripts/run.sh b/scripts/run.sh
index 2cbd115a8..c31ce7ed7 100755
--- a/scripts/run.sh
+++ b/scripts/run.sh
@@ -40,13 +40,13 @@ export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml"
 
 # 如下变量需要必须参数都有值才可以，以及会覆盖配置文件中的值
 export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317
-export RUSTFS__OBSERVABILITY__USE_STDOUT=true
+export RUSTFS__OBSERVABILITY__USE_STDOUT=false
 export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0
 export RUSTFS__OBSERVABILITY__METER_INTERVAL=30
 export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs
 export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0
 export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop
-export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info
+export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug
 export RUSTFS__SINKS__FILE__ENABLED=true
 export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log"
 export RUSTFS__SINKS__WEBHOOK__ENABLED=false

From 7926ac015afe44555dec9590839b082355bb45fd Mon Sep 17 00:00:00 2001
From: houseme <housemecn@gmail.com>
Date: Thu, 24 Apr 2025 19:14:46 +0800
Subject: [PATCH 09/13] improve code for tracing log

---
 rustfs/src/main.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs
index ba1579abd..c3ea69e94 100644
--- a/rustfs/src/main.rs
+++ b/rustfs/src/main.rs
@@ -335,20 +335,20 @@ async fn run(opt: config::Opt) -> Result<()> {
                             span
                         })
                         .on_request(|request: &HttpRequest<_>, _span: &Span| {
-                            debug!("started method: {}, url path: {}", request.method(), request.uri().path())
+                            debug!("http started method: {}, url path: {}", request.method(), request.uri().path())
                         })
                         .on_response(|response: &Response<_>, latency: Duration, _span: &Span| {
-                            _span.record("status_code", tracing::field::display(response.status()));
-                            debug!("response generated in {:?}", latency)
+                            _span.record("http response status_code", tracing::field::display(response.status()));
+                            debug!("http response generated in {:?}", latency)
                         })
                         .on_body_chunk(|chunk: &Bytes, latency: Duration, _span: &Span| {
-                            debug!("sending {} bytes in {:?}", chunk.len(), latency)
+                            debug!("http body sending {} bytes in {:?}", chunk.len(), latency)
                         })
                         .on_eos(|_trailers: Option<&HeaderMap>, stream_duration: Duration, _span: &Span| {
-                            debug!("stream closed after {:?}", stream_duration)
+                            debug!("http stream closed after {:?}", stream_duration)
                         })
                         .on_failure(|_error, latency: Duration, _span: &Span| {
-                            debug!("request error: {:?} in {:?}", _error, latency)
+                            debug!("http request failure error: {:?} in {:?}", _error, latency)
                         }),
                 )
                 .layer(CorsLayer::permissive())

From 7dae5f8ab7ca982ae6e1f6828b581e2813a0d818 Mon Sep 17 00:00:00 2001
From: houseme <housemecn@gmail.com>
Date: Fri, 25 Apr 2025 13:35:03 +0800
Subject: [PATCH 10/13] improve code

---
 crates/obs/src/telemetry.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs
index 9ffc265dd..ae9fd40e4 100644
--- a/crates/obs/src/telemetry.rs
+++ b/crates/obs/src/telemetry.rs
@@ -238,8 +238,10 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
         // configure the formatting layer
         let enable_color = std::io::stdout().is_terminal();
         let fmt_layer = tracing_subscriber::fmt::layer()
+            .with_target(true)
             .with_ansi(enable_color)
             .with_thread_names(true)
+            .with_thread_ids(true)
             .with_file(true)
             .with_line_number(true)
             .with_filter(

From 9fc4bb919ea7f3b40453d44a27aa10049b61a279 Mon Sep 17 00:00:00 2001
From: weisd <weishidavip@163.com>
Date: Fri, 25 Apr 2025 09:54:41 +0800
Subject: [PATCH 11/13] fix:#355 multi pools select error

---
 ecstore/src/disk/error.rs  |  4 +++
 ecstore/src/disk/local.rs  | 39 ++++++++++++++++++++-
 ecstore/src/disk/mod.rs    | 49 +++++++++++++++++++++++++-
 ecstore/src/disk/remote.rs | 39 +++++++++++++++++++++
 ecstore/src/erasure.rs     |  7 ++--
 ecstore/src/set_disk.rs    | 70 +++++++++++++++++++++++++++++++++-----
 ecstore/src/sets.rs        | 46 +++++++++++++++++++++++++
 ecstore/src/store.rs       | 37 ++++++++++++++++++++
 8 files changed, 275 insertions(+), 16 deletions(-)

diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs
index d7ea2854a..0d422b106 100644
--- a/ecstore/src/disk/error.rs
+++ b/ecstore/src/disk/error.rs
@@ -364,6 +364,10 @@ pub fn is_unformatted_disk(err: &Error) -> bool {
 }
 
 pub fn is_err_file_not_found(err: &Error) -> bool {
+    if let Some(ioerr) = err.downcast_ref::<io::Error>() {
+        return ioerr.kind() == ErrorKind::NotFound;
+    }
+
     matches!(err.downcast_ref::<DiskError>(), Some(DiskError::FileNotFound))
 }
 
diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs
index 4ab9bb183..74cdaa690 100644
--- a/ecstore/src/disk/local.rs
+++ b/ecstore/src/disk/local.rs
@@ -1079,30 +1079,39 @@ fn skip_access_checks(p: impl AsRef<str>) -> bool {
 
 #[async_trait::async_trait]
 impl DiskAPI for LocalDisk {
+    #[tracing::instrument(skip(self))]
     fn to_string(&self) -> String {
         self.root.to_string_lossy().to_string()
     }
+    #[tracing::instrument(skip(self))]
     fn is_local(&self) -> bool {
         true
     }
+    #[tracing::instrument(skip(self))]
     fn host_name(&self) -> String {
         self.endpoint.host_port()
     }
+    #[tracing::instrument(skip(self))]
     async fn is_online(&self) -> bool {
         self.check_format_json().await.is_ok()
     }
 
+    #[tracing::instrument(skip(self))]
     fn endpoint(&self) -> Endpoint {
         self.endpoint.clone()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn close(&self) -> Result<()> {
         Ok(())
     }
+
+    #[tracing::instrument(skip(self))]
     fn path(&self) -> PathBuf {
         self.root.clone()
     }
 
+    #[tracing::instrument(skip(self))]
     fn get_disk_location(&self) -> DiskLocation {
         DiskLocation {
             pool_idx: {
@@ -1179,6 +1188,7 @@ impl DiskAPI for LocalDisk {
         Ok(Some(disk_id))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
         // 本地不需要设置
         // TODO: add check_id_store
@@ -1188,6 +1198,7 @@ impl DiskAPI for LocalDisk {
     }
 
     #[must_use]
+    #[tracing::instrument(skip(self))]
     async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
         if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE {
             let format_info = self.format_info.read().await;
@@ -1202,10 +1213,12 @@ impl DiskAPI for LocalDisk {
         Ok(data)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
         self.write_all_public(volume, path, data).await
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
         let volume_dir = self.get_bucket_path(volume)?;
         if !skip_access_checks(volume) {
@@ -1223,6 +1236,7 @@ impl DiskAPI for LocalDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
         let volume_dir = self.get_bucket_path(volume)?;
         if !skip_access_checks(volume) {
@@ -1268,6 +1282,7 @@ impl DiskAPI for LocalDisk {
         Ok(resp)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
         let volume_dir = self.get_bucket_path(volume)?;
         check_path_length(volume_dir.join(path).to_string_lossy().as_ref())?;
@@ -1405,6 +1420,8 @@ impl DiskAPI for LocalDisk {
 
         Ok(())
     }
+
+    #[tracing::instrument(skip(self))]
     async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
         let src_volume_dir = self.get_bucket_path(src_volume)?;
         let dst_volume_dir = self.get_bucket_path(dst_volume)?;
@@ -1704,7 +1721,7 @@ impl DiskAPI for LocalDisk {
         Ok(())
     }
 
-    // #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self))]
     async fn rename_data(
         &self,
         src_volume: &str,
@@ -1910,6 +1927,7 @@ impl DiskAPI for LocalDisk {
         })
     }
 
+    #[tracing::instrument(skip(self))]
     async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
         for vol in volumes {
             if let Err(e) = self.make_volume(vol).await {
@@ -1921,6 +1939,8 @@ impl DiskAPI for LocalDisk {
         }
         Ok(())
     }
+
+    #[tracing::instrument(skip(self))]
     async fn make_volume(&self, volume: &str) -> Result<()> {
         if !Self::is_valid_volname(volume) {
             return Err(Error::msg("Invalid arguments specified"));
@@ -1944,6 +1964,8 @@ impl DiskAPI for LocalDisk {
 
         Err(Error::from(DiskError::VolumeExists))
     }
+
+    #[tracing::instrument(skip(self))]
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
         let mut volumes = Vec::new();
 
@@ -1968,6 +1990,8 @@ impl DiskAPI for LocalDisk {
 
         Ok(volumes)
     }
+
+    #[tracing::instrument(skip(self))]
     async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
         let volume_dir = self.get_bucket_path(volume)?;
         let meta = match utils::fs::lstat(&volume_dir).await {
@@ -1995,6 +2019,8 @@ impl DiskAPI for LocalDisk {
             created: modtime,
         })
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
         let volume_dir = self.get_bucket_path(volume)?;
         if !skip_access_checks(volume) {
@@ -2013,6 +2039,8 @@ impl DiskAPI for LocalDisk {
 
         Ok(())
     }
+
+    #[tracing::instrument(skip(self))]
     async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
         if fi.metadata.is_some() {
             let volume_dir = self.get_bucket_path(volume)?;
@@ -2053,6 +2081,8 @@ impl DiskAPI for LocalDisk {
 
         Err(Error::msg("Invalid Argument"))
     }
+
+    #[tracing::instrument(skip(self))]
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
         let p = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?;
 
@@ -2106,6 +2136,8 @@ impl DiskAPI for LocalDisk {
 
         Ok(RawFileInfo { buf })
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_version(
         &self,
         volume: &str,
@@ -2214,6 +2246,8 @@ impl DiskAPI for LocalDisk {
 
         Ok(errs)
     }
+
+    #[tracing::instrument(skip(self))]
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
         let mut results = Vec::new();
         let mut found = 0;
@@ -2273,6 +2307,7 @@ impl DiskAPI for LocalDisk {
         Ok(results)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_volume(&self, volume: &str) -> Result<()> {
         let p = self.get_bucket_path(volume)?;
 
@@ -2295,6 +2330,7 @@ impl DiskAPI for LocalDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn disk_info(&self, _: &DiskInfoOptions) -> Result<DiskInfo> {
         let mut info = Cache::get(self.disk_info_cache.clone()).await?;
         // TODO: nr_requests, rotational
@@ -2454,6 +2490,7 @@ impl DiskAPI for LocalDisk {
         Ok(data_usage_info)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn healing(&self) -> Option<HealingTracker> {
         let healing_file = path_join(&[
             self.path(),
diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs
index e26eebb8b..ddefd1bed 100644
--- a/ecstore/src/disk/mod.rs
+++ b/ecstore/src/disk/mod.rs
@@ -49,6 +49,7 @@ pub enum Disk {
 
 #[async_trait::async_trait]
 impl DiskAPI for Disk {
+    #[tracing::instrument(skip(self))]
     fn to_string(&self) -> String {
         match self {
             Disk::Local(local_disk) => local_disk.to_string(),
@@ -56,6 +57,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     fn is_local(&self) -> bool {
         match self {
             Disk::Local(local_disk) => local_disk.is_local(),
@@ -63,30 +65,39 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     fn host_name(&self) -> String {
         match self {
             Disk::Local(local_disk) => local_disk.host_name(),
             Disk::Remote(remote_disk) => remote_disk.host_name(),
         }
     }
+
+    #[tracing::instrument(skip(self))]
     async fn is_online(&self) -> bool {
         match self {
             Disk::Local(local_disk) => local_disk.is_online().await,
             Disk::Remote(remote_disk) => remote_disk.is_online().await,
         }
     }
+
+    #[tracing::instrument(skip(self))]
     fn endpoint(&self) -> Endpoint {
         match self {
             Disk::Local(local_disk) => local_disk.endpoint(),
             Disk::Remote(remote_disk) => remote_disk.endpoint(),
         }
     }
+
+    #[tracing::instrument(skip(self))]
     async fn close(&self) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.close().await,
             Disk::Remote(remote_disk) => remote_disk.close().await,
         }
     }
+
+    #[tracing::instrument(skip(self))]
     fn path(&self) -> PathBuf {
         match self {
             Disk::Local(local_disk) => local_disk.path(),
@@ -94,6 +105,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     fn get_disk_location(&self) -> DiskLocation {
         match self {
             Disk::Local(local_disk) => local_disk.get_disk_location(),
@@ -101,12 +113,15 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_disk_id(&self) -> Result<Option<Uuid>> {
         match self {
             Disk::Local(local_disk) => local_disk.get_disk_id().await,
             Disk::Remote(remote_disk) => remote_disk.get_disk_id().await,
         }
     }
+
+    #[tracing::instrument(skip(self))]
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.set_disk_id(id).await,
@@ -114,6 +129,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
         match self {
             Disk::Local(local_disk) => local_disk.read_all(volume, path).await,
@@ -121,6 +137,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.write_all(volume, path, data).await,
@@ -128,6 +145,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.delete(volume, path, opt).await,
@@ -135,6 +153,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
         match self {
             Disk::Local(local_disk) => local_disk.verify_file(volume, path, fi).await,
@@ -142,6 +161,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
         match self {
             Disk::Local(local_disk) => local_disk.check_parts(volume, path, fi).await,
@@ -149,6 +169,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.rename_part(src_volume, src_path, dst_volume, dst_path, meta).await,
@@ -159,6 +180,8 @@ impl DiskAPI for Disk {
             }
         }
     }
+
+    #[tracing::instrument(skip(self))]
     async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
@@ -166,6 +189,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<FileWriter> {
         match self {
             Disk::Local(local_disk) => local_disk.create_file(_origvolume, volume, path, _file_size).await,
@@ -173,6 +197,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
         match self {
             Disk::Local(local_disk) => local_disk.append_file(volume, path).await,
@@ -180,6 +205,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
         match self {
             Disk::Local(local_disk) => local_disk.read_file(volume, path).await,
@@ -187,6 +213,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
         match self {
             Disk::Local(local_disk) => local_disk.read_file_stream(volume, path, offset, length).await,
@@ -194,6 +221,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
         match self {
             Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
@@ -201,6 +229,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self, wr))]
     async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.walk_dir(opts, wr).await,
@@ -208,6 +237,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn rename_data(
         &self,
         src_volume: &str,
@@ -222,6 +252,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.make_volumes(volumes).await,
@@ -229,6 +260,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn make_volume(&self, volume: &str) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.make_volume(volume).await,
@@ -236,6 +268,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
         match self {
             Disk::Local(local_disk) => local_disk.list_volumes().await,
@@ -243,6 +276,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
         match self {
             Disk::Local(local_disk) => local_disk.stat_volume(volume).await,
@@ -250,12 +284,15 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.delete_paths(volume, paths).await,
             Disk::Remote(remote_disk) => remote_disk.delete_paths(volume, paths).await,
         }
     }
+
+    #[tracing::instrument(skip(self))]
     async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.update_metadata(volume, path, fi, opts).await,
@@ -263,6 +300,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.write_metadata(_org_volume, volume, path, fi).await,
@@ -285,13 +323,15 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument]
+    #[tracing::instrument(skip(self))]
     async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
         match self {
             Disk::Local(local_disk) => local_disk.read_xl(volume, path, read_data).await,
             Disk::Remote(remote_disk) => remote_disk.read_xl(volume, path, read_data).await,
         }
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_version(
         &self,
         volume: &str,
@@ -305,6 +345,8 @@ impl DiskAPI for Disk {
             Disk::Remote(remote_disk) => remote_disk.delete_version(volume, path, fi, force_del_marker, opts).await,
         }
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_versions(
         &self,
         volume: &str,
@@ -317,6 +359,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
         match self {
             Disk::Local(local_disk) => local_disk.read_multiple(req).await,
@@ -324,6 +367,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_volume(&self, volume: &str) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.delete_volume(volume).await,
@@ -331,6 +375,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
         match self {
             Disk::Local(local_disk) => local_disk.disk_info(opts).await,
@@ -338,6 +383,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self, cache, we_sleep, scan_mode))]
     async fn ns_scanner(
         &self,
         cache: &DataUsageCache,
@@ -351,6 +397,7 @@ impl DiskAPI for Disk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn healing(&self) -> Option<HealingTracker> {
         match self {
             Disk::Local(local_disk) => local_disk.healing().await,
diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs
index c635d744a..2b4e64dc2 100644
--- a/ecstore/src/disk/remote.rs
+++ b/ecstore/src/disk/remote.rs
@@ -70,17 +70,21 @@ impl RemoteDisk {
 // TODO: all api need to handle errors
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
+    #[tracing::instrument(skip(self))]
     fn to_string(&self) -> String {
         self.endpoint.to_string()
     }
 
+    #[tracing::instrument(skip(self))]
     fn is_local(&self) -> bool {
         false
     }
 
+    #[tracing::instrument(skip(self))]
     fn host_name(&self) -> String {
         self.endpoint.host_port()
     }
+    #[tracing::instrument(skip(self))]
     async fn is_online(&self) -> bool {
         // TODO: 连接状态
         if (node_service_time_out_client(&self.addr).await).is_ok() {
@@ -88,16 +92,20 @@ impl DiskAPI for RemoteDisk {
         }
         false
     }
+    #[tracing::instrument(skip(self))]
     fn endpoint(&self) -> Endpoint {
         self.endpoint.clone()
     }
+    #[tracing::instrument(skip(self))]
     async fn close(&self) -> Result<()> {
         Ok(())
     }
+    #[tracing::instrument(skip(self))]
     fn path(&self) -> PathBuf {
         self.root.clone()
     }
 
+    #[tracing::instrument(skip(self))]
     fn get_disk_location(&self) -> DiskLocation {
         DiskLocation {
             pool_idx: {
@@ -124,9 +132,12 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_disk_id(&self) -> Result<Option<Uuid>> {
         Ok(*self.id.lock().await)
     }
+
+    #[tracing::instrument(skip(self))]
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
         let mut lock = self.id.lock().await;
         *lock = id;
@@ -134,6 +145,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
         info!("read_all {}/{}", volume, path);
         let mut client = node_service_time_out_client(&self.addr)
@@ -154,6 +166,7 @@ impl DiskAPI for RemoteDisk {
         Ok(response.data)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
         info!("write_all");
         let mut client = node_service_time_out_client(&self.addr)
@@ -179,6 +192,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
         info!("delete {}/{}/{}", self.endpoint.to_string(), volume, path);
         let options = serde_json::to_string(&opt)?;
@@ -205,6 +219,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
         info!("verify_file");
         let file_info = serde_json::to_string(&fi)?;
@@ -233,6 +248,7 @@ impl DiskAPI for RemoteDisk {
         Ok(check_parts_resp)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
         info!("check_parts");
         let file_info = serde_json::to_string(&fi)?;
@@ -261,6 +277,7 @@ impl DiskAPI for RemoteDisk {
         Ok(check_parts_resp)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
         info!("rename_part {}/{}", src_volume, src_path);
         let mut client = node_service_time_out_client(&self.addr)
@@ -287,6 +304,7 @@ impl DiskAPI for RemoteDisk {
 
         Ok(())
     }
+
     #[tracing::instrument(level = "debug", skip(self))]
     async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
         info!("rename_file");
@@ -365,6 +383,7 @@ impl DiskAPI for RemoteDisk {
         ))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
         info!("list_dir {}/{}", volume, _dir_path);
         let mut client = node_service_time_out_client(&self.addr)
@@ -389,6 +408,7 @@ impl DiskAPI for RemoteDisk {
     }
 
     // FIXME: TODO: use writer
+    #[tracing::instrument(skip(self, wr))]
     async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
         let now = std::time::SystemTime::now();
         info!("walk_dir {}/{}/{:?}", self.endpoint.to_string(), opts.bucket, opts.filter_prefix);
@@ -429,6 +449,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn rename_data(
         &self,
         src_volume: &str,
@@ -466,6 +487,7 @@ impl DiskAPI for RemoteDisk {
         Ok(rename_data_resp)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
         info!("make_volumes");
         let mut client = node_service_time_out_client(&self.addr)
@@ -489,6 +511,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn make_volume(&self, volume: &str) -> Result<()> {
         info!("make_volume");
         let mut client = node_service_time_out_client(&self.addr)
@@ -512,6 +535,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
         info!("list_volumes");
         let mut client = node_service_time_out_client(&self.addr)
@@ -540,6 +564,7 @@ impl DiskAPI for RemoteDisk {
         Ok(infos)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
         info!("stat_volume");
         let mut client = node_service_time_out_client(&self.addr)
@@ -565,6 +590,7 @@ impl DiskAPI for RemoteDisk {
         Ok(volume_info)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
         info!("delete_paths");
         let paths = paths.to_owned();
@@ -589,6 +615,8 @@ impl DiskAPI for RemoteDisk {
 
         Ok(())
     }
+
+    #[tracing::instrument(skip(self))]
     async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
         info!("update_metadata");
         let file_info = serde_json::to_string(&fi)?;
@@ -618,6 +646,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
         info!("write_metadata {}/{}", volume, path);
         let file_info = serde_json::to_string(&fi)?;
@@ -644,6 +673,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_version(
         &self,
         _org_volume: &str,
@@ -707,6 +737,8 @@ impl DiskAPI for RemoteDisk {
 
         Ok(raw_file_info)
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_version(
         &self,
         volume: &str,
@@ -745,6 +777,8 @@ impl DiskAPI for RemoteDisk {
 
         Ok(())
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_versions(
         &self,
         volume: &str,
@@ -790,6 +824,7 @@ impl DiskAPI for RemoteDisk {
         Ok(errors)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
         info!("read_multiple {}/{}/{}", self.endpoint.to_string(), req.bucket, req.prefix);
         let read_multiple_req = serde_json::to_string(&req)?;
@@ -820,6 +855,7 @@ impl DiskAPI for RemoteDisk {
         Ok(read_multiple_resps)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_volume(&self, volume: &str) -> Result<()> {
         info!("delete_volume {}/{}", self.endpoint.to_string(), volume);
         let mut client = node_service_time_out_client(&self.addr)
@@ -843,6 +879,7 @@ impl DiskAPI for RemoteDisk {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
         let opts = serde_json::to_string(&opts)?;
         let mut client = node_service_time_out_client(&self.addr)
@@ -868,6 +905,7 @@ impl DiskAPI for RemoteDisk {
         Ok(disk_info)
     }
 
+    #[tracing::instrument(skip(self, cache, scan_mode, _we_sleep))]
     async fn ns_scanner(
         &self,
         cache: &DataUsageCache,
@@ -909,6 +947,7 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn healing(&self) -> Option<HealingTracker> {
         None
     }
diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs
index b4c45ff11..021ae7bed 100644
--- a/ecstore/src/erasure.rs
+++ b/ecstore/src/erasure.rs
@@ -102,14 +102,11 @@ impl Erasure {
             self.encode_data(&self.buf, &mut blocks)?;
 
             let write_futures = writers.iter_mut().enumerate().map(|(i, w_op)| {
-                let i_inner = i.clone();
+                let i_inner = i;
                 let blocks_inner = blocks.clone();
                 async move {
                     if let Some(w) = w_op {
-                        match w.write(blocks_inner[i_inner].clone()).await {
-                            Ok(_) => None,
-                            Err(e) => Some(e),
-                        }
+                        (w.write(blocks_inner[i_inner].clone()).await).err()
                     } else {
                         Some(Error::new(DiskError::DiskNotFound))
                     }
diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs
index 35f41b6a1..19c244190 100644
--- a/ecstore/src/set_disk.rs
+++ b/ecstore/src/set_disk.rs
@@ -33,7 +33,7 @@ use crate::{
         ListMultipartsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectIO, ObjectInfo,
         ObjectOptions, ObjectPartInfo, ObjectToDelete, PartInfo, PutObjReader, RawFileInfo, StorageAPI, DEFAULT_BITROT_ALGO,
     },
-    store_err::{to_object_err, StorageError},
+    store_err::{is_err_object_not_found, to_object_err, StorageError},
     store_init::{load_format_erasure, ErasureError},
     utils::{
         self,
@@ -345,7 +345,7 @@ impl SetDisks {
                                 false,
                                 DeleteOptions {
                                     undo_write: true,
-                                    old_data_dir: old_data_dir,
+                                    old_data_dir,
                                     ..Default::default()
                                 },
                             )
@@ -459,6 +459,7 @@ impl SetDisks {
         Ok(())
     }
 
+    #[tracing::instrument(skip(disks))]
     async fn cleanup_multipart_path(disks: &[Option<DiskStore>], paths: &[String]) {
         let mut futures = Vec::with_capacity(disks.len());
 
@@ -490,6 +491,8 @@ impl SetDisks {
             warn!("cleanup_multipart_path errs {:?}", &errs);
         }
     }
+
+    #[tracing::instrument(skip(disks, meta))]
     async fn rename_part(
         disks: &[Option<DiskStore>],
         src_bucket: &str,
@@ -580,6 +583,7 @@ impl SetDisks {
     //     errors
     // }
 
+    #[tracing::instrument(skip(disks, files))]
     async fn write_unique_file_info(
         disks: &[Option<DiskStore>],
         org_bucket: &str,
@@ -931,7 +935,15 @@ impl SetDisks {
         let (parts_metadata, errs) =
             Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await;
 
-        let (read_quorum, write_quorum) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
+        let map_err_notfound = |err: Error| {
+            if is_err_object_not_found(&err) {
+                return Error::new(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()));
+            }
+            err
+        };
+
+        let (read_quorum, write_quorum) =
+            Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count).map_err(map_err_notfound)?;
 
         if read_quorum < 0 {
             return Err(Error::new(QuorumError::Read));
@@ -946,10 +958,10 @@ impl SetDisks {
             quorum = write_quorum as usize;
 
             if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), quorum) {
-                return Err(err);
+                return Err(map_err_notfound(err));
             }
         } else if let Some(err) = reduce_read_quorum_errs(&errs, object_op_ignored_errs().as_ref(), quorum) {
-            return Err(err);
+            return Err(map_err_notfound(err));
         }
 
         let (_, mod_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, quorum);
@@ -3875,14 +3887,17 @@ impl ObjectIO for SetDisks {
 
 #[async_trait::async_trait]
 impl StorageAPI for SetDisks {
+    #[tracing::instrument(skip(self))]
     async fn backend_info(&self) -> madmin::BackendInfo {
         unimplemented!()
     }
+    #[tracing::instrument(skip(self))]
     async fn storage_info(&self) -> madmin::StorageInfo {
         let disks = self.get_disks_internal().await;
 
         get_storage_info(&disks, &self.set_endpoints).await
     }
+    #[tracing::instrument(skip(self))]
     async fn local_storage_info(&self) -> madmin::StorageInfo {
         let disks = self.get_disks_internal().await;
 
@@ -3898,17 +3913,21 @@ impl StorageAPI for SetDisks {
 
         get_storage_info(&local_disks, &local_endpoints).await
     }
+    #[tracing::instrument(skip(self))]
     async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
         unimplemented!()
     }
+    #[tracing::instrument(skip(self))]
     async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn copy_object(
         &self,
         src_bucket: &str,
@@ -4013,6 +4032,7 @@ impl StorageAPI for SetDisks {
 
         Ok(fi.to_object_info(src_bucket, src_object, src_opts.versioned || src_opts.version_suspended))
     }
+    #[tracing::instrument(skip(self))]
     async fn delete_objects(
         &self,
         bucket: &str,
@@ -4123,6 +4143,8 @@ impl StorageAPI for SetDisks {
 
         Ok((del_objects, del_errs))
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         if opts.delete_prefix {
             self.delete_prefix(bucket, object)
@@ -4134,6 +4156,7 @@ impl StorageAPI for SetDisks {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn list_objects_v2(
         self: Arc<Self>,
         _bucket: &str,
@@ -4146,6 +4169,8 @@ impl StorageAPI for SetDisks {
     ) -> Result<ListObjectsV2Info> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn list_object_versions(
         self: Arc<Self>,
         _bucket: &str,
@@ -4157,6 +4182,8 @@ impl StorageAPI for SetDisks {
     ) -> Result<ListObjectVersionsInfo> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         // let mut _ns = None;
         // if !opts.no_lock {
@@ -4198,6 +4225,7 @@ impl StorageAPI for SetDisks {
         Ok(oi)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         // TODO: nslock
 
@@ -4280,6 +4308,7 @@ impl StorageAPI for SetDisks {
         Ok(fi.to_object_info(bucket, object, opts.versioned || opts.version_suspended))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
         let oi = self.get_object_info(bucket, object, opts).await?;
         Ok(oi.user_tags)
@@ -4306,10 +4335,13 @@ impl StorageAPI for SetDisks {
         // TODO: versioned
         Ok(fi.to_object_info(bucket, object, opts.versioned || opts.version_suspended))
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.put_object_tags(bucket, object, "", opts).await
     }
 
+    #[tracing::instrument(skip(self))]
     async fn copy_object_part(
         &self,
         _src_bucket: &str,
@@ -4432,6 +4464,8 @@ impl StorageAPI for SetDisks {
 
         Ok(ret)
     }
+
+    #[tracing::instrument(skip(self))]
     async fn list_multipart_uploads(
         &self,
         bucket: &str,
@@ -4471,7 +4505,7 @@ impl StorageAPI for SetDisks {
                 Err(err) => {
                     if DiskError::DiskNotFound.is(&err) {
                         None
-                    } else if DiskError::FileNotFound.is(&err) {
+                    } else if is_err_object_not_found(&err) {
                         return Ok(ListMultipartsInfo {
                             key_marker: key_marker.to_owned(),
                             max_uploads,
@@ -4575,9 +4609,9 @@ impl StorageAPI for SetDisks {
             ..Default::default()
         })
     }
-    async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
-        warn!("new_multipart_upload opt {:?}", opts);
 
+    #[tracing::instrument(skip(self))]
+    async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
         let disks = self.disks.read().await;
 
         let disks = disks.clone();
@@ -4680,6 +4714,8 @@ impl StorageAPI for SetDisks {
 
         Ok(MultipartUploadResult { upload_id })
     }
+
+    #[tracing::instrument(skip(self))]
     async fn get_multipart_info(
         &self,
         bucket: &str,
@@ -4701,6 +4737,8 @@ impl StorageAPI for SetDisks {
             ..Default::default()
         })
     }
+
+    #[tracing::instrument(skip(self))]
     async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, _opts: &ObjectOptions) -> Result<()> {
         self.check_upload_id_exists(bucket, object, upload_id, false).await?;
         let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
@@ -4708,7 +4746,7 @@ impl StorageAPI for SetDisks {
         self.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path).await
     }
     // complete_multipart_upload 完成
-    // #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self))]
     async fn complete_multipart_upload(
         &self,
         bucket: &str,
@@ -4957,24 +4995,32 @@ impl StorageAPI for SetDisks {
         Ok(fi.to_object_info(bucket, object, opts.versioned || opts.version_suspended))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result<Vec<Option<DiskStore>>> {
         Ok(self.get_disks_internal().await)
     }
 
+    #[tracing::instrument(skip(self))]
     fn set_drive_counts(&self) -> Vec<usize> {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result<HealResultItem> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn heal_object(
         &self,
         bucket: &str,
@@ -5025,6 +5071,8 @@ impl StorageAPI for SetDisks {
         }
         return Ok((result, err));
     }
+
+    #[tracing::instrument(skip(self))]
     async fn heal_objects(
         &self,
         _bucket: &str,
@@ -5035,9 +5083,13 @@ impl StorageAPI for SetDisks {
     ) -> Result<()> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn get_pool_and_set(&self, _id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
         unimplemented!()
     }
diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs
index 0a2e716fa..7a4cf93a3 100644
--- a/ecstore/src/sets.rs
+++ b/ecstore/src/sets.rs
@@ -308,9 +308,11 @@ impl ObjectIO for Sets {
 
 #[async_trait::async_trait]
 impl StorageAPI for Sets {
+    #[tracing::instrument(skip(self))]
     async fn backend_info(&self) -> madmin::BackendInfo {
         unimplemented!()
     }
+    #[tracing::instrument(skip(self))]
     async fn storage_info(&self) -> madmin::StorageInfo {
         let mut futures = Vec::with_capacity(self.disk_set.len());
 
@@ -331,6 +333,7 @@ impl StorageAPI for Sets {
             ..Default::default()
         }
     }
+    #[tracing::instrument(skip(self))]
     async fn local_storage_info(&self) -> madmin::StorageInfo {
         let mut futures = Vec::with_capacity(self.disk_set.len());
 
@@ -350,16 +353,21 @@ impl StorageAPI for Sets {
             ..Default::default()
         }
     }
+    #[tracing::instrument(skip(self))]
     async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
         unimplemented!()
     }
+    #[tracing::instrument(skip(self))]
     async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn copy_object(
         &self,
         src_bucket: &str,
@@ -416,6 +424,8 @@ impl StorageAPI for Sets {
             "put_object_reader2 is none".to_owned(),
         )))
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_objects(
         &self,
         bucket: &str,
@@ -498,6 +508,8 @@ impl StorageAPI for Sets {
 
         Ok((del_objects, del_errs))
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         if opts.delete_prefix && !opts.delete_prefix_object {
             self.delete_prefix(bucket, object).await?;
@@ -506,6 +518,8 @@ impl StorageAPI for Sets {
 
         self.get_disks_by_key(object).delete_object(bucket, object, opts).await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn list_objects_v2(
         self: Arc<Self>,
         _bucket: &str,
@@ -518,6 +532,8 @@ impl StorageAPI for Sets {
     ) -> Result<ListObjectsV2Info> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn list_object_versions(
         self: Arc<Self>,
         _bucket: &str,
@@ -529,14 +545,18 @@ impl StorageAPI for Sets {
     ) -> Result<ListObjectVersionsInfo> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
     }
 
+    #[tracing::instrument(skip(self))]
     async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.get_disks_by_key(object).put_object_metadata(bucket, object, opts).await
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
         self.get_disks_by_key(object).get_object_tags(bucket, object, opts).await
     }
@@ -546,10 +566,13 @@ impl StorageAPI for Sets {
             .put_object_tags(bucket, object, tags, opts)
             .await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await
     }
 
+    #[tracing::instrument(skip(self))]
     async fn copy_object_part(
         &self,
         _src_bucket: &str,
@@ -566,6 +589,8 @@ impl StorageAPI for Sets {
     ) -> Result<()> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn put_object_part(
         &self,
         bucket: &str,
@@ -579,6 +604,8 @@ impl StorageAPI for Sets {
             .put_object_part(bucket, object, upload_id, part_id, data, opts)
             .await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn list_multipart_uploads(
         &self,
         bucket: &str,
@@ -592,9 +619,13 @@ impl StorageAPI for Sets {
             .list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads)
             .await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
         self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn get_multipart_info(
         &self,
         bucket: &str,
@@ -606,11 +637,15 @@ impl StorageAPI for Sets {
             .get_multipart_info(bucket, object, upload_id, opts)
             .await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> {
         self.get_disks_by_key(object)
             .abort_multipart_upload(bucket, object, upload_id, opts)
             .await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn complete_multipart_upload(
         &self,
         bucket: &str,
@@ -623,17 +658,23 @@ impl StorageAPI for Sets {
             .complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
             .await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result<Vec<Option<DiskStore>>> {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     fn set_drive_counts(&self) -> Vec<usize> {
         unimplemented!()
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
         unimplemented!()
     }
+
+    #[tracing::instrument(skip(self))]
     async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         let (disks, _) = init_storage_disks_with_errors(
             &self.endpoints.endpoints,
@@ -718,9 +759,11 @@ impl StorageAPI for Sets {
         }
         Ok((res, None))
     }
+    #[tracing::instrument(skip(self))]
     async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result<HealResultItem> {
         unimplemented!()
     }
+    #[tracing::instrument(skip(self))]
     async fn heal_object(
         &self,
         bucket: &str,
@@ -732,6 +775,7 @@ impl StorageAPI for Sets {
             .heal_object(bucket, object, version_id, opts)
             .await
     }
+    #[tracing::instrument(skip(self))]
     async fn heal_objects(
         &self,
         _bucket: &str,
@@ -742,9 +786,11 @@ impl StorageAPI for Sets {
     ) -> Result<()> {
         unimplemented!()
     }
+    #[tracing::instrument(skip(self))]
     async fn get_pool_and_set(&self, _id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
         unimplemented!()
     }
+    #[tracing::instrument(skip(self))]
     async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
         unimplemented!()
     }
diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs
index 23a17596b..5a2075775 100644
--- a/ecstore/src/store.rs
+++ b/ecstore/src/store.rs
@@ -1246,6 +1246,7 @@ lazy_static! {
 
 #[async_trait::async_trait]
 impl StorageAPI for ECStore {
+    #[tracing::instrument(skip(self))]
     async fn backend_info(&self) -> madmin::BackendInfo {
         let (standard_sc_parity, rr_sc_parity) = {
             if let Some(sc) = GLOBAL_StorageClass.get() {
@@ -1290,6 +1291,7 @@ impl StorageAPI for ECStore {
             ..Default::default()
         }
     }
+    #[tracing::instrument(skip(self))]
     async fn storage_info(&self) -> madmin::StorageInfo {
         let Some(notification_sy) = get_global_notification_sys() else {
             return madmin::StorageInfo::default();
@@ -1297,6 +1299,7 @@ impl StorageAPI for ECStore {
 
         notification_sy.storage_info(self).await
     }
+    #[tracing::instrument(skip(self))]
     async fn local_storage_info(&self) -> madmin::StorageInfo {
         let mut futures = Vec::with_capacity(self.pools.len());
 
@@ -1316,6 +1319,7 @@ impl StorageAPI for ECStore {
         madmin::StorageInfo { backend, disks }
     }
 
+    #[tracing::instrument(skip(self))]
     async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
         // TODO: opts.cached
 
@@ -1331,6 +1335,7 @@ impl StorageAPI for ECStore {
         Ok(buckets)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
         if is_meta_bucketname(bucket) {
             return Err(StorageError::BucketNameInvalid(bucket.to_string()).into());
@@ -1360,6 +1365,7 @@ impl StorageAPI for ECStore {
             .await?;
         Ok(())
     }
+    #[tracing::instrument(skip(self))]
     async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
         if !is_meta_bucketname(bucket) {
             if let Err(err) = check_valid_bucket_name_strict(bucket) {
@@ -1403,6 +1409,7 @@ impl StorageAPI for ECStore {
 
         Ok(())
     }
+    #[tracing::instrument(skip(self))]
     async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
         let mut info = self
             .peer_sys
@@ -1420,6 +1427,7 @@ impl StorageAPI for ECStore {
     }
 
     // TODO: review
+    #[tracing::instrument(skip(self))]
     async fn copy_object(
         &self,
         src_bucket: &str,
@@ -1490,6 +1498,7 @@ impl StorageAPI for ECStore {
     }
 
     // TODO: review
+    #[tracing::instrument(skip(self))]
     async fn delete_objects(
         &self,
         bucket: &str,
@@ -1643,6 +1652,7 @@ impl StorageAPI for ECStore {
         Ok((del_objects, del_errs))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         check_del_obj_args(bucket, object)?;
 
@@ -1717,6 +1727,7 @@ impl StorageAPI for ECStore {
     // @start_after as marker when continuation_token empty
     // @delimiter default="/", empty when recursive
     // @max_keys limit
+    #[tracing::instrument(skip(self))]
     async fn list_objects_v2(
         self: Arc<Self>,
         bucket: &str,
@@ -1730,6 +1741,7 @@ impl StorageAPI for ECStore {
         self.inner_list_objects_v2(bucket, prefix, continuation_token, delimiter, max_keys, fetch_owner, start_after)
             .await
     }
+    #[tracing::instrument(skip(self))]
     async fn list_object_versions(
         self: Arc<Self>,
         bucket: &str,
@@ -1742,6 +1754,7 @@ impl StorageAPI for ECStore {
         self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
             .await
     }
+    #[tracing::instrument(skip(self))]
     async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         check_object_args(bucket, object)?;
 
@@ -1758,6 +1771,7 @@ impl StorageAPI for ECStore {
         Ok(info)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
         let object = encode_dir_object(object);
 
@@ -1770,6 +1784,7 @@ impl StorageAPI for ECStore {
         Ok(oi.user_tags)
     }
 
+    #[tracing::instrument(skip(self))]
     async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         let object = encode_dir_object(object);
         if self.single_pool() {
@@ -1796,6 +1811,7 @@ impl StorageAPI for ECStore {
 
         self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
     }
+    #[tracing::instrument(skip(self))]
     async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         let object = encode_dir_object(object);
 
@@ -1807,6 +1823,8 @@ impl StorageAPI for ECStore {
 
         self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn copy_object_part(
         &self,
         src_bucket: &str,
@@ -1828,6 +1846,7 @@ impl StorageAPI for ECStore {
 
         unimplemented!()
     }
+    #[tracing::instrument(skip(self, data))]
     async fn put_object_part(
         &self,
         bucket: &str,
@@ -1859,6 +1878,7 @@ impl StorageAPI for ECStore {
             };
 
             if let Some(err) = err {
+                error!("put_object_part err: {:?}", err);
                 return Err(err);
             }
         }
@@ -1870,6 +1890,7 @@ impl StorageAPI for ECStore {
         )))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn list_multipart_uploads(
         &self,
         bucket: &str,
@@ -1917,6 +1938,8 @@ impl StorageAPI for ECStore {
             ..Default::default()
         })
     }
+
+    #[tracing::instrument(skip(self))]
     async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
         check_new_multipart_args(bucket, object)?;
 
@@ -1946,6 +1969,8 @@ impl StorageAPI for ECStore {
 
         self.pools[idx].new_multipart_upload(bucket, object, opts).await
     }
+
+    #[tracing::instrument(skip(self))]
     async fn get_multipart_info(
         &self,
         bucket: &str,
@@ -1981,6 +2006,7 @@ impl StorageAPI for ECStore {
             upload_id.to_owned(),
         )))
     }
+    #[tracing::instrument(skip(self))]
     async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> {
         check_abort_multipart_args(bucket, object, upload_id)?;
 
@@ -2016,6 +2042,7 @@ impl StorageAPI for ECStore {
             upload_id.to_owned(),
         )))
     }
+    #[tracing::instrument(skip(self))]
     async fn complete_multipart_upload(
         &self,
         bucket: &str,
@@ -2062,6 +2089,7 @@ impl StorageAPI for ECStore {
         )))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result<Vec<Option<DiskStore>>> {
         if pool_idx < self.pools.len() && set_idx < self.pools[pool_idx].disk_set.len() {
             self.pools[pool_idx].disk_set[set_idx].get_disks(0, 0).await
@@ -2070,6 +2098,7 @@ impl StorageAPI for ECStore {
         }
     }
 
+    #[tracing::instrument(skip(self))]
     fn set_drive_counts(&self) -> Vec<usize> {
         let mut counts = vec![0; self.pools.len()];
 
@@ -2078,6 +2107,8 @@ impl StorageAPI for ECStore {
         }
         counts
     }
+
+    #[tracing::instrument(skip(self))]
     async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         info!("heal_format");
         let mut r = HealResultItem {
@@ -2112,9 +2143,11 @@ impl StorageAPI for ECStore {
         Ok((r, None))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
         self.peer_sys.heal_bucket(bucket, opts).await
     }
+    #[tracing::instrument(skip(self))]
     async fn heal_object(
         &self,
         bucket: &str,
@@ -2188,6 +2221,8 @@ impl StorageAPI for ECStore {
 
         Ok((HealResultItem::default(), Some(Error::new(DiskError::FileNotFound))))
     }
+
+    #[tracing::instrument(skip(self))]
     async fn heal_objects(
         &self,
         bucket: &str,
@@ -2299,6 +2334,7 @@ impl StorageAPI for ECStore {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
         for (pool_idx, pool) in self.pools.iter().enumerate() {
             for (set_idx, set) in pool.format.erasure.sets.iter().enumerate() {
@@ -2313,6 +2349,7 @@ impl StorageAPI for ECStore {
         Err(Error::new(DiskError::DiskNotFound))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
         let object = utils::path::encode_dir_object(object);
         if self.single_pool() {

From 8e3c22b5954667e837ec63ddea77470dd4f0f259 Mon Sep 17 00:00:00 2001
From: junxiang Mu <1948535941@qq.com>
Date: Fri, 25 Apr 2025 08:59:39 +0000
Subject: [PATCH 12/13] fix

Signed-off-by: junxiang Mu <1948535941@qq.com>
---
 ecstore/src/heal/data_scanner.rs     | 14 ++++++++------
 ecstore/src/heal/data_usage_cache.rs |  6 +-----
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs
index ff4fe850b..b1b110a54 100644
--- a/ecstore/src/heal/data_scanner.rs
+++ b/ecstore/src/heal/data_scanner.rs
@@ -696,8 +696,8 @@ impl FolderScanner {
                     }
                 }
             }
-            if found_objects && *GLOBAL_IsErasure.read().await {
-                // If we found an object in erasure mode, we skip subdirs (only datadirs)...
+            // if found_objects && *GLOBAL_IsErasure.read().await {
+            if found_objects {
                 break 'outer;
             }
 
@@ -1029,11 +1029,13 @@ impl FolderScanner {
 
 #[tracing::instrument(level = "info", skip(into, folder_scanner))]
 async fn scan(folder: &CachedFolder, into: &mut DataUsageEntry, folder_scanner: &mut FolderScanner) {
-    if into.compacted {
-        *into = DataUsageEntry::default();
-    }
+    let mut dst = if !into.compacted {
+        DataUsageEntry::default()
+    } else {
+        into.clone()
+    };
 
-    if Box::pin(folder_scanner.scan_folder(folder, into)).await.is_err() {
+    if Box::pin(folder_scanner.scan_folder(folder, &mut dst)).await.is_err() {
         return;
     }
     if !into.compacted {
diff --git a/ecstore/src/heal/data_usage_cache.rs b/ecstore/src/heal/data_usage_cache.rs
index 11814c3ef..2dad9f55b 100644
--- a/ecstore/src/heal/data_usage_cache.rs
+++ b/ecstore/src/heal/data_usage_cache.rs
@@ -21,7 +21,7 @@ use tokio::time::sleep;
 use tracing::warn;
 
 use super::data_scanner::{SizeSummary, DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS};
-use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, DATA_USAGE_ROOT};
+use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo};
 
 // DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
 pub const DATA_USAGE_BUCKET_LEN: usize = 11;
@@ -858,9 +858,5 @@ impl DataUsageHash {
 }
 
 pub fn hash_path(data: &str) -> DataUsageHash {
-    let mut data = data;
-    if data != DATA_USAGE_ROOT {
-        data = data.trim_matches('/');
-    }
     DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
 }

From 05f14123230ec09ccce835b1adf73533804a5b7f Mon Sep 17 00:00:00 2001
From: houseme <housemecn@gmail.com>
Date: Sat, 26 Apr 2025 17:26:27 +0800
Subject: [PATCH 13/13] improve code for opentelemetry

---
 crates/obs/src/telemetry.rs | 83 ++++++++++++++-----------------------
 rustfs/src/main.rs          |  7 ++++
 2 files changed, 37 insertions(+), 53 deletions(-)

diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs
index ae9fd40e4..654ae4145 100644
--- a/crates/obs/src/telemetry.rs
+++ b/crates/obs/src/telemetry.rs
@@ -3,7 +3,7 @@ use crate::utils::get_local_ip_with_default;
 use crate::OtelConfig;
 use opentelemetry::trace::TracerProvider;
 use opentelemetry::{global, KeyValue};
-use opentelemetry_appender_tracing::layer;
+use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
 use opentelemetry_otlp::WithExportConfig;
 use opentelemetry_sdk::logs::SdkLoggerProvider;
 use opentelemetry_sdk::{
@@ -202,39 +202,6 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
 
     // configuring tracing
     {
-        // optimize filter configuration
-        let otel_layer = {
-            let filter_otel = match logger_level {
-                "trace" | "debug" => {
-                    info!("OpenTelemetry tracing initialized with level: {}", logger_level);
-                    EnvFilter::new(logger_level)
-                }
-                _ => {
-                    let mut filter = EnvFilter::new(logger_level);
-
-                    // use smallvec to avoid heap allocation
-                    let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
-
-                    for directive in directives {
-                        filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
-                    }
-                    filter
-                }
-            };
-            layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
-        };
-
-        let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
-
-        // Configure registry to avoid repeated calls to filter methods
-        let level_filter = switch_level(logger_level);
-        let registry = tracing_subscriber::registry()
-            .with(level_filter)
-            .with(OpenTelemetryLayer::new(tracer))
-            .with(MetricsLayer::new(meter_provider.clone()))
-            .with(otel_layer)
-            .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
-
         // configure the formatting layer
         let enable_color = std::io::stdout().is_terminal();
         let fmt_layer = tracing_subscriber::fmt::layer()
@@ -243,16 +210,25 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
             .with_thread_names(true)
             .with_thread_ids(true)
             .with_file(true)
-            .with_line_number(true)
-            .with_filter(
-                EnvFilter::new(logger_level).add_directive(
-                    format!("opentelemetry={}", if endpoint.is_empty() { logger_level } else { "off" })
-                        .parse()
-                        .unwrap(),
-                ),
-            );
+            .with_line_number(true);
 
-        registry.with(ErrorLayer::default()).with(fmt_layer).init();
+        let filter = build_env_filter(logger_level, None);
+
+        let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
+
+        let otel_filter = build_env_filter(logger_level, None);
+        let otel_layer = OpenTelemetryTracingBridge::new(&logger_provider).with_filter(otel_filter);
+
+        // Configure registry to avoid repeated calls to filter methods
+        tracing_subscriber::registry()
+            .with(filter)
+            .with(fmt_layer)
+            .with(ErrorLayer::default())
+            .with(otel_layer)
+            .with(MetricsLayer::new(meter_provider.clone()))
+            .with(OpenTelemetryLayer::new(tracer))
+            .with(ErrorLayer::default())
+            .init();
 
         if !endpoint.is_empty() {
             info!(
@@ -269,15 +245,16 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
     }
 }
 
-/// Switch log level
-fn switch_level(logger_level: &str) -> tracing_subscriber::filter::LevelFilter {
-    use tracing_subscriber::filter::LevelFilter;
-    match logger_level {
-        "error" => LevelFilter::ERROR,
-        "warn" => LevelFilter::WARN,
-        "info" => LevelFilter::INFO,
-        "debug" => LevelFilter::DEBUG,
-        "trace" => LevelFilter::TRACE,
-        _ => LevelFilter::OFF,
+fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter {
+    let level = default_level.unwrap_or(logger_level);
+    let mut filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level));
+
+    if !matches!(logger_level, "trace" | "debug") {
+        let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
+        for directive in directives {
+            filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+        }
     }
+
+    filter
 }
diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs
index c3ea69e94..ca73521f8 100644
--- a/rustfs/src/main.rs
+++ b/rustfs/src/main.rs
@@ -335,6 +335,12 @@ async fn run(opt: config::Opt) -> Result<()> {
                             span
                         })
                         .on_request(|request: &HttpRequest<_>, _span: &Span| {
+                            info!(
+                                counter.rustfs_api_requests_total = 1_u64,
+                                key_request_method = %request.method().to_string(),
+                                key_request_uri_path = %request.uri().path().to_owned(),
+                                "handle request api total",
+                            );
                             debug!("http started method: {}, url path: {}", request.method(), request.uri().path())
                         })
                         .on_response(|response: &Response<_>, latency: Duration, _span: &Span| {
@@ -342,6 +348,7 @@ async fn run(opt: config::Opt) -> Result<()> {
                             debug!("http response generated in {:?}", latency)
                         })
                         .on_body_chunk(|chunk: &Bytes, latency: Duration, _span: &Span| {
+                            info!(histogram.request.body.len = chunk.len(), "histogram request body lenght",);
                             debug!("http body sending {} bytes in {:?}", chunk.len(), latency)
                         })
                         .on_eos(|_trailers: Option<&HeaderMap>, stream_duration: Duration, _span: &Span| {