chore(obs): Improve tracing instrumentation (#2086)

Co-authored-by: loverustfs <hello@rustfs.com>

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -56,8 +56,7 @@ use tokio::sync::mpsc::Sender;
 use tokio::task::JoinHandle;
 use tokio::time::Duration;
 use tokio_util::sync::CancellationToken;
-use tracing::info;
-use tracing::warn;
+use tracing::{info, instrument, warn};
 
 // Worker limits
 pub const WORKER_MAX_LIMIT: usize = 500;
@@ -796,6 +795,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
     }
 
     /// Load bucket replication resync statuses into memory
+    #[instrument(skip(cancellation_token))]
     async fn load_resync(self: Arc<Self>, buckets: &[String], cancellation_token: CancellationToken) -> Result<(), EcstoreError> {
         // TODO: add leader_lock
         // Make sure only one node running resync on the cluster

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -73,7 +73,7 @@ use tokio::task::JoinSet;
 use tokio::time::Duration as TokioDuration;
 use tokio_util::io::ReaderStream;
 use tokio_util::sync::CancellationToken;
-use tracing::{error, info, warn};
+use tracing::{error, info, instrument, warn};
 
 const REPLICATION_DIR: &str = ".replication";
 const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -294,6 +294,7 @@ impl ReplicationResyncer {
         // TODO: Metrics
     }
 
+    #[instrument(skip(cancellation_token, storage))]
     pub async fn resync_bucket<S: StorageAPI>(
         self: Arc<Self>,
         cancellation_token: CancellationToken,

crates/ecstore/src/config/com.rs

@@ -22,7 +22,7 @@ use rustfs_utils::path::SLASH_SEPARATOR;
 use std::collections::HashSet;
 use std::sync::Arc;
 use std::sync::LazyLock;
-use tracing::{error, warn};
+use tracing::{error, instrument, warn};
 
 pub const CONFIG_PREFIX: &str = "config";
 const CONFIG_FILE: &str = "config.json";
@@ -36,6 +36,8 @@ static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
     h.insert(STORAGE_CLASS_SUB_SYS.to_owned());
     h
 });
+
+#[instrument(skip(api))]
 pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
     let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
     Ok(data)
@@ -68,6 +70,7 @@ pub async fn read_config_with_metadata<S: StorageAPI>(
     Ok((data, rd.object_info))
 }
 
+#[instrument(skip(api, data))]
 pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
     save_config_with_opts(
         api,
@@ -81,6 +84,7 @@ pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>)
     .await
 }
 
+#[instrument(skip(api))]
 pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
     match api
         .delete_object(

crates/ecstore/src/data_usage.rs

@@ -34,7 +34,7 @@ use std::{
 };
 use tokio::fs;
 use tokio::sync::RwLock;
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, instrument};
 
 // Data usage storage constants
 pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
@@ -77,6 +77,7 @@ lazy_static::lazy_static! {
 }
 
 /// Store data usage info to backend storage
+#[instrument(skip(store))]
 pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
     // Prevent older data from overwriting newer persisted stats
     if let Ok(buf) = read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await
@@ -103,6 +104,7 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
 }
 
 /// Load data usage info from backend storage
+#[instrument(skip(store))]
 pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
     let buf: Vec<u8> = match read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await {
         Ok(data) => data,
@@ -633,6 +635,7 @@ pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str
     Ok(d)
 }
 
+#[instrument(skip(cache))]
 pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate::error::Result<()> {
     use crate::config::com::save_config;
     use crate::disk::BUCKET_META_PREFIX;

crates/ecstore/src/disk/local.rs

@@ -1317,39 +1317,34 @@ fn normalize_path_components(path: impl AsRef<Path>) -> PathBuf {
 
 #[async_trait::async_trait]
 impl DiskAPI for LocalDisk {
-    #[tracing::instrument(skip(self))]
     fn to_string(&self) -> String {
         self.root.to_string_lossy().to_string()
     }
-    #[tracing::instrument(skip(self))]
+
     fn is_local(&self) -> bool {
         true
     }
-    #[tracing::instrument(skip(self))]
+
     fn host_name(&self) -> String {
         self.endpoint.host_port()
     }
-    #[tracing::instrument(skip(self))]
+
     async fn is_online(&self) -> bool {
         true
     }
 
-    #[tracing::instrument(skip(self))]
     fn endpoint(&self) -> Endpoint {
         self.endpoint.clone()
     }
 
-    #[tracing::instrument(skip(self))]
     async fn close(&self) -> Result<()> {
         Ok(())
     }
 
-    #[tracing::instrument(skip(self))]
     fn path(&self) -> PathBuf {
         self.root.clone()
     }
 
-    #[tracing::instrument(skip(self))]
     fn get_disk_location(&self) -> DiskLocation {
         DiskLocation {
             pool_idx: {
@@ -1437,7 +1432,6 @@ impl DiskAPI for LocalDisk {
         Ok(Some(disk_id))
     }
 
-    #[tracing::instrument(skip(self))]
     async fn set_disk_id(&self, _id: Option<Uuid>) -> Result<()> {
         // No setup is required locally
         Ok(())
@@ -2601,6 +2595,7 @@ impl DiskAPI for LocalDisk {
         ScanGuard(Arc::clone(&self.scanning))
     }
 
+    #[tracing::instrument(skip(self))]
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
         // Try to use cached file content reading for better performance, with safe fallback
         let file_path = self.get_object_path(volume, path)?;
@@ -2617,6 +2612,7 @@ impl DiskAPI for LocalDisk {
     }
 }
 
+#[tracing::instrument]
 async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInfo, bool)> {
     let drive_path = drive_path.to_string_lossy().to_string();
     check_path_length(&drive_path)?;

crates/ecstore/src/disk/mod.rs

@@ -60,7 +60,6 @@ pub enum Disk {
 
 #[async_trait::async_trait]
 impl DiskAPI for Disk {
-    #[tracing::instrument(skip(self))]
     fn to_string(&self) -> String {
         match self {
             Disk::Local(local_disk) => local_disk.to_string(),
@@ -68,7 +67,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     async fn is_online(&self) -> bool {
         match self {
             Disk::Local(local_disk) => local_disk.is_online().await,
@@ -76,7 +74,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     fn is_local(&self) -> bool {
         match self {
             Disk::Local(local_disk) => local_disk.is_local(),
@@ -84,7 +81,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     fn host_name(&self) -> String {
         match self {
             Disk::Local(local_disk) => local_disk.host_name(),
@@ -92,7 +88,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     fn endpoint(&self) -> Endpoint {
         match self {
             Disk::Local(local_disk) => local_disk.endpoint(),
@@ -100,7 +95,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     async fn close(&self) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.close().await,
@@ -108,7 +102,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     async fn get_disk_id(&self) -> Result<Option<Uuid>> {
         match self {
             Disk::Local(local_disk) => local_disk.get_disk_id().await,
@@ -116,7 +109,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
         match self {
             Disk::Local(local_disk) => local_disk.set_disk_id(id).await,
@@ -124,7 +116,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     fn path(&self) -> PathBuf {
         match self {
             Disk::Local(local_disk) => local_disk.path(),
@@ -132,7 +123,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     fn get_disk_location(&self) -> DiskLocation {
         match self {
             Disk::Local(local_disk) => local_disk.get_disk_location(),
@@ -164,7 +154,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
     async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
         match self {
             Disk::Local(local_disk) => local_disk.stat_volume(volume).await,

crates/ecstore/src/erasure_coding/erasure.rs

@@ -321,7 +321,7 @@ impl Erasure {
     ///
     /// # Returns
     /// A vector of encoded shards as `Bytes`.
-    #[tracing::instrument(level = "info", skip_all, fields(data_len=data.len()))]
+    #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
     pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
         // let shard_size = self.shard_size();
         // let total_size = shard_size * self.total_shard_count();

crates/ecstore/src/set_disk.rs

@@ -519,7 +519,7 @@ impl ObjectIO for SetDisks {
         Ok(reader)
     }
 
-    #[tracing::instrument(level = "debug", skip(self, data,))]
+    #[tracing::instrument(skip(self, data,))]
     async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
         let disks = self.get_disks_internal().await;
 

crates/ecstore/src/set_disk/heal.rs

@@ -680,6 +680,7 @@ impl SetDisks {
         Ok((result, None))
     }
 
+    #[tracing::instrument(skip(self))]
     pub(super) async fn heal_object_dir(
         &self,
         bucket: &str,

crates/ecstore/src/sets.rs

@@ -381,7 +381,6 @@ impl BucketOperations for Sets {
 
 #[async_trait::async_trait]
 impl ObjectOperations for Sets {
-    #[tracing::instrument(skip(self))]
     async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
     }
@@ -826,7 +825,6 @@ impl HealOperations for Sets {
 
 #[async_trait::async_trait]
 impl StorageAPI for Sets {
-    #[tracing::instrument(skip(self))]
     async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
         self.disk_set[0].new_ns_lock(bucket, object).await
     }

crates/ecstore/src/store.rs

@@ -281,7 +281,6 @@ impl BucketOperations for ECStore {
 
 #[async_trait::async_trait]
 impl ObjectOperations for ECStore {
-    #[instrument(skip(self))]
     async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         self.handle_get_object_info(bucket, object, opts).await
     }
@@ -561,7 +560,6 @@ impl HealOperations for ECStore {
 
 #[async_trait::async_trait]
 impl StorageAPI for ECStore {
-    #[instrument(skip(self))]
     async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
         self.handle_new_ns_lock(bucket, object).await
     }