feat(obs): improve telemetry stack, replication metrics, and Grafana alignment (#2672)

Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>

crates/io-core/src/pool.rs

@@ -84,6 +84,12 @@ struct PoolTier {
     available_buffers: Mutex<Vec<BytesMut>>,
     /// Metrics for tracking this tier
     metrics: Mutex<Option<Arc<BytesPoolMetrics>>>,
+    /// Total acquisitions for this tier
+    tier_total_acquires: AtomicU64,
+    /// Total hits for this tier
+    tier_pool_hits: AtomicU64,
+    /// Current allocated bytes for this tier
+    tier_current_allocated_bytes: AtomicU64,
 }
 
 /// Pool metrics for monitoring and optimization.
@@ -291,6 +297,9 @@ impl PoolTier {
             name,
             available_buffers: Mutex::new(Vec::new()),
             metrics: Mutex::new(None),
+            tier_total_acquires: AtomicU64::new(0),
+            tier_pool_hits: AtomicU64::new(0),
+            tier_current_allocated_bytes: AtomicU64::new(0),
         }
     }
 
@@ -298,6 +307,65 @@ impl PoolTier {
         *self.metrics.lock().unwrap() = Some(metrics);
     }
 
+    fn take_or_allocate_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> (BytesMut, bool) {
+        let buffer_opt = {
+            let mut available = self.available_buffers.lock().unwrap();
+            available.pop()
+        };
+        let was_reused = buffer_opt.is_some();
+
+        let buffer = if let Some(mut buf) = buffer_opt {
+            let previous_capacity = buf.capacity();
+            buf.clear();
+            if previous_capacity < size {
+                buf.reserve(size - previous_capacity);
+            }
+            let current_capacity = buf.capacity();
+            if current_capacity > previous_capacity {
+                let delta = (current_capacity - previous_capacity) as u64;
+                pool_metrics.total_bytes_allocated.fetch_add(delta, Ordering::Relaxed);
+                pool_metrics.current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+                self.tier_current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+            }
+            buf
+        } else {
+            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
+            let allocated_bytes = buf.capacity() as u64;
+            pool_metrics
+                .total_bytes_allocated
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            pool_metrics
+                .current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            self.tier_current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            buf
+        };
+
+        (buffer, was_reused)
+    }
+
+    fn record_acquire_metrics(&self, pool_metrics: &BytesPoolMetrics, buffer_capacity: usize, was_reused: bool) {
+        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
+
+        if was_reused {
+            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
+            self.tier_pool_hits.fetch_add(1, Ordering::Relaxed);
+        } else {
+            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
+        }
+
+        let tier_total_acquires = self.tier_total_acquires.load(Ordering::Relaxed);
+        let tier_pool_hits = self.tier_pool_hits.load(Ordering::Relaxed);
+        let tier_hit_rate = if tier_total_acquires == 0 {
+            0.0
+        } else {
+            tier_pool_hits as f64 / tier_total_acquires as f64
+        };
+        rustfs_io_metrics::record_bytes_pool_hit_rate(self.name, tier_hit_rate);
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
+    }
+
     async fn acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> PooledBuffer {
         // Acquire semaphore permit (owned for storage in PooledBuffer)
         let permit = Arc::clone(&self.semaphore).acquire_owned().await.unwrap();
@@ -308,45 +376,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer - clear and ensure capacity
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -365,45 +399,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         Some(PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -421,8 +421,24 @@ impl PoolTier {
             if let Some(ref metrics) = *self.metrics.lock().unwrap() {
                 metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
             }
+        } else {
+            let released_bytes = buffer.capacity() as u64;
+            self.tier_current_allocated_bytes
+                .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                    Some(current.saturating_sub(released_bytes))
+                })
+                .ok();
+            if let Some(ref metrics) = *self.metrics.lock().unwrap() {
+                metrics
+                    .current_allocated_bytes
+                    .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                        Some(current.saturating_sub(released_bytes))
+                    })
+                    .ok();
+            }
         }
         // If pool is full, buffer is dropped and memory is freed
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 }
 
@@ -617,4 +633,47 @@ mod tests {
         let delta_hits = pool.metrics().pool_hits.load(Ordering::Relaxed) - initial_hits;
         assert_eq!(delta_hits, 1);
     }
+
+    #[tokio::test]
+    async fn test_tier_allocated_bytes_tracks_real_allocations() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire allocates one small-tier buffer.
+        let buf1 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // Return and reuse should not increase allocated bytes.
+        drop(buf1);
+        let buf2 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // A second in-flight buffer forces one more allocation.
+        let _buf3 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 2048);
+
+        drop(buf2);
+    }
+
+    #[tokio::test]
+    async fn test_tier_hit_rate_counters_track_reuse() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire is miss.
+        let buf1 = pool.acquire_buffer(512).await;
+        drop(buf1);
+
+        // Second acquire reuses previous buffer and counts as hit.
+        let _buf2 = pool.acquire_buffer(512).await;
+
+        assert_eq!(pool.small_pool.tier_total_acquires.load(Ordering::Relaxed), 2);
+        assert_eq!(pool.small_pool.tier_pool_hits.load(Ordering::Relaxed), 1);
+    }
 }