fix(obs): evict retired metric cache entries (#4539)

Co-authored-by: heihutu <heihutu@gmail.com>

crates/obs/src/telemetry/mod.rs

@@ -52,7 +52,7 @@ mod rolling;
 use crate::TelemetryError;
 use crate::config::OtelConfig;
 pub use guard::OtelGuard;
-pub use recorder::Recorder;
+pub use recorder::{Recorder, retire_metric_series};
 use rustfs_config::observability::ENV_OBS_LOG_DIRECTORY;
 use rustfs_config::{DEFAULT_LOG_LEVEL, ENVIRONMENT, observability::DEFAULT_OBS_ENVIRONMENT_PRODUCTION};
 use rustfs_utils::get_env_opt_str;

crates/obs/src/telemetry/otel.rs

@@ -42,7 +42,7 @@ use crate::global::set_observability_metric_enabled;
 use crate::telemetry::filter::build_env_filter;
 use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
 use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
-use crate::telemetry::recorder::Recorder;
+use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
 use crate::telemetry::resource::build_resource;
 use crate::telemetry::rolling::{RollingAppender, Rotation};
 // Import helper functions from local.rs (sibling module)
@@ -432,7 +432,7 @@ fn build_meter_provider(
         .build();
 
     global::set_meter_provider(provider.clone());
-    metrics::set_global_recorder(recorder).map_err(|e| TelemetryError::InstallMetricsRecorder(e.to_string()))?;
+    install_process_global_recorder(recorder).map_err(|e| TelemetryError::InstallMetricsRecorder(e.to_string()))?;
     set_observability_metric_enabled(true);
     Ok(Some(provider))
 }

crates/obs/src/telemetry/recorder.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::GlobalError;
-use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit};
+use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Label, Metadata, SharedString, Unit};
 use opentelemetry::{
     InstrumentationScope, InstrumentationScopeBuilder, KeyValue, global,
     metrics::{Meter, MeterProvider},
@@ -24,7 +24,7 @@ use std::{
     collections::HashMap,
     ops::Deref,
     sync::{
-        Arc, Mutex, RwLock,
+        Arc, Mutex, OnceLock, RwLock,
         atomic::{AtomicU64, Ordering},
     },
 };
@@ -33,6 +33,7 @@ use tracing::error;
 const LOG_COMPONENT_OBS: &str = "obs";
 const LOG_SUBSYSTEM_RECORDER: &str = "recorder";
 const EVENT_RECORDER_STATE: &str = "recorder_state";
+static GLOBAL_RECORDER: OnceLock<Recorder> = OnceLock::new();
 
 macro_rules! configure_builder {
     ($builder:expr, $metadata:expr) => {{
@@ -96,6 +97,7 @@ impl Builder {
     pub fn install(self) -> Result<(SdkMeterProvider, Recorder), GlobalError> {
         let (provider, recorder) = self.build();
         metrics::set_global_recorder(recorder.clone())?;
+        remember_global_recorder(&recorder);
 
         Ok((provider, recorder))
     }
@@ -178,6 +180,18 @@ impl Recorder {
         value
     }
 
+    fn remove_cached_metric<T>(lock: &RwLock<HashMap<Key, T>>, key: &Key, metric_type: &str) -> bool {
+        let mut cache = match lock.write() {
+            Ok(g) => g,
+            Err(e) => {
+                error!(event = EVENT_RECORDER_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_RECORDER, metric_type = %metric_type, result = "cache_remove_lock_poisoned", error = %e, "recorder state changed");
+                e.into_inner()
+            }
+        };
+
+        cache.remove(key).is_some()
+    }
+
     fn with_metadata_lock<F, R>(&self, f: F) -> R
     where
         F: FnOnce(&mut HashMap<KeyName, MetricMetadata>) -> R,
@@ -198,6 +212,39 @@ impl Recorder {
     fn get_metadata_for_builder(&self, key_name: &str) -> Option<MetricMetadata> {
         self.with_metadata_lock(|metadata| metadata.get(key_name).cloned())
     }
+
+    fn retire_metric_series_key(&self, key: &Key) -> usize {
+        let mut retired = 0usize;
+        retired += usize::from(Self::remove_cached_metric(&self.cached_counters, key, "counter"));
+        retired += usize::from(Self::remove_cached_metric(&self.cached_gauges, key, "gauge"));
+        retired += usize::from(Self::remove_cached_metric(&self.cached_histograms, key, "histogram"));
+        retired
+    }
+}
+
+fn remember_global_recorder(recorder: &Recorder) {
+    let _ = GLOBAL_RECORDER.set(recorder.clone());
+}
+
+pub(crate) fn install_process_global_recorder(recorder: Recorder) -> Result<(), metrics::SetRecorderError<Recorder>> {
+    metrics::set_global_recorder(recorder.clone())?;
+    remember_global_recorder(&recorder);
+    Ok(())
+}
+
+pub fn retire_metric_series(name: &str, labels: &[(&'static str, Cow<'static, str>)]) -> usize {
+    let Some(recorder) = GLOBAL_RECORDER.get() else {
+        return 0;
+    };
+
+    let key = Key::from_parts(
+        name.to_string(),
+        labels
+            .iter()
+            .map(|(key, value)| Label::new((*key).to_string(), value.to_string()))
+            .collect::<Vec<_>>(),
+    );
+    recorder.retire_metric_series_key(&key)
 }
 
 impl Deref for Recorder {
@@ -499,4 +546,48 @@ mod tests {
 
         let _ = recorder.register_counter(&second, &meta);
     }
+
+    #[test]
+    fn retire_metric_series_key_removes_cached_counter() {
+        let recorder = test_recorder();
+        let key = Key::from_parts("retired_counter", vec![metrics::Label::new("bucket", "tmp")]);
+        let meta = test_metadata();
+
+        let _counter = recorder.register_counter(&key, &meta);
+        assert_eq!(recorder.cached_counters.read().unwrap().len(), 1);
+
+        let retired = recorder.retire_metric_series_key(&key);
+        assert_eq!(retired, 1);
+        assert!(recorder.cached_counters.read().unwrap().is_empty());
+    }
+
+    #[test]
+    fn retiring_churned_series_bounds_the_cache() {
+        // #1026: under bucket/target churn the recorder handle cache used to grow
+        // unbounded. After each churned series is retired the cache must fall back
+        // to a bounded size, and a same-named label set must re-register cleanly.
+        let recorder = test_recorder();
+        let meta = test_metadata();
+
+        let mut keys = Vec::with_capacity(3000);
+        for i in 0..3000 {
+            let key = Key::from_parts("churn_counter", vec![metrics::Label::new("bucket", format!("bucket-{i}"))]);
+            let _ = recorder.register_counter(&key, &meta);
+            keys.push(key);
+        }
+        assert_eq!(recorder.cached_counters.read().unwrap().len(), 3000);
+
+        for key in &keys {
+            assert_eq!(recorder.retire_metric_series_key(key), 1);
+        }
+        assert!(
+            recorder.cached_counters.read().unwrap().is_empty(),
+            "recorder cache must be bounded after churned series are retired"
+        );
+
+        // A previously retired label set can be registered and observed again.
+        let reused = &keys[0];
+        let _ = recorder.register_counter(reused, &meta);
+        assert_eq!(recorder.cached_counters.read().unwrap().len(), 1);
+    }
 }