fix(obs): export resettable metrics as gauges (#4432)

Co-authored-by: heihutu <heihutu@gmail.com>

crates/obs/src/metrics/collectors/audit.rs

@@ -72,6 +72,7 @@ pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::metrics::schema::MetricType;
 
     #[test]
     fn test_collect_audit_metrics() {
@@ -106,4 +107,11 @@ mod tests {
         let metrics = collect_audit_metrics(&stats);
         assert!(metrics.is_empty());
     }
+
+    #[test]
+    fn audit_target_totals_are_exported_as_gauges() {
+        assert_eq!(AUDIT_FAILED_MESSAGES_MD.metric_type, MetricType::Gauge);
+        assert_eq!(AUDIT_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge);
+        assert_eq!(AUDIT_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge);
+    }
 }

crates/obs/src/metrics/collectors/cluster_iam.rs

@@ -87,6 +87,7 @@ pub fn collect_iam_metrics(stats: &IamStats) -> Vec<PrometheusMetric> {
 mod tests {
     use super::*;
     use crate::metrics::report::report_metrics;
+    use crate::metrics::schema::MetricType;
 
     #[test]
     fn test_collect_iam_metrics() {
@@ -125,4 +126,25 @@ mod tests {
             assert!(metric.labels.is_empty());
         }
     }
+
+    #[test]
+    fn iam_metric_descriptors_preserve_gauge_vs_counter_semantics() {
+        let gauge_descriptors = [
+            &LAST_SYNC_DURATION_MILLIS_MD,
+            &PLUGIN_AUTHN_SERVICE_FAILED_REQUESTS_MINUTE_MD,
+            &PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD,
+            &PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD,
+            &PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD,
+            &PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD,
+            &PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD,
+            &SINCE_LAST_SYNC_MILLIS_MD,
+        ];
+
+        for descriptor in gauge_descriptors {
+            assert_eq!(descriptor.metric_type, MetricType::Gauge);
+        }
+
+        assert_eq!(SYNC_FAILURES_MD.metric_type, MetricType::Counter);
+        assert_eq!(SYNC_SUCCESSES_MD.metric_type, MetricType::Counter);
+    }
 }

crates/obs/src/metrics/collectors/notification_target.rs

@@ -63,6 +63,7 @@ pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) ->
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::metrics::schema::MetricType;
 
     #[test]
     fn test_collect_notification_target_metrics() {
@@ -89,4 +90,11 @@ mod tests {
                     .any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
         }));
     }
+
+    #[test]
+    fn notification_target_totals_are_exported_as_gauges() {
+        assert_eq!(NOTIFICATION_TARGET_FAILED_MESSAGES_MD.metric_type, MetricType::Gauge);
+        assert_eq!(NOTIFICATION_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge);
+        assert_eq!(NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge);
+    }
 }