refactor(obs): migrate metrics runtime/schema and tighten migration guards (#2584)

crates/targets/src/error.rs

@@ -82,6 +82,9 @@ pub enum TargetError {
     #[error("Target is disabled")]
     Disabled,
 
+    #[error("Queued payload dropped: {0}")]
+    Dropped(String),
+
     #[error("Configuration parsing error: {0}")]
     ParseError(String),
 

crates/targets/src/lib.rs

@@ -24,7 +24,7 @@ pub use error::{StoreError, TargetError};
 pub use rustfs_s3_common::EventName;
 use serde::{Deserialize, Serialize};
 pub use sys::user_agent::*;
-pub use target::Target;
+pub use target::{Target, TargetDeliverySnapshot};
 
 /// Represents a log of events for sending to targets
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/targets/src/target/mod.rs

@@ -21,12 +21,49 @@ use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use std::fmt::Formatter;
 use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 use tracing::warn;
 
 pub mod mqtt;
 pub mod webhook;
 
+/// A read-only snapshot of delivery counters for a target.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct TargetDeliverySnapshot {
+    pub failed_messages: u64,
+    pub queue_length: u64,
+    pub total_messages: u64,
+}
+
+/// Shared target delivery counters.
+#[derive(Debug, Default)]
+pub struct TargetDeliveryCounters {
+    failed_messages: AtomicU64,
+    total_messages: AtomicU64,
+}
+
+impl TargetDeliveryCounters {
+    #[inline]
+    pub fn record_success(&self) {
+        self.total_messages.fetch_add(1, Ordering::Relaxed);
+    }
+
+    #[inline]
+    pub fn record_final_failure(&self) {
+        self.failed_messages.fetch_add(1, Ordering::Relaxed);
+    }
+
+    #[inline]
+    pub fn snapshot(&self, queue_length: u64) -> TargetDeliverySnapshot {
+        TargetDeliverySnapshot {
+            failed_messages: self.failed_messages.load(Ordering::Relaxed),
+            queue_length,
+            total_messages: self.total_messages.load(Ordering::Relaxed),
+        }
+    }
+}
+
 /// Trait for notification targets
 #[async_trait]
 pub trait Target<E>: Send + Sync + 'static
@@ -70,8 +107,9 @@ where
                         "Failed to delete invalid queued payload {key} after decode error '{err}': {delete_err}"
                     ))
                 })?;
+                self.record_final_failure();
                 warn!("Dropped invalid queued payload {key}: {err}");
-                return Ok(());
+                return Err(TargetError::Dropped(format!("Dropped invalid queued payload {key}: {err}")));
             }
         };
 
@@ -96,6 +134,17 @@ where
 
     /// Check if the target is enabled
     fn is_enabled(&self) -> bool;
+
+    /// Returns a read-only delivery snapshot for metrics collection.
+    fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
+        TargetDeliverySnapshot {
+            queue_length: self.store().map_or(0, |store| store.len() as u64),
+            ..TargetDeliverySnapshot::default()
+        }
+    }
+
+    /// Records a final, non-retryable delivery failure for metrics collection.
+    fn record_final_failure(&self) {}
 }
 
 #[derive(Debug, Serialize, Clone, Deserialize)]

crates/targets/src/target/mqtt.rs

@@ -17,7 +17,10 @@ use crate::{
     arn::TargetID,
     error::TargetError,
     store::{Key, QueueStore, Store},
-    target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
+    target::{
+        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
+        TargetType,
+    },
 };
 use async_trait::async_trait;
 use hyper_rustls::ConfigBuilderExt;
@@ -494,6 +497,7 @@ where
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     connected: Arc<AtomicBool>,
     bg_task_manager: Arc<BgTaskManager>,
+    delivery_counters: Arc<TargetDeliveryCounters>,
     _phantom: PhantomData<E>,
 }
 
@@ -546,6 +550,7 @@ where
             store: queue_store,
             connected: Arc::new(AtomicBool::new(false)),
             bg_task_manager,
+            delivery_counters: Arc::new(TargetDeliveryCounters::default()),
             _phantom: PhantomData,
         })
     }
@@ -685,6 +690,7 @@ where
             })?;
 
         debug!(target_id = %self.id, topic = %self.args.topic, "Event published to MQTT topic");
+        self.delivery_counters.record_success();
         Ok(())
     }
 
@@ -696,6 +702,7 @@ where
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             connected: self.connected.clone(),
             bg_task_manager: self.bg_task_manager.clone(),
+            delivery_counters: self.delivery_counters.clone(),
             _phantom: PhantomData,
         })
     }
@@ -890,21 +897,31 @@ where
 
     #[instrument(skip(self, event), fields(target_id = %self.id))]
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
-        let queued = self.build_queued_payload(&event)?;
+        let queued = match self.build_queued_payload(&event) {
+            Ok(queued) => queued,
+            Err(err) => {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+        };
 
         if let Some(store) = &self.store {
             debug!(target_id = %self.id, "Event saved to store start");
-            match store.put_raw(
-                &queued
-                    .encode()
-                    .map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
-            ) {
+            let encoded = match queued.encode() {
+                Ok(encoded) => encoded,
+                Err(err) => {
+                    self.delivery_counters.record_final_failure();
+                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
+                }
+            };
+            match store.put_raw(&encoded) {
                 Ok(_) => {
                     debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
                     Ok(())
                 }
                 Err(e) => {
                     error!(target_id = %self.id, error = %e, "Failed to save event to store");
+                    self.delivery_counters.record_final_failure();
                     return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
                 }
             }
@@ -920,15 +937,21 @@ where
                     Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
                     Err(e) => {
                         error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
+                        self.delivery_counters.record_final_failure();
                         return Err(TargetError::NotConnected);
                     }
                 }
                 if !self.connected.load(Ordering::SeqCst) {
                     error!(target_id = %self.id, "Cannot save (send directly) as target is not active after init attempt.");
+                    self.delivery_counters.record_final_failure();
                     return Err(TargetError::NotConnected);
                 }
             }
-            self.send_body(queued.body, &queued.meta).await
+            if let Err(err) = self.send_body(queued.body, &queued.meta).await {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+            Ok(())
         }
     }
 
@@ -1017,6 +1040,15 @@ where
     fn is_enabled(&self) -> bool {
         self.args.enable
     }
+
+    fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
+        self.delivery_counters
+            .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
+    }
+
+    fn record_final_failure(&self) {
+        self.delivery_counters.record_final_failure();
+    }
 }
 
 #[cfg(test)]

crates/targets/src/target/webhook.rs

@@ -17,7 +17,10 @@ use crate::{
     arn::TargetID,
     error::TargetError,
     store::{Key, QueueStore, Store},
-    target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
+    target::{
+        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
+        TargetType,
+    },
 };
 use async_trait::async_trait;
 use reqwest::{Client, StatusCode, Url};
@@ -108,6 +111,7 @@ where
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     initialized: AtomicBool,
     cancel_sender: mpsc::Sender<()>,
+    delivery_counters: Arc<TargetDeliveryCounters>,
     _phantom: PhantomData<E>,
 }
 
@@ -124,6 +128,7 @@ where
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
             cancel_sender: self.cancel_sender.clone(),
+            delivery_counters: Arc::clone(&self.delivery_counters),
             _phantom: PhantomData,
         })
     }
@@ -172,6 +177,7 @@ where
             store: queue_store,
             initialized: AtomicBool::new(false),
             cancel_sender,
+            delivery_counters: Arc::new(TargetDeliveryCounters::default()),
             _phantom: PhantomData,
         })
     }
@@ -320,6 +326,7 @@ where
         let status = resp.status();
         if status.is_success() {
             debug!("Event sent to webhook target: {}", self.id);
+            self.delivery_counters.record_success();
             Ok(())
         } else if status == StatusCode::FORBIDDEN {
             Err(TargetError::Authentication(format!(
@@ -369,16 +376,26 @@ where
     }
 
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
-        let queued = self.build_queued_payload(&event)?;
+        let queued = match self.build_queued_payload(&event) {
+            Ok(queued) => queued,
+            Err(err) => {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+        };
 
         if let Some(store) = &self.store {
-            store
-                .put_raw(
-                    &queued
-                        .encode()
-                        .map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
-                )
-                .map_err(|e| TargetError::Storage(format!("Failed to save event to store: {e}")))?;
+            let encoded = match queued.encode() {
+                Ok(encoded) => encoded,
+                Err(err) => {
+                    self.delivery_counters.record_final_failure();
+                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
+                }
+            };
+            if let Err(e) = store.put_raw(&encoded) {
+                self.delivery_counters.record_final_failure();
+                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+            }
             debug!("Event saved to store for target: {}", self.id);
             Ok(())
         } else {
@@ -386,10 +403,15 @@ where
                 Ok(_) => (),
                 Err(e) => {
                     error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
+                    self.delivery_counters.record_final_failure();
                     return Err(TargetError::NotConnected);
                 }
             }
-            self.send_body(queued.body, &queued.meta).await
+            if let Err(err) = self.send_body(queued.body, &queued.meta).await {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+            Ok(())
         }
     }
 
@@ -443,6 +465,15 @@ where
     fn is_enabled(&self) -> bool {
         self.args.enable
     }
+
+    fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
+        self.delivery_counters
+            .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
+    }
+
+    fn record_final_failure(&self) {
+        self.delivery_counters.record_final_failure();
+    }
 }
 
 #[cfg(test)]