gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-10 15:16:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(admin): add audit target APIs and harden target source handling (#2350)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-04 09:07:22 +08:00
committed by GitHub
parent 67863630b2
commit d2901fd78c
29 changed files with 3534 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/targets/src/store.rs
+362 -110
View File
@@ -13,20 +13,34 @@
// limitations under the License.
use crate::error::StoreError;
use rustfs_config::DEFAULT_LIMIT;
use rustfs_config::notify::{COMPRESS_EXT, DEFAULT_EXT};
use rustfs_config::{DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, ENV_TARGET_STORE_COMPRESS, EnableState};
use serde::{Serialize, de::DeserializeOwned};
use snap::raw::{Decoder, Encoder};
use std::sync::{Arc, RwLock};
use std::{
collections::HashMap,
marker::PhantomData,
path::PathBuf,
sync::{
Arc, RwLock,
atomic::{AtomicU64, Ordering},
},
time::{SystemTime, UNIX_EPOCH},
};
use tracing::{debug, warn};
use uuid::Uuid;
fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool {
value
.and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled()))
.unwrap_or(DEFAULT_TARGET_STORE_COMPRESS)
}
fn queue_store_compression_enabled() -> bool {
let value = std::env::var(ENV_TARGET_STORE_COMPRESS).ok();
resolve_queue_store_compression_from_env_value(value.as_deref())
}
/// Represents a key for an entry in the store
#[derive(Debug, Clone)]
pub struct Key {
@@ -63,21 +77,7 @@ impl Key {
impl std::fmt::Display for Key {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let name_part = if self.item_count > 1 {
format!("{}:{}", self.item_count, self.name)
} else {
self.name.clone()
};
let mut file_name = name_part;
if !self.extension.is_empty() {
file_name.push_str(&self.extension);
}
if self.compress {
file_name.push_str(COMPRESS_EXT);
}
write!(f, "{file_name}")
f.write_str(&self.to_key_string())
}
}
@@ -123,6 +123,28 @@ pub fn parse_key(s: &str) -> Key {
}
}
pub fn ensure_store_entry_raw_readable<T>(
store: &(dyn Store<T, Error = StoreError, Key = Key> + Send),
key: &Key,
) -> Result<bool, StoreError>
where
T: Send + Sync + 'static + Clone + Serialize,
{
match store.get_raw(key) {
Ok(_) => Ok(true),
Err(StoreError::NotFound) => Ok(false),
Err(err) => {
match store.del(key) {
Ok(()) | Err(StoreError::NotFound) => {}
Err(del_err) => {
return Err(StoreError::Internal(format!("Failed to remove unreadable store entry {key}: {del_err}")));
}
}
Err(err)
}
}
}
/// Trait for a store that can store and retrieve items of type T
pub trait Store<T>: Send + Sync
where
@@ -142,15 +164,24 @@ where
/// Stores multiple items in a single batch
fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error>;
/// Stores raw bytes in a single entry.
fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error>;
/// Retrieves a single item by key
fn get(&self, key: &Self::Key) -> Result<T, Self::Error>;
/// Retrieves multiple items by key
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error>;
/// Retrieves the raw bytes stored for a key.
fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error>;
/// Deletes an item by key
fn del(&self, key: &Self::Key) -> Result<(), Self::Error>;
/// Deletes the underlying store directory and clears all in-memory state.
fn delete(&self) -> Result<(), Self::Error>;
/// Lists all keys in the store
fn list(&self) -> Vec<Self::Key>;
@@ -169,7 +200,10 @@ pub struct QueueStore<T> {
entry_limit: u64,
directory: PathBuf,
file_ext: String,
compress: bool,
entries: Arc<RwLock<HashMap<String, i64>>>, // key -> modtime as unix nano
pending_entries: Arc<AtomicU64>,
fs_guard: Arc<RwLock<()>>,
_phantom: PhantomData<T>,
}
@@ -179,35 +213,70 @@ impl<T> Clone for QueueStore<T> {
entry_limit: self.entry_limit,
directory: self.directory.clone(),
file_ext: self.file_ext.clone(),
compress: self.compress,
entries: Arc::clone(&self.entries),
pending_entries: Arc::clone(&self.pending_entries),
fs_guard: Arc::clone(&self.fs_guard),
_phantom: PhantomData,
}
}
}
struct EntryReservation<'a> {
pending_entries: &'a AtomicU64,
}
impl Drop for EntryReservation<'_> {
fn drop(&mut self) {
self.pending_entries.fetch_sub(1, Ordering::SeqCst);
}
}
impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
/// Creates a new QueueStore
pub fn new(directory: impl Into<PathBuf>, limit: u64, ext: &str) -> Self {
Self::new_with_compression(directory, limit, ext, queue_store_compression_enabled())
}
/// Creates a new QueueStore with an explicit compression setting.
pub fn new_with_compression(directory: impl Into<PathBuf>, limit: u64, ext: &str, compress: bool) -> Self {
let file_ext = if ext.is_empty() { DEFAULT_EXT } else { ext };
let entry_limit = if limit == 0 { DEFAULT_LIMIT } else { limit };
QueueStore {
directory: directory.into(),
entry_limit: if limit == 0 { DEFAULT_LIMIT } else { limit },
entry_limit,
file_ext: file_ext.to_string(),
entries: Arc::new(RwLock::new(HashMap::with_capacity(limit as usize))),
compress,
entries: Arc::new(RwLock::new(HashMap::with_capacity(entry_limit as usize))),
pending_entries: Arc::new(AtomicU64::new(0)),
fs_guard: Arc::new(RwLock::new(())),
_phantom: PhantomData,
}
}
/// Returns the full path for a key
fn file_path(&self, key: &Key) -> PathBuf {
self.directory.join(key.to_string())
self.directory.join(key.to_key_string())
}
fn build_key(&self, item_count: usize) -> Key {
Key {
name: Uuid::new_v4().to_string(),
extension: self.file_ext.clone(),
item_count,
compress: self.compress,
}
}
/// Reads a file for the given key
fn read_file(&self, key: &Key) -> Result<Vec<u8>, StoreError> {
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let path = self.file_path(key);
debug!("Reading file for key: {},path: {}", key.to_string(), path.display());
debug!("Reading file for key: {},path: {}", key, path.display());
let data = std::fs::read(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
StoreError::NotFound
@@ -220,41 +289,89 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
return Err(StoreError::NotFound);
}
if key.compress {
let mut decoder = Decoder::new();
decoder
.decompress_vec(&data)
.map_err(|e| StoreError::Compression(e.to_string()))
} else {
Ok(data)
if !key.compress {
return Ok(data);
}
let mut decoder = Decoder::new();
decoder
.decompress_vec(&data)
.map_err(|e| StoreError::Compression(e.to_string()))
}
fn reserve_entry_slot(&self) -> Result<EntryReservation<'_>, StoreError> {
loop {
let entries = self
.entries
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
let entries_len = entries.len() as u64;
let pending = self.pending_entries.load(Ordering::SeqCst);
if entries_len + pending >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
if self
.pending_entries
.compare_exchange(pending, pending + 1, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
return Ok(EntryReservation {
pending_entries: self.pending_entries.as_ref(),
});
}
}
}
/// Writes data to a file for the given key
fn write_file(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> {
/// Writes data to a file for the given key.
fn write_file(&self, key: &Key, data: &[u8]) -> Result<i64, StoreError> {
let path = self.file_path(key);
// Create directory if it doesn't exist
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(StoreError::Io)?;
}
let data = if key.compress {
if key.compress {
let mut encoder = Encoder::new();
encoder
let compressed = encoder
.compress_vec(data)
.map_err(|e| StoreError::Compression(e.to_string()))?
.map_err(|e| StoreError::Compression(e.to_string()))?;
std::fs::write(&path, &compressed).map_err(StoreError::Io)?;
} else {
data.to_vec()
};
std::fs::write(&path, &data).map_err(StoreError::Io)?;
std::fs::write(&path, data).map_err(StoreError::Io)?;
}
let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
debug!("Wrote event to store: {}", key);
Ok(modified)
}
fn insert_entry(&self, key: &Key, modified: i64) -> Result<(), StoreError> {
let mut entries = self
.entries
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
entries.insert(key.to_string(), modified);
debug!("Wrote event to store: {}", key.to_string());
entries.insert(key.to_key_string(), modified);
Ok(())
}
fn remove_file_if_present(&self, key: &Key) -> Result<(), StoreError> {
let path = self.file_path(key);
match std::fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(StoreError::Io(err)),
}
}
fn write_and_index(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> {
let modified = self.write_file(key, data)?;
if let Err(err) = self.insert_entry(key, modified) {
self.remove_file_if_present(key).map_err(|cleanup_err| {
StoreError::Internal(format!("Failed to index store entry {key}: {err}; cleanup failed: {cleanup_err}"))
})?;
return Err(err);
}
Ok(())
}
}
@@ -267,15 +384,20 @@ where
type Key = Key;
fn open(&self) -> Result<(), Self::Error> {
let _fs_guard = self
.fs_guard
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?;
std::fs::create_dir_all(&self.directory).map_err(StoreError::Io)?;
let entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?;
// Get the write lock to update the internal state
let dir_entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?;
let mut entries_map = self
.entries
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
for entry in entries {
self.pending_entries.store(0, Ordering::SeqCst);
entries_map.clear();
for entry in dir_entries {
let entry = entry.map_err(StoreError::Io)?;
let metadata = entry.metadata().map_err(StoreError::Io)?;
if metadata.is_file() {
@@ -292,71 +414,47 @@ where
}
fn put(&self, item: Arc<T>) -> Result<Self::Key, Self::Error> {
// Check storage limits
{
let entries = self
.entries
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
if entries.len() as u64 >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
}
let uuid = Uuid::new_v4();
let key = Key {
name: uuid.to_string(),
extension: self.file_ext.clone(),
item_count: 1,
compress: true,
};
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let _reservation = self.reserve_entry_slot()?;
let key = self.build_key(1);
let data = serde_json::to_vec(&*item).map_err(|e| StoreError::Serialization(e.to_string()))?;
self.write_file(&key, &data)?;
self.write_and_index(&key, &data)?;
Ok(key)
}
fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error> {
// Check storage limits
{
let entries = self
.entries
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
if entries.len() as u64 >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
}
if items.is_empty() {
// Or return an error, or a special key?
return Err(StoreError::Internal("Cannot put_multiple with empty items list".to_string()));
}
let uuid = Uuid::new_v4();
let key = Key {
name: uuid.to_string(),
extension: self.file_ext.clone(),
item_count: items.len(),
compress: true,
};
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let _reservation = self.reserve_entry_slot()?;
let key = self.build_key(items.len());
// Serialize all items into a single Vec<u8>
// This current approach for get_multiple/put_multiple assumes items are concatenated JSON objects.
// This might be problematic for deserialization if not handled carefully.
// A better approach for multiple items might be to store them as a JSON array `Vec<T>`.
// For now, sticking to current logic of concatenating.
let mut buffer = Vec::new();
for item in items {
// If items are Vec<Event>, and Event is large, this could be inefficient.
// The current get_multiple deserializes one by one.
let item_data = serde_json::to_vec(&item).map_err(|e| StoreError::Serialization(e.to_string()))?;
buffer.extend_from_slice(&item_data);
// If using JSON array: buffer = serde_json::to_vec(&items)?
serde_json::to_writer(&mut buffer, &item).map_err(|e| StoreError::Serialization(e.to_string()))?;
}
self.write_file(&key, &buffer)?;
self.write_and_index(&key, &buffer)?;
Ok(key)
}
fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let _reservation = self.reserve_entry_slot()?;
let key = self.build_key(1);
self.write_and_index(&key, data)?;
Ok(key)
}
@@ -373,8 +471,8 @@ where
}
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
debug!("Reading items from store for key: {}", key.to_string());
let data = self.read_file(key)?;
debug!("Reading items from store for key: {}", key);
let data = self.get_raw(key)?;
if data.is_empty() {
return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string()));
}
@@ -404,7 +502,7 @@ where
warn!(
"Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.",
key.item_count,
key.to_string(),
key,
items.len()
);
// Depending on strictness, this could be an error.
@@ -426,20 +524,24 @@ where
Ok(items)
}
fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
self.read_file(key)
}
fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let path = self.file_path(key);
std::fs::remove_file(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
// If file not found, still try to remove from entries map in case of inconsistency
warn!(
"File not found for key {} during del, but proceeding to remove from entries map.",
key.to_string()
);
StoreError::NotFound
} else {
StoreError::Io(e)
match std::fs::remove_file(&path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// File already gone — still clean up the entries map to avoid stale keys.
warn!("File not found for key {} during del, cleaning up entries map.", key);
}
})?;
Err(e) => return Err(StoreError::Io(e)),
}
// Get the write lock to update the internal state
let mut entries = self
@@ -447,15 +549,32 @@ where
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
if entries.remove(&key.to_string()).is_none() {
// Key was not in the map, could be an inconsistency or already deleted.
// This is not necessarily an error if the file deletion succeeded or was NotFound.
if entries.remove(&key.to_key_string()).is_none() {
debug!("Key {} not found in entries map during del, might have been already removed.", key);
}
debug!("Deleted event from store: {}", key.to_string());
Ok(())
}
fn delete(&self) -> Result<(), Self::Error> {
let _fs_guard = self
.fs_guard
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?;
let mut entries = self
.entries
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
entries.clear();
self.pending_entries.store(0, Ordering::SeqCst);
match std::fs::remove_dir_all(&self.directory) {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(StoreError::Io(err)),
}
}
fn list(&self) -> Vec<Self::Key> {
// Get the read lock to read the internal state
let entries = match self.entries.read() {
@@ -492,3 +611,136 @@ where
Box::new(self.clone()) as Box<dyn Store<T, Error = Self::Error, Key = Self::Key> + Send + Sync>
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::{
sync::{Arc, Barrier},
thread,
};
fn temp_store_dir(name: &str) -> PathBuf {
std::env::temp_dir().join(format!("rustfs-targets-{name}-{}", Uuid::new_v4()))
}
#[test]
fn resolve_queue_store_compression_defaults_to_true() {
assert!(resolve_queue_store_compression_from_env_value(None));
}
#[test]
fn resolve_queue_store_compression_respects_disabled_env_value() {
assert!(!resolve_queue_store_compression_from_env_value(Some("off")));
assert!(!resolve_queue_store_compression_from_env_value(Some("false")));
}
#[test]
fn put_uses_store_compression_setting_in_key() {
let dir = temp_store_dir("put-key");
let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", false);
store.open().unwrap();
let key = store.put(Arc::new("payload".to_string())).unwrap();
assert!(!key.compress);
assert!(store.file_path(&key).exists());
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn parse_key_round_trips_batch_and_compression_suffixes() {
let key = Key {
name: "event-id".to_string(),
extension: ".json".to_string(),
item_count: 3,
compress: true,
};
let parsed = parse_key(&key.to_key_string());
assert_eq!(parsed.name, key.name);
assert_eq!(parsed.extension, key.extension);
assert_eq!(parsed.item_count, key.item_count);
assert_eq!(parsed.compress, key.compress);
}
#[test]
fn put_raw_and_get_raw_round_trip_bytes() {
let dir = temp_store_dir("raw-roundtrip");
let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", true);
store.open().unwrap();
let payload = br#"{"kind":"notify","bucket":"demo","key":"alpha.txt"}"#;
let key = store.put_raw(payload).unwrap();
let raw = store.get_raw(&key).unwrap();
assert_eq!(raw, payload);
let _ = store.delete();
}
#[test]
fn delete_removes_directory_and_clears_entries() {
let dir = temp_store_dir("delete-store");
let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", false);
store.open().unwrap();
let _ = store.put(Arc::new("payload".to_string())).unwrap();
store.delete().unwrap();
assert!(store.list().is_empty());
assert!(!dir.exists());
}
#[test]
fn put_enforces_entry_limit() {
let dir = temp_store_dir("limit");
let store = QueueStore::<String>::new_with_compression(&dir, 1, ".test", false);
store.open().unwrap();
let _ = store.put(Arc::new("first".to_string())).unwrap();
let err = store.put(Arc::new("second".to_string())).unwrap_err();
assert!(matches!(err, StoreError::LimitExceeded));
let _ = store.delete();
}
#[test]
fn concurrent_put_raw_respects_entry_limit() {
let dir = temp_store_dir("concurrent-limit");
let store = Arc::new(QueueStore::<String>::new_with_compression(&dir, 1, ".test", true));
store.open().unwrap();
let start = Arc::new(Barrier::new(4));
let mut handles = Vec::new();
for idx in 0..4 {
let store = Arc::clone(&store);
let start = Arc::clone(&start);
handles.push(thread::spawn(move || {
let payload = vec![b'x'; 32 * 1024 + idx];
start.wait();
store.put_raw(&payload)
}));
}
let mut successes = 0;
let mut limit_errors = 0;
for handle in handles {
match handle.join().unwrap() {
Ok(_) => successes += 1,
Err(StoreError::LimitExceeded) => limit_errors += 1,
Err(err) => panic!("unexpected error: {err}"),
}
}
assert_eq!(successes, 1);
assert_eq!(limit_errors, 3);
assert_eq!(store.len(), 1);
let _ = store.delete();
}
}
crates/targets/src/target/mod.rs
+176 -3
View File
@@ -21,6 +21,8 @@ use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::fmt::Formatter;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::warn;
pub mod mqtt;
pub mod webhook;
@@ -45,14 +47,43 @@ where
/// Saves an event (either sends it immediately or stores it for later)
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
/// Sends an event from the store
async fn send_from_store(&self, key: Key) -> Result<(), TargetError>;
/// Sends an event from the store using the queued raw body and metadata.
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError>;
/// Sends an event from the store.
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
let store = self
.store()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
let raw = match store.get_raw(&key) {
Ok(raw) => raw,
Err(StoreError::NotFound) => return Ok(()),
Err(err) => return Err(TargetError::Storage(format!("Failed to read queued payload from store: {err}"))),
};
let queued = match QueuedPayload::decode(&raw) {
Ok(queued) => queued,
Err(err) => {
delete_stored_payload(store, &key).map_err(|delete_err| {
TargetError::Storage(format!(
"Failed to delete invalid queued payload {key} after decode error '{err}': {delete_err}"
))
})?;
warn!("Dropped invalid queued payload {key}: {err}");
return Ok(());
}
};
self.send_raw_from_store(key.clone(), queued.body, queued.meta).await?;
delete_stored_payload(store, &key)
}
/// Closes the target and releases resources
async fn close(&self) -> Result<(), TargetError>;
/// Returns the store associated with the target (if any)
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)>;
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)>;
/// Returns the type of the target
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync>;
@@ -78,6 +109,106 @@ where
pub data: E,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QueuedPayloadMeta {
pub event_name: EventName,
pub bucket_name: String,
pub object_name: String,
pub content_type: String,
pub queued_at_unix_ms: u64,
pub payload_len: usize,
}
impl QueuedPayloadMeta {
pub fn new(
event_name: EventName,
bucket_name: String,
object_name: String,
content_type: impl Into<String>,
payload_len: usize,
) -> Self {
Self {
event_name,
bucket_name,
object_name,
content_type: content_type.into(),
queued_at_unix_ms: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64,
payload_len,
}
}
pub fn best_effort_preview(&self, body: &[u8], limit: usize) -> String {
if limit == 0 || body.is_empty() {
return String::new();
}
let slice = &body[..body.len().min(limit)];
match std::str::from_utf8(slice) {
Ok(text) => {
if body.len() > limit {
format!("{text}...")
} else {
text.to_string()
}
}
Err(_) => format!("<{} bytes binary>", body.len()),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QueuedPayload {
pub meta: QueuedPayloadMeta,
pub body: Vec<u8>,
}
impl QueuedPayload {
const MAGIC: [u8; 4] = *b"RQP1";
pub fn new(meta: QueuedPayloadMeta, body: Vec<u8>) -> Self {
Self { meta, body }
}
pub fn encode(&self) -> Result<Vec<u8>, TargetError> {
let meta = serde_json::to_vec(&self.meta)
.map_err(|err| TargetError::Serialization(format!("Failed to serialize queued payload metadata: {err}")))?;
let meta_len = u32::try_from(meta.len())
.map_err(|_| TargetError::Serialization("Queued payload metadata is too large".to_string()))?;
let mut out = Vec::with_capacity(Self::MAGIC.len() + 4 + meta.len() + self.body.len());
out.extend_from_slice(&Self::MAGIC);
out.extend_from_slice(&meta_len.to_le_bytes());
out.extend_from_slice(&meta);
out.extend_from_slice(&self.body);
Ok(out)
}
pub fn decode(raw: &[u8]) -> Result<Self, TargetError> {
if raw.len() < Self::MAGIC.len() + 4 {
return Err(TargetError::Serialization("Queued payload is too short".to_string()));
}
if raw[..Self::MAGIC.len()] != Self::MAGIC {
return Err(TargetError::Serialization("Queued payload magic mismatch".to_string()));
}
let mut meta_len_bytes = [0u8; 4];
meta_len_bytes.copy_from_slice(&raw[Self::MAGIC.len()..Self::MAGIC.len() + 4]);
let meta_len = u32::from_le_bytes(meta_len_bytes) as usize;
let meta_start = Self::MAGIC.len() + 4;
let meta_end = meta_start + meta_len;
if meta_end > raw.len() {
return Err(TargetError::Serialization("Queued payload metadata length exceeds input".to_string()));
}
let meta = serde_json::from_slice(&raw[meta_start..meta_end])
.map_err(|err| TargetError::Serialization(format!("Failed to deserialize queued payload metadata: {err}")))?;
let body = raw[meta_end..].to_vec();
Ok(Self { meta, body })
}
}
/// The `ChannelTargetType` enum represents the different types of channel Target
/// used in the notification system.
///
@@ -187,3 +318,45 @@ pub fn decode_object_name(encoded: &str) -> Result<String, TargetError> {
.map(|s| s.into_owned())
.map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {e}")))
}
pub(crate) fn delete_stored_payload(
store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
key: &Key,
) -> Result<(), TargetError> {
match store.del(key) {
Ok(()) | Err(StoreError::NotFound) => Ok(()),
Err(err) => Err(TargetError::Storage(format!("Failed to delete event from store: {err}"))),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn queued_payload_round_trips_meta_and_body() {
let meta = QueuedPayloadMeta::new(
EventName::ObjectCreatedPut,
"bucket-a".to_string(),
"folder/object.txt".to_string(),
"application/json",
12,
);
let payload = QueuedPayload::new(meta.clone(), br#"{"ok":true}"#.to_vec());
let encoded = payload.encode().unwrap();
let decoded = QueuedPayload::decode(&encoded).unwrap();
assert_eq!(decoded.meta.event_name, meta.event_name);
assert_eq!(decoded.meta.bucket_name, meta.bucket_name);
assert_eq!(decoded.meta.object_name, meta.object_name);
assert_eq!(decoded.meta.content_type, meta.content_type);
assert_eq!(decoded.body, br#"{"ok":true}"#);
}
#[test]
fn queued_payload_decode_rejects_invalid_magic() {
let err = QueuedPayload::decode(b"bad-payload").unwrap_err();
assert!(err.to_string().contains("magic") || err.to_string().contains("short"));
}
}
crates/targets/src/target/mqtt.rs
+54 -72
View File
@@ -17,7 +17,7 @@ use crate::{
arn::TargetID,
error::TargetError,
store::{Key, QueueStore, Store},
target::{ChannelTargetType, EntityTarget, TargetType},
target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
};
use async_trait::async_trait;
use rumqttc::{AsyncClient, ConnectionError, EventLoop, MqttOptions, Outgoing, Packet, QoS, mqttbytes::Error as MqttBytesError};
@@ -25,6 +25,7 @@ use serde::Serialize;
use serde::de::DeserializeOwned;
use std::sync::Arc;
use std::{
marker::PhantomData,
path::PathBuf,
sync::atomic::{AtomicBool, Ordering},
time::Duration,
@@ -110,9 +111,10 @@ where
id: TargetID,
args: MQTTArgs,
client: Arc<Mutex<Option<AsyncClient>>>,
store: Option<Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: Arc<AtomicBool>,
bg_task_manager: Arc<BgTaskManager>,
_phantom: PhantomData<E>,
}
impl<E> MQTTTarget<E>
@@ -135,7 +137,7 @@ where
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
};
let store = QueueStore::<EntityTarget<E>>::new(specific_queue_path, args.queue_limit, extension);
let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
if let Err(e) = store.open() {
error!(
target_id = %target_id,
@@ -144,7 +146,7 @@ where
);
return Err(TargetError::Storage(format!("{e}")));
}
Some(Box::new(store) as Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>)
Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
@@ -157,13 +159,14 @@ where
});
info!(target_id = %target_id, "MQTT target created");
Ok(MQTTTarget {
Ok(MQTTTarget::<E> {
id: target_id,
args,
client: Arc::new(Mutex::new(None)),
store: queue_store,
connected: Arc::new(AtomicBool::new(false)),
bg_task_manager,
_phantom: PhantomData,
})
}
@@ -251,14 +254,7 @@ where
}
}
#[instrument(skip(self, event), fields(target_id = %self.id))]
async fn send(&self, event: &EntityTarget<E>) -> Result<(), TargetError> {
let client_guard = self.client.lock().await;
let client = client_guard
.as_ref()
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
// Decode form-urlencoded object name
fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
let object_name = crate::target::decode_object_name(&event.object_name)?;
let key = format!("{}/{}", event.bucket_name, object_name);
@@ -269,14 +265,35 @@ where
records: vec![event.clone()],
};
let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let meta = QueuedPayloadMeta::new(
event.event_name,
event.bucket_name.clone(),
event.object_name.clone(),
"application/json",
body.len(),
);
Ok(QueuedPayload::new(meta, body))
}
let data_string = String::from_utf8(data.clone())
.map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?;
debug!("Sending event to mqtt target: {}, event log: {}", self.id, data_string);
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
let client_guard = self.client.lock().await;
let client = client_guard
.as_ref()
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
debug!(
target = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
preview = %meta.best_effort_preview(&body, 256),
"Sending MQTT payload"
);
client
.publish(&self.args.topic, self.args.qos, false, data)
.publish(&self.args.topic, self.args.qos, false, body)
.await
.map_err(|e| {
if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
@@ -293,13 +310,14 @@ where
}
pub fn clone_target(&self) -> Box<dyn Target<E> + Send + Sync> {
Box::new(MQTTTarget {
Box::new(MQTTTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
client: self.client.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: self.connected.clone(),
bg_task_manager: self.bg_task_manager.clone(),
_phantom: PhantomData,
})
}
}
@@ -494,11 +512,15 @@ where
#[instrument(skip(self, event), fields(target_id = %self.id))]
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
let queued = self.build_queued_payload(&event)?;
if let Some(store) = &self.store {
debug!(target_id = %self.id, "Event saved to store start");
// If store is configured, ONLY put the event into the store.
// Do NOT send it directly here.
match store.put(event.clone()) {
match store.put_raw(
&queued
.encode()
.map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
) {
Ok(_) => {
debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
Ok(())
@@ -516,7 +538,7 @@ where
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init.");
// Call the struct's init method, not the trait's default
match MQTTTarget::init(self).await {
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
@@ -528,13 +550,13 @@ where
return Err(TargetError::NotConnected);
}
}
self.send(&event).await
self.send_body(queued.body, &queued.meta).await
}
}
#[instrument(skip(self), fields(target_id = %self.id))]
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send event from store with key.");
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store.");
if !self.is_enabled() {
return Err(TargetError::Disabled);
@@ -542,7 +564,7 @@ where
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Not connected; trying to init before sending from store.");
match MQTTTarget::init(self).await {
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
@@ -555,33 +577,8 @@ where
}
}
let store = self
.store
.as_ref()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
let event = match store.get(&key) {
Ok(event) => {
debug!(target_id = %self.id, ?key, "Retrieved event from store for sending.");
event
}
Err(StoreError::NotFound) => {
// Assuming NotFound takes the key
debug!(target_id = %self.id, ?key, "Event not found in store for sending.");
return Ok(());
}
Err(e) => {
error!(
target_id = %self.id,
error = %e,
"Failed to get event from store"
);
return Err(TargetError::Storage(format!("Failed to get event from store: {e}")));
}
};
debug!(target_id = %self.id, ?key, "Sending event from store.");
if let Err(e) = self.send(&event).await {
if let Err(e) = self.send_body(body, &meta).await {
if matches!(e, TargetError::NotConnected) {
warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store.");
return Err(TargetError::NotConnected);
@@ -589,22 +586,7 @@ where
error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error.");
return Err(e);
}
debug!(target_id = %self.id, ?key, "Event sent from store successfully. deleting from store. ");
match store.del(&key) {
Ok(_) => {
debug!(target_id = %self.id, ?key, "Event deleted from store after successful send.")
}
Err(StoreError::NotFound) => {
debug!(target_id = %self.id, ?key, "Event already deleted from store.");
}
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to delete event from store after send.");
return Err(TargetError::Storage(format!("Failed to delete event from store: {e}")));
}
}
debug!(target_id = %self.id, ?key, "Event deleted from store.");
debug!(target_id = %self.id, ?key, "Event sent from store successfully.");
Ok(())
}
@@ -637,7 +619,7 @@ where
Ok(())
}
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
self.store.as_deref()
}
@@ -651,7 +633,7 @@ where
return Ok(());
}
// Call the internal init logic
MQTTTarget::init(self).await
MQTTTarget::<E>::init(self).await
}
fn is_enabled(&self) -> bool {
crates/targets/src/target/webhook.rs
+97 -102
View File
@@ -17,7 +17,7 @@ use crate::{
arn::TargetID,
error::TargetError,
store::{Key, QueueStore, Store},
target::{ChannelTargetType, EntityTarget, TargetType},
target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
};
use async_trait::async_trait;
use reqwest::{Client, StatusCode, Url};
@@ -26,6 +26,7 @@ use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{
marker::PhantomData,
path::PathBuf,
sync::{
Arc,
@@ -33,7 +34,6 @@ use std::{
},
time::Duration,
};
use tokio::net::lookup_host;
use tokio::sync::mpsc;
use tracing::{debug, error, info, instrument, warn};
@@ -105,10 +105,10 @@ where
args: WebhookArgs,
http_client: Arc<Client>,
// Add Send + Sync constraints to ensure thread safety
store: Option<Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
initialized: AtomicBool,
addr: String,
cancel_sender: mpsc::Sender<()>,
_phantom: PhantomData<E>,
}
impl<E> WebhookTarget<E>
@@ -117,14 +117,14 @@ where
{
/// Clones the WebhookTarget, creating a new instance with the same configuration
pub fn clone_box(&self) -> Box<dyn Target<E> + Send + Sync> {
Box::new(WebhookTarget {
Box::new(WebhookTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
http_client: Arc::clone(&self.http_client),
store: self.store.as_ref().map(|s| s.boxed_clone()),
initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
addr: self.addr.clone(),
cancel_sender: self.cancel_sender.clone(),
_phantom: PhantomData,
})
}
@@ -149,7 +149,7 @@ where
TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
};
let store = QueueStore::<EntityTarget<E>>::new(queue_dir, args.queue_limit, extension);
let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
if let Err(e) = store.open() {
error!("Failed to open store for Webhook target {}: {}", target_id.id, e);
@@ -157,32 +157,22 @@ where
}
// Make sure that the Store trait implemented by QueueStore matches the expected error type
Some(Box::new(store) as Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>)
Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
// resolved address
let addr = {
let host = args.endpoint.host_str().unwrap_or("localhost");
let port = args
.endpoint
.port()
.unwrap_or_else(|| if args.endpoint.scheme() == "https" { 443 } else { 80 });
format!("{host}:{port}")
};
// Create a cancel channel
let (cancel_sender, _) = mpsc::channel(1);
info!(target_id = %target_id.id, "Webhook target created");
Ok(WebhookTarget {
Ok(WebhookTarget::<E> {
id: target_id,
args,
http_client,
store: queue_store,
initialized: AtomicBool::new(false),
addr,
cancel_sender,
_phantom: PhantomData,
})
}
@@ -226,53 +216,80 @@ where
.map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}")))
}
async fn init(&self) -> Result<(), TargetError> {
// Use CAS operations to ensure thread-safe initialization
if !self.initialized.load(Ordering::SeqCst) {
// Check the connection
match self.is_active().await {
Ok(true) => {
info!("Webhook target {} is active", self.id);
}
Ok(false) => {
return Err(TargetError::NotConnected);
}
Err(e) => {
error!("Failed to check if Webhook target {} is active: {}", self.id, e);
return Err(e);
async fn init_inner(&self) -> Result<(), TargetError> {
if self.initialized.load(Ordering::SeqCst) {
return Ok(());
}
// HTTP HEAD probe: verifies the full request path (proxy, TLS, firewall)
// unlike TCP connect which can't detect proxy issues.
let probe_timeout = Duration::from_secs(5);
match tokio::time::timeout(probe_timeout, self.http_client.head(self.args.endpoint.as_str()).send()).await {
Ok(Ok(resp)) => {
let status = resp.status();
if status.is_success() || status == StatusCode::NOT_FOUND {
// NOT_FOUND is acceptable for HEAD probes — the endpoint may not
// exist as a HEAD route, but the server is reachable.
debug!("Webhook target {} HEAD probe returned {}", self.id, status);
} else if status == StatusCode::METHOD_NOT_ALLOWED {
// Server is reachable but doesn't support HEAD — still valid.
debug!("Webhook target {} HEAD probe: METHOD_NOT_ALLOWED (reachable)", self.id);
} else {
warn!("Webhook target {} HEAD probe returned {}", self.id, status);
}
}
self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id);
Ok(Err(e)) => {
// Connection-level error (DNS, TLS, refused, timeout)
return Err(if e.is_timeout() || e.is_connect() {
TargetError::NotConnected
} else {
TargetError::Network(format!("Webhook HEAD probe failed: {e}"))
});
}
Err(_) => {
return Err(TargetError::Timeout("Webhook HEAD probe timed out".to_string()));
}
}
self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id);
Ok(())
}
async fn send(&self, event: &EntityTarget<E>) -> Result<(), TargetError> {
info!("Webhook Sending event to webhook target: {}", self.id);
// Decode form-urlencoded object name
fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
let object_name = crate::target::decode_object_name(&event.object_name)?;
let key = format!("{}/{}", event.bucket_name, object_name);
let log = TargetLog {
event_name: event.event_name,
key,
records: vec![event.data.clone()],
};
let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let meta = QueuedPayloadMeta::new(
event.event_name,
event.bucket_name.clone(),
event.object_name.clone(),
"application/json",
body.len(),
);
Ok(QueuedPayload::new(meta, body))
}
let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
info!("Webhook sending queued payload to target: {}", self.id);
debug!(
target = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
preview = %meta.best_effort_preview(&body, 256),
"Sending webhook payload"
);
// Vec<u8> Convert to String
let data_string = String::from_utf8(data.clone())
.map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?;
debug!("Sending event to webhook target: {}, event log: {}", self.id, data_string);
// build request
let mut req_builder = self
.http_client
.post(self.args.endpoint.as_str())
.header("Content-Type", "application/json");
.header("Content-Type", meta.content_type.as_str());
if !self.args.auth_token.is_empty() {
// Split auth_token string to check if the authentication type is included
@@ -293,7 +310,7 @@ where
}
// Send a request
let resp = req_builder.body(data).send().await.map_err(|e| {
let resp = req_builder.body(body).send().await.map_err(|e| {
if e.is_timeout() || e.is_connect() {
TargetError::NotConnected
} else {
@@ -329,34 +346,39 @@ where
}
async fn is_active(&self) -> Result<bool, TargetError> {
let socket_addr = lookup_host(&self.addr)
.await
.map_err(|e| TargetError::Network(format!("Failed to resolve host: {e}")))?
.next()
.ok_or_else(|| TargetError::Network("No address found".to_string()))?;
debug!("is_active socket addr: {},target id:{}", socket_addr, self.id.id);
match tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await {
Ok(Ok(_)) => {
debug!("Connection to {} is active", self.addr);
Ok(true)
}
Ok(Err(e)) => {
debug!("Connection to {} failed: {}", self.addr, e);
if e.kind() == std::io::ErrorKind::ConnectionRefused {
Err(TargetError::NotConnected)
match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(self.args.endpoint.as_str()).send()).await {
Ok(Ok(resp)) => {
let status = resp.status();
if status.is_server_error() {
debug!("Webhook {} server error: {}", self.id, status);
Ok(false)
} else {
Err(TargetError::Network(format!("Connection failed: {e}")))
debug!("Webhook {} is reachable (status: {})", self.id, status);
Ok(true)
}
}
Err(_) => Err(TargetError::Timeout("Connection timed out".to_string())),
Ok(Err(e)) => {
debug!("Webhook {} request failed: {}", self.id, e);
if e.is_timeout() || e.is_connect() {
Err(TargetError::NotConnected)
} else {
Err(TargetError::Network(format!("Webhook health check failed: {e}")))
}
}
Err(_) => Err(TargetError::Timeout("Webhook health check timed out".to_string())),
}
}
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
let queued = self.build_queued_payload(&event)?;
if let Some(store) = &self.store {
// Call the store method directly, no longer need to acquire the lock
store
.put(event)
.put_raw(
&queued
.encode()
.map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
)
.map_err(|e| TargetError::Storage(format!("Failed to save event to store: {e}")))?;
debug!("Event saved to store for target: {}", self.id);
Ok(())
@@ -368,12 +390,12 @@ where
return Err(TargetError::NotConnected);
}
}
self.send(&event).await
self.send_body(queued.body, &queued.meta).await
}
}
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
debug!("Sending event from store for target: {}", self.id);
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!("Sending queued payload from store for target: {}, key: {}", self.id, key);
match self.init().await {
Ok(_) => {
debug!("Event sent to store for target: {}", self.name());
@@ -384,37 +406,13 @@ where
}
}
let store = self
.store
.as_ref()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
// Get events directly from the store, no longer need to acquire locks
let event = match store.get(&key) {
Ok(event) => event,
Err(StoreError::NotFound) => return Ok(()),
Err(e) => {
return Err(TargetError::Storage(format!("Failed to get event from store: {e}")));
}
};
if let Err(e) = self.send(&event).await {
if let Err(e) = self.send_body(body, &meta).await {
if let TargetError::NotConnected = e {
return Err(TargetError::NotConnected);
}
return Err(e);
}
// Use the immutable reference of the store to delete the event content corresponding to the key
debug!("Deleting event from store for target: {}, key:{}, start", self.id, key.to_string());
match store.del(&key) {
Ok(_) => debug!("Event deleted from store for target: {}, key:{}, end", self.id, key.to_string()),
Err(e) => {
error!("Failed to delete event from store: {}", e);
return Err(TargetError::Storage(format!("Failed to delete event from store: {e}")));
}
}
debug!("Event sent from store and deleted for target: {}", self.id);
Ok(())
}
@@ -426,7 +424,7 @@ where
Ok(())
}
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
// Returns the reference to the internal store
self.store.as_deref()
}
@@ -436,14 +434,11 @@ where
}
async fn init(&self) -> Result<(), TargetError> {
// If the target is disabled, return to success directly
if !self.is_enabled() {
debug!("Webhook target {} is disabled, skipping initialization", self.id);
return Ok(());
}
// Use existing initialization logic
WebhookTarget::init(self).await
self.init_inner().await
}
fn is_enabled(&self) -> bool {