mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-25 21:46:50 +00:00
init event crate
This commit is contained in:
houseme
@@ -1,180 +0,0 @@
|
||||
use crate::config::kafka::KafkaConfig;
|
||||
use crate::config::{default_queue_limit, DEFAULT_RETRY_INTERVAL, STORE_PREFIX};
|
||||
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||
use crate::{Error, Event, QueueStore};
|
||||
use async_trait::async_trait;
|
||||
use rdkafka::error::KafkaError;
|
||||
use rdkafka::producer::{FutureProducer, FutureRecord};
|
||||
use rdkafka::types::RDKafkaErrorCode;
|
||||
use rdkafka::util::Timeout;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use ChannelAdapterType::Kafka;
|
||||
|
||||
/// Kafka adapter for sending events to a Kafka topic.
|
||||
pub struct KafkaAdapter {
|
||||
producer: FutureProducer,
|
||||
store: Option<Arc<QueueStore<Event>>>,
|
||||
config: KafkaConfig,
|
||||
}
|
||||
|
||||
impl KafkaAdapter {
|
||||
/// Creates a new Kafka adapter.
|
||||
pub fn new(config: &KafkaConfig) -> Result<Self, Error> {
|
||||
// Create a Kafka producer with the provided configuration.
|
||||
let producer = rdkafka::config::ClientConfig::new()
|
||||
.set("bootstrap.servers", &config.brokers)
|
||||
.set("message.timeout.ms", config.timeout.to_string())
|
||||
.create()
|
||||
.map_err(|e| Error::msg(format!("Failed to create a Kafka producer: {}", e)))?;
|
||||
|
||||
// create a queue store if enabled
|
||||
let store = if !config.common.queue_dir.is_empty() {
|
||||
let store_path = PathBuf::from(&config.common.queue_dir).join(format!(
|
||||
"{}-{}-{}",
|
||||
STORE_PREFIX,
|
||||
Kafka.as_str(),
|
||||
config.common.identifier
|
||||
));
|
||||
|
||||
let queue_limit = if config.queue_limit > 0 {
|
||||
config.queue_limit
|
||||
} else {
|
||||
default_queue_limit()
|
||||
};
|
||||
let store = QueueStore::new(store_path, config.queue_limit, Some(".event".to_string()));
|
||||
if let Err(e) = store.open() {
|
||||
tracing::error!("Unable to open queue storage: {}", e);
|
||||
None
|
||||
} else {
|
||||
Some(Arc::new(store))
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Self { config, producer, store })
|
||||
}
|
||||
|
||||
/// Handle backlog events in storage
|
||||
pub async fn process_backlog(&self) -> Result<(), Error> {
|
||||
if let Some(store) = &self.store {
|
||||
let keys = store.list();
|
||||
|
||||
for key in keys {
|
||||
match store.get_multiple(&key) {
|
||||
Ok(events) => {
|
||||
for event in events {
|
||||
// Use the retry interval to send events
|
||||
if let Err(e) = self.send_with_retry(&event).await {
|
||||
tracing::error!("Processing of backlog events failed: {}", e);
|
||||
// If it still fails, we remain in the queue
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// The event is deleted after it has been successfully processed
|
||||
if let Err(e) = store.del(&key) {
|
||||
tracing::error!("Failed to delete a handled event: {}", e);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Fetch events from the queue failed: {}", e);
|
||||
|
||||
// If the event cannot be read, it may be corrupted, delete it
|
||||
if let Err(del_err) = store.del(&key) {
|
||||
tracing::error!("Failed to delete a corrupted event: {}", del_err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sends an event to the Kafka topic with retry logic.
|
||||
async fn send_with_retry(&self, event: &Event) -> Result<(), Error> {
|
||||
let retry_interval = match self.config.retry_interval {
|
||||
Some(t) => Duration::from_secs(t),
|
||||
None => Duration::from_secs(DEFAULT_RETRY_INTERVAL), // Default to 3 seconds if not set
|
||||
};
|
||||
|
||||
for attempt in 0..self.max_retries {
|
||||
match self.send_request(event).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err((KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull), _)) => {
|
||||
tracing::warn!("Kafka attempt {} failed: Queue full. Retrying...", attempt + 1);
|
||||
// sleep(Duration::from_secs(2u64.pow(attempt))).await;
|
||||
sleep(retry_interval).await;
|
||||
}
|
||||
Err((e, _)) => {
|
||||
tracing::error!("Kafka send error: {}", e);
|
||||
return Err(Error::Kafka(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::Custom("Exceeded maximum retry attempts for Kafka message".to_string()))
|
||||
}
|
||||
|
||||
/// Send a single Kafka message
|
||||
async fn send_request(&self, event: &Event) -> Result<(), Error> {
|
||||
// Serialize events
|
||||
let payload = serde_json::to_string(event).map_err(|e| Error::Custom(format!("Serialization event failed: {}", e)))?;
|
||||
|
||||
// Create a Kafka record
|
||||
let record = FutureRecord::to(&self.config.topic).payload(&payload).key(&event.id); // Use the event ID as the key
|
||||
|
||||
// Send to Kafka
|
||||
let delivery_status = self
|
||||
.producer
|
||||
.send(record, Duration::from_millis(self.config.timeout))
|
||||
.await
|
||||
.map_err(|(e, _)| Error::Custom(format!("Failed to send to Kafka: {}", e)))?;
|
||||
// Check delivery status
|
||||
if let Some((err, _)) = delivery_status {
|
||||
return Err(Error::Kafka(err));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save the event to the queue
|
||||
async fn save_to_queue(&self, event: &Event) -> Result<(), Error> {
|
||||
if let Some(store) = &self.store {
|
||||
store
|
||||
.put(event.clone())
|
||||
.map_err(|e| Error::Custom(format!("Saving events to queue failed: {}", e)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChannelAdapter for KafkaAdapter {
|
||||
fn name(&self) -> String {
|
||||
ChannelAdapterType::Kafka.to_string()
|
||||
}
|
||||
|
||||
async fn send(&self, event: &Event) -> Result<(), Error> {
|
||||
// Try to deal with the backlog of events first
|
||||
let _ = self.process_backlog().await;
|
||||
|
||||
// An attempt was made to send the current event
|
||||
match self.send_with_retry(event).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
// If the send fails and the queue is enabled, save to the queue
|
||||
if let Some(_) = &self.store {
|
||||
tracing::warn!("Failed to send events to Kafka and saved to a queue: {}", e);
|
||||
self.save_to_queue(event).await?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,8 @@
|
||||
use crate::config::adapter::AdapterConfig;
|
||||
use crate::config::AdapterConfig;
|
||||
use crate::{Error, Event};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
pub(crate) mod kafka;
|
||||
#[cfg(feature = "mqtt")]
|
||||
pub(crate) mod mqtt;
|
||||
#[cfg(feature = "webhook")]
|
||||
@@ -97,10 +95,6 @@ pub fn create_adapters(configs: Vec<AdapterConfig>) -> Result<Vec<Arc<dyn Channe
|
||||
webhook_config.validate().map_err(Error::ConfigError)?;
|
||||
adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone())));
|
||||
}
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
AdapterConfig::Kafka(kafka_config) => {
|
||||
adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?));
|
||||
}
|
||||
#[cfg(feature = "mqtt")]
|
||||
AdapterConfig::Mqtt(mqtt_config) => {
|
||||
let (mqtt, mut event_loop) = mqtt::MqttAdapter::new(mqtt_config);
|
||||
@@ -109,8 +103,6 @@ pub fn create_adapters(configs: Vec<AdapterConfig>) -> Result<Vec<Arc<dyn Channe
|
||||
}
|
||||
#[cfg(not(feature = "webhook"))]
|
||||
AdapterConfig::Webhook(_) => return Err(Error::FeatureDisabled("webhook")),
|
||||
#[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
|
||||
AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")),
|
||||
#[cfg(not(feature = "mqtt"))]
|
||||
AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")),
|
||||
}
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
use crate::config::webhook::WebhookConfig;
|
||||
use crate::config::STORE_PREFIX;
|
||||
use crate::store::queue::Store;
|
||||
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||
use crate::{Error, QueueStore};
|
||||
use crate::{Event, DEFAULT_RETRY_INTERVAL};
|
||||
use async_trait::async_trait;
|
||||
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
|
||||
|
||||
@@ -1,20 +1,59 @@
|
||||
use crate::config::{adapter::AdapterConfig, kafka::KafkaConfig, mqtt::MqttConfig, webhook::WebhookConfig};
|
||||
use rustfs_config::notify::mqtt::MQTTArgs;
|
||||
use rustfs_config::notify::webhook::WebhookArgs;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use std::env;
|
||||
|
||||
/// The default configuration file name
|
||||
const DEFAULT_CONFIG_FILE: &str = "notify";
|
||||
|
||||
/// The prefix for the configuration file
|
||||
pub const STORE_PREFIX: &str = "rustfs";
|
||||
|
||||
/// The default retry interval for the webhook adapter
|
||||
pub const DEFAULT_RETRY_INTERVAL: u64 = 3;
|
||||
|
||||
/// The default maximum retry count for the webhook adapter
|
||||
pub const DEFAULT_MAX_RETRIES: u32 = 3;
|
||||
|
||||
/// The default notification queue limit
|
||||
pub const DEFAULT_NOTIFY_QUEUE_LIMIT: u64 = 10000;
|
||||
|
||||
/// Provide temporary directories as default storage paths
|
||||
pub(crate) fn default_queue_dir() -> String {
|
||||
env::var("EVENT_QUEUE_DIR").unwrap_or_else(|e| {
|
||||
tracing::info!("Failed to get `EVENT_QUEUE_DIR` failed err: {}", e.to_string());
|
||||
env::temp_dir().join(DEFAULT_CONFIG_FILE).to_string_lossy().to_string()
|
||||
})
|
||||
}
|
||||
|
||||
/// Provides the recommended default channel capacity for high concurrency systems
|
||||
pub(crate) fn default_queue_limit() -> u64 {
|
||||
env::var("EVENT_CHANNEL_CAPACITY")
|
||||
.unwrap_or_else(|_| DEFAULT_NOTIFY_QUEUE_LIMIT.to_string())
|
||||
.parse()
|
||||
.unwrap_or(DEFAULT_NOTIFY_QUEUE_LIMIT) // Default to 10000 if parsing fails
|
||||
}
|
||||
|
||||
/// Configuration for the adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum AdapterConfig {
|
||||
Webhook(WebhookArgs),
|
||||
Mqtt(MQTTArgs),
|
||||
}
|
||||
|
||||
/// Event Notifier Configuration
|
||||
/// This struct contains the configuration for the event notifier system,
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct EventNotifierConfig {
|
||||
/// A collection of webhook configurations, with the key being a unique identifier
|
||||
#[serde(default)]
|
||||
pub webhook: HashMap<String, WebhookConfig>,
|
||||
/// A collection of Kafka configurations, with the key being a unique identifier
|
||||
#[serde(default)]
|
||||
pub kafka: HashMap<String, KafkaConfig>,
|
||||
pub webhook: HashMap<String, WebhookArgs>,
|
||||
///MQTT configuration collection, with the key being a unique identifier
|
||||
#[serde(default)]
|
||||
pub mqtt: HashMap<String, MqttConfig>,
|
||||
pub mqtt: HashMap<String, MQTTArgs>,
|
||||
}
|
||||
|
||||
impl EventNotifierConfig {
|
||||
@@ -49,21 +88,14 @@ impl EventNotifierConfig {
|
||||
|
||||
// Add all enabled webhook configurations
|
||||
for webhook in self.webhook.values() {
|
||||
if webhook.common.enable {
|
||||
if webhook.enable {
|
||||
adapters.push(AdapterConfig::Webhook(webhook.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// Add all enabled Kafka configurations
|
||||
for kafka in self.kafka.values() {
|
||||
if kafka.common.enable {
|
||||
adapters.push(AdapterConfig::Kafka(kafka.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// Add all enabled MQTT configurations
|
||||
for mqtt in self.mqtt.values() {
|
||||
if mqtt.common.enable {
|
||||
if mqtt.enable {
|
||||
adapters.push(AdapterConfig::Mqtt(mqtt.clone()));
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
use crate::config::kafka::KafkaConfig;
|
||||
use crate::config::mqtt::MqttConfig;
|
||||
use crate::config::webhook::WebhookConfig;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Add a common field for the adapter configuration
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AdapterCommon {
|
||||
/// Adapter identifier for unique identification
|
||||
pub identifier: String,
|
||||
/// Adapter description information
|
||||
pub comment: String,
|
||||
/// Whether to enable this adapter
|
||||
#[serde(default)]
|
||||
pub enable: bool,
|
||||
#[serde(default = "default_queue_dir")]
|
||||
pub queue_dir: String,
|
||||
#[serde(default = "default_queue_limit")]
|
||||
pub queue_limit: u64,
|
||||
}
|
||||
|
||||
impl Default for AdapterCommon {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
identifier: String::new(),
|
||||
comment: String::new(),
|
||||
enable: false,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for the adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum AdapterConfig {
|
||||
Webhook(WebhookConfig),
|
||||
Kafka(KafkaConfig),
|
||||
Mqtt(MqttConfig),
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
use crate::config::adapter::AdapterCommon;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Configuration for the Kafka adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KafkaConfig {
|
||||
#[serde(flatten)]
|
||||
pub common: AdapterCommon,
|
||||
pub brokers: String,
|
||||
pub topic: String,
|
||||
pub max_retries: u32,
|
||||
pub timeout: u64,
|
||||
}
|
||||
|
||||
impl Default for KafkaConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
common: AdapterCommon::default(),
|
||||
brokers: String::new(),
|
||||
topic: String::new(),
|
||||
max_retries: 3,
|
||||
timeout: 5000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl KafkaConfig {
|
||||
/// Create a new Kafka configuration
|
||||
pub fn new(identifier: impl Into<String>, brokers: impl Into<String>, topic: impl Into<String>) -> Self {
|
||||
Self {
|
||||
common: AdapterCommon {
|
||||
identifier: identifier.into(),
|
||||
comment: String::new(),
|
||||
enable: true,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
},
|
||||
brokers: brokers.into(),
|
||||
topic: topic.into(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
use std::env;
|
||||
|
||||
pub mod adapter;
|
||||
pub mod kafka;
|
||||
pub mod mqtt;
|
||||
pub mod notifier;
|
||||
pub mod webhook;
|
||||
|
||||
/// The default configuration file name
|
||||
const DEFAULT_CONFIG_FILE: &str = "event";
|
||||
|
||||
/// The prefix for the configuration file
|
||||
pub const STORE_PREFIX: &str = "rustfs";
|
||||
|
||||
/// The default retry interval for the webhook adapter
|
||||
pub const DEFAULT_RETRY_INTERVAL: u64 = 3;
|
||||
|
||||
/// The default maximum retry count for the webhook adapter
|
||||
pub const DEFAULT_MAX_RETRIES: u32 = 3;
|
||||
|
||||
/// The default notification queue limit
|
||||
pub const DEFAULT_NOTIFY_QUEUE_LIMIT: u64 = 10000;
|
||||
|
||||
/// Provide temporary directories as default storage paths
|
||||
pub(crate) fn default_queue_dir() -> String {
|
||||
env::var("EVENT_QUEUE_DIR").unwrap_or_else(|e| {
|
||||
tracing::info!("Failed to get `EVENT_QUEUE_DIR` failed err: {}", e.to_string());
|
||||
env::temp_dir().join(DEFAULT_CONFIG_FILE).to_string_lossy().to_string()
|
||||
})
|
||||
}
|
||||
|
||||
/// Provides the recommended default channel capacity for high concurrency systems
|
||||
pub(crate) fn default_queue_limit() -> u64 {
|
||||
env::var("EVENT_CHANNEL_CAPACITY")
|
||||
.unwrap_or_else(|_| DEFAULT_NOTIFY_QUEUE_LIMIT.to_string())
|
||||
.parse()
|
||||
.unwrap_or(DEFAULT_NOTIFY_QUEUE_LIMIT) // Default to 10000 if parsing fails
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
use crate::config::adapter::AdapterCommon;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Configuration for the MQTT adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MqttConfig {
|
||||
#[serde(flatten)]
|
||||
pub common: AdapterCommon,
|
||||
pub broker: String,
|
||||
pub port: u16,
|
||||
pub client_id: String,
|
||||
pub topic: String,
|
||||
pub max_retries: u32,
|
||||
}
|
||||
|
||||
impl Default for MqttConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
common: AdapterCommon::default(),
|
||||
broker: String::new(),
|
||||
port: 1883,
|
||||
client_id: String::new(),
|
||||
topic: String::new(),
|
||||
max_retries: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MqttConfig {
|
||||
/// Create a new MQTT configuration
|
||||
pub fn new(identifier: impl Into<String>, broker: impl Into<String>, topic: impl Into<String>) -> Self {
|
||||
Self {
|
||||
common: AdapterCommon {
|
||||
identifier: identifier.into(),
|
||||
comment: String::new(),
|
||||
enable: true,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
},
|
||||
broker: broker.into(),
|
||||
topic: topic.into(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
use crate::config::adapter::AdapterCommon;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use tracing::info;
|
||||
|
||||
/// Configuration for the webhook adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct WebhookConfig {
|
||||
#[serde(flatten)]
|
||||
pub common: AdapterCommon,
|
||||
pub endpoint: String,
|
||||
pub auth_token: Option<String>,
|
||||
pub custom_headers: Option<HashMap<String, String>>,
|
||||
pub max_retries: u32,
|
||||
pub retry_interval: Option<u64>,
|
||||
pub timeout: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub client_cert: Option<String>,
|
||||
#[serde(default)]
|
||||
pub client_key: Option<String>,
|
||||
}
|
||||
|
||||
impl WebhookConfig {
|
||||
/// validate the configuration for the webhook adapter
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// - `Result<(), String>`: Ok if the configuration is valid, Err with a message if invalid.
|
||||
///
|
||||
/// # Errors
|
||||
/// - Returns an error if the configuration is invalid, such as empty endpoint, unreasonable timeout, or mismatched certificate and key.
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
// If not enabled, the other fields are not validated
|
||||
if !self.common.enable {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// verify that endpoint cannot be empty
|
||||
if self.endpoint.trim().is_empty() {
|
||||
return Err("Webhook endpoint cannot be empty".to_string());
|
||||
}
|
||||
|
||||
// verification timeout must be reasonable
|
||||
if self.timeout.is_some() {
|
||||
match self.timeout {
|
||||
Some(timeout) if timeout > 0 => {
|
||||
info!("Webhook timeout is set to {}", timeout);
|
||||
}
|
||||
_ => return Err("Webhook timeout must be greater than 0".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that the maximum number of retry is reasonable
|
||||
if self.max_retries > 10 {
|
||||
return Err("Maximum retry count cannot exceed 10".to_string());
|
||||
}
|
||||
|
||||
// Verify the queue directory path
|
||||
if !self.common.queue_dir.is_empty() && !Path::new(&self.common.queue_dir).is_absolute() {
|
||||
return Err("Queue directory path should be absolute".to_string());
|
||||
}
|
||||
|
||||
// The authentication certificate and key must appear in pairs
|
||||
if (self.client_cert.is_some() && self.client_key.is_none()) || (self.client_cert.is_none() && self.client_key.is_some())
|
||||
{
|
||||
return Err("Certificate and key must be specified as a pair".to_string());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a new webhook configuration
|
||||
pub fn new(identifier: impl Into<String>, endpoint: impl Into<String>) -> Self {
|
||||
Self {
|
||||
common: AdapterCommon {
|
||||
identifier: identifier.into(),
|
||||
comment: String::new(),
|
||||
enable: true,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
},
|
||||
endpoint: endpoint.into(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,8 +7,6 @@ mod store;
|
||||
mod system;
|
||||
|
||||
pub use adapter::create_adapters;
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
pub use adapter::kafka::KafkaAdapter;
|
||||
#[cfg(feature = "mqtt")]
|
||||
pub use adapter::mqtt::MqttAdapter;
|
||||
#[cfg(feature = "webhook")]
|
||||
@@ -16,18 +14,6 @@ pub use adapter::webhook::WebhookAdapter;
|
||||
|
||||
pub use adapter::ChannelAdapter;
|
||||
pub use adapter::ChannelAdapterType;
|
||||
pub use config::adapter::AdapterCommon;
|
||||
pub use config::adapter::AdapterConfig;
|
||||
pub use config::notifier::EventNotifierConfig;
|
||||
pub use config::{DEFAULT_MAX_RETRIES, DEFAULT_RETRY_INTERVAL};
|
||||
pub use config::{AdapterConfig, EventNotifierConfig, DEFAULT_MAX_RETRIES, DEFAULT_RETRY_INTERVAL};
|
||||
pub use error::Error;
|
||||
pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source};
|
||||
pub use store::queue::QueueStore;
|
||||
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
pub use config::kafka::KafkaConfig;
|
||||
#[cfg(feature = "mqtt")]
|
||||
pub use config::mqtt::MqttConfig;
|
||||
|
||||
#[cfg(feature = "webhook")]
|
||||
pub use config::webhook::WebhookConfig;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::config::notifier::EventNotifierConfig;
|
||||
use crate::config::EventNotifierConfig;
|
||||
use crate::Event;
|
||||
use common::error::{Error, Result};
|
||||
use ecstore::store::ECStore;
|
||||
|
||||
@@ -115,11 +115,6 @@ impl EventManager {
|
||||
merged.webhook.insert(id, config);
|
||||
}
|
||||
|
||||
// Merge Kafka configurations
|
||||
for (id, config) in new.kafka {
|
||||
merged.kafka.insert(id, config);
|
||||
}
|
||||
|
||||
// Merge MQTT configurations
|
||||
for (id, config) in new.mqtt {
|
||||
merged.mqtt.insert(id, config);
|
||||
|
||||
@@ -1,2 +1,314 @@
|
||||
use async_trait::async_trait;
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::fmt::Display;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time;
|
||||
|
||||
pub(crate) mod manager;
|
||||
pub(crate) mod queue;
|
||||
|
||||
// 常量定义
|
||||
pub const RETRY_INTERVAL: Duration = Duration::from_secs(3);
|
||||
pub const DEFAULT_LIMIT: u64 = 100000; // 默认存储限制
|
||||
pub const DEFAULT_EXT: &str = ".unknown";
|
||||
pub const COMPRESS_EXT: &str = ".snappy";
|
||||
|
||||
// 错误类型
|
||||
#[derive(Debug)]
|
||||
pub enum StoreError {
|
||||
NotConnected,
|
||||
LimitExceeded,
|
||||
IoError(std::io::Error),
|
||||
Utf8(std::str::Utf8Error),
|
||||
SerdeError(serde_json::Error),
|
||||
Deserialize(serde_json::Error),
|
||||
UuidError(uuid::Error),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl Display for StoreError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
StoreError::NotConnected => write!(f, "not connected to target server/service"),
|
||||
StoreError::LimitExceeded => write!(f, "the maximum store limit reached"),
|
||||
StoreError::IoError(e) => write!(f, "IO error: {}", e),
|
||||
StoreError::Utf8(e) => write!(f, "UTF-8 conversion error: {}", e),
|
||||
StoreError::SerdeError(e) => write!(f, "serialization error: {}", e),
|
||||
StoreError::Deserialize(e) => write!(f, "deserialization error: {}", e),
|
||||
StoreError::UuidError(e) => write!(f, "UUID generation error: {}", e),
|
||||
StoreError::Other(s) => write!(f, "{}", s),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for StoreError {
|
||||
fn source(&self) -> Option<&(dyn Error + 'static)> {
|
||||
match self {
|
||||
StoreError::IoError(e) => Some(e),
|
||||
StoreError::SerdeError(e) => Some(e),
|
||||
StoreError::UuidError(e) => Some(e),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for StoreError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
StoreError::IoError(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_json::Error> for StoreError {
|
||||
fn from(e: serde_json::Error) -> Self {
|
||||
StoreError::SerdeError(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<uuid::Error> for StoreError {
|
||||
fn from(e: uuid::Error) -> Self {
|
||||
StoreError::UuidError(e)
|
||||
}
|
||||
}
|
||||
|
||||
pub type StoreResult<T> = Result<T, StoreError>;
|
||||
|
||||
// 日志记录器类型
|
||||
pub type Logger = fn(ctx: Option<&str>, err: StoreError, id: &str, err_kind: &[&dyn Display]);
|
||||
|
||||
// Key 结构体定义
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Key {
|
||||
pub name: String,
|
||||
pub compress: bool,
|
||||
pub extension: String,
|
||||
pub item_count: usize,
|
||||
}
|
||||
|
||||
impl Key {
|
||||
pub fn new(name: String, extension: String) -> Self {
|
||||
Self {
|
||||
name,
|
||||
extension,
|
||||
compress: false,
|
||||
item_count: 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_compression(mut self, compress: bool) -> Self {
|
||||
self.compress = compress;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_item_count(mut self, count: usize) -> Self {
|
||||
self.item_count = count;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn to_string(&self) -> String {
|
||||
let mut key_str = self.name.clone();
|
||||
|
||||
if self.item_count > 1 {
|
||||
key_str = format!("{}:{}", self.item_count, self.name);
|
||||
}
|
||||
|
||||
let ext = if self.compress {
|
||||
format!("{}{}", self.extension, COMPRESS_EXT)
|
||||
} else {
|
||||
self.extension.clone()
|
||||
};
|
||||
|
||||
format!("{}{}", key_str, ext)
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Key {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_key(k: &str) -> Key {
|
||||
let mut key = Key {
|
||||
name: k.to_string(),
|
||||
compress: false,
|
||||
extension: String::new(),
|
||||
item_count: 1,
|
||||
};
|
||||
|
||||
// 检查压缩扩展名
|
||||
if k.ends_with(COMPRESS_EXT) {
|
||||
key.compress = true;
|
||||
key.name = key.name[..key.name.len() - COMPRESS_EXT.len()].to_string();
|
||||
}
|
||||
|
||||
// 解析项目数量
|
||||
if let Some(colon_pos) = key.name.find(':') {
|
||||
if let Ok(count) = key.name[..colon_pos].parse::<usize>() {
|
||||
key.item_count = count;
|
||||
key.name = key.name[colon_pos + 1..].to_string();
|
||||
}
|
||||
}
|
||||
|
||||
// 解析扩展名
|
||||
if let Some(dot_pos) = key.name.rfind('.') {
|
||||
key.extension = key.name[dot_pos..].to_string();
|
||||
key.name = key.name[..dot_pos].to_string();
|
||||
}
|
||||
|
||||
key
|
||||
}
|
||||
|
||||
// Target trait 定义
|
||||
#[async_trait]
|
||||
pub trait Target: Send + Sync {
|
||||
fn name(&self) -> String;
|
||||
async fn send_from_store(&self, key: Key) -> StoreResult<()>;
|
||||
}
|
||||
|
||||
// Store trait 定义
|
||||
#[async_trait]
|
||||
pub trait Store<T>: Send + Sync
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Send + Sync + 'static,
|
||||
{
|
||||
async fn put(&self, item: T) -> StoreResult<Key>;
|
||||
async fn put_multiple(&self, items: Vec<T>) -> StoreResult<Key>;
|
||||
async fn get(&self, key: Key) -> StoreResult<T>;
|
||||
async fn get_multiple(&self, key: Key) -> StoreResult<Vec<T>>;
|
||||
async fn get_raw(&self, key: Key) -> StoreResult<Vec<u8>>;
|
||||
async fn put_raw(&self, b: Vec<u8>) -> StoreResult<Key>;
|
||||
async fn len(&self) -> usize;
|
||||
async fn list(&self) -> Vec<Key>;
|
||||
async fn del(&self, key: Key) -> StoreResult<()>;
|
||||
async fn open(&self) -> StoreResult<()>;
|
||||
async fn delete(&self) -> StoreResult<()>;
|
||||
}
|
||||
|
||||
// 重播项目辅助函数
|
||||
pub async fn replay_items<T>(store: Arc<dyn Store<T>>, done_ch: mpsc::Receiver<()>, log: Logger, id: &str) -> mpsc::Receiver<Key>
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Send + Sync + 'static,
|
||||
{
|
||||
let (tx, rx) = mpsc::channel(100); // 合理的缓冲区大小
|
||||
let id = id.to_string();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut done_ch = done_ch;
|
||||
let mut retry_interval = time::interval(RETRY_INTERVAL);
|
||||
let mut retry_interval = time::interval_at(retry_interval.tick().await, RETRY_INTERVAL);
|
||||
|
||||
loop {
|
||||
let keys = store.list().await;
|
||||
|
||||
for key in keys {
|
||||
let tx = tx.clone();
|
||||
tokio::select! {
|
||||
_ = tx.send(key) => {
|
||||
// 成功发送下一个键
|
||||
}
|
||||
_ = done_ch.recv() => {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
_ = retry_interval.tick() => {
|
||||
// 重试定时器触发，继续循环
|
||||
}
|
||||
_ = done_ch.recv() => {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
rx
|
||||
}
|
||||
|
||||
// 发送项目辅助函数
|
||||
pub async fn send_items(target: &dyn Target, mut key_ch: mpsc::Receiver<Key>, mut done_ch: mpsc::Receiver<()>, logger: Logger) {
|
||||
let mut retry_interval = time::interval(RETRY_INTERVAL);
|
||||
|
||||
async fn try_send(
|
||||
target: &dyn Target,
|
||||
key: Key,
|
||||
retry_interval: &mut time::Interval,
|
||||
done_ch: &mut mpsc::Receiver<()>,
|
||||
logger: Logger,
|
||||
) -> bool {
|
||||
loop {
|
||||
match target.send_from_store(key.clone()).await {
|
||||
Ok(_) => return true,
|
||||
Err(err) => {
|
||||
logger(None, err, &target.name(), &[&format!("unable to send log entry to '{}'", target.name())]);
|
||||
|
||||
tokio::select! {
|
||||
_ = retry_interval.tick() => {
|
||||
// 重试
|
||||
}
|
||||
_ = done_ch.recv() => {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
maybe_key = key_ch.recv() => {
|
||||
match maybe_key {
|
||||
Some(key) => {
|
||||
if !try_send(target, key, &mut retry_interval, &mut done_ch, logger).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
None => return,
|
||||
}
|
||||
}
|
||||
_ = done_ch.recv() => {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 流式传输项目
|
||||
pub async fn stream_items<T>(store: Arc<dyn Store<T>>, target: &dyn Target, done_ch: mpsc::Receiver<()>, logger: Logger)
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Send + Sync + 'static,
|
||||
{
|
||||
// 创建一个 done_ch 的克隆，以便可以将其传递给 replay_items
|
||||
// let (tx, rx) = mpsc::channel::<()>(1);
|
||||
|
||||
let (tx_replay, rx_replay) = mpsc::channel::<()>(1);
|
||||
let (tx_send, rx_send) = mpsc::channel::<()>(1);
|
||||
|
||||
let mut done_ch = done_ch;
|
||||
|
||||
let key_ch = replay_items(store, rx_replay, logger, &target.name()).await;
|
||||
// let key_ch = replay_items(store, rx, logger, &target.name()).await;
|
||||
|
||||
let tx_replay_clone = tx_replay.clone();
|
||||
let tx_send_clone = tx_send.clone();
|
||||
|
||||
// 监听原始 done_ch，如果收到信号，则关闭我们创建的通道
|
||||
tokio::spawn(async move {
|
||||
// if done_ch.recv().await.is_some() {
|
||||
// let _ = tx.send(()).await;
|
||||
// }
|
||||
if done_ch.recv().await.is_some() {
|
||||
let _ = tx_replay_clone.send(()).await;
|
||||
let _ = tx_send_clone.send(()).await;
|
||||
}
|
||||
});
|
||||
|
||||
// send_items(target, key_ch, rx, logger).await;
|
||||
send_items(target, key_ch, rx_send, logger).await;
|
||||
}
|
||||
|
||||
+149
-421
@@ -1,515 +1,243 @@
|
||||
use common::error::{Error, Result};
|
||||
use crate::store::{parse_key, Key, Store, StoreError, StoreResult, DEFAULT_EXT, DEFAULT_LIMIT};
|
||||
use async_trait::async_trait;
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
use snap::raw::{Decoder, Encoder};
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
use std::marker::PhantomData;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::{fs, io};
|
||||
use tokio::fs;
|
||||
use tokio::sync::RwLock;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Keys in storage
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct Key {
|
||||
/// Key name
|
||||
pub name: String,
|
||||
/// Whether to compress
|
||||
pub compress: bool,
|
||||
/// filename extension
|
||||
pub extension: String,
|
||||
/// Number of items
|
||||
pub item_count: usize,
|
||||
}
|
||||
|
||||
impl Key {
|
||||
/// Create a new key
|
||||
pub fn new(name: impl Into<String>, extension: impl Into<String>, compress: bool) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
compress,
|
||||
extension: extension.into(),
|
||||
item_count: 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to string form
|
||||
#[allow(clippy::inherent_to_string)]
|
||||
pub fn to_string(&self) -> String {
|
||||
let mut key_str = self.name.clone();
|
||||
if self.item_count > 1 {
|
||||
key_str = format!("{}:{}", self.item_count, self.name);
|
||||
}
|
||||
|
||||
let compress_ext = if self.compress { COMPRESS_EXT } else { "" };
|
||||
format!("{}{}{}", key_str, self.extension, compress_ext)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse key from file name
|
||||
#[allow(clippy::redundant_closure)]
|
||||
pub fn parse_key(filename: &str) -> Key {
|
||||
let compress = filename.ends_with(COMPRESS_EXT);
|
||||
let filename = if compress {
|
||||
&filename[..filename.len() - 7] // 移除 ".snappy"
|
||||
} else {
|
||||
filename
|
||||
};
|
||||
|
||||
let mut parts = filename.splitn(2, '.');
|
||||
let name_part = parts.next().unwrap_or("");
|
||||
let extension = parts
|
||||
.next()
|
||||
.map_or_else(|| String::new(), |ext| format!(".{}", ext))
|
||||
.to_string();
|
||||
|
||||
let mut name = name_part.to_string();
|
||||
let mut item_count = 1;
|
||||
|
||||
if let Some(pos) = name_part.find(':') {
|
||||
if let Ok(count) = name_part[..pos].parse::<usize>() {
|
||||
item_count = count;
|
||||
name = name_part[pos + 1..].to_string();
|
||||
}
|
||||
}
|
||||
|
||||
Key {
|
||||
name,
|
||||
compress,
|
||||
extension,
|
||||
item_count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Store the characteristics of the project
|
||||
pub trait Store<T>: Send + Sync
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
|
||||
{
|
||||
/// Store a single item
|
||||
fn put(&self, item: T) -> Result<Key>;
|
||||
|
||||
/// Store multiple projects
|
||||
fn put_multiple(&self, items: Vec<T>) -> Result<Key>;
|
||||
|
||||
/// Get a single item
|
||||
fn get(&self, key: &Key) -> Result<T>;
|
||||
|
||||
/// Get multiple items
|
||||
fn get_multiple(&self, key: &Key) -> Result<Vec<T>>;
|
||||
|
||||
/// Get the raw bytes
|
||||
fn get_raw(&self, key: &Key) -> Result<Vec<u8>>;
|
||||
|
||||
/// Stores raw bytes
|
||||
fn put_raw(&self, data: &[u8]) -> Result<Key>;
|
||||
|
||||
/// Gets the number of items in storage
|
||||
fn len(&self) -> usize;
|
||||
|
||||
/// Whether it is empty or not
|
||||
fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// Lists all keys
|
||||
fn list(&self) -> Vec<Key>;
|
||||
|
||||
/// Delete the key
|
||||
fn del(&self, key: &Key) -> Result<()>;
|
||||
|
||||
/// Open Storage
|
||||
fn open(&self) -> Result<()>;
|
||||
|
||||
/// Delete the storage
|
||||
fn delete(&self) -> Result<()>;
|
||||
}
|
||||
|
||||
const DEFAULT_LIMIT: u64 = 100000;
|
||||
const DEFAULT_EXT: &str = ".unknown";
|
||||
const COMPRESS_EXT: &str = ".snappy";
|
||||
|
||||
/// Queue storage implementation
|
||||
pub struct QueueStore<T> {
|
||||
/// Project Limitations
|
||||
entry_limit: u64,
|
||||
/// Storage directory
|
||||
directory: PathBuf,
|
||||
/// filename extension
|
||||
file_ext: String,
|
||||
/// Item mapping: key -> modified time (Unix nanoseconds)
|
||||
entries: Arc<RwLock<HashMap<String, i64>>>,
|
||||
/// Type tags
|
||||
_phantom: PhantomData<T>,
|
||||
/// Whether to compress
|
||||
compress: bool,
|
||||
/// Store name
|
||||
name: String,
|
||||
entries: RwLock<BTreeMap<String, i64>>,
|
||||
_phantom: std::marker::PhantomData<T>,
|
||||
}
|
||||
|
||||
impl<T> QueueStore<T>
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
|
||||
T: Serialize + DeserializeOwned + Send + Sync + 'static,
|
||||
{
|
||||
/// Create a new queue store
|
||||
pub fn new<P: AsRef<Path>>(directory: P, name: String, limit: u64, ext: Option<String>) -> Self {
|
||||
let limit = if limit == 0 { DEFAULT_LIMIT } else { limit };
|
||||
let ext = ext.unwrap_or_else(|| DEFAULT_EXT.to_string());
|
||||
let mut path = PathBuf::from(directory.as_ref());
|
||||
path.push(&name);
|
||||
|
||||
// Create a directory (if it doesn't exist)
|
||||
if !path.exists() {
|
||||
if let Err(e) = fs::create_dir_all(&path) {
|
||||
tracing::error!("创建存储目录失败 {}: {}", path.display(), e);
|
||||
}
|
||||
}
|
||||
pub fn new<P: AsRef<Path>>(directory: P, limit: u64, ext: Option<&str>) -> Self {
|
||||
let entry_limit = if limit == 0 { DEFAULT_LIMIT } else { limit };
|
||||
let ext = ext.unwrap_or(DEFAULT_EXT).to_string();
|
||||
|
||||
Self {
|
||||
directory: directory.as_ref().to_path_buf(),
|
||||
name,
|
||||
entry_limit: limit,
|
||||
entry_limit,
|
||||
file_ext: ext,
|
||||
compress: true, // Default to compressing
|
||||
entries: Arc::new(RwLock::new(HashMap::with_capacity(limit as usize))),
|
||||
_phantom: PhantomData,
|
||||
entries: RwLock::new(BTreeMap::new()),
|
||||
_phantom: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the file extension
|
||||
pub fn with_file_ext(mut self, file_ext: &str) -> Self {
|
||||
self.file_ext = file_ext.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether to compress or not
|
||||
pub fn with_compression(mut self, compress: bool) -> Self {
|
||||
self.compress = compress;
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the file path
|
||||
fn get_file_path(&self, key: &Key) -> PathBuf {
|
||||
let mut filename = key.to_string();
|
||||
filename.push_str(if self.compress { COMPRESS_EXT } else { &self.file_ext });
|
||||
self.directory.join(filename)
|
||||
}
|
||||
|
||||
/// Serialize the project
|
||||
fn serialize_item(&self, item: &T) -> Result<Vec<u8>> {
|
||||
let data = serde_json::to_vec(item).map_err(|e| Error::msg(format!("Serialization failed: {}", e)))?;
|
||||
|
||||
if self.compress {
|
||||
let mut encoder = Encoder::new();
|
||||
Ok(encoder
|
||||
.compress_vec(&data)
|
||||
.map_err(|e| Error::msg(format!("Compression failed: {}", e)))?)
|
||||
} else {
|
||||
Ok(data)
|
||||
}
|
||||
}
|
||||
|
||||
/// Deserialize the project
|
||||
fn deserialize_item(&self, data: &[u8], is_compressed: bool) -> Result<T> {
|
||||
let data = if is_compressed {
|
||||
let mut decoder = Decoder::new();
|
||||
decoder
|
||||
.decompress_vec(data)
|
||||
.map_err(|e| Error::msg(format!("Unzipping failed: {}", e)))?
|
||||
} else {
|
||||
data.to_vec()
|
||||
};
|
||||
|
||||
serde_json::from_slice(&data).map_err(|e| Error::msg(format!("Deserialization failed: {}", e)))
|
||||
}
|
||||
|
||||
/// Lists all files in the directory, sorted by modification time (oldest takes precedence.)）
|
||||
fn list_files(&self) -> Result<Vec<fs::DirEntry>> {
|
||||
let mut files = Vec::new();
|
||||
|
||||
for entry in fs::read_dir(&self.directory)? {
|
||||
let entry = entry?;
|
||||
let metadata = entry.metadata()?;
|
||||
if metadata.is_file() {
|
||||
files.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by modification time
|
||||
files.sort_by(|a, b| {
|
||||
let a_time = a
|
||||
.metadata()
|
||||
.map(|m| m.modified())
|
||||
.unwrap_or(Ok(UNIX_EPOCH))
|
||||
.unwrap_or(UNIX_EPOCH);
|
||||
let b_time = b
|
||||
.metadata()
|
||||
.map(|m| m.modified())
|
||||
.unwrap_or(Ok(UNIX_EPOCH))
|
||||
.unwrap_or(UNIX_EPOCH);
|
||||
a_time.cmp(&b_time)
|
||||
});
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Write the object to a file
|
||||
fn write_object(&self, key: &Key, item: &T) -> Result<()> {
|
||||
// Serialize the object
|
||||
let data = serde_json::to_vec(item)?;
|
||||
self.write_bytes(key, &data)
|
||||
}
|
||||
|
||||
/// Write multiple objects to a file
|
||||
fn write_multiple_objects(&self, key: &Key, items: &[T]) -> Result<()> {
|
||||
let mut data = Vec::new();
|
||||
for item in items {
|
||||
let item_data = serde_json::to_vec(item)?;
|
||||
data.extend_from_slice(&item_data);
|
||||
data.push(b'\n');
|
||||
}
|
||||
self.write_bytes(key, &data)
|
||||
}
|
||||
|
||||
/// Write bytes to a file
|
||||
fn write_bytes(&self, key: &Key, data: &[u8]) -> Result<()> {
|
||||
async fn write_bytes(&self, key: Key, data: Vec<u8>) -> StoreResult<()> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
|
||||
let file_data = if key.compress {
|
||||
// Use snap to compress data
|
||||
let data = if key.compress {
|
||||
let mut encoder = Encoder::new();
|
||||
encoder
|
||||
.compress_vec(data)
|
||||
.map_err(|e| Error::msg(format!("Compression failed:{}", e)))?
|
||||
encoder.compress_vec(&data).map_err(|e| StoreError::Other(e.to_string()))?
|
||||
} else {
|
||||
data.to_vec()
|
||||
data
|
||||
};
|
||||
|
||||
// Make sure the directory exists
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
fs::write(&path, &data).await?;
|
||||
|
||||
// Write to the file
|
||||
fs::write(&path, &file_data)?;
|
||||
// 更新条目映射
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_err(|e| StoreError::Other(e.to_string()))?
|
||||
.as_nanos() as i64;
|
||||
|
||||
// Update the item mapping
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
|
||||
|
||||
let mut entries = self
|
||||
.entries
|
||||
.write()
|
||||
.map_err(|_| Error::msg("Failed to obtain a write lock"))?;
|
||||
entries.insert(key.to_string(), now);
|
||||
self.entries.write().await.insert(key.to_string(), now);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read bytes from a file
|
||||
fn read_bytes(&self, key: &Key) -> Result<Vec<u8>> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
let data = fs::read(&path)?;
|
||||
|
||||
if data.is_empty() {
|
||||
return Err(Error::msg("The file is empty"));
|
||||
}
|
||||
|
||||
if key.compress {
|
||||
// Use Snap to extract the data
|
||||
let mut decoder = Decoder::new();
|
||||
decoder
|
||||
.decompress_vec(&data)
|
||||
.map_err(|e| Error::msg(format!("Failed to decompress:{}", e)))
|
||||
} else {
|
||||
Ok(data)
|
||||
}
|
||||
async fn write(&self, key: Key, item: T) -> StoreResult<()> {
|
||||
let data = serde_json::to_vec(&item)?;
|
||||
self.write_bytes(key, data).await
|
||||
}
|
||||
|
||||
/// Check whether the storage limit is reached
|
||||
fn check_entry_limit(&self) -> Result<()> {
|
||||
let entries = self.entries.read().map_err(|_| Error::msg("Failed to obtain a read lock"))?;
|
||||
if entries.len() as u64 >= self.entry_limit {
|
||||
return Err(Error::msg("The storage limit has been reached"));
|
||||
async fn multi_write(&self, key: Key, items: Vec<T>) -> StoreResult<()> {
|
||||
let mut buffer = Vec::new();
|
||||
|
||||
for item in items {
|
||||
let item_data = serde_json::to_vec(&item)?;
|
||||
buffer.extend_from_slice(&item_data);
|
||||
buffer.push(b'\n'); // 使用换行符分隔项目
|
||||
}
|
||||
|
||||
self.write_bytes(key, buffer).await
|
||||
}
|
||||
|
||||
async fn del_internal(&self, key: &Key) -> StoreResult<()> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
|
||||
if let Err(e) = fs::remove_file(&path).await {
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
self.entries.write().await.remove(&key.to_string());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<T> Store<T> for QueueStore<T>
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
|
||||
T: Serialize + DeserializeOwned + Send + Sync + 'static,
|
||||
{
|
||||
fn put(&self, item: T) -> Result<Key> {
|
||||
{
|
||||
self.check_entry_limit()?;
|
||||
async fn put(&self, item: T) -> StoreResult<Key> {
|
||||
let entries_len = self.entries.read().await.len() as u64;
|
||||
if entries_len >= self.entry_limit {
|
||||
return Err(StoreError::LimitExceeded);
|
||||
}
|
||||
|
||||
// generate a new uuid
|
||||
// 生成 UUID 作为键
|
||||
let uuid = Uuid::new_v4();
|
||||
let key = Key::new(uuid.to_string(), &self.file_ext, true);
|
||||
let key = Key::new(uuid.to_string(), self.file_ext.clone());
|
||||
|
||||
self.write_object(&key, &item)?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
fn put_multiple(&self, items: Vec<T>) -> Result<Key> {
|
||||
if items.is_empty() {
|
||||
return Err(Error::msg("The list of items is empty"));
|
||||
}
|
||||
|
||||
{
|
||||
self.check_entry_limit()?;
|
||||
}
|
||||
|
||||
// Generate a new UUID
|
||||
let uuid = Uuid::new_v4();
|
||||
let mut key = Key::new(uuid.to_string(), &self.file_ext, true);
|
||||
key.item_count = items.len();
|
||||
|
||||
self.write_multiple_objects(&key, &items)?;
|
||||
self.write(key.clone(), item).await?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
fn get(&self, key: &Key) -> Result<T> {
|
||||
let items = self.get_multiple(key)?;
|
||||
if items.is_empty() {
|
||||
return Err(Error::msg("Item not found"));
|
||||
async fn put_multiple(&self, items: Vec<T>) -> StoreResult<Key> {
|
||||
let entries_len = self.entries.read().await.len() as u64;
|
||||
if entries_len >= self.entry_limit {
|
||||
return Err(StoreError::LimitExceeded);
|
||||
}
|
||||
|
||||
Ok(items[0].clone())
|
||||
if items.is_empty() {
|
||||
return Err(StoreError::Other("Cannot store empty item list".into()));
|
||||
}
|
||||
|
||||
// 生成 UUID 作为键
|
||||
let uuid = Uuid::new_v4();
|
||||
let key = Key::new(uuid.to_string(), self.file_ext.clone())
|
||||
.with_item_count(items.len())
|
||||
.with_compression(true);
|
||||
|
||||
self.multi_write(key.clone(), items).await?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
fn get_multiple(&self, key: &Key) -> Result<Vec<T>> {
|
||||
let data = self.get_raw(key)?;
|
||||
async fn get(&self, key: Key) -> StoreResult<T> {
|
||||
let items = self.get_multiple(key).await?;
|
||||
items
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| StoreError::Other("No items found".into()))
|
||||
}
|
||||
|
||||
let mut items = Vec::with_capacity(key.item_count);
|
||||
let mut reader = io::Cursor::new(&data);
|
||||
async fn get_multiple(&self, key: Key) -> StoreResult<Vec<T>> {
|
||||
let data = self.get_raw(key).await?;
|
||||
|
||||
// Try to read each JSON object
|
||||
let mut buffer = Vec::new();
|
||||
|
||||
// if the read fails try parsing it once
|
||||
if reader.read_to_end(&mut buffer).is_err() {
|
||||
// Try to parse the entire data as a single object
|
||||
return match serde_json::from_slice::<T>(&data) {
|
||||
Ok(item) => {
|
||||
items.push(item);
|
||||
Ok(items)
|
||||
}
|
||||
Err(_) => {
|
||||
// An attempt was made to resolve to an array of objects
|
||||
match serde_json::from_slice::<Vec<T>>(&data) {
|
||||
Ok(array_items) => Ok(array_items),
|
||||
Err(e) => Err(Error::msg(format!("Failed to parse the data:{}", e))),
|
||||
}
|
||||
}
|
||||
};
|
||||
// 尝试解析为 JSON 数组
|
||||
match serde_json::from_slice::<Vec<T>>(&data) {
|
||||
Ok(items) if !items.is_empty() => return Ok(items),
|
||||
Ok(_) => return Err(StoreError::Other("No items deserialized".into())),
|
||||
Err(_) => {} // 失败则尝试按行解析
|
||||
}
|
||||
|
||||
// Read JSON objects by row
|
||||
for line in buffer.split(|&b| b == b'\n') {
|
||||
if !line.is_empty() {
|
||||
match serde_json::from_slice::<T>(line) {
|
||||
Ok(item) => items.push(item),
|
||||
Err(e) => tracing::warn!("Failed to parse row data:{}", e),
|
||||
}
|
||||
// 如果直接解析为 Vec<T> 失败，则尝试按行解析
|
||||
// 转换为字符串并按行解析
|
||||
let data_str = std::str::from_utf8(&data).map_err(StoreError::Utf8)?;
|
||||
// 按行解析（JSON Lines）
|
||||
let mut items = Vec::new();
|
||||
for line in data_str.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let item = serde_json::from_str::<T>(line).map_err(StoreError::Deserialize)?;
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
if items.is_empty() {
|
||||
return Err(Error::msg("Failed to resolve any items"));
|
||||
return Err(StoreError::Other("Failed to deserialize items".into()));
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
fn get_raw(&self, key: &Key) -> Result<Vec<u8>> {
|
||||
let data = self.read_bytes(key)?;
|
||||
async fn get_raw(&self, key: Key) -> StoreResult<Vec<u8>> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
let data = fs::read(&path).await?;
|
||||
|
||||
// Delete the wrong file
|
||||
if data.is_empty() {
|
||||
let _ = self.del(key);
|
||||
return Err(Error::msg("the file is empty"));
|
||||
return Err(StoreError::Other("Empty file".into()));
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
if key.compress {
|
||||
let mut decoder = Decoder::new();
|
||||
decoder.decompress_vec(&data).map_err(|e| StoreError::Other(e.to_string()))
|
||||
} else {
|
||||
Ok(data)
|
||||
}
|
||||
}
|
||||
|
||||
fn put_raw(&self, data: &[u8]) -> Result<Key> {
|
||||
{
|
||||
let entries = self.entries.read().map_err(|_| Error::msg("Failed to obtain a read lock"))?;
|
||||
if entries.len() as u64 >= self.entry_limit {
|
||||
return Err(Error::msg("the storage limit has been reached"));
|
||||
}
|
||||
async fn put_raw(&self, data: Vec<u8>) -> StoreResult<Key> {
|
||||
let entries_len = self.entries.read().await.len() as u64;
|
||||
if entries_len >= self.entry_limit {
|
||||
return Err(StoreError::LimitExceeded);
|
||||
}
|
||||
|
||||
// Generate a new UUID
|
||||
// 生成 UUID 作为键
|
||||
let uuid = Uuid::new_v4();
|
||||
let key = Key::new(uuid.to_string(), &self.file_ext, true);
|
||||
let key = Key::new(uuid.to_string(), self.file_ext.clone());
|
||||
|
||||
self.write_bytes(&key, data)?;
|
||||
self.write_bytes(key.clone(), data).await?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.entries.read().map(|e| e.len()).unwrap_or(0)
|
||||
async fn len(&self) -> usize {
|
||||
self.entries.read().await.len()
|
||||
}
|
||||
|
||||
fn list(&self) -> Vec<Key> {
|
||||
let entries = match self.entries.read() {
|
||||
Ok(guard) => guard,
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
async fn list(&self) -> Vec<Key> {
|
||||
let entries = self.entries.read().await;
|
||||
|
||||
// Convert entries to vectors and sort by timestamp
|
||||
let mut entries_vec: Vec<_> = entries.iter().collect();
|
||||
entries_vec.sort_by(|a, b| a.1.cmp(b.1));
|
||||
// 将条目转换为 (key, timestamp) 元组并排序
|
||||
let mut entries_vec: Vec<(&String, &i64)> = entries.iter().collect();
|
||||
entries_vec.sort_by_key(|(_k, &v)| v);
|
||||
|
||||
// Parsing key
|
||||
entries_vec.iter().map(|(filename, _)| parse_key(filename)).collect()
|
||||
// 将排序后的键解析为 Key 结构体
|
||||
entries_vec.into_iter().map(|(k, _)| parse_key(k)).collect()
|
||||
}
|
||||
|
||||
fn del(&self, key: &Key) -> Result<()> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
|
||||
// Delete the file
|
||||
if let Err(e) = fs::remove_file(&path) {
|
||||
if e.kind() != io::ErrorKind::NotFound {
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the entry from the map
|
||||
let mut entries = self
|
||||
.entries
|
||||
.write()
|
||||
.map_err(|_| Error::msg("Failed to obtain a write lock"))?;
|
||||
entries.remove(&key.to_string());
|
||||
|
||||
Ok(())
|
||||
async fn del(&self, key: Key) -> StoreResult<()> {
|
||||
self.del_internal(&key).await
|
||||
}
|
||||
|
||||
fn open(&self) -> Result<()> {
|
||||
// Create a directory (if it doesn't exist)
|
||||
fs::create_dir_all(&self.directory)?;
|
||||
async fn open(&self) -> StoreResult<()> {
|
||||
// 创建目录（如果不存在）
|
||||
fs::create_dir_all(&self.directory).await?;
|
||||
|
||||
// Read existing files
|
||||
let files = self.list_files()?;
|
||||
// 读取已经存在的文件
|
||||
let entries = self.entries.write();
|
||||
let mut entries = entries.await;
|
||||
entries.clear();
|
||||
|
||||
let mut entries = self
|
||||
.entries
|
||||
.write()
|
||||
.map_err(|_| Error::msg("Failed to obtain a write lock"))?;
|
||||
let mut dir_entries = fs::read_dir(&self.directory).await?;
|
||||
while let Some(entry) = dir_entries.next_entry().await? {
|
||||
if let Ok(metadata) = entry.metadata().await {
|
||||
if metadata.is_file() {
|
||||
let modified = metadata
|
||||
.modified()?
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_err(|e| StoreError::Other(e.to_string()))?
|
||||
.as_nanos() as i64;
|
||||
|
||||
for file in files {
|
||||
if let Ok(meta) = file.metadata() {
|
||||
if let Ok(modified) = meta.modified() {
|
||||
if let Ok(since_epoch) = modified.duration_since(UNIX_EPOCH) {
|
||||
entries.insert(file.file_name().to_string_lossy().to_string(), since_epoch.as_nanos() as i64);
|
||||
}
|
||||
entries.insert(entry.file_name().to_string_lossy().to_string(), modified);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -517,8 +245,8 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete(&self) -> Result<()> {
|
||||
fs::remove_dir_all(&self.directory)?;
|
||||
async fn delete(&self) -> StoreResult<()> {
|
||||
fs::remove_dir_all(&self.directory).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::config::notifier::EventNotifierConfig;
|
||||
use crate::config::EventNotifierConfig;
|
||||
use crate::notifier::EventNotifier;
|
||||
use common::error::Result;
|
||||
use ecstore::store::ECStore;
|
||||
|
||||
Reference in New Issue
Block a user