mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-25 05:26:50 +00:00
improve channel adapter
This commit is contained in:
@@ -74,6 +74,7 @@ datafusion = "46.0.1"
|
|||||||
derive_builder = "0.20.2"
|
derive_builder = "0.20.2"
|
||||||
dioxus = { version = "0.6.3", features = ["router"] }
|
dioxus = { version = "0.6.3", features = ["router"] }
|
||||||
dirs = "6.0.0"
|
dirs = "6.0.0"
|
||||||
|
dotenvy = "0.15.7"
|
||||||
flatbuffers = "25.2.10"
|
flatbuffers = "25.2.10"
|
||||||
futures = "0.3.31"
|
futures = "0.3.31"
|
||||||
futures-core = "0.3.31"
|
futures-core = "0.3.31"
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ rdkafka = { workspace = true, features = ["tokio"], optional = true }
|
|||||||
tokio = { workspace = true, features = ["test-util"] }
|
tokio = { workspace = true, features = ["test-util"] }
|
||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
axum = { workspace = true }
|
axum = { workspace = true }
|
||||||
dotenvy = "0.15.7"
|
dotenvy = { workspace = true }
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use rustfs_event::{
|
use rustfs_event::{
|
||||||
AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotifierConfig, Object, Source, WebhookConfig,
|
AdapterConfig, Bucket, ChannelAdapterType, Error as NotifierError, Event, Identity, Metadata, Name, NotifierConfig, Object,
|
||||||
|
Source, WebhookConfig,
|
||||||
};
|
};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use tokio::signal;
|
use tokio::signal;
|
||||||
@@ -103,7 +104,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
})
|
})
|
||||||
.s3(metadata)
|
.s3(metadata)
|
||||||
.source(source)
|
.source(source)
|
||||||
.channels(vec!["webhook".to_string()])
|
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||||
.build()
|
.build()
|
||||||
.expect("failed to create event");
|
.expect("failed to create event");
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use rustfs_event::create_adapters;
|
|
||||||
use rustfs_event::NotifierSystem;
|
use rustfs_event::NotifierSystem;
|
||||||
|
use rustfs_event::{create_adapters, ChannelAdapterType};
|
||||||
use rustfs_event::{AdapterConfig, NotifierConfig, WebhookConfig};
|
use rustfs_event::{AdapterConfig, NotifierConfig, WebhookConfig};
|
||||||
use rustfs_event::{Bucket, Event, Identity, Metadata, Name, Object, Source};
|
use rustfs_event::{Bucket, Event, Identity, Metadata, Name, Object, Source};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -31,7 +31,7 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
|
|||||||
|
|
||||||
// event_load_config
|
// event_load_config
|
||||||
// loading configuration from environment variables
|
// loading configuration from environment variables
|
||||||
let _config = NotifierConfig::event_load_config(Some("./crates/event-notifier/examples/event.toml".to_string()));
|
let _config = NotifierConfig::event_load_config(Some("./crates/event/examples/event.toml".to_string()));
|
||||||
tracing::info!("event_load_config config: {:?} \n", _config);
|
tracing::info!("event_load_config config: {:?} \n", _config);
|
||||||
dotenvy::dotenv()?;
|
dotenvy::dotenv()?;
|
||||||
let _config = NotifierConfig::event_load_config(None);
|
let _config = NotifierConfig::event_load_config(None);
|
||||||
@@ -77,7 +77,7 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
|
|||||||
})
|
})
|
||||||
.s3(metadata)
|
.s3(metadata)
|
||||||
.source(source)
|
.source(source)
|
||||||
.channels(vec!["webhook".to_string()])
|
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||||
.build()
|
.build()
|
||||||
.expect("failed to create event");
|
.expect("failed to create event");
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use crate::ChannelAdapter;
|
|
||||||
use crate::Error;
|
use crate::Error;
|
||||||
use crate::Event;
|
use crate::Event;
|
||||||
use crate::KafkaConfig;
|
use crate::KafkaConfig;
|
||||||
|
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use rdkafka::error::KafkaError;
|
use rdkafka::error::KafkaError;
|
||||||
use rdkafka::producer::{FutureProducer, FutureRecord};
|
use rdkafka::producer::{FutureProducer, FutureRecord};
|
||||||
@@ -60,7 +60,7 @@ impl KafkaAdapter {
|
|||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl ChannelAdapter for KafkaAdapter {
|
impl ChannelAdapter for KafkaAdapter {
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
"kafka".to_string()
|
ChannelAdapterType::Kafka.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn send(&self, event: &Event) -> Result<(), Error> {
|
async fn send(&self, event: &Event) -> Result<(), Error> {
|
||||||
|
|||||||
@@ -11,6 +11,53 @@ pub(crate) mod mqtt;
|
|||||||
#[cfg(feature = "webhook")]
|
#[cfg(feature = "webhook")]
|
||||||
pub(crate) mod webhook;
|
pub(crate) mod webhook;
|
||||||
|
|
||||||
|
/// The `ChannelAdapterType` enum represents the different types of channel adapters.
|
||||||
|
///
|
||||||
|
/// It is used to identify the type of adapter being used in the system.
|
||||||
|
///
|
||||||
|
/// # Variants
|
||||||
|
///
|
||||||
|
/// - `Webhook`: Represents a webhook adapter.
|
||||||
|
/// - `Kafka`: Represents a Kafka adapter.
|
||||||
|
/// - `Mqtt`: Represents an MQTT adapter.
|
||||||
|
///
|
||||||
|
/// # Example
|
||||||
|
///
|
||||||
|
/// ```
|
||||||
|
/// use rustfs_event::ChannelAdapterType;
|
||||||
|
///
|
||||||
|
/// let adapter_type = ChannelAdapterType::Webhook;
|
||||||
|
/// match adapter_type {
|
||||||
|
/// ChannelAdapterType::Webhook => println!("Using webhook adapter"),
|
||||||
|
/// ChannelAdapterType::Kafka => println!("Using Kafka adapter"),
|
||||||
|
/// ChannelAdapterType::Mqtt => println!("Using MQTT adapter"),
|
||||||
|
/// }
|
||||||
|
pub enum ChannelAdapterType {
|
||||||
|
Webhook,
|
||||||
|
Kafka,
|
||||||
|
Mqtt,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChannelAdapterType {
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
ChannelAdapterType::Webhook => "webhook",
|
||||||
|
ChannelAdapterType::Kafka => "kafka",
|
||||||
|
ChannelAdapterType::Mqtt => "mqtt",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for ChannelAdapterType {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
ChannelAdapterType::Webhook => write!(f, "webhook"),
|
||||||
|
ChannelAdapterType::Kafka => write!(f, "kafka"),
|
||||||
|
ChannelAdapterType::Mqtt => write!(f, "mqtt"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The `ChannelAdapter` trait defines the interface for all channel adapters.
|
/// The `ChannelAdapter` trait defines the interface for all channel adapters.
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait ChannelAdapter: Send + Sync + 'static {
|
pub trait ChannelAdapter: Send + Sync + 'static {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use crate::ChannelAdapter;
|
|
||||||
use crate::Error;
|
use crate::Error;
|
||||||
use crate::Event;
|
use crate::Event;
|
||||||
use crate::MqttConfig;
|
use crate::MqttConfig;
|
||||||
|
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use rumqttc::{AsyncClient, MqttOptions, QoS};
|
use rumqttc::{AsyncClient, MqttOptions, QoS};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -33,7 +33,7 @@ impl MqttAdapter {
|
|||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl ChannelAdapter for MqttAdapter {
|
impl ChannelAdapter for MqttAdapter {
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
"mqtt".to_string()
|
ChannelAdapterType::Mqtt.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn send(&self, event: &Event) -> Result<(), Error> {
|
async fn send(&self, event: &Event) -> Result<(), Error> {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use crate::ChannelAdapter;
|
|
||||||
use crate::Error;
|
use crate::Error;
|
||||||
use crate::Event;
|
use crate::Event;
|
||||||
use crate::WebhookConfig;
|
use crate::WebhookConfig;
|
||||||
|
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use reqwest::{Client, RequestBuilder};
|
use reqwest::{Client, RequestBuilder};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -16,10 +16,11 @@ pub struct WebhookAdapter {
|
|||||||
impl WebhookAdapter {
|
impl WebhookAdapter {
|
||||||
/// Creates a new Webhook adapter.
|
/// Creates a new Webhook adapter.
|
||||||
pub fn new(config: WebhookConfig) -> Self {
|
pub fn new(config: WebhookConfig) -> Self {
|
||||||
let client = Client::builder()
|
let mut builder = Client::builder();
|
||||||
.timeout(Duration::from_secs(config.timeout))
|
if config.timeout > 0 {
|
||||||
.build()
|
builder = builder.timeout(Duration::from_secs(config.timeout));
|
||||||
.expect("Failed to build reqwest client");
|
}
|
||||||
|
let client = builder.build().expect("Failed to build reqwest client");
|
||||||
Self { config, client }
|
Self { config, client }
|
||||||
}
|
}
|
||||||
/// Builds the request to send the event.
|
/// Builds the request to send the event.
|
||||||
@@ -40,7 +41,7 @@ impl WebhookAdapter {
|
|||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl ChannelAdapter for WebhookAdapter {
|
impl ChannelAdapter for WebhookAdapter {
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
"webhook".to_string()
|
ChannelAdapterType::Webhook.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn send(&self, event: &Event) -> Result<(), Error> {
|
async fn send(&self, event: &Event) -> Result<(), Error> {
|
||||||
|
|||||||
@@ -21,17 +21,10 @@ pub async fn event_bus(
|
|||||||
shutdown: CancellationToken,
|
shutdown: CancellationToken,
|
||||||
shutdown_complete: Option<tokio::sync::oneshot::Sender<()>>,
|
shutdown_complete: Option<tokio::sync::oneshot::Sender<()>>,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
let mut current_log = Log {
|
|
||||||
event_name: crate::event::Name::Everything,
|
|
||||||
key: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(),
|
|
||||||
records: Vec::new(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut unprocessed_events = Vec::new();
|
let mut unprocessed_events = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
Some(event) = rx.recv() => {
|
Some(event) = rx.recv() => {
|
||||||
current_log.records.push(event.clone());
|
|
||||||
let mut send_tasks = Vec::new();
|
let mut send_tasks = Vec::new();
|
||||||
for adapter in &adapters {
|
for adapter in &adapters {
|
||||||
if event.channels.contains(&adapter.name()) {
|
if event.channels.contains(&adapter.name()) {
|
||||||
@@ -54,9 +47,6 @@ pub async fn event_bus(
|
|||||||
unprocessed_events.push(failed_event);
|
unprocessed_events.push(failed_event);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear the current log because we only care about unprocessed events
|
|
||||||
current_log.records.clear();
|
|
||||||
}
|
}
|
||||||
_ = shutdown.cancelled() => {
|
_ = shutdown.cancelled() => {
|
||||||
tracing::info!("Shutting down event bus, saving pending logs...");
|
tracing::info!("Shutting down event bus, saving pending logs...");
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ use serde::{Deserialize, Serialize};
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::env;
|
use std::env;
|
||||||
|
|
||||||
|
const DEFAULT_CONFIG_FILE: &str = "event";
|
||||||
|
|
||||||
/// Configuration for the webhook adapter.
|
/// Configuration for the webhook adapter.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct WebhookConfig {
|
pub struct WebhookConfig {
|
||||||
@@ -206,17 +208,18 @@ impl NotifierConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_CONFIG_FILE: &str = "event";
|
|
||||||
|
|
||||||
/// Provide temporary directories as default storage paths
|
/// Provide temporary directories as default storage paths
|
||||||
fn default_store_path() -> String {
|
fn default_store_path() -> String {
|
||||||
env::var("EVENT_STORE_PATH").unwrap_or_else(|e| {
|
env::var("EVENT_STORE_PATH").unwrap_or_else(|e| {
|
||||||
tracing::error!("Failed to get EVENT_STORE_PATH: {}", e);
|
tracing::info!("Failed to get `EVENT_STORE_PATH` failed err: {}", e.to_string());
|
||||||
env::temp_dir().join(DEFAULT_CONFIG_FILE).to_string_lossy().to_string()
|
env::temp_dir().join(DEFAULT_CONFIG_FILE).to_string_lossy().to_string()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Provides the recommended default channel capacity for high concurrency systems
|
/// Provides the recommended default channel capacity for high concurrency systems
|
||||||
fn default_channel_capacity() -> usize {
|
fn default_channel_capacity() -> usize {
|
||||||
10000 // Reasonable default values for high concurrency systems
|
env::var("EVENT_CHANNEL_CAPACITY")
|
||||||
|
.unwrap_or_else(|_| "10000".to_string())
|
||||||
|
.parse()
|
||||||
|
.unwrap_or(10000) // Default to 10000 if parsing fails
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,267 +0,0 @@
|
|||||||
use crate::NotifierConfig;
|
|
||||||
use common::error::{Error, Result};
|
|
||||||
use ecstore::config::com::CONFIG_PREFIX;
|
|
||||||
use ecstore::config::error::{is_err_config_not_found, ConfigError};
|
|
||||||
use ecstore::disk::RUSTFS_META_BUCKET;
|
|
||||||
use ecstore::store::ECStore;
|
|
||||||
use ecstore::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
|
||||||
use ecstore::store_err::is_err_object_not_found;
|
|
||||||
use ecstore::utils::path::SLASH_SEPARATOR;
|
|
||||||
use ecstore::StorageAPI;
|
|
||||||
use http::HeaderMap;
|
|
||||||
use lazy_static::lazy_static;
|
|
||||||
use std::io::Cursor;
|
|
||||||
use std::sync::{Arc, OnceLock};
|
|
||||||
use tracing::{error, instrument, warn};
|
|
||||||
|
|
||||||
lazy_static! {
|
|
||||||
pub static ref GLOBAL_EventSys: EventSys = EventSys::new();
|
|
||||||
pub static ref GLOBAL_EventSysConfig: OnceLock<NotifierConfig> = OnceLock::new();
|
|
||||||
}
|
|
||||||
/// * config file
|
|
||||||
const CONFIG_FILE: &str = "event.json";
|
|
||||||
|
|
||||||
/// event sys config
|
|
||||||
const EVENT: &str = "event";
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct EventSys {}
|
|
||||||
|
|
||||||
impl Default for EventSys {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EventSys {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {}
|
|
||||||
}
|
|
||||||
#[instrument(skip_all)]
|
|
||||||
pub async fn init(&self, api: Arc<ECStore>) -> Result<()> {
|
|
||||||
tracing::info!("event sys config init start");
|
|
||||||
let cfg = read_config_without_migrate(api.clone().clone()).await?;
|
|
||||||
let _ = GLOBAL_EventSysConfig.set(cfg);
|
|
||||||
tracing::info!("event sys config init done");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// get event sys config file
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// NotifierConfig
|
|
||||||
pub fn get_event_notifier_config() -> &'static NotifierConfig {
|
|
||||||
GLOBAL_EventSysConfig.get_or_init(NotifierConfig::default)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_event_sys_file() -> String {
|
|
||||||
format!("{}{}{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, EVENT, SLASH_SEPARATOR, CONFIG_FILE)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// read config without migrate
|
|
||||||
///
|
|
||||||
/// # Parameters
|
|
||||||
/// - `api`: StorageAPI
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// Configuration information
|
|
||||||
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
|
|
||||||
let config_file = get_event_sys_file();
|
|
||||||
let data = match read_config(api.clone(), config_file.as_str()).await {
|
|
||||||
Ok(res) => res,
|
|
||||||
Err(err) => {
|
|
||||||
return if is_err_config_not_found(&err) {
|
|
||||||
warn!("config not found, start to init");
|
|
||||||
let cfg = new_and_save_server_config(api).await?;
|
|
||||||
warn!("config init done");
|
|
||||||
Ok(cfg)
|
|
||||||
} else {
|
|
||||||
error!("read config err {:?}", &err);
|
|
||||||
Err(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
read_server_config(api, data.as_slice()).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// save config with options
|
|
||||||
///
|
|
||||||
/// # Parameters
|
|
||||||
/// - `api`: StorageAPI
|
|
||||||
/// - `file`: file name
|
|
||||||
/// - `data`: data to save
|
|
||||||
/// - `opts`: object options
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// Result
|
|
||||||
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
|
|
||||||
let size = data.len();
|
|
||||||
let _ = api
|
|
||||||
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::new(Box::new(Cursor::new(data)), size), opts)
|
|
||||||
.await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// new server config
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// NotifierConfig
|
|
||||||
fn new_server_config() -> NotifierConfig {
|
|
||||||
NotifierConfig::new()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
|
|
||||||
let cfg = new_server_config();
|
|
||||||
save_server_config(api, &cfg).await?;
|
|
||||||
|
|
||||||
Ok(cfg)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<NotifierConfig> {
|
|
||||||
let cfg = {
|
|
||||||
if data.is_empty() {
|
|
||||||
let config_file = get_event_sys_file();
|
|
||||||
let cfg_data = match read_config(api.clone(), config_file.as_str()).await {
|
|
||||||
Ok(res) => res,
|
|
||||||
Err(err) => {
|
|
||||||
return if is_err_config_not_found(&err) {
|
|
||||||
warn!("config not found init start");
|
|
||||||
let cfg = new_and_save_server_config(api).await?;
|
|
||||||
warn!("config not found init done");
|
|
||||||
Ok(cfg)
|
|
||||||
} else {
|
|
||||||
error!("read config err {:?}", &err);
|
|
||||||
Err(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
// TODO: decrypt
|
|
||||||
|
|
||||||
NotifierConfig::unmarshal(cfg_data.as_slice())?
|
|
||||||
} else {
|
|
||||||
NotifierConfig::unmarshal(data)?
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(cfg.merge())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// save server config
|
|
||||||
///
|
|
||||||
/// # Parameters
|
|
||||||
/// - `api`: StorageAPI
|
|
||||||
/// - `cfg`: configuration to save
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// Result
|
|
||||||
async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &NotifierConfig) -> Result<()> {
|
|
||||||
let data = cfg.marshal()?;
|
|
||||||
|
|
||||||
let config_file = get_event_sys_file();
|
|
||||||
|
|
||||||
save_config(api, &config_file, data).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// save config
|
|
||||||
///
|
|
||||||
/// # Parameters
|
|
||||||
/// - `api`: StorageAPI
|
|
||||||
/// - `file`: file name
|
|
||||||
/// - `data`: data to save
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// Result
|
|
||||||
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
|
|
||||||
save_config_with_opts(
|
|
||||||
api,
|
|
||||||
file,
|
|
||||||
data,
|
|
||||||
&ObjectOptions {
|
|
||||||
max_parity: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// delete config
|
|
||||||
///
|
|
||||||
/// # Parameters
|
|
||||||
/// - `api`: StorageAPI
|
|
||||||
/// - `file`: file name
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// Result
|
|
||||||
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
|
|
||||||
match api
|
|
||||||
.delete_object(
|
|
||||||
RUSTFS_META_BUCKET,
|
|
||||||
file,
|
|
||||||
ObjectOptions {
|
|
||||||
delete_prefix: true,
|
|
||||||
delete_prefix_object: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(_) => Ok(()),
|
|
||||||
Err(err) => {
|
|
||||||
if is_err_object_not_found(&err) {
|
|
||||||
Err(Error::new(ConfigError::NotFound))
|
|
||||||
} else {
|
|
||||||
Err(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// read config
|
|
||||||
///
|
|
||||||
/// # Parameters
|
|
||||||
/// - `api`: StorageAPI
|
|
||||||
/// - `file`: file name
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// Configuration data
|
|
||||||
pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
|
|
||||||
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
|
|
||||||
Ok(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// read config with metadata
|
|
||||||
///
|
|
||||||
/// # Parameters
|
|
||||||
/// - `api`: StorageAPI
|
|
||||||
/// - `file`: file name
|
|
||||||
/// - `opts`: object options
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// Configuration data and object info
|
|
||||||
pub async fn read_config_with_metadata<S: StorageAPI>(
|
|
||||||
api: Arc<S>,
|
|
||||||
file: &str,
|
|
||||||
opts: &ObjectOptions,
|
|
||||||
) -> Result<(Vec<u8>, ObjectInfo)> {
|
|
||||||
let h = HeaderMap::new();
|
|
||||||
let mut rd = api
|
|
||||||
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
|
|
||||||
.await
|
|
||||||
.map_err(|err| {
|
|
||||||
if is_err_object_not_found(&err) {
|
|
||||||
Error::new(ConfigError::NotFound)
|
|
||||||
} else {
|
|
||||||
err
|
|
||||||
}
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let data = rd.read_all().await?;
|
|
||||||
|
|
||||||
if data.is_empty() {
|
|
||||||
return Err(Error::new(ConfigError::NotFound));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok((data, rd.object_info))
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,6 @@ mod bus;
|
|||||||
mod config;
|
mod config;
|
||||||
mod error;
|
mod error;
|
||||||
mod event;
|
mod event;
|
||||||
mod event_sys;
|
|
||||||
mod global;
|
mod global;
|
||||||
mod notifier;
|
mod notifier;
|
||||||
mod store;
|
mod store;
|
||||||
@@ -16,6 +15,7 @@ pub use adapter::mqtt::MqttAdapter;
|
|||||||
#[cfg(feature = "webhook")]
|
#[cfg(feature = "webhook")]
|
||||||
pub use adapter::webhook::WebhookAdapter;
|
pub use adapter::webhook::WebhookAdapter;
|
||||||
pub use adapter::ChannelAdapter;
|
pub use adapter::ChannelAdapter;
|
||||||
|
pub use adapter::ChannelAdapterType;
|
||||||
pub use bus::event_bus;
|
pub use bus::event_bus;
|
||||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||||
pub use config::KafkaConfig;
|
pub use config::KafkaConfig;
|
||||||
@@ -29,12 +29,9 @@ pub use error::Error;
|
|||||||
pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source};
|
pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source};
|
||||||
pub use global::{initialize, is_initialized, is_ready, send_event, shutdown};
|
pub use global::{initialize, is_initialized, is_ready, send_event, shutdown};
|
||||||
pub use notifier::NotifierSystem;
|
pub use notifier::NotifierSystem;
|
||||||
pub use store::EventStore;
|
pub use store::event::EventStore;
|
||||||
|
|
||||||
pub use event_sys::delete_config;
|
pub use store::get_event_notifier_config;
|
||||||
pub use event_sys::get_event_notifier_config;
|
|
||||||
pub use event_sys::read_config;
|
|
||||||
pub use event_sys::save_config;
|
|
||||||
|
|
||||||
pub use event_sys::EventSys;
|
pub use store::EventSys;
|
||||||
pub use event_sys::GLOBAL_EventSys;
|
pub use store::GLOBAL_EventSys;
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
pub(crate) mod event;
|
||||||
|
|
||||||
|
use crate::NotifierConfig;
|
||||||
|
use common::error::Result;
|
||||||
|
use ecstore::config::com::{read_config, save_config, CONFIG_PREFIX};
|
||||||
|
use ecstore::config::error::is_err_config_not_found;
|
||||||
|
use ecstore::store::ECStore;
|
||||||
|
use ecstore::utils::path::SLASH_SEPARATOR;
|
||||||
|
use ecstore::StorageAPI;
|
||||||
|
use lazy_static::lazy_static;
|
||||||
|
use std::sync::{Arc, OnceLock};
|
||||||
|
use tracing::{error, instrument, warn};
|
||||||
|
|
||||||
|
lazy_static! {
|
||||||
|
pub static ref GLOBAL_EventSys: EventSys = EventSys::new();
|
||||||
|
pub static ref GLOBAL_EventSysConfig: OnceLock<NotifierConfig> = OnceLock::new();
|
||||||
|
}
|
||||||
|
/// * config file
|
||||||
|
const CONFIG_FILE: &str = "event.json";
|
||||||
|
|
||||||
|
/// event sys config
|
||||||
|
const EVENT: &str = "event";
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct EventSys {}
|
||||||
|
|
||||||
|
impl Default for EventSys {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EventSys {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {}
|
||||||
|
}
|
||||||
|
#[instrument(skip_all)]
|
||||||
|
pub async fn init(&self, api: Arc<ECStore>) -> Result<()> {
|
||||||
|
tracing::info!("event sys config init start");
|
||||||
|
let cfg = read_config_without_migrate(api.clone().clone()).await?;
|
||||||
|
let _ = GLOBAL_EventSysConfig.set(cfg);
|
||||||
|
tracing::info!("event sys config init done");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// get event sys config file
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// NotifierConfig
|
||||||
|
pub fn get_event_notifier_config() -> &'static NotifierConfig {
|
||||||
|
GLOBAL_EventSysConfig.get_or_init(NotifierConfig::default)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_event_sys_file() -> String {
|
||||||
|
format!("{}{}{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, EVENT, SLASH_SEPARATOR, CONFIG_FILE)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// new server config
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// NotifierConfig
|
||||||
|
fn new_server_config() -> NotifierConfig {
|
||||||
|
NotifierConfig::new()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
|
||||||
|
let cfg = new_server_config();
|
||||||
|
save_server_config(api, &cfg).await?;
|
||||||
|
|
||||||
|
Ok(cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// save server config
|
||||||
|
///
|
||||||
|
/// # Parameters
|
||||||
|
/// - `api`: StorageAPI
|
||||||
|
/// - `cfg`: configuration to save
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// Result
|
||||||
|
async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &NotifierConfig) -> Result<()> {
|
||||||
|
let data = cfg.marshal()?;
|
||||||
|
|
||||||
|
let config_file = get_event_sys_file();
|
||||||
|
|
||||||
|
save_config(api, &config_file, data).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// read config without migrate
|
||||||
|
///
|
||||||
|
/// # Parameters
|
||||||
|
/// - `api`: StorageAPI
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// Configuration information
|
||||||
|
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
|
||||||
|
let config_file = get_event_sys_file();
|
||||||
|
let data = match read_config(api.clone(), config_file.as_str()).await {
|
||||||
|
Ok(data) => {
|
||||||
|
if data.is_empty() {
|
||||||
|
return new_and_save_server_config(api).await;
|
||||||
|
}
|
||||||
|
data
|
||||||
|
}
|
||||||
|
Err(err) if is_err_config_not_found(&err) => {
|
||||||
|
warn!("config not found, start to init");
|
||||||
|
return new_and_save_server_config(api).await;
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
error!("read config error: {:?}", err);
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// TODO: decrypt if needed
|
||||||
|
let cfg = NotifierConfig::unmarshal(data.as_slice())?;
|
||||||
|
Ok(cfg.merge())
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
use rustfs_event::{AdapterConfig, NotifierSystem, WebhookConfig};
|
use rustfs_event::{AdapterConfig, ChannelAdapterType, NotifierSystem, WebhookConfig};
|
||||||
use rustfs_event::{Bucket, Event, EventBuilder, Identity, Metadata, Name, Object, Source};
|
use rustfs_event::{Bucket, Event, EventBuilder, Identity, Metadata, Name, Object, Source};
|
||||||
use rustfs_event::{ChannelAdapter, WebhookAdapter};
|
use rustfs_event::{ChannelAdapter, WebhookAdapter};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -57,7 +57,7 @@ async fn test_webhook_adapter() {
|
|||||||
.response_elements(HashMap::new())
|
.response_elements(HashMap::new())
|
||||||
.s3(metadata)
|
.s3(metadata)
|
||||||
.source(source)
|
.source(source)
|
||||||
.channels(vec!["webhook".to_string()])
|
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||||
.build()
|
.build()
|
||||||
.expect("failed to create event");
|
.expect("failed to create event");
|
||||||
|
|
||||||
@@ -122,7 +122,7 @@ async fn test_notification_system() {
|
|||||||
principal_id: "user123".to_string(),
|
principal_id: "user123".to_string(),
|
||||||
})
|
})
|
||||||
.event_time("2023-10-01T12:00:00.000Z")
|
.event_time("2023-10-01T12:00:00.000Z")
|
||||||
.channels(vec!["webhook".to_string()])
|
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||||
.build()
|
.build()
|
||||||
.expect("failed to create event");
|
.expect("failed to create event");
|
||||||
|
|
||||||
|
|||||||
+50
-39
@@ -33,33 +33,41 @@ lazy_static! {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// * read config
|
/// Reads configuration file content from the storage system
|
||||||
///
|
///
|
||||||
/// * @param api
|
/// This function reads a specified configuration file through the provided storage API interface
|
||||||
/// * @param file
|
/// and returns its raw byte content. It's a basic configuration reading function that returns
|
||||||
|
/// only the configuration data without any metadata.
|
||||||
|
///
|
||||||
|
/// # Parameters
|
||||||
|
/// * `api` - An Arc smart pointer containing an implementation of the StorageAPI trait
|
||||||
|
/// * `file` - The name of the configuration file to read
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// * `Result<Vec<u8>>` - Returns the configuration file contents as bytes on success, or an error on failure
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// May return the following errors:
|
||||||
|
/// * `ConfigError::NotFound` - When the requested configuration file does not exist
|
||||||
|
/// * Other storage operation related errors
|
||||||
///
|
///
|
||||||
/// * @return
|
|
||||||
/// * @description
|
|
||||||
/// * read config
|
|
||||||
/// * @error
|
|
||||||
/// * * ConfigError::NotFound
|
|
||||||
pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
|
pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
|
||||||
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
|
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
|
||||||
Ok(data)
|
Ok(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// * read_config_with_metadata
|
/// read config with metadata with api,file and opts
|
||||||
/// read config with metadata
|
|
||||||
///
|
///
|
||||||
/// * @param api
|
/// # Parameters
|
||||||
/// * @param file
|
/// - `api`: StorageAPI
|
||||||
/// * @param opts
|
/// - `file`: file name
|
||||||
|
/// - `opts`: object options
|
||||||
///
|
///
|
||||||
/// * @return
|
/// # Returns
|
||||||
/// * @description
|
/// Result<(Vec<u8>, ObjectInfo)>
|
||||||
/// * read config with metadata
|
///
|
||||||
/// * @error
|
/// # Errors
|
||||||
/// * * ConfigError::NotFound
|
/// - ConfigError::NotFound
|
||||||
pub async fn read_config_with_metadata<S: StorageAPI>(
|
pub async fn read_config_with_metadata<S: StorageAPI>(
|
||||||
api: Arc<S>,
|
api: Arc<S>,
|
||||||
file: &str,
|
file: &str,
|
||||||
@@ -86,15 +94,16 @@ pub async fn read_config_with_metadata<S: StorageAPI>(
|
|||||||
Ok((data, rd.object_info))
|
Ok((data, rd.object_info))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// * save_config
|
/// save config with api,file and data
|
||||||
///
|
///
|
||||||
/// * @param api
|
/// # Parameters
|
||||||
/// * @param file
|
|
||||||
/// * @param data
|
|
||||||
///
|
///
|
||||||
/// * @return
|
/// - `api`: StorageAPI
|
||||||
/// * @description
|
/// - `file`: file name
|
||||||
/// * save config
|
/// - `data`: data to save
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// Result
|
||||||
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
|
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
|
||||||
save_config_with_opts(
|
save_config_with_opts(
|
||||||
api,
|
api,
|
||||||
@@ -108,13 +117,15 @@ pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>)
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// * delete_config
|
/// delete config with api and file
|
||||||
|
///
|
||||||
|
/// # Parameters
|
||||||
|
/// - `api`: StorageAPI
|
||||||
|
/// - `file`: file name
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// Result
|
||||||
///
|
///
|
||||||
/// * @param api
|
|
||||||
/// * @param file
|
|
||||||
/// * @return
|
|
||||||
/// * @description
|
|
||||||
/// * delete config
|
|
||||||
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
|
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
|
||||||
match api
|
match api
|
||||||
.delete_object(
|
.delete_object(
|
||||||
@@ -139,15 +150,15 @@ pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// * save_config_with_opts
|
|
||||||
/// save config with opts
|
/// save config with opts
|
||||||
/// * @param api
|
///
|
||||||
/// * @param file
|
/// # Parameters
|
||||||
/// * @param data
|
/// - `api`: StorageAPI
|
||||||
/// * @param opts
|
/// - `file`: file name
|
||||||
/// * @return
|
/// - `data`: data to save
|
||||||
/// * @description
|
///
|
||||||
/// * save config with opts
|
/// # Returns
|
||||||
|
/// Result
|
||||||
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
|
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
|
||||||
let size = data.len();
|
let size = data.len();
|
||||||
let _ = api
|
let _ = api
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ use lazy_static::lazy_static;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
// default_parity_count 默认配置,根据磁盘总数分配校验磁盘数量
|
/// Default parity count for a given drive count
|
||||||
|
/// The default configuration allocates the number of check disks based on the total number of disks
|
||||||
pub fn default_parity_count(drive: usize) -> usize {
|
pub fn default_parity_count(drive: usize) -> usize {
|
||||||
match drive {
|
match drive {
|
||||||
1 => 0,
|
1 => 0,
|
||||||
|
|||||||
+23
-23
@@ -3,29 +3,29 @@ use tracing::{error, info, instrument};
|
|||||||
|
|
||||||
#[instrument]
|
#[instrument]
|
||||||
pub(crate) async fn init_event_notifier(notifier_config: Option<String>) {
|
pub(crate) async fn init_event_notifier(notifier_config: Option<String>) {
|
||||||
// Initialize event notifier
|
info!("Initializing event notifier...");
|
||||||
if notifier_config.is_some() {
|
let notifier_config_present = notifier_config.is_some();
|
||||||
info!("event_config is not empty");
|
let config = if notifier_config_present {
|
||||||
tokio::spawn(async move {
|
info!("event_config is not empty, path: {:?}", notifier_config);
|
||||||
let config = NotifierConfig::event_load_config(notifier_config);
|
NotifierConfig::event_load_config(notifier_config)
|
||||||
let result = rustfs_event::initialize(&config).await;
|
|
||||||
if let Err(e) = result {
|
|
||||||
error!("Failed to initialize event notifier: {}", e);
|
|
||||||
} else {
|
|
||||||
info!("Event notifier initialized successfully");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} else {
|
} else {
|
||||||
info!("event_config is empty");
|
info!("event_config is empty");
|
||||||
tokio::spawn(async move {
|
rustfs_event::get_event_notifier_config().clone()
|
||||||
let config = rustfs_event::get_event_notifier_config();
|
};
|
||||||
info!("event_config is {:?}", config);
|
|
||||||
let result = rustfs_event::initialize(config).await;
|
info!("using event_config: {:?}", config);
|
||||||
if let Err(e) = result {
|
tokio::spawn(async move {
|
||||||
error!("Failed to initialize event notifier: {}", e);
|
let result = rustfs_event::initialize(&config).await;
|
||||||
} else {
|
match result {
|
||||||
info!("Event notifier initialized successfully");
|
Ok(_) => info!(
|
||||||
}
|
"event notifier initialized successfully {}",
|
||||||
});
|
if notifier_config_present {
|
||||||
}
|
"by config file"
|
||||||
|
} else {
|
||||||
|
"by sys config"
|
||||||
|
}
|
||||||
|
),
|
||||||
|
Err(e) => error!("Failed to initialize event notifier: {}", e),
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -506,8 +506,10 @@ async fn run(opt: config::Opt) -> Result<()> {
|
|||||||
})?;
|
})?;
|
||||||
|
|
||||||
ecconfig::init();
|
ecconfig::init();
|
||||||
|
// config system configuration
|
||||||
GLOBAL_ConfigSys.init(store.clone()).await?;
|
GLOBAL_ConfigSys.init(store.clone()).await?;
|
||||||
|
|
||||||
|
// event system configuration
|
||||||
GLOBAL_EventSys.init(store.clone()).await?;
|
GLOBAL_EventSys.init(store.clone()).await?;
|
||||||
|
|
||||||
// Initialize event notifier
|
// Initialize event notifier
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
pub mod access;
|
pub mod access;
|
||||||
pub mod ecfs;
|
pub mod ecfs;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
mod event_notifier;
|
mod event;
|
||||||
pub mod options;
|
pub mod options;
|
||||||
|
|||||||
Reference in New Issue
Block a user