gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #498 from rustfs/feature/observability-metrics

Browse Source 
Feat(notify): Notify module
This commit is contained in:
loverustfs
2025-06-23 09:32:16 +08:00
committed by GitHub
parent 3721c93a55 5155a3d544
commit 1722780560
137 changed files with 9504 additions and 4050 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.docker/mqtt/config/emqx.conf
+37
View File
@@ -0,0 +1,37 @@
# 节点配置
node.name = "emqx@127.0.0.1"
node.cookie = "aBcDeFgHiJkLmNoPqRsTuVwXyZ012345"
node.data_dir = "/opt/emqx/data"
# 日志配置
log.console = {level = info, enable = true}
log.file = {path = "/opt/emqx/log/emqx.log", enable = true, level = info}
# MQTT TCP 监听器
listeners.tcp.default = {bind = "0.0.0.0:1883", max_connections = 1000000, enable = true}
# MQTT SSL 监听器
listeners.ssl.default = {bind = "0.0.0.0:8883", enable = false}
# MQTT WebSocket 监听器
listeners.ws.default = {bind = "0.0.0.0:8083", enable = true}
# MQTT WebSocket SSL 监听器
listeners.wss.default = {bind = "0.0.0.0:8084", enable = false}
# 管理控制台
dashboard.listeners.http = {bind = "0.0.0.0:18083", enable = true}
# HTTP API
management.listeners.http = {bind = "0.0.0.0:8081", enable = true}
# 认证配置
authentication = [
{enable = true, mechanism = password_based, backend = built_in_database, user_id_type = username}
]
# 授权配置
authorization.sources = [{type = built_in_database, enable = true}]
# 持久化消息存储
message.storage.backend = built_in_database
.docker/mqtt/config/vm.args
+9
View File
@@ -0,0 +1,9 @@
-name emqx@127.0.0.1
-setcookie aBcDeFgHiJkLmNoPqRsTuVwXyZ012345
+P 2097152
+t 1048576
+zdbbl 32768
-kernel inet_dist_listen_min 6000
-kernel inet_dist_listen_max 6100
-smp enable
-mnesia dir "/opt/emqx/data/mnesia"
.docker/mqtt/docker-compose-more.yml
+60
View File
@@ -0,0 +1,60 @@
services:
emqx:
image: emqx/emqx:latest
container_name: emqx
restart: unless-stopped
environment:
- EMQX_NODE__NAME=emqx@127.0.0.1
- EMQX_NODE__COOKIE=aBcDeFgHiJkLmNoPqRsTuVwXyZ012345
- EMQX_NODE__DATA_DIR=/opt/emqx/data
- EMQX_LOG__CONSOLE__LEVEL=info
- EMQX_LOG__CONSOLE__ENABLE=true
- EMQX_LOG__FILE__PATH=/opt/emqx/log/emqx.log
- EMQX_LOG__FILE__LEVEL=info
- EMQX_LOG__FILE__ENABLE=true
- EMQX_LISTENERS__TCP__DEFAULT__BIND=0.0.0.0:1883
- EMQX_LISTENERS__TCP__DEFAULT__MAX_CONNECTIONS=1000000
- EMQX_LISTENERS__TCP__DEFAULT__ENABLE=true
- EMQX_LISTENERS__SSL__DEFAULT__BIND=0.0.0.0:8883
- EMQX_LISTENERS__SSL__DEFAULT__ENABLE=false
- EMQX_LISTENERS__WS__DEFAULT__BIND=0.0.0.0:8083
- EMQX_LISTENERS__WS__DEFAULT__ENABLE=true
- EMQX_LISTENERS__WSS__DEFAULT__BIND=0.0.0.0:8084
- EMQX_LISTENERS__WSS__DEFAULT__ENABLE=false
- EMQX_DASHBOARD__LISTENERS__HTTP__BIND=0.0.0.0:18083
- EMQX_DASHBOARD__LISTENERS__HTTP__ENABLE=true
- EMQX_MANAGEMENT__LISTENERS__HTTP__BIND=0.0.0.0:8081
- EMQX_MANAGEMENT__LISTENERS__HTTP__ENABLE=true
- EMQX_AUTHENTICATION__1__ENABLE=true
- EMQX_AUTHENTICATION__1__MECHANISM=password_based
- EMQX_AUTHENTICATION__1__BACKEND=built_in_database
- EMQX_AUTHENTICATION__1__USER_ID_TYPE=username
- EMQX_AUTHORIZATION__SOURCES__1__TYPE=built_in_database
- EMQX_AUTHORIZATION__SOURCES__1__ENABLE=true
ports:
- "1883:1883" # MQTT TCP
- "8883:8883" # MQTT SSL
- "8083:8083" # MQTT WebSocket
- "8084:8084" # MQTT WebSocket SSL
- "18083:18083" # Web 管理控制台
- "8081:8081" # HTTP API
volumes:
- ./data:/opt/emqx/data
- ./log:/opt/emqx/log
- ./config:/opt/emqx/etc
networks:
- mqtt-net
healthcheck:
test: [ "CMD", "/opt/emqx/bin/emqx_ctl", "status" ]
interval: 30s
timeout: 10s
retries: 3
logging:
driver: "json-file"
options:
max-size: "100m"
max-file: "3"
networks:
mqtt-net:
driver: bridge
.docker/mqtt/docker-compose.yml
+15
View File
@@ -0,0 +1,15 @@
services:
emqx:
image: emqx/emqx:latest
container_name: emqx
ports:
- "1883:1883"
- "8083:8083"
- "8084:8084"
- "8883:8883"
- "18083:18083"
restart: unless-stopped
networks:
default:
driver: bridge
.github/workflows/build.yml
+15 -15
View File
@@ -7,7 +7,7 @@ on:
push:
branches:
- main
tags: ["v*", "*"]
tags: [ "v*", "*" ]
jobs:
build-rustfs:
@@ -15,25 +15,25 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
os: [ ubuntu-latest, macos-latest, windows-latest ]
variant:
- {
profile: release,
target: x86_64-unknown-linux-musl,
glibc: "default",
}
profile: release,
target: x86_64-unknown-linux-musl,
glibc: "default",
}
- {
profile: release,
target: x86_64-unknown-linux-gnu,
glibc: "default",
}
profile: release,
target: x86_64-unknown-linux-gnu,
glibc: "default",
}
- { profile: release, target: aarch64-apple-darwin, glibc: "default" }
#- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" }
- {
profile: release,
target: aarch64-unknown-linux-musl,
glibc: "default",
}
profile: release,
target: aarch64-unknown-linux-musl,
glibc: "default",
}
#- { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" }
exclude:
# Linux targets on non-Linux systems
@@ -518,7 +518,7 @@ jobs:
merge:
runs-on: ubuntu-latest
needs: [build-rustfs]
needs: [ build-rustfs ]
if: startsWith(github.ref, 'refs/tags/')
steps:
- uses: actions/upload-artifact/merge@v4
.github/workflows/ci.yml
+1 -1
View File
@@ -75,4 +75,4 @@ jobs:
- uses: actions/upload-artifact@v4
with:
name: s3s-e2e.logs
path: /tmp/rustfs.log
path: /tmp/rustfs.log
.gitignore
+1 -1
View File
@@ -17,4 +17,4 @@ deploy/certs/*
.rustfs.sys
.cargo
profile.json
.docker/openobserve-otel/data
.docker/openobserve-otel/data
Cargo.lock
Generated
+429 -404
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+17 -11
View File
@@ -3,13 +3,16 @@ members = [
"appauth", # Application authentication and authorization
"cli/rustfs-gui", # Graphical user interface client
"common/common", # Shared utilities and data structures
"crates/filemeta", # File metadata management
"common/lock", # Distributed locking implementation
"common/protos", # Protocol buffer definitions
"common/workers", # Worker thread pools and task scheduling
"crates/config", # Configuration management
"crates/event-notifier", # Event notification system
"crates/notify", # Notification system for events
"crates/obs", # Observability utilities
"crates/rio", # Rust I/O utilities and abstractions
"crates/utils", # Utility functions and helpers
"crates/zip", # ZIP file handling and compression
"crypto", # Cryptography and security features
"ecstore", # Erasure coding storage implementation
"e2e_test", # End-to-end test suite
@@ -18,9 +21,7 @@ members = [
"rustfs", # Core file system implementation
"s3select/api", # S3 Select API interface
"s3select/query", # S3 Select query engine
"crates/zip",
"crates/filemeta",
"crates/rio",
]
resolver = "2"
@@ -54,7 +55,7 @@ rustfs = { path = "./rustfs", version = "0.0.1" }
rustfs-zip = { path = "./crates/zip", version = "0.0.1" }
rustfs-config = { path = "./crates/config", version = "0.0.1" }
rustfs-obs = { path = "crates/obs", version = "0.0.1" }
rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" }
rustfs-notify = { path = "crates/notify", version = "0.0.1" }
rustfs-utils = { path = "crates/utils", version = "0.0.1" }
rustfs-rio = { path = "crates/rio", version = "0.0.1" }
rustfs-filemeta = { path = "crates/filemeta", version = "0.0.1" }
@@ -88,11 +89,12 @@ const-str = { version = "0.6.2", features = ["std", "proc"] }
crc32fast = "1.4.2"
datafusion = "46.0.1"
derive_builder = "0.20.2"
dotenvy = "0.15.7"
dioxus = { version = "0.6.3", features = ["router"] }
dirs = "6.0.0"
dotenvy = "0.15.7"
flatbuffers = "25.2.10"
flexi_logger = { version = "0.30.2", features = ["trc","dont_minimize_extra_stacks"] }
flexi_logger = { version = "0.30.2", features = ["trc", "dont_minimize_extra_stacks"] }
form_urlencoded = "1.2.1"
futures = "0.3.31"
futures-core = "0.3.31"
futures-util = "0.3.31"
@@ -158,17 +160,17 @@ pin-project-lite = "0.2.16"
prost = "0.13.5"
prost-build = "0.13.5"
protobuf = "3.7"
quick-xml = "0.37.5"
rand = "0.9.1"
brotli = "8.0.1"
flate2 = "1.1.1"
zstd = "0.13.3"
lz4 = "1.28.1"
snap = "1.1.1"
rdkafka = { version = "0.37.0", features = ["tokio"] }
reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] }
reed-solomon-simd = { version = "3.0.0" }
regex = { version = "1.11.1" }
reqwest = { version = "0.12.19", default-features = false, features = [
reqwest = { version = "0.12.20", default-features = false, features = [
"rustls-tls",
"charset",
"http2",
@@ -186,6 +188,7 @@ rmp-serde = "1.3.0"
rsa = "0.9.8"
rumqttc = { version = "0.24" }
rust-embed = { version = "8.7.2" }
rust-i18n = { version = "3.1.4" }
rustfs-rsc = "2025.506.1"
rustls = { version = "0.23.27" }
rustls-pki-types = "1.12.0"
@@ -202,6 +205,7 @@ sha2 = "0.10.9"
siphasher = "1.0.1"
smallvec = { version = "1.15.1", features = ["serde"] }
snafu = "0.8.6"
snap = "1.1.1"
socket2 = "0.5.10"
strum = { version = "0.27.1", features = ["derive"] }
sysinfo = "0.35.2"
@@ -215,13 +219,14 @@ time = { version = "0.3.41", features = [
"macros",
"serde",
] }
tokio = { version = "1.45.1", features = ["fs", "rt-multi-thread"] }
tonic = { version = "0.13.1", features = ["gzip"] }
tonic-build = { version = "0.13.1" }
tokio-rustls = { version = "0.26.2", default-features = false }
tokio-stream = { version = "0.1.17" }
tokio-tar = "0.3.1"
tokio-util = { version = "0.7.15", features = ["io", "compat"] }
tonic = { version = "0.13.1", features = ["gzip"] }
tonic-build = { version = "0.13.1" }
tower = { version = "0.5.2", features = ["timeout"] }
tower-http = { version = "0.6.6", features = ["cors"] }
tracing = "0.1.41"
@@ -238,6 +243,7 @@ uuid = { version = "1.17.0", features = [
"fast-rng",
"macro-diagnostics",
] }
wildmatch = { version = "2.4.0", features = ["serde"] }
winapi = { version = "0.3.9" }
xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
README.md
+29 -2
View File
@@ -73,9 +73,9 @@ export RUSTFS_OBS_ENDPOINT="http://localhost:4317"
./rustfs /data/rustfs
```
### Observability Stack
## Observability Stack Otel and OpenObserve
#### Deployment
### OpenTelemetry Collector 和 Jaeger、Grafana、Prometheus、Loki
1. Navigate to the observability directory:
```bash
@@ -92,3 +92,30 @@ export RUSTFS_OBS_ENDPOINT="http://localhost:4317"
- Grafana: `http://localhost:3000` (credentials: `admin`/`admin`)
- Jaeger: `http://localhost:16686`
- Prometheus: `http://localhost:9090`
#### Configure observability
```
OpenTelemetry Collector address(endpoint): http://localhost:4317
```
---
### OpenObserve and OpenTelemetry Collector
1. Navigate to the OpenObserve and OpenTelemetry directory:
```bash
cd .docker/openobserve-otel
```
2. Start the OpenObserve and OpenTelemetry Collector services:
```bash
docker compose -f docker-compose.yml up -d
```
3. Access the OpenObserve UI:
OpenObserve UI: `http://localhost:5080`
- Default credentials:
- Username: `root@rustfs.com`
- Password: `rustfs123`
- Exposed ports:
- 5080: HTTP API and UI
- 5081: OTLP gRPC
README_ZH.md
+25 -3
View File
@@ -72,9 +72,9 @@ export RUSTFS_OBS_ENDPOINT="http://localhost:4317"
./rustfs /data/rustfs
```
### 可观测性系统
## 可观测性系统 Otel 和 OpenObserve
#### 部署
### OpenTelemetry Collector 和 Jaeger、Grafana、Prometheus、Loki
1. 进入可观测性目录：
```bash
@@ -96,4 +96,26 @@ export RUSTFS_OBS_ENDPOINT="http://localhost:4317"
```
OpenTelemetry Collector 地址(endpoint): http://localhost:4317
```
```
---
### OpenObserve 和 OpenTelemetry Collector
1. 进入 OpenObserve 和 OpenTelemetry 目录：
```bash
cd .docker/openobserve-otel
```
2. 启动 OpenObserve 和 OpenTelemetry Collector 服务：
```bash
docker compose -f docker-compose.yml up -d
```
3. 访问 OpenObserve UI：
OpenObserve UI: `http://localhost:5080`
- 默认凭据：
- 用户名：`root@rustfs.com`
- 密码：`rustfs123`
- 开放端口：
- 5080：HTTP API 和 UI
- 5081：OTLP gRPC
appauth/src/token.rs
+2 -2
View File
@@ -1,8 +1,8 @@
use rsa::Pkcs1v15Encrypt;
use rsa::{
RsaPrivateKey, RsaPublicKey,
pkcs8::{DecodePrivateKey, DecodePublicKey},
rand_core::OsRng,
RsaPrivateKey, RsaPublicKey,
};
use serde::{Deserialize, Serialize};
use std::io::{Error, Result};
@@ -58,8 +58,8 @@ static TEST_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhk
mod tests {
use super::*;
use rsa::{
RsaPrivateKey,
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
RsaPrivateKey,
};
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
cli/rustfs-gui/Cargo.toml
+1
View File
@@ -15,6 +15,7 @@ keyring = { workspace = true }
lazy_static = { workspace = true }
rfd = { workspace = true }
rust-embed = { workspace = true, features = ["interpolate-folder-path"] }
rust-i18n = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
crates/config/Cargo.toml
+7 -2
View File
@@ -7,11 +7,16 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
config = { workspace = true }
const-str = { workspace = true }
const-str = { workspace = true, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
[lints]
workspace = true
[features]
default = []
constants = ["dep:const-str"]
observability = []
crates/config/src/config.rs
-200
View File
@@ -1,200 +0,0 @@
use crate::ObservabilityConfig;
use crate::event::config::NotifierConfig;
/// RustFs configuration
pub struct RustFsConfig {
pub observability: ObservabilityConfig,
pub event: NotifierConfig,
}
impl RustFsConfig {
pub fn new() -> Self {
Self {
observability: ObservabilityConfig::new(),
event: NotifierConfig::new(),
}
}
}
impl Default for RustFsConfig {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_rustfs_config_new() {
let config = RustFsConfig::new();
// Verify that observability config is properly initialized
assert!(!config.observability.sinks.is_empty(), "Observability sinks should not be empty");
assert!(config.observability.logger.is_some(), "Logger config should be present");
// Verify that event config is properly initialized
assert!(!config.event.store_path.is_empty(), "Event store path should not be empty");
assert!(config.event.channel_capacity > 0, "Channel capacity should be positive");
assert!(!config.event.adapters.is_empty(), "Event adapters should not be empty");
}
#[test]
fn test_rustfs_config_default() {
let config = RustFsConfig::default();
// Default should be equivalent to new()
let new_config = RustFsConfig::new();
// Compare observability config
assert_eq!(config.observability.sinks.len(), new_config.observability.sinks.len());
assert_eq!(config.observability.logger.is_some(), new_config.observability.logger.is_some());
// Compare event config
assert_eq!(config.event.store_path, new_config.event.store_path);
assert_eq!(config.event.channel_capacity, new_config.event.channel_capacity);
assert_eq!(config.event.adapters.len(), new_config.event.adapters.len());
}
#[test]
fn test_rustfs_config_components_independence() {
let mut config = RustFsConfig::new();
// Modify observability config
config.observability.sinks.clear();
// Event config should remain unchanged
assert!(!config.event.adapters.is_empty(), "Event adapters should remain unchanged");
assert!(config.event.channel_capacity > 0, "Channel capacity should remain unchanged");
// Create new config to verify independence
let new_config = RustFsConfig::new();
assert!(!new_config.observability.sinks.is_empty(), "New config should have default sinks");
}
#[test]
fn test_rustfs_config_observability_integration() {
let config = RustFsConfig::new();
// Test observability config properties
assert!(config.observability.otel.endpoint.is_empty() || !config.observability.otel.endpoint.is_empty());
assert!(config.observability.otel.use_stdout.is_some());
assert!(config.observability.otel.sample_ratio.is_some());
assert!(config.observability.otel.meter_interval.is_some());
assert!(config.observability.otel.service_name.is_some());
assert!(config.observability.otel.service_version.is_some());
assert!(config.observability.otel.environment.is_some());
assert!(config.observability.otel.logger_level.is_some());
}
#[test]
fn test_rustfs_config_event_integration() {
let config = RustFsConfig::new();
// Test event config properties
assert!(!config.event.store_path.is_empty(), "Store path should not be empty");
assert!(
config.event.channel_capacity >= 1000,
"Channel capacity should be reasonable for production"
);
// Test that store path is a valid path format
let store_path = &config.event.store_path;
assert!(!store_path.contains('\0'), "Store path should not contain null characters");
// Test adapters configuration
for adapter in &config.event.adapters {
// Each adapter should have a valid configuration
match adapter {
crate::event::adapters::AdapterConfig::Webhook(_) => {
// Webhook adapter should be properly configured
}
crate::event::adapters::AdapterConfig::Kafka(_) => {
// Kafka adapter should be properly configured
}
crate::event::adapters::AdapterConfig::Mqtt(_) => {
// MQTT adapter should be properly configured
}
}
}
}
#[test]
fn test_rustfs_config_memory_usage() {
// Test that config doesn't use excessive memory
let config = RustFsConfig::new();
// Basic memory usage checks
assert!(std::mem::size_of_val(&config) < 10000, "Config should not use excessive memory");
// Test that strings are not excessively long
assert!(config.event.store_path.len() < 1000, "Store path should not be excessively long");
// Test that collections are reasonably sized
assert!(config.observability.sinks.len() < 100, "Sinks collection should be reasonably sized");
assert!(config.event.adapters.len() < 100, "Adapters collection should be reasonably sized");
}
#[test]
fn test_rustfs_config_serialization_compatibility() {
let config = RustFsConfig::new();
// Test that observability config can be serialized (it has Serialize trait)
let observability_json = serde_json::to_string(&config.observability);
assert!(observability_json.is_ok(), "Observability config should be serializable");
// Test that event config can be serialized (it has Serialize trait)
let event_json = serde_json::to_string(&config.event);
assert!(event_json.is_ok(), "Event config should be serializable");
}
#[test]
fn test_rustfs_config_debug_format() {
let config = RustFsConfig::new();
// Test that observability config has Debug trait
let observability_debug = format!("{:?}", config.observability);
assert!(!observability_debug.is_empty(), "Observability config should have debug output");
assert!(
observability_debug.contains("ObservabilityConfig"),
"Debug output should contain type name"
);
// Test that event config has Debug trait
let event_debug = format!("{:?}", config.event);
assert!(!event_debug.is_empty(), "Event config should have debug output");
assert!(event_debug.contains("NotifierConfig"), "Debug output should contain type name");
}
#[test]
fn test_rustfs_config_clone_behavior() {
let config = RustFsConfig::new();
// Test that observability config can be cloned
let observability_clone = config.observability.clone();
assert_eq!(observability_clone.sinks.len(), config.observability.sinks.len());
// Test that event config can be cloned
let event_clone = config.event.clone();
assert_eq!(event_clone.store_path, config.event.store_path);
assert_eq!(event_clone.channel_capacity, config.event.channel_capacity);
}
#[test]
fn test_rustfs_config_environment_independence() {
// Test that config creation doesn't depend on specific environment variables
// This test ensures the config can be created in any environment
let config1 = RustFsConfig::new();
let config2 = RustFsConfig::new();
// Both configs should have the same structure
assert_eq!(config1.observability.sinks.len(), config2.observability.sinks.len());
assert_eq!(config1.event.adapters.len(), config2.event.adapters.len());
// Store paths should be consistent
assert_eq!(config1.event.store_path, config2.event.store_path);
assert_eq!(config1.event.channel_capacity, config2.event.channel_capacity);
}
}
crates/config/src/event/adapters.rs
-27
View File
@@ -1,27 +0,0 @@
use crate::event::kafka::KafkaAdapter;
use crate::event::mqtt::MqttAdapter;
use crate::event::webhook::WebhookAdapter;
use serde::{Deserialize, Serialize};
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum AdapterConfig {
Webhook(WebhookAdapter),
Kafka(KafkaAdapter),
Mqtt(MqttAdapter),
}
impl AdapterConfig {
/// create a new configuration with default values
pub fn new() -> Self {
Self::Webhook(WebhookAdapter::new())
}
}
impl Default for AdapterConfig {
/// create a new configuration with default values
fn default() -> Self {
Self::new()
}
}
crates/config/src/event/config.rs
-334
View File
@@ -1,334 +0,0 @@
use crate::event::adapters::AdapterConfig;
use serde::{Deserialize, Serialize};
use std::env;
#[allow(dead_code)]
const DEFAULT_CONFIG_FILE: &str = "event";
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NotifierConfig {
#[serde(default = "default_store_path")]
pub store_path: String,
#[serde(default = "default_channel_capacity")]
pub channel_capacity: usize,
pub adapters: Vec<AdapterConfig>,
}
impl Default for NotifierConfig {
fn default() -> Self {
Self::new()
}
}
impl NotifierConfig {
/// create a new configuration with default values
pub fn new() -> Self {
Self {
store_path: default_store_path(),
channel_capacity: default_channel_capacity(),
adapters: vec![AdapterConfig::new()],
}
}
}
/// Provide temporary directories as default storage paths
fn default_store_path() -> String {
env::temp_dir().join("event-notification").to_string_lossy().to_string()
}
/// Provides the recommended default channel capacity for high concurrency systems
fn default_channel_capacity() -> usize {
10000 // Reasonable default values for high concurrency systems
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::Path;
#[test]
fn test_notifier_config_new() {
let config = NotifierConfig::new();
// Verify store path is set
assert!(!config.store_path.is_empty(), "Store path should not be empty");
assert!(
config.store_path.contains("event-notification"),
"Store path should contain event-notification"
);
// Verify channel capacity is reasonable
assert_eq!(config.channel_capacity, 10000, "Channel capacity should be 10000");
assert!(config.channel_capacity > 0, "Channel capacity should be positive");
// Verify adapters are initialized
assert!(!config.adapters.is_empty(), "Adapters should not be empty");
assert_eq!(config.adapters.len(), 1, "Should have exactly one default adapter");
}
#[test]
fn test_notifier_config_default() {
let config = NotifierConfig::default();
let new_config = NotifierConfig::new();
// Default should be equivalent to new()
assert_eq!(config.store_path, new_config.store_path);
assert_eq!(config.channel_capacity, new_config.channel_capacity);
assert_eq!(config.adapters.len(), new_config.adapters.len());
}
#[test]
fn test_default_store_path() {
let store_path = default_store_path();
// Verify store path properties
assert!(!store_path.is_empty(), "Store path should not be empty");
assert!(store_path.contains("event-notification"), "Store path should contain event-notification");
// Verify it's a valid path format
let path = Path::new(&store_path);
assert!(path.is_absolute() || path.is_relative(), "Store path should be a valid path");
// Verify it doesn't contain invalid characters
assert!(!store_path.contains('\0'), "Store path should not contain null characters");
// Verify it's based on temp directory
let temp_dir = env::temp_dir();
let expected_path = temp_dir.join("event-notification");
assert_eq!(store_path, expected_path.to_string_lossy().to_string());
}
#[test]
fn test_default_channel_capacity() {
let capacity = default_channel_capacity();
// Verify capacity is reasonable
assert_eq!(capacity, 10000, "Default capacity should be 10000");
assert!(capacity > 0, "Capacity should be positive");
assert!(capacity >= 1000, "Capacity should be at least 1000 for production use");
assert!(capacity <= 1_000_000, "Capacity should not be excessively large");
}
#[test]
fn test_notifier_config_serialization() {
let config = NotifierConfig::new();
// Test serialization to JSON
let json_result = serde_json::to_string(&config);
assert!(json_result.is_ok(), "Config should be serializable to JSON");
let json_str = json_result.unwrap();
assert!(!json_str.is_empty(), "Serialized JSON should not be empty");
assert!(json_str.contains("store_path"), "JSON should contain store_path");
assert!(json_str.contains("channel_capacity"), "JSON should contain channel_capacity");
assert!(json_str.contains("adapters"), "JSON should contain adapters");
// Test deserialization from JSON
let deserialized_result: Result<NotifierConfig, _> = serde_json::from_str(&json_str);
assert!(deserialized_result.is_ok(), "Config should be deserializable from JSON");
let deserialized_config = deserialized_result.unwrap();
assert_eq!(deserialized_config.store_path, config.store_path);
assert_eq!(deserialized_config.channel_capacity, config.channel_capacity);
assert_eq!(deserialized_config.adapters.len(), config.adapters.len());
}
#[test]
fn test_notifier_config_serialization_with_defaults() {
// Test serialization with minimal JSON (using serde defaults)
let minimal_json = r#"{"adapters": []}"#;
let deserialized_result: Result<NotifierConfig, _> = serde_json::from_str(minimal_json);
assert!(deserialized_result.is_ok(), "Config should deserialize with defaults");
let config = deserialized_result.unwrap();
assert_eq!(config.store_path, default_store_path(), "Should use default store path");
assert_eq!(config.channel_capacity, default_channel_capacity(), "Should use default channel capacity");
assert!(config.adapters.is_empty(), "Should have empty adapters as specified");
}
#[test]
fn test_notifier_config_debug_format() {
let config = NotifierConfig::new();
let debug_str = format!("{:?}", config);
assert!(!debug_str.is_empty(), "Debug output should not be empty");
assert!(debug_str.contains("NotifierConfig"), "Debug output should contain struct name");
assert!(debug_str.contains("store_path"), "Debug output should contain store_path field");
assert!(
debug_str.contains("channel_capacity"),
"Debug output should contain channel_capacity field"
);
assert!(debug_str.contains("adapters"), "Debug output should contain adapters field");
}
#[test]
fn test_notifier_config_clone() {
let config = NotifierConfig::new();
let cloned_config = config.clone();
// Test that clone creates an independent copy
assert_eq!(cloned_config.store_path, config.store_path);
assert_eq!(cloned_config.channel_capacity, config.channel_capacity);
assert_eq!(cloned_config.adapters.len(), config.adapters.len());
// Verify they are independent (modifying one doesn't affect the other)
let mut modified_config = config.clone();
modified_config.channel_capacity = 5000;
assert_ne!(modified_config.channel_capacity, config.channel_capacity);
assert_eq!(cloned_config.channel_capacity, config.channel_capacity);
}
#[test]
fn test_notifier_config_modification() {
let mut config = NotifierConfig::new();
// Test modifying store path
let original_store_path = config.store_path.clone();
config.store_path = "/custom/path".to_string();
assert_ne!(config.store_path, original_store_path);
assert_eq!(config.store_path, "/custom/path");
// Test modifying channel capacity
let original_capacity = config.channel_capacity;
config.channel_capacity = 5000;
assert_ne!(config.channel_capacity, original_capacity);
assert_eq!(config.channel_capacity, 5000);
// Test modifying adapters
let original_adapters_len = config.adapters.len();
config.adapters.push(AdapterConfig::new());
assert_eq!(config.adapters.len(), original_adapters_len + 1);
// Test clearing adapters
config.adapters.clear();
assert!(config.adapters.is_empty());
}
#[test]
fn test_notifier_config_adapters() {
let config = NotifierConfig::new();
// Test default adapter configuration
assert_eq!(config.adapters.len(), 1, "Should have exactly one default adapter");
// Test that we can add more adapters
let mut config_mut = config.clone();
config_mut.adapters.push(AdapterConfig::new());
assert_eq!(config_mut.adapters.len(), 2, "Should be able to add more adapters");
// Test adapter types
for adapter in &config.adapters {
match adapter {
AdapterConfig::Webhook(_) => {
// Webhook adapter should be properly configured
}
AdapterConfig::Kafka(_) => {
// Kafka adapter should be properly configured
}
AdapterConfig::Mqtt(_) => {
// MQTT adapter should be properly configured
}
}
}
}
#[test]
fn test_notifier_config_edge_cases() {
// Test with empty adapters
let mut config = NotifierConfig::new();
config.adapters.clear();
assert!(config.adapters.is_empty(), "Adapters should be empty after clearing");
// Test serialization with empty adapters
let json_result = serde_json::to_string(&config);
assert!(json_result.is_ok(), "Config with empty adapters should be serializable");
// Test with very large channel capacity
config.channel_capacity = 1_000_000;
assert_eq!(config.channel_capacity, 1_000_000);
// Test with minimum channel capacity
config.channel_capacity = 1;
assert_eq!(config.channel_capacity, 1);
// Test with empty store path
config.store_path = String::new();
assert!(config.store_path.is_empty());
}
#[test]
fn test_notifier_config_memory_efficiency() {
let config = NotifierConfig::new();
// Test that config doesn't use excessive memory
let config_size = std::mem::size_of_val(&config);
assert!(config_size < 5000, "Config should not use excessive memory");
// Test that store path is not excessively long
assert!(config.store_path.len() < 1000, "Store path should not be excessively long");
// Test that adapters collection is reasonably sized
assert!(config.adapters.len() < 100, "Adapters collection should be reasonably sized");
}
#[test]
fn test_notifier_config_consistency() {
// Create multiple configs and ensure they're consistent
let config1 = NotifierConfig::new();
let config2 = NotifierConfig::new();
// Both configs should have the same default values
assert_eq!(config1.store_path, config2.store_path);
assert_eq!(config1.channel_capacity, config2.channel_capacity);
assert_eq!(config1.adapters.len(), config2.adapters.len());
}
#[test]
fn test_notifier_config_path_validation() {
let config = NotifierConfig::new();
// Test that store path is a valid path
let path = Path::new(&config.store_path);
// Path should be valid
assert!(path.components().count() > 0, "Path should have components");
// Path should not contain invalid characters for most filesystems
assert!(!config.store_path.contains('\0'), "Path should not contain null characters");
assert!(!config.store_path.contains('\x01'), "Path should not contain control characters");
// Path should be reasonable length
assert!(config.store_path.len() < 260, "Path should be shorter than Windows MAX_PATH");
}
#[test]
fn test_notifier_config_production_readiness() {
let config = NotifierConfig::new();
// Test production readiness criteria
assert!(config.channel_capacity >= 1000, "Channel capacity should be sufficient for production");
assert!(!config.store_path.is_empty(), "Store path should be configured");
assert!(!config.adapters.is_empty(), "At least one adapter should be configured");
// Test that configuration is reasonable for high-load scenarios
assert!(config.channel_capacity <= 10_000_000, "Channel capacity should not be excessive");
// Test that store path is in a reasonable location (temp directory)
assert!(config.store_path.contains("event-notification"), "Store path should be identifiable");
}
#[test]
fn test_default_config_file_constant() {
// Test that the constant is properly defined
assert_eq!(DEFAULT_CONFIG_FILE, "event");
// DEFAULT_CONFIG_FILE is a const, so is_empty() check is redundant
// assert!(!DEFAULT_CONFIG_FILE.is_empty(), "Config file name should not be empty");
assert!(!DEFAULT_CONFIG_FILE.contains('/'), "Config file name should not contain path separators");
assert!(
!DEFAULT_CONFIG_FILE.contains('\\'),
"Config file name should not contain Windows path separators"
);
}
}
crates/config/src/event/kafka.rs
-29
View File
@@ -1,29 +0,0 @@
use serde::{Deserialize, Serialize};
/// Configuration for the Kafka adapter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KafkaAdapter {
pub brokers: String,
pub topic: String,
pub max_retries: u32,
pub timeout: u64,
}
impl KafkaAdapter {
/// create a new configuration with default values
pub fn new() -> Self {
Self {
brokers: "localhost:9092".to_string(),
topic: "kafka_topic".to_string(),
max_retries: 3,
timeout: 1000,
}
}
}
impl Default for KafkaAdapter {
/// create a new configuration with default values
fn default() -> Self {
Self::new()
}
}
crates/config/src/event/mod.rs
-5
View File
@@ -1,5 +0,0 @@
pub(crate) mod adapters;
pub(crate) mod config;
pub(crate) mod kafka;
pub(crate) mod mqtt;
pub(crate) mod webhook;
crates/config/src/event/mqtt.rs
-31
View File
@@ -1,31 +0,0 @@
use serde::{Deserialize, Serialize};
/// Configuration for the MQTT adapter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MqttAdapter {
pub broker: String,
pub port: u16,
pub client_id: String,
pub topic: String,
pub max_retries: u32,
}
impl MqttAdapter {
/// create a new configuration with default values
pub fn new() -> Self {
Self {
broker: "localhost".to_string(),
port: 1883,
client_id: "mqtt_client".to_string(),
topic: "mqtt_topic".to_string(),
max_retries: 3,
}
}
}
impl Default for MqttAdapter {
/// create a new configuration with default values
fn default() -> Self {
Self::new()
}
}
crates/config/src/event/webhook.rs
-51
View File
@@ -1,51 +0,0 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebhookAdapter {
pub endpoint: String,
pub auth_token: Option<String>,
pub custom_headers: Option<HashMap<String, String>>,
pub max_retries: u32,
pub timeout: u64,
}
impl WebhookAdapter {
/// verify that the configuration is valid
pub fn validate(&self) -> Result<(), String> {
// verify that endpoint cannot be empty
if self.endpoint.trim().is_empty() {
return Err("Webhook endpoint cannot be empty".to_string());
}
// verification timeout must be reasonable
if self.timeout == 0 {
return Err("Webhook timeout must be greater than 0".to_string());
}
// Verify that the maximum number of retry is reasonable
if self.max_retries > 10 {
return Err("Maximum retry count cannot exceed 10".to_string());
}
Ok(())
}
/// Get the default configuration
pub fn new() -> Self {
Self {
endpoint: "".to_string(),
auth_token: None,
custom_headers: Some(HashMap::new()),
max_retries: 3,
timeout: 1000,
}
}
}
impl Default for WebhookAdapter {
fn default() -> Self {
Self::new()
}
}
crates/config/src/lib.rs
+5 -9
View File
@@ -1,11 +1,7 @@
use crate::observability::config::ObservabilityConfig;
mod config;
mod constants;
mod event;
mod observability;
pub use config::RustFsConfig;
#[cfg(feature = "constants")]
pub mod constants;
#[cfg(feature = "constants")]
pub use constants::app::*;
pub use event::config::NotifierConfig;
#[cfg(feature = "observability")]
pub mod observability;
crates/event-notifier/Cargo.toml
-43
View File
@@ -1,43 +0,0 @@
[package]
name = "rustfs-event-notifier"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[features]
default = ["webhook"]
webhook = ["dep:reqwest"]
mqtt = ["rumqttc"]
kafka = ["dep:rdkafka"]
[dependencies]
async-trait = { workspace = true }
config = { workspace = true }
reqwest = { workspace = true, optional = true }
rumqttc = { workspace = true, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_with = { workspace = true }
smallvec = { workspace = true, features = ["serde"] }
strum = { workspace = true, features = ["derive"] }
tracing = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-multi-thread"] }
tokio-util = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde"] }
# Only enable kafka features and related dependencies on Linux
[target.'cfg(target_os = "linux")'.dependencies]
rdkafka = { workspace = true, features = ["tokio"], optional = true }
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
tracing-subscriber = { workspace = true }
http = { workspace = true }
axum = { workspace = true }
dotenvy = { workspace = true }
[lints]
workspace = true
crates/event-notifier/examples/.env.example
-28
View File
@@ -1,28 +0,0 @@
## ===== global configuration =====
#NOTIFIER__STORE_PATH=/var/log/event-notification
#NOTIFIER__CHANNEL_CAPACITY=5000
#
## ===== adapter configuration array format =====
## webhook adapter index 0
#NOTIFIER__ADAPTERS_0__type=Webhook
#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
#NOTIFIER__ADAPTERS_0__max_retries=3
#NOTIFIER__ADAPTERS_0__timeout=50
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
#
## kafka adapter index 1
#NOTIFIER__ADAPTERS_1__type=Kafka
#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
#NOTIFIER__ADAPTERS_1__topic=notifications
#NOTIFIER__ADAPTERS_1__max_retries=3
#NOTIFIER__ADAPTERS_1__timeout=60
#
## mqtt adapter index 2
#NOTIFIER__ADAPTERS_2__type=Mqtt
#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
#NOTIFIER__ADAPTERS_2__port=1883
#NOTIFIER__ADAPTERS_2__client_id=event-notifier
#NOTIFIER__ADAPTERS_2__topic=events
#NOTIFIER__ADAPTERS_2__max_retries=3
crates/event-notifier/examples/.env.zh.example
-28
View File
@@ -1,28 +0,0 @@
## ===== 全局配置 =====
#NOTIFIER__STORE_PATH=/var/log/event-notification
#NOTIFIER__CHANNEL_CAPACITY=5000
#
## ===== 适配器配置（数组格式） =====
## Webhook 适配器（索引 0）
#NOTIFIER__ADAPTERS_0__type=Webhook
#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
#NOTIFIER__ADAPTERS_0__max_retries=3
#NOTIFIER__ADAPTERS_0__timeout=50
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
#
## Kafka 适配器（索引 1）
#NOTIFIER__ADAPTERS_1__type=Kafka
#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
#NOTIFIER__ADAPTERS_1__topic=notifications
#NOTIFIER__ADAPTERS_1__max_retries=3
#NOTIFIER__ADAPTERS_1__timeout=60
#
## MQTT 适配器（索引 2）
#NOTIFIER__ADAPTERS_2__type=Mqtt
#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
#NOTIFIER__ADAPTERS_2__port=1883
#NOTIFIER__ADAPTERS_2__client_id=event-notifier
#NOTIFIER__ADAPTERS_2__topic=events
#NOTIFIER__ADAPTERS_2__max_retries=3
crates/event-notifier/examples/event.toml
-29
View File
@@ -1,29 +0,0 @@
# config.toml
store_path = "/var/log/event-notifier"
channel_capacity = 5000
[[adapters]]
type = "Webhook"
endpoint = "http://127.0.0.1:3020/webhook"
auth_token = "your-auth-token"
max_retries = 3
timeout = 50
[adapters.custom_headers]
custom_server = "value_server"
custom_client = "value_client"
[[adapters]]
type = "Kafka"
brokers = "localhost:9092"
topic = "notifications"
max_retries = 3
timeout = 60
[[adapters]]
type = "Mqtt"
broker = "mqtt.example.com"
port = 1883
client_id = "event-notifier"
topic = "events"
max_retries = 3
crates/event-notifier/examples/full.rs
-133
View File
@@ -1,133 +0,0 @@
use rustfs_event_notifier::{
AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotifierConfig, Object, Source, WebhookConfig,
};
use std::collections::HashMap;
use tokio::signal;
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
async fn setup_notification_system() -> Result<(), NotifierError> {
let config = NotifierConfig {
store_path: "./deploy/logs/event_store".into(),
channel_capacity: 100,
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
endpoint: "http://127.0.0.1:3020/webhook".into(),
auth_token: Some("your-auth-token".into()),
custom_headers: Some(HashMap::new()),
max_retries: 3,
timeout: 30,
})],
};
rustfs_event_notifier::initialize(config).await?;
// wait for the system to be ready
for _ in 0..50 {
// wait up to 5 seconds
if rustfs_event_notifier::is_ready() {
return Ok(());
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
Err(NotifierError::custom("notify the system of initialization timeout"))
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// initialization log
// tracing_subscriber::fmt::init();
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG) // set to debug or lower level
.with_target(false) // simplify output
.finish();
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
// set up notification system
if let Err(e) = setup_notification_system().await {
eprintln!("unable to initialize notification system:{}", e);
return Err(e.into());
}
// create a shutdown signal processing
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
// start signal processing task
tokio::spawn(async move {
let _ = signal::ctrl_c().await;
println!("Received the shutdown signal and prepared to exit...");
let _ = shutdown_tx.send(());
});
// main application logic
tokio::select! {
_ = async {
loop {
// application logic
// create an s3 metadata object
let metadata = Metadata {
schema_version: "1.0".to_string(),
configuration_id: "test-config".to_string(),
bucket: Bucket {
name: "my-bucket".to_string(),
owner_identity: Identity {
principal_id: "owner123".to_string(),
},
arn: "arn:aws:s3:::my-bucket".to_string(),
},
object: Object {
key: "test.txt".to_string(),
size: Some(1024),
etag: Some("abc123".to_string()),
content_type: Some("text/plain".to_string()),
user_metadata: None,
version_id: None,
sequencer: "1234567890".to_string(),
},
};
// create source object
let source = Source {
host: "localhost".to_string(),
port: "80".to_string(),
user_agent: "curl/7.68.0".to_string(),
};
// create events using builder mode
let event = Event::builder()
.event_time("2023-10-01T12:00:00.000Z")
.event_name(Name::ObjectCreatedPut)
.user_identity(Identity {
principal_id: "user123".to_string(),
})
.s3(metadata)
.source(source)
.channels(vec!["webhook".to_string()])
.build()
.expect("failed to create event");
if let Err(e) = rustfs_event_notifier::send_event(event).await {
eprintln!("send event failed:{}", e);
}
tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
}
} => {},
_ = &mut shutdown_rx => {
println!("close the app");
}
}
// 优雅关闭通知系统
println!("turn off the notification system");
if let Err(e) = rustfs_event_notifier::shutdown().await {
eprintln!("An error occurred while shutting down the notification system:{}", e);
} else {
println!("the notification system has been closed safely");
}
println!("the application has been closed safely");
Ok(())
}
crates/event-notifier/examples/simple.rs
-110
View File
@@ -1,110 +0,0 @@
use rustfs_event_notifier::NotifierSystem;
use rustfs_event_notifier::create_adapters;
use rustfs_event_notifier::{AdapterConfig, NotifierConfig, WebhookConfig};
use rustfs_event_notifier::{Bucket, Event, Identity, Metadata, Name, Object, Source};
use std::collections::HashMap;
use std::error;
use std::sync::Arc;
use tokio::signal;
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
#[tokio::main]
async fn main() -> Result<(), Box<dyn error::Error>> {
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG) // set to debug or lower level
.with_target(false) // simplify output
.finish();
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
let config = NotifierConfig {
store_path: "./events".to_string(),
channel_capacity: 100,
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
endpoint: "http://127.0.0.1:3020/webhook".to_string(),
auth_token: Some("secret-token".to_string()),
custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])),
max_retries: 3,
timeout: 10,
})],
};
// event_load_config
// loading configuration from environment variables
let _config = NotifierConfig::event_load_config(Some("./crates/event-notifier/examples/event.toml".to_string()));
tracing::info!("event_load_config config: {:?} \n", _config);
dotenvy::dotenv()?;
let _config = NotifierConfig::event_load_config(None);
tracing::info!("event_load_config config: {:?} \n", _config);
let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?));
let adapters = create_adapters(&config.adapters)?;
// create an s3 metadata object
let metadata = Metadata {
schema_version: "1.0".to_string(),
configuration_id: "test-config".to_string(),
bucket: Bucket {
name: "my-bucket".to_string(),
owner_identity: Identity {
principal_id: "owner123".to_string(),
},
arn: "arn:aws:s3:::my-bucket".to_string(),
},
object: Object {
key: "test.txt".to_string(),
size: Some(1024),
etag: Some("abc123".to_string()),
content_type: Some("text/plain".to_string()),
user_metadata: None,
version_id: None,
sequencer: "1234567890".to_string(),
},
};
// create source object
let source = Source {
host: "localhost".to_string(),
port: "80".to_string(),
user_agent: "curl/7.68.0".to_string(),
};
// create events using builder mode
let event = Event::builder()
.event_time("2023-10-01T12:00:00.000Z")
.event_name(Name::ObjectCreatedPut)
.user_identity(Identity {
principal_id: "user123".to_string(),
})
.s3(metadata)
.source(source)
.channels(vec!["webhook".to_string()])
.build()
.expect("failed to create event");
{
let system = system.lock().await;
system.send_event(event).await?;
}
let system_clone = Arc::clone(&system);
let system_handle = tokio::spawn(async move {
let mut system = system_clone.lock().await;
system.start(adapters).await
});
signal::ctrl_c().await?;
tracing::info!("Received shutdown signal");
let result = {
let mut system = system.lock().await;
system.shutdown().await
};
if let Err(e) = result {
tracing::error!("Failed to shut down the notification system: {}", e);
} else {
tracing::info!("Notification system shut down successfully");
}
system_handle.await??;
Ok(())
}
crates/event-notifier/examples/webhook.rs
-94
View File
@@ -1,94 +0,0 @@
use axum::{Router, extract::Json, http::StatusCode, routing::post};
use serde_json::Value;
use std::time::{SystemTime, UNIX_EPOCH};
#[tokio::main]
async fn main() {
// 构建应用
let app = Router::new().route("/webhook", post(receive_webhook));
// 启动服务器
let listener = tokio::net::TcpListener::bind("0.0.0.0:3020").await.unwrap();
println!("Server running on http://0.0.0.0:3020");
// 创建关闭信号处理
tokio::select! {
result = axum::serve(listener, app) => {
if let Err(e) = result {
eprintln!("Server error: {}", e);
}
}
_ = tokio::signal::ctrl_c() => {
println!("Shutting down server...");
}
}
}
async fn receive_webhook(Json(payload): Json<Value>) -> StatusCode {
let start = SystemTime::now();
let since_the_epoch = start.duration_since(UNIX_EPOCH).expect("Time went backwards");
// get the number of seconds since the unix era
let seconds = since_the_epoch.as_secs();
// Manually calculate year, month, day, hour, minute, and second
let (year, month, day, hour, minute, second) = convert_seconds_to_date(seconds);
// output result
println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
println!(
"received a webhook request time:{} content:\n {}",
seconds,
serde_json::to_string_pretty(&payload).unwrap()
);
StatusCode::OK
}
fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) {
// assume that the time zone is utc
let seconds_per_minute = 60;
let seconds_per_hour = 3600;
let seconds_per_day = 86400;
// Calculate the year, month, day, hour, minute, and second corresponding to the number of seconds
let mut total_seconds = seconds;
let mut year = 1970;
let mut month = 1;
let mut day = 1;
let mut hour = 0;
let mut minute = 0;
let mut second = 0;
// calculate year
while total_seconds >= 31536000 {
year += 1;
total_seconds -= 31536000; // simplified processing no leap year considered
}
// calculate month
let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
for m in &days_in_month {
if total_seconds >= m * seconds_per_day {
month += 1;
total_seconds -= m * seconds_per_day;
} else {
break;
}
}
// calculate the number of days
day += total_seconds / seconds_per_day;
total_seconds %= seconds_per_day;
// calculate hours
hour += total_seconds / seconds_per_hour;
total_seconds %= seconds_per_hour;
// calculate minutes
minute += total_seconds / seconds_per_minute;
total_seconds %= seconds_per_minute;
// calculate the number of seconds
second += total_seconds;
(year as u32, month as u32, day as u32, hour as u32, minute as u32, second as u32)
}
crates/event-notifier/src/adapter/kafka.rs
-69
View File
@@ -1,69 +0,0 @@
use crate::ChannelAdapter;
use crate::Error;
use crate::Event;
use crate::KafkaConfig;
use async_trait::async_trait;
use rdkafka::error::KafkaError;
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::types::RDKafkaErrorCode;
use rdkafka::util::Timeout;
use std::time::Duration;
use tokio::time::sleep;
/// Kafka adapter for sending events to a Kafka topic.
pub struct KafkaAdapter {
producer: FutureProducer,
topic: String,
max_retries: u32,
}
impl KafkaAdapter {
/// Creates a new Kafka adapter.
pub fn new(config: &KafkaConfig) -> Result<Self, Error> {
// Create a Kafka producer with the provided configuration.
let producer = rdkafka::config::ClientConfig::new()
.set("bootstrap.servers", &config.brokers)
.set("message.timeout.ms", config.timeout.to_string())
.create()?;
Ok(Self {
producer,
topic: config.topic.clone(),
max_retries: config.max_retries,
})
}
/// Sends an event to the Kafka topic with retry logic.
async fn send_with_retry(&self, event: &Event) -> Result<(), Error> {
let event_id = event.id.to_string();
let payload = serde_json::to_string(&event)?;
for attempt in 0..self.max_retries {
let record = FutureRecord::to(&self.topic).key(&event_id).payload(&payload);
match self.producer.send(record, Timeout::Never).await {
Ok(_) => return Ok(()),
Err((KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull), _)) => {
tracing::warn!("Kafka attempt {} failed: Queue full. Retrying...", attempt + 1);
sleep(Duration::from_secs(2u64.pow(attempt))).await;
}
Err((e, _)) => {
tracing::error!("Kafka send error: {}", e);
return Err(Error::Kafka(e));
}
}
}
Err(Error::Custom("Exceeded maximum retry attempts for Kafka message".to_string()))
}
}
#[async_trait]
impl ChannelAdapter for KafkaAdapter {
fn name(&self) -> String {
"kafka".to_string()
}
async fn send(&self, event: &Event) -> Result<(), Error> {
self.send_with_retry(event).await
}
}
crates/event-notifier/src/adapter/mod.rs
-54
View File
@@ -1,54 +0,0 @@
use crate::AdapterConfig;
use crate::Error;
use crate::Event;
use async_trait::async_trait;
use std::sync::Arc;
#[cfg(all(feature = "kafka", target_os = "linux"))]
pub(crate) mod kafka;
#[cfg(feature = "mqtt")]
pub(crate) mod mqtt;
#[cfg(feature = "webhook")]
pub(crate) mod webhook;
/// The `ChannelAdapter` trait defines the interface for all channel adapters.
#[async_trait]
pub trait ChannelAdapter: Send + Sync + 'static {
/// Sends an event to the channel.
fn name(&self) -> String;
/// Sends an event to the channel.
async fn send(&self, event: &Event) -> Result<(), Error>;
}
/// Creates channel adapters based on the provided configuration.
pub fn create_adapters(configs: &[AdapterConfig]) -> Result<Vec<Arc<dyn ChannelAdapter>>, Error> {
let mut adapters: Vec<Arc<dyn ChannelAdapter>> = Vec::new();
for config in configs {
match config {
#[cfg(feature = "webhook")]
AdapterConfig::Webhook(webhook_config) => {
webhook_config.validate().map_err(Error::ConfigError)?;
adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone())));
}
#[cfg(all(feature = "kafka", target_os = "linux"))]
AdapterConfig::Kafka(kafka_config) => {
adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?));
}
#[cfg(feature = "mqtt")]
AdapterConfig::Mqtt(mqtt_config) => {
let (mqtt, mut event_loop) = mqtt::MqttAdapter::new(mqtt_config);
tokio::spawn(async move { while event_loop.poll().await.is_ok() {} });
adapters.push(Arc::new(mqtt));
}
#[cfg(not(feature = "webhook"))]
AdapterConfig::Webhook(_) => return Err(Error::FeatureDisabled("webhook")),
#[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")),
#[cfg(not(feature = "mqtt"))]
AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")),
}
}
Ok(adapters)
}
crates/event-notifier/src/adapter/mqtt.rs
-58
View File
@@ -1,58 +0,0 @@
use crate::ChannelAdapter;
use crate::Error;
use crate::Event;
use crate::MqttConfig;
use async_trait::async_trait;
use rumqttc::{AsyncClient, MqttOptions, QoS};
use std::time::Duration;
use tokio::time::sleep;
/// MQTT adapter for sending events to an MQTT broker.
pub struct MqttAdapter {
client: AsyncClient,
topic: String,
max_retries: u32,
}
impl MqttAdapter {
/// Creates a new MQTT adapter.
pub fn new(config: &MqttConfig) -> (Self, rumqttc::EventLoop) {
let mqtt_options = MqttOptions::new(&config.client_id, &config.broker, config.port);
let (client, event_loop) = rumqttc::AsyncClient::new(mqtt_options, 10);
(
Self {
client,
topic: config.topic.clone(),
max_retries: config.max_retries,
},
event_loop,
)
}
}
#[async_trait]
impl ChannelAdapter for MqttAdapter {
fn name(&self) -> String {
"mqtt".to_string()
}
async fn send(&self, event: &Event) -> Result<(), Error> {
let payload = serde_json::to_string(event).map_err(Error::Serde)?;
let mut attempt = 0;
loop {
match self
.client
.publish(&self.topic, QoS::AtLeastOnce, false, payload.clone())
.await
{
Ok(()) => return Ok(()),
Err(e) if attempt < self.max_retries => {
attempt += 1;
tracing::warn!("MQTT attempt {} failed: {}. Retrying...", attempt, e);
sleep(Duration::from_secs(2u64.pow(attempt))).await;
}
Err(e) => return Err(Error::Mqtt(e)),
}
}
}
}
crates/event-notifier/src/adapter/webhook.rs
-64
View File
@@ -1,64 +0,0 @@
use crate::ChannelAdapter;
use crate::Error;
use crate::Event;
use crate::WebhookConfig;
use async_trait::async_trait;
use reqwest::{Client, RequestBuilder};
use std::time::Duration;
use tokio::time::sleep;
/// Webhook adapter for sending events to a webhook endpoint.
pub struct WebhookAdapter {
config: WebhookConfig,
client: Client,
}
impl WebhookAdapter {
/// Creates a new Webhook adapter.
pub fn new(config: WebhookConfig) -> Self {
let client = Client::builder()
.timeout(Duration::from_secs(config.timeout))
.build()
.expect("Failed to build reqwest client");
Self { config, client }
}
/// Builds the request to send the event.
fn build_request(&self, event: &Event) -> RequestBuilder {
let mut request = self.client.post(&self.config.endpoint).json(event);
if let Some(token) = &self.config.auth_token {
request = request.header("Authorization", format!("Bearer {}", token));
}
if let Some(headers) = &self.config.custom_headers {
for (key, value) in headers {
request = request.header(key, value);
}
}
request
}
}
#[async_trait]
impl ChannelAdapter for WebhookAdapter {
fn name(&self) -> String {
"webhook".to_string()
}
async fn send(&self, event: &Event) -> Result<(), Error> {
let mut attempt = 0;
tracing::info!("Attempting to send webhook request: {:?}", event);
loop {
match self.build_request(event).send().await {
Ok(response) => {
response.error_for_status().map_err(Error::Http)?;
return Ok(());
}
Err(e) if attempt < self.config.max_retries => {
attempt += 1;
tracing::warn!("Webhook attempt {} failed: {}. Retrying...", attempt, e);
sleep(Duration::from_secs(2u64.pow(attempt))).await;
}
Err(e) => return Err(Error::Http(e)),
}
}
}
}
crates/event-notifier/src/bus.rs
-102
View File
@@ -1,102 +0,0 @@
use crate::ChannelAdapter;
use crate::Error;
use crate::EventStore;
use crate::{Event, Log};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::mpsc;
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::instrument;
/// Handles incoming events from the producer.
///
/// This function is responsible for receiving events from the producer and sending them to the appropriate adapters.
/// It also handles the shutdown process and saves any pending logs to the event store.
#[instrument(skip_all)]
pub async fn event_bus(
mut rx: mpsc::Receiver<Event>,
adapters: Vec<Arc<dyn ChannelAdapter>>,
store: Arc<EventStore>,
shutdown: CancellationToken,
shutdown_complete: Option<tokio::sync::oneshot::Sender<()>>,
) -> Result<(), Error> {
let mut current_log = Log {
event_name: crate::event::Name::Everything,
key: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(),
records: Vec::new(),
};
let mut unprocessed_events = Vec::new();
loop {
tokio::select! {
Some(event) = rx.recv() => {
current_log.records.push(event.clone());
let mut send_tasks = Vec::new();
for adapter in &adapters {
if event.channels.contains(&adapter.name()) {
let adapter = adapter.clone();
let event = event.clone();
send_tasks.push(tokio::spawn(async move {
if let Err(e) = adapter.send(&event).await {
tracing::error!("Failed to send event to {}: {}", adapter.name(), e);
Err(e)
} else {
Ok(())
}
}));
}
}
for task in send_tasks {
if task.await?.is_err() {
// If sending fails, add the event to the unprocessed list
let failed_event = event.clone();
unprocessed_events.push(failed_event);
}
}
// Clear the current log because we only care about unprocessed events
current_log.records.clear();
}
_ = shutdown.cancelled() => {
tracing::info!("Shutting down event bus, saving pending logs...");
// Check if there are still unprocessed messages in the channel
while let Ok(Some(event)) = tokio::time::timeout(
Duration::from_millis(100),
rx.recv()
).await {
unprocessed_events.push(event);
}
// save only if there are unprocessed events
if !unprocessed_events.is_empty() {
tracing::info!("Save {} unhandled events", unprocessed_events.len());
// create and save logging
let shutdown_log = Log {
event_name: crate::event::Name::Everything,
key: format!("shutdown_{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()),
records: unprocessed_events,
};
store.save_logs(&[shutdown_log]).await?;
} else {
tracing::info!("no unhandled events need to be saved");
}
tracing::debug!("shutdown_complete is Some: {}", shutdown_complete.is_some());
if let Some(complete_sender) = shutdown_complete {
// send a completion signal
let result = complete_sender.send(());
match result {
Ok(_) => tracing::info!("Event bus shutdown signal sent"),
Err(e) => tracing::error!("Failed to send event bus shutdown signal: {:?}", e),
}
tracing::info!("Shutting down event bus");
}
tracing::info!("Event bus shutdown complete");
break;
}
}
}
Ok(())
}
crates/event-notifier/src/config.rs
-166
View File
@@ -1,166 +0,0 @@
use config::{Config, File, FileFormat};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::env;
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebhookConfig {
pub endpoint: String,
pub auth_token: Option<String>,
pub custom_headers: Option<HashMap<String, String>>,
pub max_retries: u32,
pub timeout: u64,
}
impl WebhookConfig {
/// verify that the configuration is valid
pub fn validate(&self) -> Result<(), String> {
// verify that endpoint cannot be empty
if self.endpoint.trim().is_empty() {
return Err("Webhook endpoint cannot be empty".to_string());
}
// verification timeout must be reasonable
if self.timeout == 0 {
return Err("Webhook timeout must be greater than 0".to_string());
}
// Verify that the maximum number of retry is reasonable
if self.max_retries > 10 {
return Err("Maximum retry count cannot exceed 10".to_string());
}
Ok(())
}
}
/// Configuration for the Kafka adapter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KafkaConfig {
pub brokers: String,
pub topic: String,
pub max_retries: u32,
pub timeout: u64,
}
/// Configuration for the MQTT adapter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MqttConfig {
pub broker: String,
pub port: u16,
pub client_id: String,
pub topic: String,
pub max_retries: u32,
}
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum AdapterConfig {
Webhook(WebhookConfig),
Kafka(KafkaConfig),
Mqtt(MqttConfig),
}
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NotifierConfig {
#[serde(default = "default_store_path")]
pub store_path: String,
#[serde(default = "default_channel_capacity")]
pub channel_capacity: usize,
pub adapters: Vec<AdapterConfig>,
}
impl Default for NotifierConfig {
fn default() -> Self {
Self {
store_path: default_store_path(),
channel_capacity: default_channel_capacity(),
adapters: Vec::new(),
}
}
}
impl NotifierConfig {
/// create a new configuration with default values
pub fn new() -> Self {
Self::default()
}
/// Loading the configuration file
/// Supports TOML, YAML and .env formats, read in order by priority
///
/// # Parameters
/// - `config_dir`: Configuration file path
///
/// # Returns
/// Configuration information
///
/// # Example
/// ```
/// use rustfs_event_notifier::NotifierConfig;
///
/// let config = NotifierConfig::event_load_config(None);
/// ```
pub fn event_load_config(config_dir: Option<String>) -> NotifierConfig {
let config_dir = if let Some(path) = config_dir {
// If a path is provided, check if it's empty
if path.is_empty() {
// If empty, use the default config file name
DEFAULT_CONFIG_FILE.to_string()
} else {
// Use the provided path
let path = std::path::Path::new(&path);
if path.extension().is_some() {
// If path has extension, use it as is (extension will be added by Config::builder)
path.with_extension("").to_string_lossy().into_owned()
} else {
// If path is a directory, append the default config file name
path.to_string_lossy().into_owned()
}
}
} else {
// If no path provided, use current directory + default config file
match env::current_dir() {
Ok(dir) => dir.join(DEFAULT_CONFIG_FILE).to_string_lossy().into_owned(),
Err(_) => {
eprintln!("Warning: Failed to get current directory, using default config file");
DEFAULT_CONFIG_FILE.to_string()
}
}
};
// Log using proper logging instead of println when possible
println!("Using config file base: {}", config_dir);
let app_config = Config::builder()
.add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false))
.add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false))
.build()
.unwrap_or_default();
match app_config.try_deserialize::<NotifierConfig>() {
Ok(app_config) => {
println!("Parsed AppConfig: {:?} \n", app_config);
app_config
}
Err(e) => {
println!("Failed to deserialize config: {}", e);
NotifierConfig::default()
}
}
}
}
const DEFAULT_CONFIG_FILE: &str = "event";
/// Provide temporary directories as default storage paths
fn default_store_path() -> String {
std::env::temp_dir().join("event-notification").to_string_lossy().to_string()
}
/// Provides the recommended default channel capacity for high concurrency systems
fn default_channel_capacity() -> usize {
10000 // Reasonable default values for high concurrency systems
}
crates/event-notifier/src/error.rs
-418
View File
@@ -1,418 +0,0 @@
use config::ConfigError;
use thiserror::Error;
use tokio::sync::mpsc::error;
use tokio::task::JoinError;
/// The `Error` enum represents all possible errors that can occur in the application.
/// It implements the `std::error::Error` trait and provides a way to convert various error types into a single error type.
#[derive(Error, Debug)]
pub enum Error {
#[error("Join error: {0}")]
JoinError(#[from] JoinError),
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("Serialization error: {0}")]
Serde(#[from] serde_json::Error),
#[error("HTTP error: {0}")]
Http(#[from] reqwest::Error),
#[cfg(all(feature = "kafka", target_os = "linux"))]
#[error("Kafka error: {0}")]
Kafka(#[from] rdkafka::error::KafkaError),
#[cfg(feature = "mqtt")]
#[error("MQTT error: {0}")]
Mqtt(#[from] rumqttc::ClientError),
#[error("Channel send error: {0}")]
ChannelSend(#[from] Box<error::SendError<crate::event::Event>>),
#[error("Feature disabled: {0}")]
FeatureDisabled(&'static str),
#[error("Event bus already started")]
EventBusStarted,
#[error("necessary fields are missing:{0}")]
MissingField(&'static str),
#[error("field verification failed:{0}")]
ValidationError(&'static str),
#[error("Custom error: {0}")]
Custom(String),
#[error("Configuration error: {0}")]
ConfigError(String),
#[error("Configuration loading error: {0}")]
Config(#[from] ConfigError),
}
impl Error {
pub fn custom(msg: &str) -> Error {
Self::Custom(msg.to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::error::Error as StdError;
use std::io;
use tokio::sync::mpsc;
#[test]
fn test_error_display() {
// Test error message display
let custom_error = Error::custom("test message");
assert_eq!(custom_error.to_string(), "Custom error: test message");
let feature_error = Error::FeatureDisabled("test feature");
assert_eq!(feature_error.to_string(), "Feature disabled: test feature");
let event_bus_error = Error::EventBusStarted;
assert_eq!(event_bus_error.to_string(), "Event bus already started");
let missing_field_error = Error::MissingField("required_field");
assert_eq!(missing_field_error.to_string(), "necessary fields are missing:required_field");
let validation_error = Error::ValidationError("invalid format");
assert_eq!(validation_error.to_string(), "field verification failed:invalid format");
let config_error = Error::ConfigError("invalid config".to_string());
assert_eq!(config_error.to_string(), "Configuration error: invalid config");
}
#[test]
fn test_error_debug() {
// Test Debug trait implementation
let custom_error = Error::custom("debug test");
let debug_str = format!("{:?}", custom_error);
assert!(debug_str.contains("Custom"));
assert!(debug_str.contains("debug test"));
let feature_error = Error::FeatureDisabled("debug feature");
let debug_str = format!("{:?}", feature_error);
assert!(debug_str.contains("FeatureDisabled"));
assert!(debug_str.contains("debug feature"));
}
#[test]
fn test_custom_error_creation() {
// Test custom error creation
let error = Error::custom("test custom error");
match error {
Error::Custom(msg) => assert_eq!(msg, "test custom error"),
_ => panic!("Expected Custom error variant"),
}
// Test empty string
let empty_error = Error::custom("");
match empty_error {
Error::Custom(msg) => assert_eq!(msg, ""),
_ => panic!("Expected Custom error variant"),
}
// Test special characters
let special_error = Error::custom("Test Chinese 中文 & special chars: !@#$%");
match special_error {
Error::Custom(msg) => assert_eq!(msg, "Test Chinese 中文 & special chars: !@#$%"),
_ => panic!("Expected Custom error variant"),
}
}
#[test]
fn test_io_error_conversion() {
// Test IO error conversion
let io_error = io::Error::new(io::ErrorKind::NotFound, "file not found");
let converted_error: Error = io_error.into();
match converted_error {
Error::Io(err) => {
assert_eq!(err.kind(), io::ErrorKind::NotFound);
assert_eq!(err.to_string(), "file not found");
}
_ => panic!("Expected Io error variant"),
}
// Test different types of IO errors
let permission_error = io::Error::new(io::ErrorKind::PermissionDenied, "access denied");
let converted: Error = permission_error.into();
assert!(matches!(converted, Error::Io(_)));
}
#[test]
fn test_serde_error_conversion() {
// Test serialization error conversion
let invalid_json = r#"{"invalid": json}"#;
let serde_error = serde_json::from_str::<serde_json::Value>(invalid_json).unwrap_err();
let converted_error: Error = serde_error.into();
match converted_error {
Error::Serde(_) => {
// Verify error type is correct
assert!(converted_error.to_string().contains("Serialization error"));
}
_ => panic!("Expected Serde error variant"),
}
}
#[test]
fn test_config_error_conversion() {
// Test configuration error conversion
let config_error = ConfigError::Message("invalid configuration".to_string());
let converted_error: Error = config_error.into();
match converted_error {
Error::Config(_) => {
assert!(converted_error.to_string().contains("Configuration loading error"));
}
_ => panic!("Expected Config error variant"),
}
}
#[tokio::test]
async fn test_channel_send_error_conversion() {
// Test channel send error conversion
let (tx, rx) = mpsc::channel::<crate::event::Event>(1);
drop(rx); // Close receiver
// Create a test event
use crate::event::{Bucket, Identity, Metadata, Name, Object, Source};
use std::collections::HashMap;
let identity = Identity::new("test-user".to_string());
let bucket = Bucket::new("test-bucket".to_string(), identity.clone(), "arn:aws:s3:::test-bucket".to_string());
let object = Object::new(
"test-key".to_string(),
Some(1024),
Some("etag123".to_string()),
Some("text/plain".to_string()),
Some(HashMap::new()),
None,
"sequencer123".to_string(),
);
let metadata = Metadata::create("1.0".to_string(), "config1".to_string(), bucket, object);
let source = Source::new("localhost".to_string(), "8080".to_string(), "test-agent".to_string());
let test_event = crate::event::Event::builder()
.event_name(Name::ObjectCreatedPut)
.s3(metadata)
.source(source)
.build()
.unwrap();
let send_result = tx.send(test_event).await;
assert!(send_result.is_err());
let send_error = send_result.unwrap_err();
let boxed_error = Box::new(send_error);
let converted_error: Error = boxed_error.into();
match converted_error {
Error::ChannelSend(_) => {
assert!(converted_error.to_string().contains("Channel send error"));
}
_ => panic!("Expected ChannelSend error variant"),
}
}
#[test]
fn test_error_source_chain() {
// 测试错误源链
let io_error = io::Error::new(io::ErrorKind::InvalidData, "invalid data");
let converted_error: Error = io_error.into();
// 验证错误源
assert!(converted_error.source().is_some());
let source = converted_error.source().unwrap();
assert_eq!(source.to_string(), "invalid data");
}
#[test]
fn test_error_variants_exhaustive() {
// 测试所有错误变体的创建
let errors = vec![
Error::FeatureDisabled("test"),
Error::EventBusStarted,
Error::MissingField("field"),
Error::ValidationError("validation"),
Error::Custom("custom".to_string()),
Error::ConfigError("config".to_string()),
];
for error in errors {
// 验证每个错误都能正确显示
let error_str = error.to_string();
assert!(!error_str.is_empty());
// 验证每个错误都能正确调试
let debug_str = format!("{:?}", error);
assert!(!debug_str.is_empty());
}
}
#[test]
fn test_error_equality_and_matching() {
// 测试错误的模式匹配
let custom_error = Error::custom("test");
match custom_error {
Error::Custom(msg) => assert_eq!(msg, "test"),
_ => panic!("Pattern matching failed"),
}
let feature_error = Error::FeatureDisabled("feature");
match feature_error {
Error::FeatureDisabled(feature) => assert_eq!(feature, "feature"),
_ => panic!("Pattern matching failed"),
}
let event_bus_error = Error::EventBusStarted;
match event_bus_error {
Error::EventBusStarted => {} // 正确匹配
_ => panic!("Pattern matching failed"),
}
}
#[test]
fn test_error_message_formatting() {
// 测试错误消息格式化
let test_cases = vec![
(Error::FeatureDisabled("kafka"), "Feature disabled: kafka"),
(Error::MissingField("bucket_name"), "necessary fields are missing:bucket_name"),
(Error::ValidationError("invalid email"), "field verification failed:invalid email"),
(Error::ConfigError("missing file".to_string()), "Configuration error: missing file"),
];
for (error, expected_message) in test_cases {
assert_eq!(error.to_string(), expected_message);
}
}
#[test]
fn test_error_memory_efficiency() {
// 测试错误类型的内存效率
use std::mem;
let size = mem::size_of::<Error>();
// 错误类型应该相对紧凑，考虑到包含多种错误类型，96 字节是合理的
assert!(size <= 128, "Error size should be reasonable, got {} bytes", size);
// 测试 Option<Error>的大小
let option_size = mem::size_of::<Option<Error>>();
assert!(option_size <= 136, "Option<Error> should be efficient, got {} bytes", option_size);
}
#[test]
fn test_error_thread_safety() {
// 测试错误类型的线程安全性
fn assert_send<T: Send>() {}
fn assert_sync<T: Sync>() {}
assert_send::<Error>();
assert_sync::<Error>();
}
#[test]
fn test_custom_error_edge_cases() {
// 测试自定义错误的边界情况
let long_message = "a".repeat(1000);
let long_error = Error::custom(&long_message);
match long_error {
Error::Custom(msg) => assert_eq!(msg.len(), 1000),
_ => panic!("Expected Custom error variant"),
}
// 测试包含换行符的消息
let multiline_error = Error::custom("line1\nline2\nline3");
match multiline_error {
Error::Custom(msg) => assert!(msg.contains('\n')),
_ => panic!("Expected Custom error variant"),
}
// 测试包含 Unicode 字符的消息
let unicode_error = Error::custom("🚀 Unicode test 测试 🎉");
match unicode_error {
Error::Custom(msg) => assert!(msg.contains('🚀')),
_ => panic!("Expected Custom error variant"),
}
}
#[test]
fn test_error_conversion_consistency() {
// 测试错误转换的一致性
let original_io_error = io::Error::new(io::ErrorKind::TimedOut, "timeout");
let error_message = original_io_error.to_string();
let converted: Error = original_io_error.into();
// 验证转换后的错误包含原始错误信息
assert!(converted.to_string().contains(&error_message));
}
#[test]
fn test_error_downcast() {
// 测试错误的向下转型
let io_error = io::Error::other("test error");
let converted: Error = io_error.into();
// 验证可以获取源错误
if let Error::Io(ref inner) = converted {
assert_eq!(inner.to_string(), "test error");
assert_eq!(inner.kind(), io::ErrorKind::Other);
} else {
panic!("Expected Io error variant");
}
}
#[test]
fn test_error_chain_depth() {
// 测试错误链的深度
let root_cause = io::Error::other("root cause");
let converted: Error = root_cause.into();
let mut depth = 0;
let mut current_error: &dyn StdError = &converted;
while let Some(source) = current_error.source() {
depth += 1;
current_error = source;
// 防止无限循环
if depth > 10 {
break;
}
}
assert!(depth > 0, "Error should have at least one source");
assert!(depth <= 3, "Error chain should not be too deep");
}
#[test]
fn test_static_str_lifetime() {
// 测试静态字符串生命周期
fn create_feature_error() -> Error {
Error::FeatureDisabled("static_feature")
}
let error = create_feature_error();
match error {
Error::FeatureDisabled(feature) => assert_eq!(feature, "static_feature"),
_ => panic!("Expected FeatureDisabled error variant"),
}
}
#[test]
fn test_error_formatting_consistency() {
// 测试错误格式化的一致性
let errors = vec![
Error::FeatureDisabled("test"),
Error::MissingField("field"),
Error::ValidationError("validation"),
Error::Custom("custom".to_string()),
];
for error in errors {
let display_str = error.to_string();
let debug_str = format!("{:?}", error);
// Display 和 Debug 都不应该为空
assert!(!display_str.is_empty());
assert!(!debug_str.is_empty());
// Debug 输出通常包含更多信息，但不是绝对的
// 这里我们只验证两者都有内容即可
assert!(!debug_str.is_empty());
assert!(!display_str.is_empty());
}
}
}
crates/event-notifier/src/event.rs
-616
View File
@@ -1,616 +0,0 @@
use crate::Error;
use serde::{Deserialize, Serialize};
use serde_with::{DeserializeFromStr, SerializeDisplay};
use smallvec::{SmallVec, smallvec};
use std::borrow::Cow;
use std::collections::HashMap;
use std::time::{SystemTime, UNIX_EPOCH};
use strum::{Display, EnumString};
use uuid::Uuid;
/// A struct representing the identity of the user
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Identity {
#[serde(rename = "principalId")]
pub principal_id: String,
}
impl Identity {
/// Create a new Identity instance
pub fn new(principal_id: String) -> Self {
Self { principal_id }
}
/// Set the principal ID
pub fn set_principal_id(&mut self, principal_id: String) {
self.principal_id = principal_id;
}
}
/// A struct representing the bucket information
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Bucket {
pub name: String,
#[serde(rename = "ownerIdentity")]
pub owner_identity: Identity,
pub arn: String,
}
impl Bucket {
/// Create a new Bucket instance
pub fn new(name: String, owner_identity: Identity, arn: String) -> Self {
Self {
name,
owner_identity,
arn,
}
}
/// Set the name of the bucket
pub fn set_name(&mut self, name: String) {
self.name = name;
}
/// Set the ARN of the bucket
pub fn set_arn(&mut self, arn: String) {
self.arn = arn;
}
/// Set the owner identity of the bucket
pub fn set_owner_identity(&mut self, owner_identity: Identity) {
self.owner_identity = owner_identity;
}
}
/// A struct representing the object information
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Object {
pub key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub size: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none", rename = "eTag")]
pub etag: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none", rename = "contentType")]
pub content_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none", rename = "userMetadata")]
pub user_metadata: Option<HashMap<String, String>>,
#[serde(default, skip_serializing_if = "Option::is_none", rename = "versionId")]
pub version_id: Option<String>,
pub sequencer: String,
}
impl Object {
/// Create a new Object instance
pub fn new(
key: String,
size: Option<i64>,
etag: Option<String>,
content_type: Option<String>,
user_metadata: Option<HashMap<String, String>>,
version_id: Option<String>,
sequencer: String,
) -> Self {
Self {
key,
size,
etag,
content_type,
user_metadata,
version_id,
sequencer,
}
}
/// Set the key
pub fn set_key(&mut self, key: String) {
self.key = key;
}
/// Set the size
pub fn set_size(&mut self, size: Option<i64>) {
self.size = size;
}
/// Set the etag
pub fn set_etag(&mut self, etag: Option<String>) {
self.etag = etag;
}
/// Set the content type
pub fn set_content_type(&mut self, content_type: Option<String>) {
self.content_type = content_type;
}
/// Set the user metadata
pub fn set_user_metadata(&mut self, user_metadata: Option<HashMap<String, String>>) {
self.user_metadata = user_metadata;
}
/// Set the version ID
pub fn set_version_id(&mut self, version_id: Option<String>) {
self.version_id = version_id;
}
/// Set the sequencer
pub fn set_sequencer(&mut self, sequencer: String) {
self.sequencer = sequencer;
}
}
/// A struct representing the metadata of the event
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Metadata {
#[serde(rename = "s3SchemaVersion")]
pub schema_version: String,
#[serde(rename = "configurationId")]
pub configuration_id: String,
pub bucket: Bucket,
pub object: Object,
}
impl Default for Metadata {
fn default() -> Self {
Self::new()
}
}
impl Metadata {
/// Create a new Metadata instance with default values
pub fn new() -> Self {
Self {
schema_version: "1.0".to_string(),
configuration_id: "default".to_string(),
bucket: Bucket::new(
"default".to_string(),
Identity::new("default".to_string()),
"arn:aws:s3:::default".to_string(),
),
object: Object::new("default".to_string(), None, None, None, None, None, "default".to_string()),
}
}
/// Create a new Metadata instance
pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self {
Self {
schema_version,
configuration_id,
bucket,
object,
}
}
/// Set the schema version
pub fn set_schema_version(&mut self, schema_version: String) {
self.schema_version = schema_version;
}
/// Set the configuration ID
pub fn set_configuration_id(&mut self, configuration_id: String) {
self.configuration_id = configuration_id;
}
/// Set the bucket
pub fn set_bucket(&mut self, bucket: Bucket) {
self.bucket = bucket;
}
/// Set the object
pub fn set_object(&mut self, object: Object) {
self.object = object;
}
}
/// A struct representing the source of the event
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Source {
pub host: String,
pub port: String,
#[serde(rename = "userAgent")]
pub user_agent: String,
}
impl Source {
/// Create a new Source instance
pub fn new(host: String, port: String, user_agent: String) -> Self {
Self { host, port, user_agent }
}
/// Set the host
pub fn set_host(&mut self, host: String) {
self.host = host;
}
/// Set the port
pub fn set_port(&mut self, port: String) {
self.port = port;
}
/// Set the user agent
pub fn set_user_agent(&mut self, user_agent: String) {
self.user_agent = user_agent;
}
}
/// Builder for creating an Event.
///
/// This struct is used to build an Event object with various parameters.
/// It provides methods to set each parameter and a build method to create the Event.
#[derive(Default, Clone)]
pub struct EventBuilder {
event_version: Option<String>,
event_source: Option<String>,
aws_region: Option<String>,
event_time: Option<String>,
event_name: Option<Name>,
user_identity: Option<Identity>,
request_parameters: Option<HashMap<String, String>>,
response_elements: Option<HashMap<String, String>>,
s3: Option<Metadata>,
source: Option<Source>,
channels: Option<SmallVec<[String; 2]>>,
}
impl EventBuilder {
/// create a builder that pre filled default values
pub fn new() -> Self {
Self {
event_version: Some(Cow::Borrowed("2.0").to_string()),
event_source: Some(Cow::Borrowed("aws:s3").to_string()),
aws_region: Some("us-east-1".to_string()),
event_time: Some(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string()),
event_name: None,
user_identity: Some(Identity {
principal_id: "anonymous".to_string(),
}),
request_parameters: Some(HashMap::new()),
response_elements: Some(HashMap::new()),
s3: None,
source: None,
channels: Some(Vec::new().into()),
}
}
/// verify and set the event version
pub fn event_version(mut self, event_version: impl Into<String>) -> Self {
let event_version = event_version.into();
if !event_version.is_empty() {
self.event_version = Some(event_version);
}
self
}
/// verify and set the event source
pub fn event_source(mut self, event_source: impl Into<String>) -> Self {
let event_source = event_source.into();
if !event_source.is_empty() {
self.event_source = Some(event_source);
}
self
}
/// set up aws regions
pub fn aws_region(mut self, aws_region: impl Into<String>) -> Self {
self.aws_region = Some(aws_region.into());
self
}
/// set event time
pub fn event_time(mut self, event_time: impl Into<String>) -> Self {
self.event_time = Some(event_time.into());
self
}
/// set event name
pub fn event_name(mut self, event_name: Name) -> Self {
self.event_name = Some(event_name);
self
}
/// set user identity
pub fn user_identity(mut self, user_identity: Identity) -> Self {
self.user_identity = Some(user_identity);
self
}
/// set request parameters
pub fn request_parameters(mut self, request_parameters: HashMap<String, String>) -> Self {
self.request_parameters = Some(request_parameters);
self
}
/// set response elements
pub fn response_elements(mut self, response_elements: HashMap<String, String>) -> Self {
self.response_elements = Some(response_elements);
self
}
/// setting up s3 metadata
pub fn s3(mut self, s3: Metadata) -> Self {
self.s3 = Some(s3);
self
}
/// set event source information
pub fn source(mut self, source: Source) -> Self {
self.source = Some(source);
self
}
/// set up the sending channel
pub fn channels(mut self, channels: Vec<String>) -> Self {
self.channels = Some(channels.into());
self
}
/// Create a preconfigured builder for common object event scenarios
pub fn for_object_creation(s3: Metadata, source: Source) -> Self {
Self::new().event_name(Name::ObjectCreatedPut).s3(s3).source(source)
}
/// Create a preconfigured builder for object deletion events
pub fn for_object_removal(s3: Metadata, source: Source) -> Self {
Self::new().event_name(Name::ObjectRemovedDelete).s3(s3).source(source)
}
/// build event instance
///
/// Verify the required fields and create a complete Event object
pub fn build(self) -> Result<Event, Error> {
let event_version = self.event_version.ok_or(Error::MissingField("event_version"))?;
let event_source = self.event_source.ok_or(Error::MissingField("event_source"))?;
let aws_region = self.aws_region.ok_or(Error::MissingField("aws_region"))?;
let event_time = self.event_time.ok_or(Error::MissingField("event_time"))?;
let event_name = self.event_name.ok_or(Error::MissingField("event_name"))?;
let user_identity = self.user_identity.ok_or(Error::MissingField("user_identity"))?;
let request_parameters = self.request_parameters.unwrap_or_default();
let response_elements = self.response_elements.unwrap_or_default();
let s3 = self.s3.ok_or(Error::MissingField("s3"))?;
let source = self.source.ok_or(Error::MissingField("source"))?;
let channels = self.channels.unwrap_or_else(|| smallvec![]);
Ok(Event {
event_version,
event_source,
aws_region,
event_time,
event_name,
user_identity,
request_parameters,
response_elements,
s3,
source,
id: Uuid::new_v4(),
timestamp: SystemTime::now(),
channels,
})
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Event {
#[serde(rename = "eventVersion")]
pub event_version: String,
#[serde(rename = "eventSource")]
pub event_source: String,
#[serde(rename = "awsRegion")]
pub aws_region: String,
#[serde(rename = "eventTime")]
pub event_time: String,
#[serde(rename = "eventName")]
pub event_name: Name,
#[serde(rename = "userIdentity")]
pub user_identity: Identity,
#[serde(rename = "requestParameters")]
pub request_parameters: HashMap<String, String>,
#[serde(rename = "responseElements")]
pub response_elements: HashMap<String, String>,
pub s3: Metadata,
pub source: Source,
pub id: Uuid,
pub timestamp: SystemTime,
pub channels: SmallVec<[String; 2]>,
}
impl Event {
/// create a new event builder
///
/// Returns an EventBuilder instance pre-filled with default values
pub fn builder() -> EventBuilder {
EventBuilder::new()
}
/// Quickly create Event instances with necessary fields
///
/// suitable for common s3 event scenarios
pub fn create(event_name: Name, s3: Metadata, source: Source, channels: Vec<String>) -> Self {
Self::builder()
.event_name(event_name)
.s3(s3)
.source(source)
.channels(channels)
.build()
.expect("Failed to create event, missing necessary parameters")
}
/// a convenient way to create a preconfigured builder
pub fn for_object_creation(s3: Metadata, source: Source) -> EventBuilder {
EventBuilder::for_object_creation(s3, source)
}
/// a convenient way to create a preconfigured builder
pub fn for_object_removal(s3: Metadata, source: Source) -> EventBuilder {
EventBuilder::for_object_removal(s3, source)
}
/// Determine whether an event belongs to a specific type
pub fn is_type(&self, event_type: Name) -> bool {
let mask = event_type.mask();
(self.event_name.mask() & mask) != 0
}
/// Determine whether an event needs to be sent to a specific channel
pub fn is_for_channel(&self, channel: &str) -> bool {
self.channels.iter().any(|c| c == channel)
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Log {
#[serde(rename = "eventName")]
pub event_name: Name,
pub key: String,
pub records: Vec<Event>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr, Display, EnumString)]
#[strum(serialize_all = "SCREAMING_SNAKE_CASE")]
pub enum Name {
ObjectAccessedGet,
ObjectAccessedGetRetention,
ObjectAccessedGetLegalHold,
ObjectAccessedHead,
ObjectAccessedAttributes,
ObjectCreatedCompleteMultipartUpload,
ObjectCreatedCopy,
ObjectCreatedPost,
ObjectCreatedPut,
ObjectCreatedPutRetention,
ObjectCreatedPutLegalHold,
ObjectCreatedPutTagging,
ObjectCreatedDeleteTagging,
ObjectRemovedDelete,
ObjectRemovedDeleteMarkerCreated,
ObjectRemovedDeleteAllVersions,
ObjectRemovedNoOp,
BucketCreated,
BucketRemoved,
ObjectReplicationFailed,
ObjectReplicationComplete,
ObjectReplicationMissedThreshold,
ObjectReplicationReplicatedAfterThreshold,
ObjectReplicationNotTracked,
ObjectRestorePost,
ObjectRestoreCompleted,
ObjectTransitionFailed,
ObjectTransitionComplete,
ObjectManyVersions,
ObjectLargeVersions,
PrefixManyFolders,
IlmDelMarkerExpirationDelete,
ObjectAccessedAll,
ObjectCreatedAll,
ObjectRemovedAll,
ObjectReplicationAll,
ObjectRestoreAll,
ObjectTransitionAll,
ObjectScannerAll,
Everything,
}
impl Name {
pub fn expand(&self) -> Vec<Name> {
match self {
Name::ObjectAccessedAll => vec![
Name::ObjectAccessedGet,
Name::ObjectAccessedHead,
Name::ObjectAccessedGetRetention,
Name::ObjectAccessedGetLegalHold,
Name::ObjectAccessedAttributes,
],
Name::ObjectCreatedAll => vec![
Name::ObjectCreatedCompleteMultipartUpload,
Name::ObjectCreatedCopy,
Name::ObjectCreatedPost,
Name::ObjectCreatedPut,
Name::ObjectCreatedPutRetention,
Name::ObjectCreatedPutLegalHold,
Name::ObjectCreatedPutTagging,
Name::ObjectCreatedDeleteTagging,
],
Name::ObjectRemovedAll => vec![
Name::ObjectRemovedDelete,
Name::ObjectRemovedDeleteMarkerCreated,
Name::ObjectRemovedNoOp,
Name::ObjectRemovedDeleteAllVersions,
],
Name::ObjectReplicationAll => vec![
Name::ObjectReplicationFailed,
Name::ObjectReplicationComplete,
Name::ObjectReplicationNotTracked,
Name::ObjectReplicationMissedThreshold,
Name::ObjectReplicationReplicatedAfterThreshold,
],
Name::ObjectRestoreAll => vec![Name::ObjectRestorePost, Name::ObjectRestoreCompleted],
Name::ObjectTransitionAll => {
vec![Name::ObjectTransitionFailed, Name::ObjectTransitionComplete]
}
Name::ObjectScannerAll => vec![Name::ObjectManyVersions, Name::ObjectLargeVersions, Name::PrefixManyFolders],
Name::Everything => (1..=Name::IlmDelMarkerExpirationDelete as u32)
.map(|i| Name::from_repr(i).unwrap())
.collect(),
_ => vec![*self],
}
}
pub fn mask(&self) -> u64 {
if (*self as u32) < Name::ObjectAccessedAll as u32 {
1 << (*self as u32 - 1)
} else {
self.expand().iter().fold(0, |acc, n| acc | (1 << (*n as u32 - 1)))
}
}
fn from_repr(discriminant: u32) -> Option<Self> {
match discriminant {
1 => Some(Name::ObjectAccessedGet),
2 => Some(Name::ObjectAccessedGetRetention),
3 => Some(Name::ObjectAccessedGetLegalHold),
4 => Some(Name::ObjectAccessedHead),
5 => Some(Name::ObjectAccessedAttributes),
6 => Some(Name::ObjectCreatedCompleteMultipartUpload),
7 => Some(Name::ObjectCreatedCopy),
8 => Some(Name::ObjectCreatedPost),
9 => Some(Name::ObjectCreatedPut),
10 => Some(Name::ObjectCreatedPutRetention),
11 => Some(Name::ObjectCreatedPutLegalHold),
12 => Some(Name::ObjectCreatedPutTagging),
13 => Some(Name::ObjectCreatedDeleteTagging),
14 => Some(Name::ObjectRemovedDelete),
15 => Some(Name::ObjectRemovedDeleteMarkerCreated),
16 => Some(Name::ObjectRemovedDeleteAllVersions),
17 => Some(Name::ObjectRemovedNoOp),
18 => Some(Name::BucketCreated),
19 => Some(Name::BucketRemoved),
20 => Some(Name::ObjectReplicationFailed),
21 => Some(Name::ObjectReplicationComplete),
22 => Some(Name::ObjectReplicationMissedThreshold),
23 => Some(Name::ObjectReplicationReplicatedAfterThreshold),
24 => Some(Name::ObjectReplicationNotTracked),
25 => Some(Name::ObjectRestorePost),
26 => Some(Name::ObjectRestoreCompleted),
27 => Some(Name::ObjectTransitionFailed),
28 => Some(Name::ObjectTransitionComplete),
29 => Some(Name::ObjectManyVersions),
30 => Some(Name::ObjectLargeVersions),
31 => Some(Name::PrefixManyFolders),
32 => Some(Name::IlmDelMarkerExpirationDelete),
33 => Some(Name::ObjectAccessedAll),
34 => Some(Name::ObjectCreatedAll),
35 => Some(Name::ObjectRemovedAll),
36 => Some(Name::ObjectReplicationAll),
37 => Some(Name::ObjectRestoreAll),
38 => Some(Name::ObjectTransitionAll),
39 => Some(Name::ObjectScannerAll),
40 => Some(Name::Everything),
_ => None,
}
}
}
crates/event-notifier/src/global.rs
-234
View File
@@ -1,234 +0,0 @@
use crate::{Error, Event, NotifierConfig, NotifierSystem, create_adapters};
use std::sync::{Arc, atomic};
use tokio::sync::{Mutex, OnceCell};
use tracing::instrument;
static GLOBAL_SYSTEM: OnceCell<Arc<Mutex<NotifierSystem>>> = OnceCell::const_new();
static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false);
static READY: atomic::AtomicBool = atomic::AtomicBool::new(false);
static INIT_LOCK: Mutex<()> = Mutex::const_new(());
/// Initializes the global notification system.
///
/// This function performs the following steps:
/// 1. Checks if the system is already initialized.
/// 2. Creates a new `NotificationSystem` instance.
/// 3. Creates adapters based on the provided configuration.
/// 4. Starts the notification system with the created adapters.
/// 5. Sets the global system instance.
///
/// # Errors
///
/// Returns an error if:
/// - The system is already initialized.
/// - Creating the `NotificationSystem` fails.
/// - Creating adapters fails.
/// - Starting the notification system fails.
/// - Setting the global system instance fails.
pub async fn initialize(config: NotifierConfig) -> Result<(), Error> {
let _lock = INIT_LOCK.lock().await;
// Check if the system is already initialized.
if INITIALIZED.load(atomic::Ordering::SeqCst) {
return Err(Error::custom("Notification system has already been initialized"));
}
// Check if the system is already ready.
if READY.load(atomic::Ordering::SeqCst) {
return Err(Error::custom("Notification system is already ready"));
}
// Check if the system is shutting down.
if let Some(system) = GLOBAL_SYSTEM.get() {
let system_guard = system.lock().await;
if system_guard.shutdown_cancelled() {
return Err(Error::custom("Notification system is shutting down"));
}
}
// check if config adapters len is than 0
if config.adapters.is_empty() {
return Err(Error::custom("No adapters configured"));
}
// Attempt to initialize, and reset the INITIALIZED flag if it fails.
let result: Result<(), Error> = async {
let system = NotifierSystem::new(config.clone()).await.map_err(|e| {
tracing::error!("Failed to create NotificationSystem: {:?}", e);
e
})?;
let adapters = create_adapters(&config.adapters).map_err(|e| {
tracing::error!("Failed to create adapters: {:?}", e);
e
})?;
tracing::info!("adapters len:{:?}", adapters.len());
let system_clone = Arc::new(Mutex::new(system));
let adapters_clone = adapters.clone();
GLOBAL_SYSTEM.set(system_clone.clone()).map_err(|_| {
let err = Error::custom("Unable to set up global notification system");
tracing::error!("{:?}", err);
err
})?;
tokio::spawn(async move {
if let Err(e) = system_clone.lock().await.start(adapters_clone).await {
tracing::error!("Notification system failed to start: {}", e);
}
tracing::info!("Notification system started in background");
});
tracing::info!("system start success,start set READY value");
READY.store(true, atomic::Ordering::SeqCst);
tracing::info!("Notification system is ready to process events");
Ok(())
}
.await;
if result.is_err() {
INITIALIZED.store(false, atomic::Ordering::SeqCst);
READY.store(false, atomic::Ordering::SeqCst);
return result;
}
INITIALIZED.store(true, atomic::Ordering::SeqCst);
Ok(())
}
/// Checks if the notification system is initialized.
pub fn is_initialized() -> bool {
INITIALIZED.load(atomic::Ordering::SeqCst)
}
/// Checks if the notification system is ready.
pub fn is_ready() -> bool {
READY.load(atomic::Ordering::SeqCst)
}
/// Sends an event to the notification system.
///
/// # Errors
///
/// Returns an error if:
/// - The system is not initialized.
/// - The system is not ready.
/// - Sending the event fails.
#[instrument(fields(event))]
pub async fn send_event(event: Event) -> Result<(), Error> {
if !READY.load(atomic::Ordering::SeqCst) {
return Err(Error::custom("Notification system not ready, please wait for initialization to complete"));
}
let system = get_system().await?;
let system_guard = system.lock().await;
system_guard.send_event(event).await
}
/// Shuts down the notification system.
#[instrument]
pub async fn shutdown() -> Result<(), Error> {
if let Some(system) = GLOBAL_SYSTEM.get() {
tracing::info!("Shutting down notification system start");
let result = {
let mut system_guard = system.lock().await;
system_guard.shutdown().await
};
if let Err(e) = &result {
tracing::error!("Notification system shutdown failed: {}", e);
} else {
tracing::info!("Event bus shutdown completed");
}
tracing::info!(
"Shutdown method called set static value start, READY: {}, INITIALIZED: {}",
READY.load(atomic::Ordering::SeqCst),
INITIALIZED.load(atomic::Ordering::SeqCst)
);
READY.store(false, atomic::Ordering::SeqCst);
INITIALIZED.store(false, atomic::Ordering::SeqCst);
tracing::info!(
"Shutdown method called set static value end, READY: {}, INITIALIZED: {}",
READY.load(atomic::Ordering::SeqCst),
INITIALIZED.load(atomic::Ordering::SeqCst)
);
result
} else {
Err(Error::custom("Notification system not initialized"))
}
}
/// Retrieves the global notification system instance.
///
/// # Errors
///
/// Returns an error if the system is not initialized.
async fn get_system() -> Result<Arc<Mutex<NotifierSystem>>, Error> {
GLOBAL_SYSTEM
.get()
.cloned()
.ok_or_else(|| Error::custom("Notification system not initialized"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::NotifierConfig;
fn init_tracing() {
// Use try_init to avoid panic if already initialized
let _ = tracing_subscriber::fmt::try_init();
}
#[tokio::test]
async fn test_initialize_success() {
init_tracing();
let config = NotifierConfig::default(); // assume there is a default configuration
let result = initialize(config).await;
assert!(result.is_err(), "Initialization should not succeed");
assert!(!is_initialized(), "System should not be marked as initialized");
assert!(!is_ready(), "System should not be marked as ready");
}
#[tokio::test]
async fn test_initialize_twice() {
init_tracing();
let config = NotifierConfig::default();
let _ = initialize(config.clone()).await; // first initialization
let result = initialize(config).await; // second initialization
assert!(result.is_err(), "Initialization should succeed");
assert!(result.is_err(), "Re-initialization should fail");
}
#[tokio::test]
async fn test_initialize_failure_resets_state() {
init_tracing();
// Test with empty adapters to force failure
let config = NotifierConfig {
adapters: Vec::new(),
..Default::default()
};
let result = initialize(config).await;
assert!(result.is_err(), "Initialization should fail with empty adapters");
assert!(!is_initialized(), "System should not be marked as initialized after failure");
assert!(!is_ready(), "System should not be marked as ready after failure");
}
#[tokio::test]
async fn test_is_initialized_and_is_ready() {
init_tracing();
// Initially, the system should not be initialized or ready
assert!(!is_initialized(), "System should not be initialized initially");
assert!(!is_ready(), "System should not be ready initially");
// Test with empty adapters to ensure failure
let config = NotifierConfig {
adapters: Vec::new(),
..Default::default()
};
let result = initialize(config).await;
assert!(result.is_err(), "Initialization should fail with empty adapters");
assert!(!is_initialized(), "System should not be initialized after failed init");
assert!(!is_ready(), "System should not be ready after failed init");
}
}
crates/event-notifier/src/lib.rs
-31
View File
@@ -1,31 +0,0 @@
mod adapter;
mod bus;
mod config;
mod error;
mod event;
mod global;
mod notifier;
mod store;
pub use adapter::ChannelAdapter;
pub use adapter::create_adapters;
#[cfg(all(feature = "kafka", target_os = "linux"))]
pub use adapter::kafka::KafkaAdapter;
#[cfg(feature = "mqtt")]
pub use adapter::mqtt::MqttAdapter;
#[cfg(feature = "webhook")]
pub use adapter::webhook::WebhookAdapter;
pub use bus::event_bus;
#[cfg(all(feature = "kafka", target_os = "linux"))]
pub use config::KafkaConfig;
#[cfg(feature = "mqtt")]
pub use config::MqttConfig;
#[cfg(feature = "webhook")]
pub use config::WebhookConfig;
pub use config::{AdapterConfig, NotifierConfig};
pub use error::Error;
pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source};
pub use global::{initialize, is_initialized, is_ready, send_event, shutdown};
pub use notifier::NotifierSystem;
pub use store::EventStore;
crates/event-notifier/src/notifier.rs
-136
View File
@@ -1,136 +0,0 @@
use crate::{ChannelAdapter, Error, Event, EventStore, NotifierConfig, event_bus};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::instrument;
/// The `NotificationSystem` struct represents the notification system.
/// It manages the event bus and the adapters.
/// It is responsible for sending and receiving events.
/// It also handles the shutdown process.
pub struct NotifierSystem {
tx: mpsc::Sender<Event>,
rx: Option<mpsc::Receiver<Event>>,
store: Arc<EventStore>,
shutdown: CancellationToken,
shutdown_complete: Option<tokio::sync::oneshot::Sender<()>>,
shutdown_receiver: Option<tokio::sync::oneshot::Receiver<()>>,
}
impl NotifierSystem {
/// Creates a new `NotificationSystem` instance.
#[instrument(skip(config))]
pub async fn new(config: NotifierConfig) -> Result<Self, Error> {
let (tx, rx) = mpsc::channel::<Event>(config.channel_capacity);
let store = Arc::new(EventStore::new(&config.store_path).await?);
let shutdown = CancellationToken::new();
let restored_logs = store.load_logs().await?;
for log in restored_logs {
for event in log.records {
// For example, where the send method may return a SendError when calling it
tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?;
}
}
// Initialize shutdown_complete to Some(tx)
let (complete_tx, complete_rx) = tokio::sync::oneshot::channel();
Ok(Self {
tx,
rx: Some(rx),
store,
shutdown,
shutdown_complete: Some(complete_tx),
shutdown_receiver: Some(complete_rx),
})
}
/// Starts the notification system.
/// It initializes the event bus and the producer.
#[instrument(skip_all)]
pub async fn start(&mut self, adapters: Vec<Arc<dyn ChannelAdapter>>) -> Result<(), Error> {
if self.shutdown.is_cancelled() {
let error = Error::custom("System is shutting down");
self.handle_error("start", &error);
return Err(error);
}
self.log(tracing::Level::INFO, "start", "Starting the notification system");
let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?;
let shutdown_clone = self.shutdown.clone();
let store_clone = self.store.clone();
let shutdown_complete = self.shutdown_complete.take();
tokio::spawn(async move {
if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone, shutdown_complete).await {
tracing::error!("Event bus failed: {}", e);
}
});
self.log(tracing::Level::INFO, "start", "Notification system started successfully");
Ok(())
}
/// Sends an event to the notification system.
/// This method is used to send events to the event bus.
#[instrument(skip(self))]
pub async fn send_event(&self, event: Event) -> Result<(), Error> {
self.log(tracing::Level::DEBUG, "send_event", &format!("Sending event: {:?}", event));
if self.shutdown.is_cancelled() {
let error = Error::custom("System is shutting down");
self.handle_error("send_event", &error);
return Err(error);
}
if let Err(e) = self.tx.send(event).await {
let error = Error::ChannelSend(Box::new(e));
self.handle_error("send_event", &error);
return Err(error);
}
self.log(tracing::Level::INFO, "send_event", "Event sent successfully");
Ok(())
}
/// Shuts down the notification system.
/// This method is used to cancel the event bus and producer tasks.
#[instrument(skip(self))]
pub async fn shutdown(&mut self) -> Result<(), Error> {
tracing::info!("Shutting down the notification system");
self.shutdown.cancel();
// wait for the event bus to be completely closed
if let Some(receiver) = self.shutdown_receiver.take() {
match receiver.await {
Ok(_) => {
tracing::info!("Event bus shutdown completed successfully");
Ok(())
}
Err(e) => {
let error = Error::custom(format!("Failed to receive shutdown completion: {}", e).as_str());
self.handle_error("shutdown", &error);
Err(error)
}
}
} else {
tracing::warn!("Shutdown receiver not available, the event bus might still be running");
Err(Error::custom("Shutdown receiver not available"))
}
}
/// shutdown state
pub fn shutdown_cancelled(&self) -> bool {
self.shutdown.is_cancelled()
}
#[instrument(skip(self))]
pub fn handle_error(&self, context: &str, error: &Error) {
self.log(tracing::Level::ERROR, context, &format!("{:?}", error));
// TODO Can be extended to record to files or send to monitoring systems
}
#[instrument(skip(self))]
fn log(&self, level: tracing::Level, context: &str, message: &str) {
match level {
tracing::Level::ERROR => tracing::error!("[{}] {}", context, message),
tracing::Level::WARN => tracing::warn!("[{}] {}", context, message),
tracing::Level::INFO => tracing::info!("[{}] {}", context, message),
tracing::Level::DEBUG => tracing::debug!("[{}] {}", context, message),
tracing::Level::TRACE => tracing::trace!("[{}] {}", context, message),
}
}
}
crates/event-notifier/src/store.rs
-60
View File
@@ -1,60 +0,0 @@
use crate::Error;
use crate::Log;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::fs::{File, OpenOptions, create_dir_all};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter};
use tokio::sync::RwLock;
use tracing::instrument;
/// `EventStore` is a struct that manages the storage of event logs.
pub struct EventStore {
path: String,
lock: Arc<RwLock<()>>,
}
impl EventStore {
pub async fn new(path: &str) -> Result<Self, Error> {
create_dir_all(path).await?;
Ok(Self {
path: path.to_string(),
lock: Arc::new(RwLock::new(())),
})
}
#[instrument(skip(self))]
pub async fn save_logs(&self, logs: &[Log]) -> Result<(), Error> {
let _guard = self.lock.write().await;
let file_path = format!(
"{}/events_{}.jsonl",
self.path,
SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()
);
let file = OpenOptions::new().create(true).append(true).open(&file_path).await?;
let mut writer = BufWriter::new(file);
for log in logs {
let line = serde_json::to_string(log)?;
writer.write_all(line.as_bytes()).await?;
writer.write_all(b"\n").await?;
}
writer.flush().await?;
tracing::info!("Saved logs to {} end", file_path);
Ok(())
}
pub async fn load_logs(&self) -> Result<Vec<Log>, Error> {
let _guard = self.lock.read().await;
let mut logs = Vec::new();
let mut entries = tokio::fs::read_dir(&self.path).await?;
while let Some(entry) = entries.next_entry().await? {
let file = File::open(entry.path()).await?;
let reader = BufReader::new(file);
let mut lines = reader.lines();
while let Some(line) = lines.next_line().await? {
let log: Log = serde_json::from_str(&line)?;
logs.push(log);
}
}
Ok(logs)
}
}
crates/event-notifier/tests/integration.rs
-159
View File
@@ -1,159 +0,0 @@
use rustfs_event_notifier::{AdapterConfig, NotifierSystem, WebhookConfig};
use rustfs_event_notifier::{Bucket, Event, EventBuilder, Identity, Metadata, Name, Object, Source};
use rustfs_event_notifier::{ChannelAdapter, WebhookAdapter};
use std::collections::HashMap;
use std::sync::Arc;
#[tokio::test]
async fn test_webhook_adapter() {
let adapter = WebhookAdapter::new(WebhookConfig {
endpoint: "http://localhost:8080/webhook".to_string(),
auth_token: None,
custom_headers: None,
max_retries: 1,
timeout: 5,
});
// create an s3 metadata object
let metadata = Metadata {
schema_version: "1.0".to_string(),
configuration_id: "test-config".to_string(),
bucket: Bucket {
name: "my-bucket".to_string(),
owner_identity: Identity {
principal_id: "owner123".to_string(),
},
arn: "arn:aws:s3:::my-bucket".to_string(),
},
object: Object {
key: "test.txt".to_string(),
size: Some(1024),
etag: Some("abc123".to_string()),
content_type: Some("text/plain".to_string()),
user_metadata: None,
version_id: None,
sequencer: "1234567890".to_string(),
},
};
// create source object
let source = Source {
host: "localhost".to_string(),
port: "80".to_string(),
user_agent: "curl/7.68.0".to_string(),
};
// Create events using builder mode
let event = Event::builder()
.event_version("2.0")
.event_source("aws:s3")
.aws_region("us-east-1")
.event_time("2023-10-01T12:00:00.000Z")
.event_name(Name::ObjectCreatedPut)
.user_identity(Identity {
principal_id: "user123".to_string(),
})
.request_parameters(HashMap::new())
.response_elements(HashMap::new())
.s3(metadata)
.source(source)
.channels(vec!["webhook".to_string()])
.build()
.expect("failed to create event");
let result = adapter.send(&event).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_notification_system() {
let config = rustfs_event_notifier::NotifierConfig {
store_path: "./test_events".to_string(),
channel_capacity: 100,
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
endpoint: "http://localhost:8080/webhook".to_string(),
auth_token: None,
custom_headers: None,
max_retries: 1,
timeout: 5,
})],
};
let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await.unwrap()));
let adapters: Vec<Arc<dyn ChannelAdapter>> = vec![Arc::new(WebhookAdapter::new(WebhookConfig {
endpoint: "http://localhost:8080/webhook".to_string(),
auth_token: None,
custom_headers: None,
max_retries: 1,
timeout: 5,
}))];
// create an s3 metadata object
let metadata = Metadata {
schema_version: "1.0".to_string(),
configuration_id: "test-config".to_string(),
bucket: Bucket {
name: "my-bucket".to_string(),
owner_identity: Identity {
principal_id: "owner123".to_string(),
},
arn: "arn:aws:s3:::my-bucket".to_string(),
},
object: Object {
key: "test.txt".to_string(),
size: Some(1024),
etag: Some("abc123".to_string()),
content_type: Some("text/plain".to_string()),
user_metadata: None,
version_id: None,
sequencer: "1234567890".to_string(),
},
};
// create source object
let source = Source {
host: "localhost".to_string(),
port: "80".to_string(),
user_agent: "curl/7.68.0".to_string(),
};
// create a preconfigured builder with objects
let event = EventBuilder::for_object_creation(metadata, source)
.user_identity(Identity {
principal_id: "user123".to_string(),
})
.event_time("2023-10-01T12:00:00.000Z")
.channels(vec!["webhook".to_string()])
.build()
.expect("failed to create event");
{
let system_lock = system.lock().await;
system_lock.send_event(event).await.unwrap();
}
let system_clone = Arc::clone(&system);
let system_handle = tokio::spawn(async move {
let mut system = system_clone.lock().await;
system.start(adapters).await
});
// set 10 seconds timeout
match tokio::time::timeout(std::time::Duration::from_secs(10), system_handle).await {
Ok(result) => {
println!("System started successfully");
assert!(result.is_ok());
}
Err(_) => {
println!("System operation timed out, forcing shutdown");
// create a new task to handle the timeout
let system = Arc::clone(&system);
tokio::spawn(async move {
if let Ok(mut guard) = system.try_lock() {
guard.shutdown().await.unwrap();
}
});
// give the system some time to clean up resources
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
}
crates/filemeta/src/filemeta.rs
+27 -27
View File
@@ -10,6 +10,7 @@ use bytes::Bytes;
use rmp::Marker;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::convert::TryFrom;
use std::hash::Hasher;
use std::io::{Read, Write};
use std::{collections::HashMap, io::Cursor};
@@ -2458,7 +2459,6 @@ pub async fn read_xl_meta_no_data<R: AsyncRead + Unpin>(reader: &mut R, size: us
}
#[cfg(test)]
mod test {
use super::*;
use crate::test_data::*;
@@ -2578,7 +2578,7 @@ mod test {
// 验证基本属性
assert_eq!(fm.meta_ver, XL_META_VERSION);
assert_eq!(fm.versions.len(), 3, "应该有3个版本（1个对象，1个删除标记，1个Legacy）");
assert_eq!(fm.versions.len(), 3, "应该有 3 个版本（1 个对象，1 个删除标记，1 个 Legacy）");
// 验证版本类型
let mut object_count = 0;
@@ -2594,9 +2594,9 @@ mod test {
}
}
assert_eq!(object_count, 1, "应该有1个对象版本");
assert_eq!(delete_count, 1, "应该有1个删除标记");
assert_eq!(legacy_count, 1, "应该有1个Legacy版本");
assert_eq!(object_count, 1, "应该有 1 个对象版本");
assert_eq!(delete_count, 1, "应该有 1 个删除标记");
assert_eq!(legacy_count, 1, "应该有 1 个 Legacy 版本");
// 验证兼容性
assert!(fm.is_compatible_with_meta(), "应该与 xl 格式兼容");
@@ -2619,7 +2619,7 @@ mod test {
let fm = FileMeta::load(&data).expect("解析复杂数据失败");
// 验证版本数量
assert!(fm.versions.len() >= 10, "应该有至少10个版本");
assert!(fm.versions.len() >= 10, "应该有至少 10 个版本");
// 验证版本排序
assert!(fm.is_sorted_by_mod_time(), "版本应该按修改时间排序");
@@ -2640,7 +2640,7 @@ mod test {
let data = create_xlmeta_with_inline_data().expect("创建内联数据测试失败");
let fm = FileMeta::load(&data).expect("解析内联数据失败");
assert_eq!(fm.versions.len(), 1, "应该有1个版本");
assert_eq!(fm.versions.len(), 1, "应该有 1 个版本");
assert!(!fm.data.as_slice().is_empty(), "应该包含内联数据");
// 验证内联数据内容
@@ -2687,7 +2687,7 @@ mod test {
for version in &fm.versions {
let signature = version.header.get_signature();
assert_eq!(signature.len(), 4, "签名应该是4字节");
assert_eq!(signature.len(), 4, "签名应该是 4 字节");
// 验证相同版本的签名一致性
let signature2 = version.header.get_signature();
@@ -2730,7 +2730,7 @@ mod test {
// 验证版本内容一致性
for (v1, v2) in fm.versions.iter().zip(fm2.versions.iter()) {
assert_eq!(v1.header.version_type, v2.header.version_type, "版本类型应该一致");
assert_eq!(v1.header.version_id, v2.header.version_id, "版本ID应该一致");
assert_eq!(v1.header.version_id, v2.header.version_id, "版本 ID 应该一致");
}
}
@@ -2751,26 +2751,26 @@ mod test {
let _serialized = fm.marshal_msg().expect("序列化失败");
let serialization_time = start.elapsed();
println!("性能测试结果:");
println!(" 创建时间: {:?}", creation_time);
println!(" 解析时间: {:?}", parsing_time);
println!(" 序列化时间: {:?}", serialization_time);
println!("性能测试结果：");
println!(" 创建时间：{:?}", creation_time);
println!(" 解析时间：{:?}", parsing_time);
println!(" 序列化时间：{:?}", serialization_time);
// 基本性能断言（这些值可能需要根据实际性能调整）
assert!(parsing_time.as_millis() < 100, "解析时间应该小于100ms");
assert!(serialization_time.as_millis() < 100, "序列化时间应该小于100ms");
assert!(parsing_time.as_millis() < 100, "解析时间应该小于 100ms");
assert!(serialization_time.as_millis() < 100, "序列化时间应该小于 100ms");
}
#[test]
fn test_edge_cases() {
// 测试边界情况
// 1. 测试空版本ID
// 1. 测试空版本 ID
let mut fm = FileMeta::new();
let version = FileMetaVersion {
version_type: VersionType::Object,
object: Some(MetaObject {
version_id: None, // 空版本ID
version_id: None, // 空版本 ID
data_dir: None,
erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
erasure_m: 1,
@@ -2866,9 +2866,9 @@ mod test {
let large_size = mem::size_of_val(&large_fm);
println!("Large FileMeta size: {} bytes", large_size);
// 验证内存使用是合理的（注意：size_of_val只计算栈上的大小，不包括堆分配）
// 对于包含Vec的结构体，size_of_val可能相同，因为Vec的容量在堆上
println!("版本数量: {}", large_fm.versions.len());
// 验证内存使用是合理的（注意：size_of_val 只计算栈上的大小，不包括堆分配）
// 对于包含 Vec 的结构体，size_of_val 可能相同，因为 Vec 的容量在堆上
println!("版本数量：{}", large_fm.versions.len());
assert!(!large_fm.versions.is_empty(), "应该有版本数据");
}
@@ -2939,8 +2939,8 @@ mod test {
};
// 验证参数的合理性
assert!(obj.erasure_m > 0, "数据块数量必须大于0");
assert!(obj.erasure_n > 0, "校验块数量必须大于0");
assert!(obj.erasure_m > 0, "数据块数量必须大于 0");
assert!(obj.erasure_n > 0, "校验块数量必须大于 0");
assert_eq!(obj.erasure_dist.len(), data_blocks + parity_blocks);
// 验证序列化和反序列化
@@ -3101,7 +3101,7 @@ mod test {
// 测试多个版本列表的合并
let merged = merge_file_meta_versions(1, false, 0, &[versions1.clone(), versions2.clone()]);
// 合并结果可能为空，这取决于版本的兼容性，这是正常的
println!("合并结果数量: {}", merged.len());
println!("合并结果数量：{}", merged.len());
}
#[test]
@@ -3111,12 +3111,12 @@ mod test {
for flag in flags {
let flag_value = flag as u8;
assert!(flag_value > 0, "标志位值应该大于0");
assert!(flag_value > 0, "标志位值应该大于 0");
// 测试标志位组合
let combined = Flags::FreeVersion as u8 | Flags::UsesDataDir as u8;
// 对于位运算，组合值可能不总是大于单个值，这是正常的
assert!(combined > 0, "组合标志位应该大于0");
assert!(combined > 0, "组合标志位应该大于 0");
}
}
@@ -3252,7 +3252,7 @@ mod test {
("tabs", "col1\tcol2\tcol3"),
("quotes", "\"quoted\" and 'single'"),
("backslashes", "path\\to\\file"),
("mixed", "Mixed: 中文, English, 123, !@#$%"),
("mixed", "Mixed: 中文，English, 123, !@#$%"),
];
for (key, value) in special_cases {
@@ -3274,7 +3274,7 @@ mod test {
("tabs", "col1\tcol2\tcol3"),
("quotes", "\"quoted\" and 'single'"),
("backslashes", "path\\to\\file"),
("mixed", "Mixed: 中文, English, 123, !@#$%"),
("mixed", "Mixed: 中文，English, 123, !@#$%"),
] {
assert_eq!(obj2.meta_user.get(key), Some(&expected_value.to_string()));
}
crates/notify/Cargo.toml
+40
View File
@@ -0,0 +1,40 @@
[package]
name = "rustfs-notify"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
rustfs-utils = { workspace = true, features = ["path", "sys"] }
async-trait = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
const-str = { workspace = true }
ecstore = { workspace = true }
form_urlencoded = { workspace = true }
once_cell = { workspace = true }
quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
reqwest = { workspace = true }
rumqttc = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
snap = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter"] }
uuid = { workspace = true, features = ["v4", "serde"] }
url = { workspace = true }
urlencoding = { workspace = true }
wildmatch = { workspace = true, features = ["serde"] }
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
axum = { workspace = true }
[lints]
workspace = true
crates/notify/examples/full_demo.rs
+172
View File
@@ -0,0 +1,172 @@
use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS};
use rustfs_notify::arn::TargetID;
use rustfs_notify::factory::{
DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME,
NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
};
use rustfs_notify::store::DEFAULT_LIMIT;
use rustfs_notify::{init_logger, BucketNotificationConfig, Event, EventName, LogLevel, NotificationError};
use rustfs_notify::{initialize, notification_system};
use std::time::Duration;
use tracing::info;
#[tokio::main]
async fn main() -> Result<(), NotificationError> {
init_logger(LogLevel::Debug);
let system = match notification_system() {
Some(sys) => sys,
None => {
let config = Config::new();
initialize(config).await?;
notification_system().expect("Failed to initialize notification system")
}
};
// --- Initial configuration (Webhook and MQTT) ---
let mut config = Config::new();
let current_root = rustfs_utils::dirs::get_project_root().expect("failed to get project root");
println!("Current project root: {}", current_root.display());
let webhook_kvs_vec = vec![
KV {
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_ENDPOINT.to_string(),
value: "http://127.0.0.1:3020/webhook".to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_AUTH_TOKEN.to_string(),
value: "secret-token".to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_QUEUE_DIR.to_string(),
value: current_root
.clone()
.join("../../deploy/logs/notify/webhook")
.to_str()
.unwrap()
.to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_QUEUE_LIMIT.to_string(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
];
let webhook_kvs = KVS(webhook_kvs_vec);
let mut webhook_targets = std::collections::HashMap::new();
webhook_targets.insert(DEFAULT_TARGET.to_string(), webhook_kvs);
config.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets);
// MQTT target configuration
let mqtt_kvs_vec = vec![
KV {
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_BROKER.to_string(),
value: "mqtt://localhost:1883".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_TOPIC.to_string(),
value: "rustfs/events".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_QOS.to_string(),
value: "1".to_string(), // AtLeastOnce
hidden_if_empty: false,
},
KV {
key: MQTT_USERNAME.to_string(),
value: "test".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_PASSWORD.to_string(),
value: "123456".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_QUEUE_DIR.to_string(),
value: current_root
.join("../../deploy/logs/notify/mqtt")
.to_str()
.unwrap()
.to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_QUEUE_LIMIT.to_string(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
];
let mqtt_kvs = KVS(mqtt_kvs_vec);
let mut mqtt_targets = std::collections::HashMap::new();
mqtt_targets.insert(DEFAULT_TARGET.to_string(), mqtt_kvs);
config.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_targets);
// Load the configuration and initialize the system
*system.config.write().await = config;
system.init().await?;
info!("✅ System initialized with Webhook and MQTT targets.");
// --- Query the currently active Target ---
let active_targets = system.get_active_targets().await;
info!("\n---> Currently active targets: {:?}", active_targets);
assert_eq!(active_targets.len(), 2);
tokio::time::sleep(Duration::from_secs(1)).await;
// --- Exactly delete a Target (e.g. MQTT) ---
info!("\n---> Removing MQTT target...");
let mqtt_target_id = TargetID::new(DEFAULT_TARGET.to_string(), "mqtt".to_string());
system.remove_target(&mqtt_target_id, NOTIFY_MQTT_SUB_SYS).await?;
info!("✅ MQTT target removed.");
// --- Query the activity's Target again ---
let active_targets_after_removal = system.get_active_targets().await;
info!("\n---> Active targets after removal: {:?}", active_targets_after_removal);
assert_eq!(active_targets_after_removal.len(), 1);
assert_eq!(active_targets_after_removal[0].id, DEFAULT_TARGET.to_string());
// --- Send events for verification ---
// Configure a rule to point to the Webhook and deleted MQTT
let mut bucket_config = BucketNotificationConfig::new("us-east-1");
bucket_config.add_rule(
&[EventName::ObjectCreatedPut],
"*".to_string(),
TargetID::new(DEFAULT_TARGET.to_string(), "webhook".to_string()),
);
bucket_config.add_rule(
&[EventName::ObjectCreatedPut],
"*".to_string(),
TargetID::new(DEFAULT_TARGET.to_string(), "mqtt".to_string()), // This rule will match, but the Target cannot be found
);
system.load_bucket_notification_config("my-bucket", &bucket_config).await?;
info!("\n---> Sending an event...");
let event = Event::new_test_event("my-bucket", "document.pdf", EventName::ObjectCreatedPut);
system
.send_event("my-bucket", "s3:ObjectCreated:Put", "document.pdf", event)
.await;
info!("✅ Event sent. Only the Webhook target should receive it. Check logs for warnings about the missing MQTT target.");
tokio::time::sleep(Duration::from_secs(2)).await;
info!("\nDemo completed successfully");
Ok(())
}
crates/notify/examples/full_demo_one.rs
+175
View File
@@ -0,0 +1,175 @@
use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS};
// Using Global Accessories
use rustfs_notify::arn::TargetID;
use rustfs_notify::factory::{
DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME,
NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
};
use rustfs_notify::store::DEFAULT_LIMIT;
use rustfs_notify::{init_logger, BucketNotificationConfig, Event, EventName, LogLevel, NotificationError};
use rustfs_notify::{initialize, notification_system};
use std::time::Duration;
use tracing::info;
#[tokio::main]
async fn main() -> Result<(), NotificationError> {
init_logger(LogLevel::Debug);
// Get global NotificationSystem instance
let system = match notification_system() {
Some(sys) => sys,
None => {
let config = Config::new();
initialize(config).await?;
notification_system().expect("Failed to initialize notification system")
}
};
// --- Initial configuration ---
let mut config = Config::new();
let current_root = rustfs_utils::dirs::get_project_root().expect("failed to get project root");
// Webhook target
let webhook_kvs_vec = vec![
KV {
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_ENDPOINT.to_string(),
value: "http://127.0.0.1:3020/webhook".to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_AUTH_TOKEN.to_string(),
value: "secret-token".to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_QUEUE_DIR.to_string(),
value: current_root
.clone()
.join("../../deploy/logs/notify/webhook")
.to_str()
.unwrap()
.to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_QUEUE_LIMIT.to_string(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
];
let webhook_kvs = KVS(webhook_kvs_vec);
let mut webhook_targets = std::collections::HashMap::new();
webhook_targets.insert(DEFAULT_TARGET.to_string(), webhook_kvs);
config.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets);
// Load the initial configuration and initialize the system
*system.config.write().await = config;
system.init().await?;
info!("✅ System initialized with Webhook target.");
tokio::time::sleep(Duration::from_secs(1)).await;
// --- Dynamically update system configuration: Add an MQTT Target ---
info!("\n---> Dynamically adding MQTT target...");
let mqtt_kvs_vec = vec![
KV {
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_BROKER.to_string(),
value: "mqtt://localhost:1883".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_TOPIC.to_string(),
value: "rustfs/events".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_QOS.to_string(),
value: "1".to_string(), // AtLeastOnce
hidden_if_empty: false,
},
KV {
key: MQTT_USERNAME.to_string(),
value: "test".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_PASSWORD.to_string(),
value: "123456".to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_QUEUE_DIR.to_string(),
value: current_root
.join("../../deploy/logs/notify/mqtt")
.to_str()
.unwrap()
.to_string(),
hidden_if_empty: false,
},
KV {
key: MQTT_QUEUE_LIMIT.to_string(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
];
let mqtt_kvs = KVS(mqtt_kvs_vec);
// let mut mqtt_targets = std::collections::HashMap::new();
// mqtt_targets.insert(DEFAULT_TARGET.to_string(), mqtt_kvs.clone());
system
.set_target_config(NOTIFY_MQTT_SUB_SYS, DEFAULT_TARGET, mqtt_kvs)
.await?;
info!("✅ MQTT target added and system reloaded.");
tokio::time::sleep(Duration::from_secs(1)).await;
// --- Loading and managing Bucket configurations ---
info!("\n---> Loading bucket notification config...");
let mut bucket_config = BucketNotificationConfig::new("us-east-1");
bucket_config.add_rule(
&[EventName::ObjectCreatedPut],
"*".to_string(),
TargetID::new(DEFAULT_TARGET.to_string(), "webhook".to_string()),
);
bucket_config.add_rule(
&[EventName::ObjectCreatedPut],
"*".to_string(),
TargetID::new(DEFAULT_TARGET.to_string(), "mqtt".to_string()),
);
system.load_bucket_notification_config("my-bucket", &bucket_config).await?;
info!("✅ Bucket 'my-bucket' config loaded.");
// --- Send events ---
info!("\n---> Sending an event...");
let event = Event::new_test_event("my-bucket", "document.pdf", EventName::ObjectCreatedPut);
system
.send_event("my-bucket", "s3:ObjectCreated:Put", "document.pdf", event)
.await;
info!("✅ Event sent. Both Webhook and MQTT targets should receive it.");
tokio::time::sleep(Duration::from_secs(2)).await;
// --- Dynamically remove configuration ---
info!("\n---> Dynamically removing Webhook target...");
system.remove_target_config("notify_webhook", "1").await?;
info!("✅ Webhook target removed and system reloaded.");
info!("\n---> Removing bucket notification config...");
system.remove_bucket_notification_config("my-bucket").await;
info!("✅ Bucket 'my-bucket' config removed.");
info!("\nDemo completed successfully");
Ok(())
}
crates/notify/examples/webhook.rs
+201
View File
@@ -0,0 +1,201 @@
use axum::routing::get;
use axum::{
extract::Json,
http::{HeaderMap, Response, StatusCode},
routing::post,
Router,
};
use serde_json::Value;
use std::time::{SystemTime, UNIX_EPOCH};
use axum::extract::Query;
use serde::Deserialize;
#[derive(Deserialize)]
struct ResetParams {
reason: Option<String>,
}
// Define a global variable and count the number of data received
use std::sync::atomic::{AtomicU64, Ordering};
static WEBHOOK_COUNT: AtomicU64 = AtomicU64::new(0);
#[tokio::main]
async fn main() {
// Build an application
let app = Router::new()
.route("/webhook", post(receive_webhook))
.route("/webhook/reset/{reason}", get(reset_webhook_count_with_path))
.route("/webhook/reset", get(reset_webhook_count))
.route("/webhook", get(receive_webhook));
// Start the server
let addr = "0.0.0.0:3020";
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
println!("Server running on {}", addr);
// Self-checking after the service is started
tokio::spawn(async move {
// Give the server some time to start
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
match is_service_active(addr).await {
Ok(true) => println!("Service health check: Successful - Service is running normally"),
Ok(false) => eprintln!("Service Health Check: Failed - Service Not Responded"),
Err(e) => eprintln!("Service health check errors:{}", e),
}
});
// Create a shutdown signal processing
tokio::select! {
result = axum::serve(listener, app) => {
if let Err(e) = result {
eprintln!("Server error: {}", e);
}
}
_ = tokio::signal::ctrl_c() => {
println!("Shutting down server...");
}
}
}
/// Create a method to reset the value of WEBHOOK_COUNT
async fn reset_webhook_count_with_path(axum::extract::Path(reason): axum::extract::Path<String>) -> Response<String> {
// Output the value of the current counter
let current_count = WEBHOOK_COUNT.load(Ordering::SeqCst);
println!("Current webhook count: {}", current_count);
println!("Reset webhook count, reason: {}", reason);
// Reset the counter to 0
WEBHOOK_COUNT.store(0, Ordering::SeqCst);
println!("Webhook count has been reset to 0.");
Response::builder()
.header("Foo", "Bar")
.status(StatusCode::OK)
.body(format!(
"Webhook count reset successfully. Previous count: {}. Reason: {}",
current_count, reason
))
.unwrap()
}
/// Create a method to reset the value of WEBHOOK_COUNT
/// You can reset the counter by calling this method
async fn reset_webhook_count(Query(params): Query<ResetParams>, headers: HeaderMap) -> Response<String> {
// Output the value of the current counter
let current_count = WEBHOOK_COUNT.load(Ordering::SeqCst);
println!("Current webhook count: {}", current_count);
let reason = params.reason.unwrap_or_else(|| "Reason not provided".to_string());
println!("Reset webhook count, reason: {}", reason);
for header in headers {
let (key, value) = header;
println!("Header: {:?}: {:?}", key, value);
}
println!("Reset webhook count printed headers");
// Reset the counter to 0
WEBHOOK_COUNT.store(0, Ordering::SeqCst);
println!("Webhook count has been reset to 0.");
Response::builder()
.header("Foo", "Bar")
.status(StatusCode::OK)
.body(format!("Webhook count reset successfully current_count:{}", current_count))
.unwrap()
}
async fn is_service_active(addr: &str) -> Result<bool, String> {
let socket_addr = tokio::net::lookup_host(addr)
.await
.map_err(|e| format!("Unable to resolve host:{}", e))?
.next()
.ok_or_else(|| "Address not found".to_string())?;
println!("Checking service status:{}", socket_addr);
match tokio::time::timeout(std::time::Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await {
Ok(Ok(_)) => Ok(true),
Ok(Err(e)) => {
if e.kind() == std::io::ErrorKind::ConnectionRefused {
Ok(false)
} else {
Err(format!("Connection failed:{}", e))
}
}
Err(_) => Err("Connection timeout".to_string()),
}
}
async fn receive_webhook(Json(payload): Json<Value>) -> StatusCode {
let start = SystemTime::now();
let since_the_epoch = start.duration_since(UNIX_EPOCH).expect("Time went backwards");
// get the number of seconds since the unix era
let seconds = since_the_epoch.as_secs();
// Manually calculate year, month, day, hour, minute, and second
let (year, month, day, hour, minute, second) = convert_seconds_to_date(seconds);
// output result
println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
println!(
"received a webhook request time:{} content:\n {}",
seconds,
serde_json::to_string_pretty(&payload).unwrap()
);
WEBHOOK_COUNT.fetch_add(1, Ordering::SeqCst);
println!("Total webhook requests received: {}", WEBHOOK_COUNT.load(Ordering::SeqCst));
StatusCode::OK
}
fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) {
// assume that the time zone is utc
let seconds_per_minute = 60;
let seconds_per_hour = 3600;
let seconds_per_day = 86400;
// Calculate the year, month, day, hour, minute, and second corresponding to the number of seconds
let mut total_seconds = seconds;
let mut year = 1970;
let mut month = 1;
let mut day = 1;
let mut hour = 0;
let mut minute = 0;
let mut second = 0;
// calculate year
while total_seconds >= 31536000 {
year += 1;
total_seconds -= 31536000; // simplified processing no leap year considered
}
// calculate month
let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
for m in &days_in_month {
if total_seconds >= m * seconds_per_day {
month += 1;
total_seconds -= m * seconds_per_day;
} else {
break;
}
}
// calculate the number of days
day += total_seconds / seconds_per_day;
total_seconds %= seconds_per_day;
// calculate hours
hour += total_seconds / seconds_per_hour;
total_seconds %= seconds_per_hour;
// calculate minutes
minute += total_seconds / seconds_per_minute;
total_seconds %= seconds_per_minute;
// calculate the number of seconds
second += total_seconds;
(year as u32, month as u32, day as u32, hour as u32, minute as u32, second as u32)
}
crates/notify/src/arn.rs
+243
View File
@@ -0,0 +1,243 @@
use crate::TargetError;
use const_str::concat;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use std::str::FromStr;
use thiserror::Error;
pub(crate) const DEFAULT_ARN_PARTITION: &str = "rustfs";
pub(crate) const DEFAULT_ARN_SERVICE: &str = "sqs";
/// Default ARN prefix for SQS
/// "arn:rustfs:sqs:"
const ARN_PREFIX: &str = concat!("arn:", DEFAULT_ARN_PARTITION, ":", DEFAULT_ARN_SERVICE, ":");
#[derive(Debug, Error)]
pub enum TargetIDError {
#[error("Invalid TargetID format '{0}', expect 'ID:Name'")]
InvalidFormat(String),
}
/// Target ID, used to identify notification targets
#[derive(Debug, Clone, Eq, PartialEq, Hash, PartialOrd, Ord)]
pub struct TargetID {
pub id: String,
pub name: String,
}
impl TargetID {
pub fn new(id: String, name: String) -> Self {
Self { id, name }
}
/// Convert to string representation
pub fn to_id_string(&self) -> String {
format!("{}:{}", self.id, self.name)
}
/// Create an ARN
pub fn to_arn(&self, region: &str) -> ARN {
ARN {
target_id: self.clone(),
region: region.to_string(),
service: DEFAULT_ARN_SERVICE.to_string(), // Default Service
partition: DEFAULT_ARN_PARTITION.to_string(), // Default partition
}
}
}
impl fmt::Display for TargetID {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}:{}", self.id, self.name)
}
}
impl FromStr for TargetID {
type Err = TargetIDError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let parts: Vec<&str> = s.splitn(2, ':').collect();
if parts.len() == 2 {
Ok(TargetID {
id: parts[0].to_string(),
name: parts[1].to_string(),
})
} else {
Err(TargetIDError::InvalidFormat(s.to_string()))
}
}
}
impl Serialize for TargetID {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&self.to_id_string())
}
}
impl<'de> Deserialize<'de> for TargetID {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
TargetID::from_str(&s).map_err(serde::de::Error::custom)
}
}
#[derive(Debug, Error)]
pub enum ArnError {
#[error("Invalid ARN format '{0}'")]
InvalidFormat(String),
#[error("ARN component missing")]
MissingComponents,
}
/// ARN - AWS resource name representation
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct ARN {
pub target_id: TargetID,
pub region: String,
// Service types, such as "sqs", "sns", "lambda", etc. This defaults to "sqs" to match the Go example.
pub service: String,
// Partitions such as "aws", "aws-cn", or customizations such as "rustfs","rustfs", etc.
pub partition: String,
}
impl ARN {
pub fn new(target_id: TargetID, region: String) -> Self {
ARN {
target_id,
region,
service: DEFAULT_ARN_SERVICE.to_string(), // Default is sqs
partition: DEFAULT_ARN_PARTITION.to_string(), // Default is rustfs partition
}
}
/// Returns the string representation of ARN
/// Returns the ARN string in the format "{ARN_PREFIX}:{region}:{target_id}"
#[allow(clippy::inherent_to_string)]
pub fn to_arn_string(&self) -> String {
if self.target_id.id.is_empty() && self.target_id.name.is_empty() && self.region.is_empty()
{
return String::new();
}
format!(
"{}:{}:{}",
ARN_PREFIX,
self.region,
self.target_id.to_id_string()
)
}
/// Parsing ARN from string
pub fn parse(s: &str) -> Result<Self, TargetError> {
if !s.starts_with(ARN_PREFIX) {
return Err(TargetError::InvalidARN(s.to_string()));
}
let tokens: Vec<&str> = s.split(':').collect();
if tokens.len() != 6 {
return Err(TargetError::InvalidARN(s.to_string()));
}
if tokens[4].is_empty() || tokens[5].is_empty() {
return Err(TargetError::InvalidARN(s.to_string()));
}
Ok(ARN {
region: tokens[3].to_string(),
target_id: TargetID {
id: tokens[4].to_string(),
name: tokens[5].to_string(),
},
service: tokens[2].to_string(), // Service Type
partition: tokens[1].to_string(), // Partition
})
}
}
impl fmt::Display for ARN {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.target_id.id.is_empty() && self.target_id.name.is_empty() && self.region.is_empty()
{
// Returns an empty string if all parts are empty
return Ok(());
}
write!(
f,
"arn:{}:{}:{}:{}:{}",
self.partition, self.service, self.region, self.target_id.id, self.target_id.name
)
}
}
impl FromStr for ARN {
type Err = ArnError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let parts: Vec<&str> = s.split(':').collect();
if parts.len() < 6 {
return Err(ArnError::InvalidFormat(s.to_string()));
}
if parts[0] != "arn" {
return Err(ArnError::InvalidFormat(s.to_string()));
}
let partition = parts[1].to_string();
let service = parts[2].to_string();
let region = parts[3].to_string();
let id = parts[4].to_string();
let name = parts[5..].join(":"); // The name section may contain colons, although this is not usually the case in SQS ARN
if id.is_empty() || name.is_empty() {
return Err(ArnError::MissingComponents);
}
Ok(ARN {
target_id: TargetID { id, name },
region,
service,
partition,
})
}
}
// Serialization implementation
impl Serialize for ARN {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&self.to_arn_string())
}
}
impl<'de> Deserialize<'de> for ARN {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
// deserializer.deserialize_str(ARNVisitor)
let s = String::deserialize(deserializer)?;
if s.is_empty() {
// Handle an empty ARN string, for example, creating an empty or default Arn instance
// Or return an error based on business logic
// Here we create an empty TargetID and region Arn
return Ok(ARN {
target_id: TargetID {
id: String::new(),
name: String::new(),
},
region: String::new(),
service: DEFAULT_ARN_SERVICE.to_string(),
partition: DEFAULT_ARN_PARTITION.to_string(),
});
}
ARN::from_str(&s).map_err(serde::de::Error::custom)
}
}
crates/notify/src/error.rs
+107
View File
@@ -0,0 +1,107 @@
use std::io;
use thiserror::Error;
/// Error types for the store
#[derive(Debug, Error)]
pub enum StoreError {
#[error("I/O error: {0}")]
Io(#[from] io::Error),
#[error("Serialization error: {0}")]
Serialization(String),
#[error("Deserialization error: {0}")]
Deserialization(String),
#[error("Compression error: {0}")]
Compression(String),
#[error("Entry limit exceeded")]
LimitExceeded,
#[error("Entry not found")]
NotFound,
#[error("Invalid entry: {0}")]
Internal(String), // Added internal error type
}
/// Error types for targets
#[derive(Debug, Error)]
pub enum TargetError {
#[error("Storage error: {0}")]
Storage(String),
#[error("Network error: {0}")]
Network(String),
#[error("Request error: {0}")]
Request(String),
#[error("Timeout error: {0}")]
Timeout(String),
#[error("Authentication error: {0}")]
Authentication(String),
#[error("Configuration error: {0}")]
Configuration(String),
#[error("Encoding error: {0}")]
Encoding(String),
#[error("Serialization error: {0}")]
Serialization(String),
#[error("Target not connected")]
NotConnected,
#[error("Target initialization failed: {0}")]
Initialization(String),
#[error("Invalid ARN: {0}")]
InvalidARN(String),
#[error("Unknown error: {0}")]
Unknown(String),
#[error("Target is disabled")]
Disabled,
}
/// Error types for the notification system
#[derive(Debug, Error)]
pub enum NotificationError {
#[error("Target error: {0}")]
Target(#[from] TargetError),
#[error("Configuration error: {0}")]
Configuration(String),
#[error("ARN not found: {0}")]
ARNNotFound(String),
#[error("Invalid ARN: {0}")]
InvalidARN(String),
#[error("Bucket notification error: {0}")]
BucketNotification(String),
#[error("Rule configuration error: {0}")]
RuleConfiguration(String),
#[error("System initialization error: {0}")]
Initialization(String),
#[error("Notification system has already been initialized")]
AlreadyInitialized,
#[error("Io error: {0}")]
Io(std::io::Error),
}
impl From<url::ParseError> for TargetError {
fn from(err: url::ParseError) -> Self {
TargetError::Configuration(format!("URL parse error: {}", err))
}
}
crates/notify/src/event.rs
+546
View File
@@ -0,0 +1,546 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
use url::form_urlencoded;
/// Error returned when parsing event name string fails。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ParseEventNameError(String);
impl fmt::Display for ParseEventNameError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Invalid event name:{}", self.0)
}
}
impl std::error::Error for ParseEventNameError {}
/// Represents the type of event that occurs on the object.
/// Based on AWS S3 event type and includes RustFS extension.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub enum EventName {
// Single event type (values are 1-32 for compatible mask logic)
ObjectAccessedGet = 1,
ObjectAccessedGetRetention = 2,
ObjectAccessedGetLegalHold = 3,
ObjectAccessedHead = 4,
ObjectAccessedAttributes = 5,
ObjectCreatedCompleteMultipartUpload = 6,
ObjectCreatedCopy = 7,
ObjectCreatedPost = 8,
ObjectCreatedPut = 9,
ObjectCreatedPutRetention = 10,
ObjectCreatedPutLegalHold = 11,
ObjectCreatedPutTagging = 12,
ObjectCreatedDeleteTagging = 13,
ObjectRemovedDelete = 14,
ObjectRemovedDeleteMarkerCreated = 15,
ObjectRemovedDeleteAllVersions = 16,
ObjectRemovedNoOP = 17,
BucketCreated = 18,
BucketRemoved = 19,
ObjectReplicationFailed = 20,
ObjectReplicationComplete = 21,
ObjectReplicationMissedThreshold = 22,
ObjectReplicationReplicatedAfterThreshold = 23,
ObjectReplicationNotTracked = 24,
ObjectRestorePost = 25,
ObjectRestoreCompleted = 26,
ObjectTransitionFailed = 27,
ObjectTransitionComplete = 28,
ScannerManyVersions = 29, // ObjectManyVersions corresponding to Go
ScannerLargeVersions = 30, // ObjectLargeVersions corresponding to Go
ScannerBigPrefix = 31, // PrefixManyFolders corresponding to Go
LifecycleDelMarkerExpirationDelete = 32, // ILMDelMarkerExpirationDelete corresponding to Go
// Compound "All" event type (no sequential value for mask)
ObjectAccessedAll,
ObjectCreatedAll,
ObjectRemovedAll,
ObjectReplicationAll,
ObjectRestoreAll,
ObjectTransitionAll,
ObjectScannerAll, // New, from Go
Everything, // New, from Go
}
// Single event type sequential array for Everything.expand()
const SINGLE_EVENT_NAMES_IN_ORDER: [EventName; 32] = [
EventName::ObjectAccessedGet,
EventName::ObjectAccessedGetRetention,
EventName::ObjectAccessedGetLegalHold,
EventName::ObjectAccessedHead,
EventName::ObjectAccessedAttributes,
EventName::ObjectCreatedCompleteMultipartUpload,
EventName::ObjectCreatedCopy,
EventName::ObjectCreatedPost,
EventName::ObjectCreatedPut,
EventName::ObjectCreatedPutRetention,
EventName::ObjectCreatedPutLegalHold,
EventName::ObjectCreatedPutTagging,
EventName::ObjectCreatedDeleteTagging,
EventName::ObjectRemovedDelete,
EventName::ObjectRemovedDeleteMarkerCreated,
EventName::ObjectRemovedDeleteAllVersions,
EventName::ObjectRemovedNoOP,
EventName::BucketCreated,
EventName::BucketRemoved,
EventName::ObjectReplicationFailed,
EventName::ObjectReplicationComplete,
EventName::ObjectReplicationMissedThreshold,
EventName::ObjectReplicationReplicatedAfterThreshold,
EventName::ObjectReplicationNotTracked,
EventName::ObjectRestorePost,
EventName::ObjectRestoreCompleted,
EventName::ObjectTransitionFailed,
EventName::ObjectTransitionComplete,
EventName::ScannerManyVersions,
EventName::ScannerLargeVersions,
EventName::ScannerBigPrefix,
EventName::LifecycleDelMarkerExpirationDelete,
];
const LAST_SINGLE_TYPE_VALUE: u32 = EventName::LifecycleDelMarkerExpirationDelete as u32;
impl EventName {
/// The parsed string is EventName.
pub fn parse(s: &str) -> Result<Self, ParseEventNameError> {
match s {
"s3:BucketCreated:*" => Ok(EventName::BucketCreated),
"s3:BucketRemoved:*" => Ok(EventName::BucketRemoved),
"s3:ObjectAccessed:*" => Ok(EventName::ObjectAccessedAll),
"s3:ObjectAccessed:Get" => Ok(EventName::ObjectAccessedGet),
"s3:ObjectAccessed:GetRetention" => Ok(EventName::ObjectAccessedGetRetention),
"s3:ObjectAccessed:GetLegalHold" => Ok(EventName::ObjectAccessedGetLegalHold),
"s3:ObjectAccessed:Head" => Ok(EventName::ObjectAccessedHead),
"s3:ObjectAccessed:Attributes" => Ok(EventName::ObjectAccessedAttributes),
"s3:ObjectCreated:*" => Ok(EventName::ObjectCreatedAll),
"s3:ObjectCreated:CompleteMultipartUpload" => Ok(EventName::ObjectCreatedCompleteMultipartUpload),
"s3:ObjectCreated:Copy" => Ok(EventName::ObjectCreatedCopy),
"s3:ObjectCreated:Post" => Ok(EventName::ObjectCreatedPost),
"s3:ObjectCreated:Put" => Ok(EventName::ObjectCreatedPut),
"s3:ObjectCreated:PutRetention" => Ok(EventName::ObjectCreatedPutRetention),
"s3:ObjectCreated:PutLegalHold" => Ok(EventName::ObjectCreatedPutLegalHold),
"s3:ObjectCreated:PutTagging" => Ok(EventName::ObjectCreatedPutTagging),
"s3:ObjectCreated:DeleteTagging" => Ok(EventName::ObjectCreatedDeleteTagging),
"s3:ObjectRemoved:*" => Ok(EventName::ObjectRemovedAll),
"s3:ObjectRemoved:Delete" => Ok(EventName::ObjectRemovedDelete),
"s3:ObjectRemoved:DeleteMarkerCreated" => Ok(EventName::ObjectRemovedDeleteMarkerCreated),
"s3:ObjectRemoved:NoOP" => Ok(EventName::ObjectRemovedNoOP),
"s3:ObjectRemoved:DeleteAllVersions" => Ok(EventName::ObjectRemovedDeleteAllVersions),
"s3:LifecycleDelMarkerExpiration:Delete" => Ok(EventName::LifecycleDelMarkerExpirationDelete),
"s3:Replication:*" => Ok(EventName::ObjectReplicationAll),
"s3:Replication:OperationFailedReplication" => Ok(EventName::ObjectReplicationFailed),
"s3:Replication:OperationCompletedReplication" => Ok(EventName::ObjectReplicationComplete),
"s3:Replication:OperationMissedThreshold" => Ok(EventName::ObjectReplicationMissedThreshold),
"s3:Replication:OperationReplicatedAfterThreshold" => Ok(EventName::ObjectReplicationReplicatedAfterThreshold),
"s3:Replication:OperationNotTracked" => Ok(EventName::ObjectReplicationNotTracked),
"s3:ObjectRestore:*" => Ok(EventName::ObjectRestoreAll),
"s3:ObjectRestore:Post" => Ok(EventName::ObjectRestorePost),
"s3:ObjectRestore:Completed" => Ok(EventName::ObjectRestoreCompleted),
"s3:ObjectTransition:Failed" => Ok(EventName::ObjectTransitionFailed),
"s3:ObjectTransition:Complete" => Ok(EventName::ObjectTransitionComplete),
"s3:ObjectTransition:*" => Ok(EventName::ObjectTransitionAll),
"s3:Scanner:ManyVersions" => Ok(EventName::ScannerManyVersions),
"s3:Scanner:LargeVersions" => Ok(EventName::ScannerLargeVersions),
"s3:Scanner:BigPrefix" => Ok(EventName::ScannerBigPrefix),
// ObjectScannerAll and Everything cannot be parsed from strings, because the Go version also does not define their string representation.
_ => Err(ParseEventNameError(s.to_string())),
}
}
/// Returns a string representation of the event type.
pub fn as_str(&self) -> &'static str {
match self {
EventName::BucketCreated => "s3:BucketCreated:*",
EventName::BucketRemoved => "s3:BucketRemoved:*",
EventName::ObjectAccessedAll => "s3:ObjectAccessed:*",
EventName::ObjectAccessedGet => "s3:ObjectAccessed:Get",
EventName::ObjectAccessedGetRetention => "s3:ObjectAccessed:GetRetention",
EventName::ObjectAccessedGetLegalHold => "s3:ObjectAccessed:GetLegalHold",
EventName::ObjectAccessedHead => "s3:ObjectAccessed:Head",
EventName::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes",
EventName::ObjectCreatedAll => "s3:ObjectCreated:*",
EventName::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload",
EventName::ObjectCreatedCopy => "s3:ObjectCreated:Copy",
EventName::ObjectCreatedPost => "s3:ObjectCreated:Post",
EventName::ObjectCreatedPut => "s3:ObjectCreated:Put",
EventName::ObjectCreatedPutTagging => "s3:ObjectCreated:PutTagging",
EventName::ObjectCreatedDeleteTagging => "s3:ObjectCreated:DeleteTagging",
EventName::ObjectCreatedPutRetention => "s3:ObjectCreated:PutRetention",
EventName::ObjectCreatedPutLegalHold => "s3:ObjectCreated:PutLegalHold",
EventName::ObjectRemovedAll => "s3:ObjectRemoved:*",
EventName::ObjectRemovedDelete => "s3:ObjectRemoved:Delete",
EventName::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated",
EventName::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP",
EventName::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions",
EventName::LifecycleDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete",
EventName::ObjectReplicationAll => "s3:Replication:*",
EventName::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication",
EventName::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication",
EventName::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked",
EventName::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold",
EventName::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold",
EventName::ObjectRestoreAll => "s3:ObjectRestore:*",
EventName::ObjectRestorePost => "s3:ObjectRestore:Post",
EventName::ObjectRestoreCompleted => "s3:ObjectRestore:Completed",
EventName::ObjectTransitionAll => "s3:ObjectTransition:*",
EventName::ObjectTransitionFailed => "s3:ObjectTransition:Failed",
EventName::ObjectTransitionComplete => "s3:ObjectTransition:Complete",
EventName::ScannerManyVersions => "s3:Scanner:ManyVersions",
EventName::ScannerLargeVersions => "s3:Scanner:LargeVersions",
EventName::ScannerBigPrefix => "s3:Scanner:BigPrefix",
// Go's String() returns "" for ObjectScannerAll and Everything
EventName::ObjectScannerAll => "s3:Scanner:*", // Follow the pattern in Go Expand
EventName::Everything => "", // Go String() returns "" to unprocessed
}
}
/// Returns the extended value of the abbreviation event type.
pub fn expand(&self) -> Vec<Self> {
match self {
EventName::ObjectAccessedAll => vec![
EventName::ObjectAccessedGet,
EventName::ObjectAccessedHead,
EventName::ObjectAccessedGetRetention,
EventName::ObjectAccessedGetLegalHold,
EventName::ObjectAccessedAttributes,
],
EventName::ObjectCreatedAll => vec![
EventName::ObjectCreatedCompleteMultipartUpload,
EventName::ObjectCreatedCopy,
EventName::ObjectCreatedPost,
EventName::ObjectCreatedPut,
EventName::ObjectCreatedPutRetention,
EventName::ObjectCreatedPutLegalHold,
EventName::ObjectCreatedPutTagging,
EventName::ObjectCreatedDeleteTagging,
],
EventName::ObjectRemovedAll => vec![
EventName::ObjectRemovedDelete,
EventName::ObjectRemovedDeleteMarkerCreated,
EventName::ObjectRemovedNoOP,
EventName::ObjectRemovedDeleteAllVersions,
],
EventName::ObjectReplicationAll => vec![
EventName::ObjectReplicationFailed,
EventName::ObjectReplicationComplete,
EventName::ObjectReplicationNotTracked,
EventName::ObjectReplicationMissedThreshold,
EventName::ObjectReplicationReplicatedAfterThreshold,
],
EventName::ObjectRestoreAll => vec![EventName::ObjectRestorePost, EventName::ObjectRestoreCompleted],
EventName::ObjectTransitionAll => vec![EventName::ObjectTransitionFailed, EventName::ObjectTransitionComplete],
EventName::ObjectScannerAll => vec![
// New
EventName::ScannerManyVersions,
EventName::ScannerLargeVersions,
EventName::ScannerBigPrefix,
],
EventName::Everything => {
// New
SINGLE_EVENT_NAMES_IN_ORDER.to_vec()
}
// A single type returns to itself directly
_ => vec![*self],
}
}
/// Returns the mask of type.
/// The compound "All" type will be expanded.
pub fn mask(&self) -> u64 {
let value = *self as u32;
if value > 0 && value <= LAST_SINGLE_TYPE_VALUE {
// It's a single type
1u64 << (value - 1)
} else {
// It's a compound type
let mut mask = 0u64;
for n in self.expand() {
mask |= n.mask(); // Recursively call mask
}
mask
}
}
}
impl fmt::Display for EventName {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.as_str())
}
}
/// Convert to `EventName` according to string
impl From<&str> for EventName {
fn from(event_str: &str) -> Self {
EventName::parse(event_str).unwrap_or_else(|e| panic!("{}", e))
}
}
/// Represents the identity of the user who triggered the event
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Identity {
/// The principal ID of the user
pub principal_id: String,
}
/// Represents the bucket that the object is in
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Bucket {
/// The name of the bucket
pub name: String,
/// The owner identity of the bucket
pub owner_identity: Identity,
/// The Amazon Resource Name (ARN) of the bucket
pub arn: String,
}
/// Represents the object that the event occurred on
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct Object {
/// The key (name) of the object
pub key: String,
/// The size of the object in bytes
#[serde(skip_serializing_if = "Option::is_none")]
pub size: Option<i64>,
/// The entity tag (ETag) of the object
#[serde(skip_serializing_if = "Option::is_none")]
pub etag: Option<String>,
/// The content type of the object
#[serde(skip_serializing_if = "Option::is_none")]
pub content_type: Option<String>,
/// User-defined metadata associated with the object
#[serde(skip_serializing_if = "Option::is_none")]
pub user_metadata: Option<HashMap<String, String>>,
/// The version ID of the object (if versioning is enabled)
#[serde(skip_serializing_if = "Option::is_none")]
pub version_id: Option<String>,
/// A unique identifier for the event
pub sequencer: String,
}
/// Metadata about the event
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Metadata {
/// The schema version of the event
#[serde(rename = "s3SchemaVersion")]
pub schema_version: String,
/// The ID of the configuration that triggered the event
pub configuration_id: String,
/// Information about the bucket
pub bucket: Bucket,
/// Information about the object
pub object: Object,
}
/// Information about the source of the event
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Source {
/// The host where the event originated
pub host: String,
/// The port on the host
pub port: String,
/// The user agent that caused the event
#[serde(rename = "userAgent")]
pub user_agent: String,
}
/// Represents a storage event
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Event {
/// The version of the event
pub event_version: String,
/// The source of the event
pub event_source: String,
/// The AWS region where the event occurred
pub aws_region: String,
/// The time when the event occurred
pub event_time: DateTime<Utc>,
/// The name of the event
pub event_name: EventName,
/// The identity of the user who triggered the event
pub user_identity: Identity,
/// Parameters from the request that caused the event
pub request_parameters: HashMap<String, String>,
/// Elements from the response
pub response_elements: HashMap<String, String>,
/// Metadata about the event
pub s3: Metadata,
/// Information about the source of the event
pub source: Source,
}
impl Event {
/// Creates a test event for a given bucket and object
pub fn new_test_event(bucket: &str, key: &str, event_name: EventName) -> Self {
let mut user_metadata = HashMap::new();
user_metadata.insert("x-amz-meta-test".to_string(), "value".to_string());
user_metadata.insert("x-amz-storage-storage-options".to_string(), "value".to_string());
user_metadata.insert("x-amz-meta-".to_string(), "value".to_string());
user_metadata.insert("x-rustfs-meta-".to_string(), "rustfs-value".to_string());
user_metadata.insert("x-request-id".to_string(), "request-id-123".to_string());
user_metadata.insert("x-bucket".to_string(), "bucket".to_string());
user_metadata.insert("x-object".to_string(), "object".to_string());
user_metadata.insert("x-rustfs-origin-endpoint".to_string(), "http://127.0.0.1".to_string());
user_metadata.insert("x-rustfs-user-metadata".to_string(), "metadata".to_string());
user_metadata.insert("x-rustfs-deployment-id".to_string(), "deployment-id-123".to_string());
user_metadata.insert("x-rustfs-origin-endpoint-code".to_string(), "http://127.0.0.1".to_string());
user_metadata.insert("x-rustfs-bucket-name".to_string(), "bucket".to_string());
user_metadata.insert("x-rustfs-object-key".to_string(), key.to_string());
user_metadata.insert("x-rustfs-object-size".to_string(), "1024".to_string());
user_metadata.insert("x-rustfs-object-etag".to_string(), "etag123".to_string());
user_metadata.insert("x-rustfs-object-version-id".to_string(), "1".to_string());
user_metadata.insert("x-request-time".to_string(), Utc::now().to_rfc3339());
Event {
event_version: "2.1".to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: "us-east-1".to_string(),
event_time: Utc::now(),
event_name,
user_identity: Identity {
principal_id: "rustfs".to_string(),
},
request_parameters: HashMap::new(),
response_elements: HashMap::new(),
s3: Metadata {
schema_version: "1.0".to_string(),
configuration_id: "test-config".to_string(),
bucket: Bucket {
name: bucket.to_string(),
owner_identity: Identity {
principal_id: "rustfs".to_string(),
},
arn: format!("arn:rustfs:s3:::{}", bucket),
},
object: Object {
key: key.to_string(),
size: Some(1024),
etag: Some("etag123".to_string()),
content_type: Some("application/octet-stream".to_string()),
user_metadata: Some(user_metadata),
version_id: Some("1".to_string()),
sequencer: "0055AED6DCD90281E5".to_string(),
},
},
source: Source {
host: "127.0.0.1".to_string(),
port: "9000".to_string(),
user_agent: "RustFS (linux; amd64) rustfs-rs/0.1".to_string(),
},
}
}
/// Return event mask
pub fn mask(&self) -> u64 {
self.event_name.mask()
}
pub fn new(args: EventArgs) -> Self {
let event_time = Utc::now().naive_local();
let unique_id = match args.object.mod_time {
Some(t) => format!("{:X}", t.unix_timestamp_nanos()),
None => format!("{:X}", event_time.and_utc().timestamp_nanos_opt().unwrap_or(0)),
};
let mut resp_elements = args.resp_elements.clone();
resp_elements
.entry("x-amz-request-id".to_string())
.or_insert_with(|| "".to_string());
resp_elements
.entry("x-amz-id-2".to_string())
.or_insert_with(|| "".to_string());
// ... Filling of other response elements
// URL encoding of object keys
let key_name = form_urlencoded::byte_serialize(args.object.name.as_bytes()).collect::<String>();
let principal_id = args.req_params.get("principalId").cloned().unwrap_or_default();
let owner_identity = Identity {
principal_id: principal_id.clone(),
};
let user_identity = Identity { principal_id };
let mut s3_metadata = Metadata {
schema_version: "1.0".to_string(),
configuration_id: "Config".to_string(), // or from args
bucket: Bucket {
name: args.bucket_name.clone(),
owner_identity,
arn: format!("arn:aws:s3:::{}", args.bucket_name),
},
object: Object {
key: key_name,
version_id: Some(args.object.version_id.unwrap().to_string()),
sequencer: unique_id,
..Default::default()
},
};
let is_removed_event = matches!(
args.event_name,
EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated
);
if !is_removed_event {
s3_metadata.object.size = Some(args.object.size);
s3_metadata.object.etag = args.object.etag.clone();
s3_metadata.object.content_type = args.object.content_type.clone();
// Filter out internal reserved metadata
let mut user_metadata = HashMap::new();
for (k, v) in &args.object.user_defined.unwrap_or_default() {
if !k.to_lowercase().starts_with("x-amz-meta-internal-") {
user_metadata.insert(k.clone(), v.clone());
}
}
s3_metadata.object.user_metadata = Some(user_metadata);
}
Self {
event_version: "2.1".to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
event_time: event_time.and_utc(),
event_name: args.event_name,
user_identity,
request_parameters: args.req_params,
response_elements: resp_elements,
s3: s3_metadata,
source: Source {
host: args.host,
port: "".to_string(),
user_agent: args.user_agent,
},
}
}
}
/// Represents a log of events for sending to targets
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventLog {
/// The event name
pub event_name: EventName,
/// The object key
pub key: String,
/// The list of events
pub records: Vec<Event>,
}
#[derive(Debug, Clone)]
pub struct EventArgs {
pub event_name: EventName,
pub bucket_name: String,
pub object: ecstore::store_api::ObjectInfo,
pub req_params: HashMap<String, String>,
pub resp_elements: HashMap<String, String>,
pub host: String,
pub user_agent: String,
}
impl EventArgs {
// Helper function to check if it is a copy request
pub fn is_replication_request(&self) -> bool {
self.req_params.contains_key("x-rustfs-source-replication-request")
}
}
crates/notify/src/factory.rs
+303
View File
@@ -0,0 +1,303 @@
use crate::store::DEFAULT_LIMIT;
use crate::{
error::TargetError,
target::{mqtt::MQTTArgs, webhook::WebhookArgs, Target},
};
use async_trait::async_trait;
use ecstore::config::{ENABLE_KEY, ENABLE_ON, KVS};
use rumqttc::QoS;
use std::time::Duration;
use tracing::warn;
use url::Url;
// --- Configuration Constants ---
// General
pub const DEFAULT_TARGET: &str = "1";
#[allow(dead_code)]
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
#[allow(dead_code)]
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
#[allow(dead_code)]
pub const NOTIFY_MY_SQL_SUB_SYS: &str = "notify_mysql";
#[allow(dead_code)]
pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
#[allow(dead_code)]
pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
#[allow(dead_code)]
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
#[allow(dead_code)]
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
#[allow(dead_code)]
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
#[allow(dead_code)]
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";
#[allow(dead_code)]
pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
// Webhook Keys
pub const WEBHOOK_ENDPOINT: &str = "endpoint";
pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
pub const WEBHOOK_QUEUE_LIMIT: &str = "queue_limit";
pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";
pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
// Webhook Environment Variables
const ENV_WEBHOOK_ENABLE: &str = "RUSTFS_NOTIFY_WEBHOOK_ENABLE";
const ENV_WEBHOOK_ENDPOINT: &str = "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT";
const ENV_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_WEBHOOK_AUTH_TOKEN";
const ENV_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT";
const ENV_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
const ENV_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
const ENV_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
// MQTT Keys
pub const MQTT_BROKER: &str = "broker";
pub const MQTT_TOPIC: &str = "topic";
pub const MQTT_QOS: &str = "qos";
pub const MQTT_USERNAME: &str = "username";
pub const MQTT_PASSWORD: &str = "password";
pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval";
pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
pub const MQTT_QUEUE_DIR: &str = "queue_dir";
pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";
// MQTT Environment Variables
const ENV_MQTT_ENABLE: &str = "RUSTFS_NOTIFY_MQTT_ENABLE";
const ENV_MQTT_BROKER: &str = "RUSTFS_NOTIFY_MQTT_BROKER";
const ENV_MQTT_TOPIC: &str = "RUSTFS_NOTIFY_MQTT_TOPIC";
const ENV_MQTT_QOS: &str = "RUSTFS_NOTIFY_MQTT_QOS";
const ENV_MQTT_USERNAME: &str = "RUSTFS_NOTIFY_MQTT_USERNAME";
const ENV_MQTT_PASSWORD: &str = "RUSTFS_NOTIFY_MQTT_PASSWORD";
const ENV_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_RECONNECT_INTERVAL";
const ENV_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL";
const ENV_MQTT_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_DIR";
const ENV_MQTT_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_LIMIT";
/// Helper function to get values from environment variables or KVS configurations.
///
/// It will give priority to reading from environment variables such as `BASE_ENV_KEY_ID` and fall back to the KVS configuration if it fails.
fn get_config_value(id: &str, base_env_key: &str, config_key: &str, config: &KVS) -> Option<String> {
let env_key = if id != DEFAULT_TARGET {
format!("{}_{}", base_env_key, id.to_uppercase().replace('-', "_"))
} else {
base_env_key.to_string()
};
match std::env::var(&env_key) {
Ok(val) => Some(val),
Err(_) => config.lookup(config_key),
}
}
/// Trait for creating targets from configuration
#[async_trait]
pub trait TargetFactory: Send + Sync {
/// Creates a target from configuration
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target + Send + Sync>, TargetError>;
/// Validates target configuration
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError>;
}
/// Factory for creating Webhook targets
pub struct WebhookTargetFactory;
#[async_trait]
impl TargetFactory for WebhookTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target + Send + Sync>, TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(&id, base_env_key, config_key, config);
let enable = get(ENV_WEBHOOK_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
return Err(TargetError::Configuration("Target is disabled".to_string()));
}
let endpoint = get(ENV_WEBHOOK_ENDPOINT, WEBHOOK_ENDPOINT)
.ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
let endpoint_url =
Url::parse(&endpoint).map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {}", e)))?;
let auth_token = get(ENV_WEBHOOK_AUTH_TOKEN, WEBHOOK_AUTH_TOKEN).unwrap_or_default();
let queue_dir = get(ENV_WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_DIR).unwrap_or_default();
let queue_limit = get(ENV_WEBHOOK_QUEUE_LIMIT, WEBHOOK_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT);
let client_cert = get(ENV_WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_CERT).unwrap_or_default();
let client_key = get(ENV_WEBHOOK_CLIENT_KEY, WEBHOOK_CLIENT_KEY).unwrap_or_default();
let args = WebhookArgs {
enable,
endpoint: endpoint_url,
auth_token,
queue_dir,
queue_limit,
client_cert,
client_key,
};
let target = crate::target::webhook::WebhookTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(id, base_env_key, config_key, config);
let enable = get(ENV_WEBHOOK_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
return Ok(());
}
let endpoint = get(ENV_WEBHOOK_ENDPOINT, WEBHOOK_ENDPOINT)
.ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
Url::parse(&endpoint).map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {}", e)))?;
let client_cert = get(ENV_WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_CERT).unwrap_or_default();
let client_key = get(ENV_WEBHOOK_CLIENT_KEY, WEBHOOK_CLIENT_KEY).unwrap_or_default();
if client_cert.is_empty() != client_key.is_empty() {
return Err(TargetError::Configuration(
"Both client_cert and client_key must be specified together".to_string(),
));
}
let queue_dir = get(ENV_WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_DIR).unwrap_or_default();
if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
}
Ok(())
}
}
/// Factory for creating MQTT targets
pub struct MQTTTargetFactory;
#[async_trait]
impl TargetFactory for MQTTTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target + Send + Sync>, TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(&id, base_env_key, config_key, config);
let enable = get(ENV_MQTT_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
return Err(TargetError::Configuration("Target is disabled".to_string()));
}
let broker =
get(ENV_MQTT_BROKER, MQTT_BROKER).ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
let broker_url = Url::parse(&broker).map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {}", e)))?;
let topic =
get(ENV_MQTT_TOPIC, MQTT_TOPIC).ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
let qos = get(ENV_MQTT_QOS, MQTT_QOS)
.and_then(|v| v.parse::<u8>().ok())
.map(|q| match q {
0 => QoS::AtMostOnce,
1 => QoS::AtLeastOnce,
2 => QoS::ExactlyOnce,
_ => QoS::AtLeastOnce,
})
.unwrap_or(QoS::AtLeastOnce);
let username = get(ENV_MQTT_USERNAME, MQTT_USERNAME).unwrap_or_default();
let password = get(ENV_MQTT_PASSWORD, MQTT_PASSWORD).unwrap_or_default();
let reconnect_interval = get(ENV_MQTT_RECONNECT_INTERVAL, MQTT_RECONNECT_INTERVAL)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or_else(|| Duration::from_secs(5));
let keep_alive = get(ENV_MQTT_KEEP_ALIVE_INTERVAL, MQTT_KEEP_ALIVE_INTERVAL)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or_else(|| Duration::from_secs(30));
let queue_dir = get(ENV_MQTT_QUEUE_DIR, MQTT_QUEUE_DIR).unwrap_or_default();
let queue_limit = get(ENV_MQTT_QUEUE_LIMIT, MQTT_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT);
let args = MQTTArgs {
enable,
broker: broker_url,
topic,
qos,
username,
password,
max_reconnect_interval: reconnect_interval,
keep_alive,
queue_dir,
queue_limit,
};
let target = crate::target::mqtt::MQTTTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(id, base_env_key, config_key, config);
let enable = get(ENV_MQTT_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
return Ok(());
}
let broker =
get(ENV_MQTT_BROKER, MQTT_BROKER).ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
let url = Url::parse(&broker).map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {}", e)))?;
match url.scheme() {
"tcp" | "ssl" | "ws" | "wss" | "mqtt" | "mqtts" => {}
_ => {
return Err(TargetError::Configuration("Unsupported broker URL scheme".to_string()));
}
}
if get(ENV_MQTT_TOPIC, MQTT_TOPIC).is_none() {
return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
}
if let Some(qos_str) = get(ENV_MQTT_QOS, MQTT_QOS) {
let qos = qos_str
.parse::<u8>()
.map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
if qos > 2 {
return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
}
}
let queue_dir = get(ENV_MQTT_QUEUE_DIR, MQTT_QUEUE_DIR).unwrap_or_default();
if !queue_dir.is_empty() {
if !std::path::Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
}
if let Some(qos_str) = get(ENV_MQTT_QOS, MQTT_QOS) {
if qos_str == "0" {
warn!("Using queue_dir with QoS 0 may result in event loss");
}
}
}
Ok(())
}
}
crates/notify/src/global.rs
+60
View File
@@ -0,0 +1,60 @@
use crate::{Event, EventArgs, NotificationError, NotificationSystem};
use ecstore::config::Config;
use once_cell::sync::Lazy;
use std::sync::{Arc, OnceLock};
static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
// Create a globally unique Notifier instance
pub static GLOBAL_NOTIFIER: Lazy<Notifier> = Lazy::new(|| Notifier {});
/// Initialize the global notification system with the given configuration.
/// This function should only be called once throughout the application life cycle.
pub async fn initialize(config: Config) -> Result<(), NotificationError> {
// `new` is synchronous and responsible for creating instances
let system = NotificationSystem::new(config);
// `init` is asynchronous and responsible for performing I/O-intensive initialization
system.init().await?;
match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
Ok(_) => Ok(()),
Err(_) => Err(NotificationError::AlreadyInitialized),
}
}
/// Returns a handle to the global NotificationSystem instance.
/// Return None if the system has not been initialized.
pub fn notification_system() -> Option<Arc<NotificationSystem>> {
NOTIFICATION_SYSTEM.get().cloned()
}
pub struct Notifier {
// Notifier can hold state, but in this design we make it stateless,
// Rely on getting an instance of NotificationSystem from the outside.
}
impl Notifier {
/// Notify an event asynchronously.
/// This is the only entry point for all event notifications in the system.
pub async fn notify(&self, args: EventArgs) {
// Dependency injection or service positioning mode obtain NotificationSystem instance
let notification_sys = match notification_system() {
// If the notification system itself cannot be retrieved, it will be returned directly
Some(sys) => sys,
None => {
tracing::error!("Notification system is not initialized.");
return;
}
};
// Avoid generating notifications for replica creation events
if args.is_replication_request() {
return;
}
// Create an event and send it
let event = Event::new(args.clone());
notification_sys
.send_event(&args.bucket_name, &args.event_name.as_str(), &args.object.name.clone(), event)
.await;
}
}
crates/notify/src/integration.rs
+520
View File
@@ -0,0 +1,520 @@
use crate::arn::TargetID;
use crate::store::{Key, Store};
use crate::{
error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream, Event,
StoreError, Target,
};
use ecstore::config::{Config, KVS};
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{mpsc, RwLock, Semaphore};
use tracing::{debug, error, info, warn};
/// Notify the system of monitoring indicators
pub struct NotificationMetrics {
/// The number of events currently being processed
processing_events: AtomicUsize,
/// Number of events that have been successfully processed
processed_events: AtomicUsize,
/// Number of events that failed to handle
failed_events: AtomicUsize,
/// System startup time
start_time: Instant,
}
impl Default for NotificationMetrics {
fn default() -> Self {
Self::new()
}
}
impl NotificationMetrics {
pub fn new() -> Self {
NotificationMetrics {
processing_events: AtomicUsize::new(0),
processed_events: AtomicUsize::new(0),
failed_events: AtomicUsize::new(0),
start_time: Instant::now(),
}
}
// Provide public methods to increase count
pub fn increment_processing(&self) {
self.processing_events.fetch_add(1, Ordering::Relaxed);
}
pub fn increment_processed(&self) {
self.processing_events.fetch_sub(1, Ordering::Relaxed);
self.processed_events.fetch_add(1, Ordering::Relaxed);
}
pub fn increment_failed(&self) {
self.processing_events.fetch_sub(1, Ordering::Relaxed);
self.failed_events.fetch_add(1, Ordering::Relaxed);
}
// Provide public methods to get count
pub fn processing_count(&self) -> usize {
self.processing_events.load(Ordering::Relaxed)
}
pub fn processed_count(&self) -> usize {
self.processed_events.load(Ordering::Relaxed)
}
pub fn failed_count(&self) -> usize {
self.failed_events.load(Ordering::Relaxed)
}
pub fn uptime(&self) -> Duration {
self.start_time.elapsed()
}
}
/// The notification system that integrates all components
pub struct NotificationSystem {
/// The event notifier
pub notifier: Arc<EventNotifier>,
/// The target registry
pub registry: Arc<TargetRegistry>,
/// The current configuration
pub config: Arc<RwLock<Config>>,
/// Cancel sender for managing stream processing tasks
stream_cancellers: Arc<RwLock<HashMap<TargetID, mpsc::Sender<()>>>>,
/// Concurrent control signal quantity
concurrency_limiter: Arc<Semaphore>,
/// Monitoring indicators
metrics: Arc<NotificationMetrics>,
}
impl NotificationSystem {
/// Creates a new NotificationSystem
pub fn new(config: Config) -> Self {
NotificationSystem {
notifier: Arc::new(EventNotifier::new()),
registry: Arc::new(TargetRegistry::new()),
config: Arc::new(RwLock::new(config)),
stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
concurrency_limiter: Arc::new(Semaphore::new(
std::env::var("RUSTFS_TARGET_STREAM_CONCURRENCY")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(20),
)), // Limit the maximum number of concurrent processing events to 20
metrics: Arc::new(NotificationMetrics::new()),
}
}
/// Initializes the notification system
pub async fn init(&self) -> Result<(), NotificationError> {
info!("Initialize notification system...");
let config = self.config.read().await;
debug!("Initializing notification system with config: {:?}", *config);
let targets: Vec<Box<dyn Target + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
info!("{} notification targets were created", targets.len());
// Initiate event stream processing for each storage enabled target
let mut cancellers = HashMap::new();
for target in &targets {
let target_id = target.id();
info!("Initializing target: {}", target.id());
// Initialize the target
if let Err(e) = target.init().await {
error!("Target {} Initialization failed:{}", target.id(), e);
continue;
}
debug!("Target {} initialized successfully,enabled:{}", target_id, target.is_enabled());
// Check if the target is enabled and has storage
if target.is_enabled() {
if let Some(store) = target.store() {
info!("Start event stream processing for target {}", target.id());
// The storage of the cloned target and the target itself
let store_clone = store.boxed_clone();
let target_box = target.clone_dyn();
let target_arc = Arc::from(target_box);
// Add a reference to the monitoring metrics
let metrics = self.metrics.clone();
let semaphore = self.concurrency_limiter.clone();
// Encapsulated enhanced version of start_event_stream
let cancel_tx = self.enhanced_start_event_stream(store_clone, target_arc, metrics, semaphore);
// Start event stream processing and save cancel sender
let target_id_clone = target_id.clone();
cancellers.insert(target_id, cancel_tx);
info!("Event stream processing for target {} is started successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
} else {
info!("Target {} is not enabled, event stream processing is skipped", target_id);
}
}
// Update canceler collection
*self.stream_cancellers.write().await = cancellers;
// Initialize the bucket target
self.notifier.init_bucket_targets(targets).await?;
info!("Notification system initialized");
Ok(())
}
/// Gets a list of Targets for all currently active (initialized).
///
/// # Return
/// A Vec containing all active Targets `TargetID`.
pub async fn get_active_targets(&self) -> Vec<TargetID> {
self.notifier.target_list().read().await.keys()
}
/// Accurately remove a Target and its related resources through TargetID.
///
/// This process includes:
/// 1. Stop the event stream associated with the Target (if present).
/// 2. Remove the Target instance from the activity list of Notifier.
/// 3. Remove the configuration item of the Target from the system configuration.
///
/// # Parameters
/// * `target_id` - The unique identifier of the Target to be removed.
///
/// # return
/// If successful, return `Ok(())`.
pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
info!("Attempting to remove target: {}", target_id);
let Some(store) = ecstore::global::new_object_layer_fn() else {
return Err(NotificationError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"errServerNotInitialized",
)));
};
let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?;
let mut changed = false;
if let Some(targets_of_type) = new_config.0.get_mut(target_type) {
if targets_of_type.remove(&target_id.name).is_some() {
info!("Removed target {} from the configuration.", target_id);
changed = true;
}
if targets_of_type.is_empty() {
new_config.0.remove(target_type);
}
}
if !changed {
warn!("Target {} was not found in the configuration.", target_id);
return Ok(());
}
if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await {
error!("Failed to save config for target removal: {}", e);
return Err(NotificationError::Configuration(format!("Failed to save config: {}", e)));
}
info!(
"Configuration updated and persisted for target {} removal. Reloading system...",
target_id
);
self.reload_config(new_config).await
}
/// Set or update a Target configuration.
/// If the configuration is changed, the entire notification system will be automatically reloaded to apply the changes.
///
/// # Arguments
/// * `target_type` - Target type, such as "notify_webhook" or "notify_mqtt".
/// * `target_name` - A unique name for a Target, such as "1".
/// * `kvs` - The full configuration of the Target.
///
/// # Returns
/// Result<(), NotificationError>
/// If the target configuration is successfully set, it returns Ok(()).
/// If the target configuration is invalid, it returns Err(NotificationError::Configuration).
pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
info!("Setting config for target {} of type {}", target_name, target_type);
// 1. Get the storage handle
let Some(store) = ecstore::global::new_object_layer_fn() else {
return Err(NotificationError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"errServerNotInitialized",
)));
};
// 2. Read the latest configuration from storage
let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?;
// 3. Modify the configuration copy
new_config
.0
.entry(target_type.to_string())
.or_default()
.insert(target_name.to_string(), kvs);
// 4. Persist the new configuration
if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await {
error!("Failed to save notification config: {}", e);
return Err(NotificationError::Configuration(format!("Failed to save notification config: {}", e)));
}
// 5. After the persistence is successful, the system will be reloaded to apply changes.
match self.reload_config(new_config).await {
Ok(_) => {
info!(
"Target {} of type {} configuration updated and reloaded successfully",
target_name, target_type
);
Ok(())
}
Err(e) => {
error!("Failed to reload config for target {} of type {}: {}", target_name, target_type, e);
Err(NotificationError::Configuration(format!(
"Configuration saved, but failed to reload: {}",
e
)))
}
}
}
/// Removes all notification configurations for a bucket.
pub async fn remove_bucket_notification_config(&self, bucket_name: &str) {
self.notifier.remove_rules_map(bucket_name).await;
}
/// Removes a Target configuration.
/// If the configuration is successfully removed, the entire notification system will be automatically reloaded.
///
/// # Arguments
/// * `target_type` - Target type, such as "notify_webhook" or "notify_mqtt".
/// * `target_name` - A unique name for a Target, such as "1".
///
/// # Returns
/// Result<(), NotificationError>
///
/// If the target configuration is successfully removed, it returns Ok(()).
/// If the target configuration does not exist, it returns Ok(()) without making any changes.
pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
info!("Removing config for target {} of type {}", target_name, target_type);
let Some(store) = ecstore::global::new_object_layer_fn() else {
return Err(NotificationError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"errServerNotInitialized",
)));
};
let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?;
let mut changed = false;
if let Some(targets) = new_config.0.get_mut(target_type) {
if targets.remove(target_name).is_some() {
changed = true;
}
if targets.is_empty() {
new_config.0.remove(target_type);
}
}
if !changed {
info!("Target {} of type {} not found, no changes made.", target_name, target_type);
return Ok(());
}
if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await {
error!("Failed to save config for target removal: {}", e);
return Err(NotificationError::Configuration(format!("Failed to save config: {}", e)));
}
info!(
"Configuration updated and persisted for target {} removal. Reloading system...",
target_name
);
self.reload_config(new_config).await
}
/// Enhanced event stream startup function, including monitoring and concurrency control
fn enhanced_start_event_stream(
&self,
store: Box<dyn Store<Event, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target + Send + Sync>,
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
) -> mpsc::Sender<()> {
// Event Stream Processing Using Batch Version
stream::start_event_stream_with_batching(store, target, metrics, semaphore)
}
/// Reloads the configuration
pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
info!("Reload notification configuration starts");
// Stop all existing streaming services
let mut cancellers = self.stream_cancellers.write().await;
for (target_id, cancel_tx) in cancellers.drain() {
info!("Stop event stream processing for target {}", target_id);
let _ = cancel_tx.send(()).await;
}
// Update the config
{
let mut config = self.config.write().await;
*config = new_config.clone();
}
// Create a new target from configuration
let targets: Vec<Box<dyn Target + Send + Sync>> = self
.registry
.create_targets_from_config(&new_config)
.await
.map_err(NotificationError::Target)?;
info!("{} notification targets were created from the new configuration", targets.len());
// Start new event stream processing for each storage enabled target
let mut new_cancellers = HashMap::new();
for target in &targets {
let target_id = target.id();
// Initialize the target
if let Err(e) = target.init().await {
error!("Target {} Initialization failed:{}", target_id, e);
continue;
}
// Check if the target is enabled and has storage
if target.is_enabled() {
if let Some(store) = target.store() {
info!("Start new event stream processing for target {}", target_id);
// The storage of the cloned target and the target itself
let store_clone = store.boxed_clone();
let target_box = target.clone_dyn();
let target_arc = Arc::from(target_box);
// Add a reference to the monitoring metrics
let metrics = self.metrics.clone();
let semaphore = self.concurrency_limiter.clone();
// Encapsulated enhanced version of start_event_stream
let cancel_tx = self.enhanced_start_event_stream(store_clone, target_arc, metrics, semaphore);
// Start event stream processing and save cancel sender
// let cancel_tx = start_event_stream(store_clone, target_clone);
let target_id_clone = target_id.clone();
new_cancellers.insert(target_id, cancel_tx);
info!("Event stream processing of target {} is restarted successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
} else {
info!("Target {} disabled, event stream processing is skipped", target_id);
}
}
// Update canceler collection
*cancellers = new_cancellers;
// Initialize the bucket target
self.notifier.init_bucket_targets(targets).await?;
info!("Configuration reloaded end");
Ok(())
}
/// Loads the bucket notification configuration
pub async fn load_bucket_notification_config(
&self,
bucket_name: &str,
config: &BucketNotificationConfig,
) -> Result<(), NotificationError> {
let arn_list = self.notifier.get_arn_list(&config.region).await;
if arn_list.is_empty() {
return Err(NotificationError::Configuration("No targets configured".to_string()));
}
info!("Available ARNs: {:?}", arn_list);
// Validate the configuration against the available ARNs
if let Err(e) = config.validate(&config.region, &arn_list) {
debug!("Bucket notification config validation region:{} failed: {}", &config.region, e);
if !e.to_string().contains("ARN not found") {
return Err(NotificationError::BucketNotification(e.to_string()));
} else {
error!("{}", e);
}
}
// let rules_map = config.to_rules_map();
let rules_map = config.get_rules_map();
self.notifier.add_rules_map(bucket_name, rules_map.clone()).await;
info!("Loaded notification config for bucket: {}", bucket_name);
Ok(())
}
/// Sends an event
pub async fn send_event(&self, bucket_name: &str, event_name: &str, object_key: &str, event: Event) {
self.notifier.send(bucket_name, event_name, object_key, event).await;
}
/// Obtain system status information
pub fn get_status(&self) -> HashMap<String, String> {
let mut status = HashMap::new();
status.insert("uptime_seconds".to_string(), self.metrics.uptime().as_secs().to_string());
status.insert("processing_events".to_string(), self.metrics.processing_count().to_string());
status.insert("processed_events".to_string(), self.metrics.processed_count().to_string());
status.insert("failed_events".to_string(), self.metrics.failed_count().to_string());
status
}
// Add a method to shut down the system
pub async fn shutdown(&self) {
info!("Turn off the notification system");
// Get the number of active targets
let active_targets = self.stream_cancellers.read().await.len();
info!("Stops {} active event stream processing tasks", active_targets);
let mut cancellers = self.stream_cancellers.write().await;
for (target_id, cancel_tx) in cancellers.drain() {
info!("Stop event stream processing for target {}", target_id);
let _ = cancel_tx.send(()).await;
}
// Wait for a short while to make sure the task has a chance to complete
tokio::time::sleep(Duration::from_millis(500)).await;
info!("Notify the system to be shut down completed");
}
}
impl Drop for NotificationSystem {
fn drop(&mut self) {
// Asynchronous operation cannot be used here, but logs can be recorded.
info!("Notify the system instance to be destroyed");
let status = self.get_status();
for (key, value) in status {
info!("key:{}, value:{}", key, value);
}
info!("Notification system status at shutdown:");
}
}
/// Loads configuration from a file
pub async fn load_config_from_file(path: &str, system: &NotificationSystem) -> Result<(), NotificationError> {
let config_data = tokio::fs::read(path)
.await
.map_err(|e| NotificationError::Configuration(format!("Failed to read config file: {}", e)))?;
let config = Config::unmarshal(config_data.as_slice())
.map_err(|e| NotificationError::Configuration(format!("Failed to parse config: {}", e)))?;
system.reload_config(config).await
}
crates/notify/src/lib.rs
+71
View File
@@ -0,0 +1,71 @@
//! RustFS Notify - A flexible and extensible event notification system for object storage.
//!
//! This library provides a Rust implementation of a storage bucket notification system,
//! similar to RustFS's notification system. It supports sending events to various targets
//! (like Webhook and MQTT) and includes features like event persistence and retry on failure.
pub mod arn;
pub mod error;
pub mod event;
pub mod factory;
pub mod global;
pub mod integration;
pub mod notifier;
pub mod registry;
pub mod rules;
pub mod store;
pub mod stream;
pub mod target;
// Re-exports
pub use error::{NotificationError, StoreError, TargetError};
pub use event::{Event, EventArgs, EventLog, EventName};
pub use global::{initialize, notification_system};
pub use integration::NotificationSystem;
pub use rules::BucketNotificationConfig;
use std::io::IsTerminal;
pub use target::Target;
use tracing_subscriber::{fmt, prelude::*, util::SubscriberInitExt, EnvFilter};
/// Initialize the tracing log system
///
/// # Example
/// ```
/// rustfs_notify::init_logger(rustfs_notify::LogLevel::Info);
/// ```
pub fn init_logger(level: LogLevel) {
let filter = EnvFilter::default().add_directive(level.into());
tracing_subscriber::registry()
.with(filter)
.with(
fmt::layer()
.with_target(true)
.with_target(true)
.with_ansi(std::io::stdout().is_terminal())
.with_thread_names(true)
.with_thread_ids(true)
.with_file(true)
.with_line_number(true),
)
.init();
}
/// Log level definition
pub enum LogLevel {
Debug,
Info,
Warn,
Error,
}
impl From<LogLevel> for tracing_subscriber::filter::Directive {
fn from(level: LogLevel) -> Self {
match level {
LogLevel::Debug => "debug".parse().unwrap(),
LogLevel::Info => "info".parse().unwrap(),
LogLevel::Warn => "warn".parse().unwrap(),
LogLevel::Error => "error".parse().unwrap(),
}
}
}
crates/notify/src/notifier.rs
+240
View File
@@ -0,0 +1,240 @@
use crate::arn::TargetID;
use crate::{error::NotificationError, event::Event, rules::RulesMap, target::Target, EventName};
use std::{collections::HashMap, sync::Arc};
use tokio::sync::RwLock;
use tracing::{debug, error, info, instrument, warn};
/// Manages event notification to targets based on rules
pub struct EventNotifier {
target_list: Arc<RwLock<TargetList>>,
bucket_rules_map: Arc<RwLock<HashMap<String, RulesMap>>>,
}
impl Default for EventNotifier {
fn default() -> Self {
Self::new()
}
}
impl EventNotifier {
/// Creates a new EventNotifier
pub fn new() -> Self {
EventNotifier {
target_list: Arc::new(RwLock::new(TargetList::new())),
bucket_rules_map: Arc::new(RwLock::new(HashMap::new())),
}
}
/// Returns a reference to the target list
/// This method provides access to the target list for external use.
///
pub fn target_list(&self) -> Arc<RwLock<TargetList>> {
Arc::clone(&self.target_list)
}
/// Removes all notification rules for a bucket
///
/// # Arguments
/// * `bucket_name` - The name of the bucket for which to remove rules
///
/// This method removes all rules associated with the specified bucket name.
/// It will log a message indicating the removal of rules.
pub async fn remove_rules_map(&self, bucket_name: &str) {
let mut rules_map = self.bucket_rules_map.write().await;
if rules_map.remove(bucket_name).is_some() {
info!("Removed all notification rules for bucket: {}", bucket_name);
}
}
/// Returns a list of ARNs for the registered targets
pub async fn get_arn_list(&self, region: &str) -> Vec<String> {
let target_list_guard = self.target_list.read().await;
target_list_guard
.keys()
.iter()
.map(|target_id| target_id.to_arn(region).to_arn_string())
.collect()
}
/// Adds a rules map for a bucket
pub async fn add_rules_map(&self, bucket_name: &str, rules_map: RulesMap) {
let mut bucket_rules_guard = self.bucket_rules_map.write().await;
if rules_map.is_empty() {
bucket_rules_guard.remove(bucket_name);
} else {
bucket_rules_guard.insert(bucket_name.to_string(), rules_map);
}
info!("Added rules for bucket: {}", bucket_name);
}
/// Removes notification rules for a bucket
pub async fn remove_notification(&self, bucket_name: &str) {
let mut bucket_rules_guard = self.bucket_rules_map.write().await;
bucket_rules_guard.remove(bucket_name);
info!("Removed notification rules for bucket: {}", bucket_name);
}
/// Removes all targets
pub async fn remove_all_bucket_targets(&self) {
let mut target_list_guard = self.target_list.write().await;
// The logic for sending cancel signals via stream_cancel_senders would be removed.
// TargetList::clear_targets_only already handles calling target.close().
target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel
info!("Removed all targets and their streams");
}
/// Sends an event to the appropriate targets based on the bucket rules
#[instrument(skip(self, event))]
pub async fn send(&self, bucket_name: &str, event_name: &str, object_key: &str, event: Event) {
let bucket_rules_guard = self.bucket_rules_map.read().await;
if let Some(rules) = bucket_rules_guard.get(bucket_name) {
let target_ids = rules.match_rules(EventName::from(event_name), object_key);
if target_ids.is_empty() {
debug!("No matching targets for event in bucket: {}", bucket_name);
return;
}
let target_ids_len = target_ids.len();
let mut handles = vec![];
// Use scope to limit the borrow scope of target_list
{
let target_list_guard = self.target_list.read().await;
info!("Sending event to targets: {:?}", target_ids);
for target_id in target_ids {
// `get` now returns Option<Arc<dyn Target + Send + Sync>>
if let Some(target_arc) = target_list_guard.get(&target_id) {
// Clone an Arc<Box<dyn Target>> (which is where target_list is stored) to move into an asynchronous task
// target_arc is already Arc, clone it for the async task
let cloned_target_for_task = target_arc.clone();
let event_clone = event.clone();
let target_name_for_task = cloned_target_for_task.name(); // Get the name before generating the task
debug!("Preparing to send event to target: {}", target_name_for_task);
// Use cloned data in closures to avoid borrowing conflicts
let handle = tokio::spawn(async move {
if let Err(e) = cloned_target_for_task.save(event_clone).await {
error!("Failed to send event to target {}: {}", target_name_for_task, e);
} else {
debug!("Successfully saved event to target {}", target_name_for_task);
}
});
handles.push(handle);
} else {
warn!("Target ID {:?} found in rules but not in target list.", target_id);
}
}
// target_list is automatically released here
}
// Wait for all tasks to be completed
for handle in handles {
if let Err(e) = handle.await {
error!("Task for sending/saving event failed: {}", e);
}
}
info!("Event processing initiated for {} targets for bucket: {}", target_ids_len, bucket_name);
} else {
debug!("No rules found for bucket: {}", bucket_name);
}
}
/// Initializes the targets for buckets
#[instrument(skip(self, targets_to_init))]
pub async fn init_bucket_targets(
&self,
targets_to_init: Vec<Box<dyn Target + Send + Sync>>,
) -> Result<(), NotificationError> {
// Currently active, simpler logic
let mut target_list_guard = self.target_list.write().await; //Gets a write lock for the TargetList
for target_boxed in targets_to_init {
// Traverse the incoming Box<dyn Target >
debug!("init bucket target: {}", target_boxed.name());
// TargetList::add method expectations Arc<dyn Target + Send + Sync>
// Therefore, you need to convert Box<dyn Target + Send + Sync> to Arc<dyn Target + Send + Sync>
let target_arc: Arc<dyn Target + Send + Sync> = Arc::from(target_boxed);
target_list_guard.add(target_arc)?; // Add Arc<dyn Target> to the list
}
info!(
"Initialized {} targets, list size: {}", // Clearer logs
target_list_guard.len(),
target_list_guard.len()
);
Ok(()) // Make sure to return a Result
}
}
/// A thread-safe list of targets
pub struct TargetList {
targets: HashMap<TargetID, Arc<dyn Target + Send + Sync>>,
}
impl Default for TargetList {
fn default() -> Self {
Self::new()
}
}
impl TargetList {
/// Creates a new TargetList
pub fn new() -> Self {
TargetList { targets: HashMap::new() }
}
/// Adds a target to the list
pub fn add(&mut self, target: Arc<dyn Target + Send + Sync>) -> Result<(), NotificationError> {
let id = target.id();
if self.targets.contains_key(&id) {
// Potentially update or log a warning/error if replacing an existing target.
warn!("Target with ID {} already exists in TargetList. It will be overwritten.", id);
}
self.targets.insert(id, target);
Ok(())
}
/// Removes a target by ID. Note: This does not stop its associated event stream.
/// Stream cancellation should be handled by EventNotifier.
pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<Arc<dyn Target + Send + Sync>> {
if let Some(target_arc) = self.targets.remove(id) {
if let Err(e) = target_arc.close().await {
// Target's own close logic
error!("Failed to close target {} during removal: {}", id, e);
}
Some(target_arc)
} else {
None
}
}
/// Clears all targets from the list. Note: This does not stop their associated event streams.
/// Stream cancellation should be handled by EventNotifier.
pub async fn clear_targets_only(&mut self) {
let target_ids_to_clear: Vec<TargetID> = self.targets.keys().cloned().collect();
for id in target_ids_to_clear {
if let Some(target_arc) = self.targets.remove(&id) {
if let Err(e) = target_arc.close().await {
error!("Failed to close target {} during clear: {}", id, e);
}
}
}
self.targets.clear();
}
/// Returns a target by ID
pub fn get(&self, id: &TargetID) -> Option<Arc<dyn Target + Send + Sync>> {
self.targets.get(id).cloned()
}
/// Returns all target IDs
pub fn keys(&self) -> Vec<TargetID> {
self.targets.keys().cloned().collect()
}
/// Returns the number of targets
pub fn len(&self) -> usize {
self.targets.len()
}
// is_empty can be derived from len()
pub fn is_empty(&self) -> bool {
self.targets.is_empty()
}
}
crates/notify/src/registry.rs
+96
View File
@@ -0,0 +1,96 @@
use crate::target::ChannelTargetType;
use crate::{
error::TargetError,
factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory},
target::Target,
};
use ecstore::config::{Config, ENABLE_KEY, ENABLE_OFF, ENABLE_ON, KVS};
use std::collections::HashMap;
use tracing::{error, info};
/// Registry for managing target factories
pub struct TargetRegistry {
factories: HashMap<String, Box<dyn TargetFactory>>,
}
impl Default for TargetRegistry {
fn default() -> Self {
Self::new()
}
}
impl TargetRegistry {
/// Creates a new TargetRegistry with built-in factories
pub fn new() -> Self {
let mut registry = TargetRegistry {
factories: HashMap::new(),
};
// Register built-in factories
registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
registry
}
/// Registers a new factory for a target type
pub fn register(&mut self, target_type: &str, factory: Box<dyn TargetFactory>) {
self.factories.insert(target_type.to_string(), factory);
}
/// Creates a target from configuration
pub async fn create_target(
&self,
target_type: &str,
id: String,
config: &KVS,
) -> Result<Box<dyn Target + Send + Sync>, TargetError> {
let factory = self
.factories
.get(target_type)
.ok_or_else(|| TargetError::Configuration(format!("Unknown target type: {}", target_type)))?;
// Validate configuration before creating target
factory.validate_config(&id, config)?;
// Create target
factory.create_target(id, config).await
}
/// Creates all targets from a configuration
pub async fn create_targets_from_config(&self, config: &Config) -> Result<Vec<Box<dyn Target + Send + Sync>>, TargetError> {
let mut targets: Vec<Box<dyn Target + Send + Sync>> = Vec::new();
// Iterate through configuration sections
for (section, subsections) in &config.0 {
// Only process notification sections
if !section.starts_with("notify_") {
continue;
}
// Extract target type from section name
let target_type = section.trim_start_matches("notify_");
// Iterate through subsections (each representing a target instance)
for (target_id, target_config) in subsections {
// Skip disabled targets
if target_config.lookup(ENABLE_KEY).unwrap_or_else(|| ENABLE_OFF.to_string()) != ENABLE_ON {
continue;
}
// Create target
match self.create_target(target_type, target_id.clone(), target_config).await {
Ok(target) => {
info!("Created target: {}/{}", target_type, target_id);
targets.push(target);
}
Err(e) => {
error!("Failed to create target {}/{}: {}", target_type, target_id, e);
}
}
}
}
Ok(targets)
}
}
crates/notify/src/rules/config.rs
+120
View File
@@ -0,0 +1,120 @@
use super::rules_map::RulesMap;
// Keep for existing structure if any, or remove if not used
use super::xml_config::ParseConfigError as BucketNotificationConfigError;
use crate::arn::TargetID;
use crate::rules::pattern_rules;
use crate::rules::target_id_set;
use crate::rules::NotificationConfiguration;
use crate::EventName;
use std::collections::HashMap;
use std::io::Read;
// Assuming this is the XML config structure
/// Configuration for bucket notifications.
/// This struct now holds the parsed and validated rules in the new RulesMap format.
#[derive(Debug, Clone, Default)]
pub struct BucketNotificationConfig {
pub region: String, // Region where this config is applicable
pub rules: RulesMap, // The new, more detailed RulesMap
}
impl BucketNotificationConfig {
pub fn new(region: &str) -> Self {
BucketNotificationConfig {
region: region.to_string(),
rules: RulesMap::new(),
}
}
/// Adds a rule to the configuration.
/// This method allows adding a rule with a specific event and target ID.
pub fn add_rule(
&mut self,
event_names: &[EventName], // Assuming event_names is a list of event names
pattern: String, // The object key pattern for the rule
target_id: TargetID, // The target ID for the notification
) {
self.rules.add_rule_config(event_names, pattern, target_id);
}
/// Parses notification configuration from XML.
/// `arn_list` is a list of valid ARN strings for validation.
pub fn from_xml<R: Read + std::io::BufRead>(
reader: R,
current_region: &str,
arn_list: &[String],
) -> Result<Self, BucketNotificationConfigError> {
let mut parsed_config = NotificationConfiguration::from_reader(reader)?;
// Set defaults (region in ARNs if empty, xmlns) before validation
parsed_config.set_defaults(current_region);
// Validate the parsed configuration
parsed_config.validate(current_region, arn_list)?;
let mut rules_map = RulesMap::new();
for queue_conf in parsed_config.queue_list {
// The ARN in queue_conf should now have its region set if it was originally empty.
// Ensure TargetID can be cloned or extracted correctly.
let target_id = queue_conf.arn.target_id.clone();
let pattern_str = queue_conf.filter.filter_rule_list.pattern();
rules_map.add_rule_config(&queue_conf.events, pattern_str, target_id);
}
Ok(BucketNotificationConfig {
region: current_region.to_string(), // Config is for the current_region
rules: rules_map,
})
}
/// Validates the *current* BucketNotificationConfig.
/// This might be redundant if construction always implies validation.
/// However, Go's Config has a Validate method.
/// The primary validation now happens during `from_xml` via `NotificationConfiguration::validate`.
/// This method could re-check against an updated arn_list or region if needed.
pub fn validate(&self, current_region: &str, arn_list: &[String]) -> Result<(), BucketNotificationConfigError> {
if self.region != current_region {
return Err(BucketNotificationConfigError::RegionMismatch {
config_region: self.region.clone(),
current_region: current_region.to_string(),
});
}
// Iterate through the rules in self.rules and validate their TargetIDs against arn_list
// This requires RulesMap to expose its internal structure or provide an iterator
for (_event_name, pattern_rules) in self.rules.inner().iter() {
for (_pattern, target_id_set) in pattern_rules.inner().iter() {
// Assuming PatternRules has inner()
for target_id in target_id_set {
// Construct the ARN string for this target_id and self.region
let arn_to_check = target_id.to_arn(&self.region); // Assuming TargetID has to_arn
if !arn_list.contains(&arn_to_check.to_arn_string()) {
return Err(BucketNotificationConfigError::ArnNotFound(arn_to_check.to_arn_string()));
}
}
}
}
Ok(())
}
// Expose the RulesMap for the notifier
pub fn get_rules_map(&self) -> &RulesMap {
&self.rules
}
pub fn to_rules_map(&self) -> RulesMap {
self.rules.clone()
}
/// Sets the region for the configuration
pub fn set_region(&mut self, region: &str) {
self.region = region.to_string();
}
}
// Add a helper to PatternRules if not already present
impl pattern_rules::PatternRules {
pub fn inner(&self) -> &HashMap<String, target_id_set::TargetIdSet> {
&self.rules
}
}
crates/notify/src/rules/mod.rs
+19
View File
@@ -0,0 +1,19 @@
pub mod pattern;
pub mod pattern_rules;
pub mod rules_map;
pub mod target_id_set;
pub mod xml_config; // For XML structure definition and parsing
pub mod config; // Definition and parsing for BucketNotificationConfig
// Re-export key types from submodules for easy access to `crate::rules::TypeName`
// Re-export key types from submodules for external use
pub use config::BucketNotificationConfig;
// Assume that BucketNotificationConfigError is also defined in config.rs
// Or if it is still an alias for xml_config::ParseConfigError , adjust accordingly
pub use xml_config::ParseConfigError as BucketNotificationConfigError;
pub use pattern_rules::PatternRules;
pub use rules_map::RulesMap;
pub use target_id_set::TargetIdSet;
pub use xml_config::{NotificationConfiguration, ParseConfigError};
crates/notify/src/rules/pattern.rs
+99
View File
@@ -0,0 +1,99 @@
use wildmatch::WildMatch;
/// Create new pattern string based on prefix and suffix。
///
/// The rule is similar to event.NewPattern in the Go version:
/// - If a prefix is provided and does not end with '*', '*' is appended.
/// - If a suffix is provided and does not start with '*', then prefix '*'.
/// - Replace "**" with "*".
pub fn new_pattern(prefix: Option<&str>, suffix: Option<&str>) -> String {
let mut pattern = String::new();
// Process the prefix part
if let Some(p) = prefix {
if !p.is_empty() {
pattern.push_str(p);
if !p.ends_with('*') {
pattern.push('*');
}
}
}
// Process the suffix part
if let Some(s) = suffix {
if !s.is_empty() {
let mut s_to_append = s.to_string();
if !s.starts_with('*') {
s_to_append.insert(0, '*');
}
// If the pattern is empty (only suffixes are provided), then the pattern is the suffix
// Otherwise, append the suffix to the pattern
if pattern.is_empty() {
pattern = s_to_append;
} else {
pattern.push_str(&s_to_append);
}
}
}
// Replace "**" with "*"
pattern = pattern.replace("**", "*");
pattern
}
/// Simple matching object names and patterns。
pub fn match_simple(pattern_str: &str, object_name: &str) -> bool {
if pattern_str == "*" {
// AWS S3 docs: A single asterisk (*) in the rule matches all objects.
return true;
}
// WildMatch considers an empty pattern to not match anything, which is usually desired.
// If pattern_str is empty, it means no specific filter, so it depends on interpretation.
// Go's wildcard.MatchSimple might treat empty pattern differently.
// For now, assume empty pattern means no match unless it's explicitly "*".
if pattern_str.is_empty() {
return false; // Or true if an empty pattern means "match all" in some contexts.
// Given Go's NewRulesMap defaults to "*", an empty pattern from Filter is unlikely to mean "match all".
}
WildMatch::new(pattern_str).matches(object_name)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_new_pattern() {
assert_eq!(new_pattern(Some("images/"), Some(".jpg")), "images/*.jpg");
assert_eq!(new_pattern(Some("images/"), None), "images/*");
assert_eq!(new_pattern(None, Some(".jpg")), "*.jpg");
assert_eq!(new_pattern(Some("foo"), Some("bar")), "foo*bar"); // foo* + *bar -> foo**bar -> foo*bar
assert_eq!(new_pattern(Some("foo*"), Some("bar")), "foo*bar"); // foo* + *bar -> foo**bar -> foo*bar
assert_eq!(new_pattern(Some("foo"), Some("*bar")), "foo*bar"); // foo* + *bar -> foo**bar -> foo*bar
assert_eq!(new_pattern(Some("foo*"), Some("*bar")), "foo*bar"); // foo* + *bar -> foo**bar -> foo*bar
assert_eq!(new_pattern(Some("*"), Some("*")), "*"); // * + * -> ** -> *
assert_eq!(new_pattern(Some("a"), Some("")), "a*");
assert_eq!(new_pattern(Some(""), Some("b")), "*b");
assert_eq!(new_pattern(None, None), "");
assert_eq!(new_pattern(Some("prefix"), Some("suffix")), "prefix*suffix");
assert_eq!(
new_pattern(Some("prefix/"), Some("/suffix")),
"prefix/*suffix"
); // prefix/* + */suffix -> prefix/**/suffix -> prefix/*/suffix
}
#[test]
fn test_match_simple() {
assert!(match_simple("foo*", "foobar"));
assert!(!match_simple("foo*", "barfoo"));
assert!(match_simple("*.jpg", "photo.jpg"));
assert!(!match_simple("*.jpg", "photo.png"));
assert!(match_simple("*", "anything.anything"));
assert!(match_simple("foo*bar", "foobazbar"));
assert!(!match_simple("foo*bar", "foobar_baz"));
assert!(match_simple("a*b*c", "axbyc"));
assert!(!match_simple("a*b*c", "axbc"));
}
}
crates/notify/src/rules/pattern_rules.rs
+80
View File
@@ -0,0 +1,80 @@
use super::pattern;
use super::target_id_set::TargetIdSet;
use crate::arn::TargetID;
use std::collections::HashMap;
/// PatternRules - Event rule that maps object name patterns to TargetID collections.
/// `event.Rules` (map[string]TargetIDSet) in the Go code
#[derive(Debug, Clone, Default)]
pub struct PatternRules {
pub(crate) rules: HashMap<String, TargetIdSet>,
}
impl PatternRules {
pub fn new() -> Self {
Default::default()
}
/// Add rules: Pattern and Target ID.
/// If the schema already exists, add target_id to the existing TargetIdSet.
pub fn add(&mut self, pattern: String, target_id: TargetID) {
self.rules.entry(pattern).or_default().insert(target_id);
}
/// Checks if there are any rules that match the given object name.
pub fn match_simple(&self, object_name: &str) -> bool {
self.rules
.keys()
.any(|p| pattern::match_simple(p, object_name))
}
/// Returns all TargetIDs that match the object name.
pub fn match_targets(&self, object_name: &str) -> TargetIdSet {
let mut matched_targets = TargetIdSet::new();
for (pattern_str, target_set) in &self.rules {
if pattern::match_simple(pattern_str, object_name) {
matched_targets.extend(target_set.iter().cloned());
}
}
matched_targets
}
pub fn is_empty(&self) -> bool {
self.rules.is_empty()
}
/// Merge another PatternRules.
/// Corresponding to Go's `Rules.Union`.
pub fn union(&self, other: &Self) -> Self {
let mut new_rules = self.clone();
for (pattern, their_targets) in &other.rules {
let our_targets = new_rules.rules.entry(pattern.clone()).or_default();
our_targets.extend(their_targets.iter().cloned());
}
new_rules
}
/// Calculate the difference from another PatternRules.
/// Corresponding to Go's `Rules.Difference`.
pub fn difference(&self, other: &Self) -> Self {
let mut result_rules = HashMap::new();
for (pattern, self_targets) in &self.rules {
match other.rules.get(pattern) {
Some(other_targets) => {
let diff_targets: TargetIdSet =
self_targets.difference(other_targets).cloned().collect();
if !diff_targets.is_empty() {
result_rules.insert(pattern.clone(), diff_targets);
}
}
None => {
// If there is no pattern in other, self_targets are all retained
result_rules.insert(pattern.clone(), self_targets.clone());
}
}
}
PatternRules {
rules: result_rules,
}
}
}
crates/notify/src/rules/rules_map.rs
+100
View File
@@ -0,0 +1,100 @@
use super::pattern_rules::PatternRules;
use super::target_id_set::TargetIdSet;
use crate::arn::TargetID;
use crate::event::EventName;
use std::collections::HashMap;
/// RulesMap - Rule mapping organized by event name。
/// `event.RulesMap` (map[Name]Rules) in the corresponding Go code
#[derive(Debug, Clone, Default)]
pub struct RulesMap {
map: HashMap<EventName, PatternRules>,
}
impl RulesMap {
pub fn new() -> Self {
Default::default()
}
/// Add rule configuration.
/// event_names: A set of event names。
/// pattern: Object key pattern.
/// target_id: Notify the target.
///
/// This method expands the composite event name.
pub fn add_rule_config(&mut self, event_names: &[EventName], pattern: String, target_id: TargetID) {
let mut effective_pattern = pattern;
if effective_pattern.is_empty() {
effective_pattern = "*".to_string(); // Match all by default
}
for event_name_spec in event_names {
for expanded_event_name in event_name_spec.expand() {
// Make sure EventName::expand() returns Vec<EventName>
self.map
.entry(expanded_event_name)
.or_default()
.add(effective_pattern.clone(), target_id.clone());
}
}
}
/// Merge another RulesMap.
/// `RulesMap.Add(rulesMap2 RulesMap) corresponding to Go
pub fn add_map(&mut self, other_map: &Self) {
for (event_name, other_pattern_rules) in &other_map.map {
let self_pattern_rules = self.map.entry(*event_name).or_default();
// PatternRules::union 返回新的 PatternRules，我们需要修改现有的
let merged_rules = self_pattern_rules.union(other_pattern_rules);
*self_pattern_rules = merged_rules;
}
}
/// Remove another rule defined in the RulesMap from the current RulesMap.
pub fn remove_map(&mut self, other_map: &Self) {
let mut events_to_remove = Vec::new();
for (event_name, self_pattern_rules) in &mut self.map {
if let Some(other_pattern_rules) = other_map.map.get(event_name) {
*self_pattern_rules = self_pattern_rules.difference(other_pattern_rules);
if self_pattern_rules.is_empty() {
events_to_remove.push(*event_name);
}
}
}
for event_name in events_to_remove {
self.map.remove(&event_name);
}
}
///Rules matching the given event name and object key, returning all matching TargetIDs.
pub fn match_rules(&self, event_name: EventName, object_key: &str) -> TargetIdSet {
// First try to directly match the event name
if let Some(pattern_rules) = self.map.get(&event_name) {
let targets = pattern_rules.match_targets(object_key);
if !targets.is_empty() {
return targets;
}
}
// Go's RulesMap[eventName] is directly retrieved, and if it does not exist, it is empty Rules.
// Rust's HashMap::get returns Option. If the event name does not exist, there is no rule.
// Compound events (such as ObjectCreatedAll) have been expanded as a single event when add_rule_config.
// Therefore, a single event name should be used when querying.
// If event_name itself is a single type, look it up directly.
// If event_name is a compound type, Go's logic is expanded when added.
// Here match_rules should receive events that may already be single.
// If the caller passes in a compound event, it should expand itself or handle this function first.
// Assume that event_name is already a specific event that can be used for searching.
self.map
.get(&event_name)
.map_or_else(TargetIdSet::new, |pr| pr.match_targets(object_key))
}
pub fn is_empty(&self) -> bool {
self.map.is_empty()
}
/// Returns a clone of internal rules for use in scenarios such as BucketNotificationConfig::validate.
pub fn inner(&self) -> &HashMap<EventName, PatternRules> {
&self.map
}
}
crates/notify/src/rules/target_id_set.rs
+15
View File
@@ -0,0 +1,15 @@
use crate::arn::TargetID;
use std::collections::HashSet;
/// TargetIDSet - A collection representation of TargetID.
pub type TargetIdSet = HashSet<TargetID>;
/// Provides a Go-like method for TargetIdSet (can be implemented as trait if needed)
#[allow(dead_code)]
pub(crate) fn new_target_id_set(target_ids: Vec<TargetID>) -> TargetIdSet {
target_ids.into_iter().collect()
}
// HashSet has built-in clone, union, difference and other operations.
// But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself.
// If you need to exactly match Go's API style, you can add wrapper functions.
crates/notify/src/rules/xml_config.rs
+274
View File
@@ -0,0 +1,274 @@
use super::pattern;
use crate::arn::{ArnError, TargetIDError, ARN};
use crate::event::EventName;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::io::Read;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ParseConfigError {
#[error("XML parsing error:{0}")]
XmlError(#[from] quick_xml::errors::serialize::DeError),
#[error("Invalid filter value:{0}")]
InvalidFilterValue(String),
#[error("Invalid filter name: {0}, only 'prefix' or 'suffix' is allowed")]
InvalidFilterName(String),
#[error("There can only be one 'prefix' in the filter rule")]
DuplicatePrefixFilter,
#[error("There can only be one 'suffix' in the filter rule")]
DuplicateSuffixFilter,
#[error("Missing event name")]
MissingEventName,
#[error("Duplicate event name:{0}")]
DuplicateEventName(String), // EventName is usually an enum, and here String is used to represent its text
#[error("Repeated queue configuration: ID={0:?}, ARN={1}")]
DuplicateQueueConfiguration(Option<String>, String),
#[error("Unsupported configuration types (e.g. Lambda, Topic)")]
UnsupportedConfiguration,
#[error("ARN not found:{0}")]
ArnNotFound(String),
#[error("Unknown area:{0}")]
UnknownRegion(String),
#[error("ARN parsing error:{0}")]
ArnParseError(#[from] ArnError),
#[error("TargetID parsing error:{0}")]
TargetIDParseError(#[from] TargetIDError),
#[error("IO Error:{0}")]
IoError(#[from] std::io::Error),
#[error("Region mismatch: Configure region {config_region}, current region {current_region}")]
RegionMismatch { config_region: String, current_region: String },
#[error("ARN {0} Not found in the provided list")]
ArnValidation(String),
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct FilterRule {
#[serde(rename = "Name")]
pub name: String,
#[serde(rename = "Value")]
pub value: String,
}
impl FilterRule {
fn validate(&self) -> Result<(), ParseConfigError> {
if self.name != "prefix" && self.name != "suffix" {
return Err(ParseConfigError::InvalidFilterName(self.name.clone()));
}
// ValidateFilterRuleValue from Go:
// no "." or ".." path segments, <= 1024 chars, valid UTF-8, no '\'.
for segment in self.value.split('/') {
if segment == "." || segment == ".." {
return Err(ParseConfigError::InvalidFilterValue(self.value.clone()));
}
}
if self.value.len() > 1024 || self.value.contains('\\') || std::str::from_utf8(self.value.as_bytes()).is_err() {
return Err(ParseConfigError::InvalidFilterValue(self.value.clone()));
}
Ok(())
}
}
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
pub struct FilterRuleList {
#[serde(rename = "FilterRule", default, skip_serializing_if = "Vec::is_empty")]
pub rules: Vec<FilterRule>,
}
impl FilterRuleList {
pub fn validate(&self) -> Result<(), ParseConfigError> {
let mut has_prefix = false;
let mut has_suffix = false;
for rule in &self.rules {
rule.validate()?;
if rule.name == "prefix" {
if has_prefix {
return Err(ParseConfigError::DuplicatePrefixFilter);
}
has_prefix = true;
} else if rule.name == "suffix" {
if has_suffix {
return Err(ParseConfigError::DuplicateSuffixFilter);
}
has_suffix = true;
}
}
Ok(())
}
pub fn pattern(&self) -> String {
let mut prefix_val: Option<&str> = None;
let mut suffix_val: Option<&str> = None;
for rule in &self.rules {
if rule.name == "prefix" {
prefix_val = Some(&rule.value);
} else if rule.name == "suffix" {
suffix_val = Some(&rule.value);
}
}
pattern::new_pattern(prefix_val, suffix_val)
}
pub fn is_empty(&self) -> bool {
self.rules.is_empty()
}
}
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
pub struct S3KeyFilter {
#[serde(rename = "FilterRuleList", default, skip_serializing_if = "FilterRuleList::is_empty")]
pub filter_rule_list: FilterRuleList,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
pub struct QueueConfig {
#[serde(rename = "Id", skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(rename = "Queue")] // This is ARN in XML
pub arn: ARN,
#[serde(rename = "Event", default)] // XML has multiple <Event> tags
pub events: Vec<EventName>, // EventName needs to handle XML (de)serialization if not string
#[serde(rename = "Filter", default, skip_serializing_if = "s3key_filter_is_empty")]
pub filter: S3KeyFilter,
}
fn s3key_filter_is_empty(f: &S3KeyFilter) -> bool {
f.filter_rule_list.is_empty()
}
impl QueueConfig {
pub fn validate(&self, region: &str, arn_list: &[String]) -> Result<(), ParseConfigError> {
if self.events.is_empty() {
return Err(ParseConfigError::MissingEventName);
}
let mut event_set = HashSet::new();
for event in &self.events {
// EventName::to_string() or similar for uniqueness check
if !event_set.insert(event.to_string()) {
return Err(ParseConfigError::DuplicateEventName(event.to_string()));
}
}
self.filter.filter_rule_list.validate()?;
// Validate ARN (similar to Go's Queue.Validate)
// The Go code checks targetList.Exists(q.ARN.TargetID)
// Here we check against a provided arn_list
let _config_arn_str = self.arn.to_arn_string();
if !self.arn.region.is_empty() && self.arn.region != region {
return Err(ParseConfigError::UnknownRegion(self.arn.region.clone()));
}
// Construct the ARN string that would be in arn_list
// The arn_list contains ARNs like "arn:rustfs:sqs:REGION:ID:NAME"
// We need to ensure self.arn (potentially with region adjusted) is in arn_list
let effective_arn = ARN {
target_id: self.arn.target_id.clone(),
region: if self.arn.region.is_empty() {
region.to_string()
} else {
self.arn.region.clone()
},
service: self.arn.service.clone(), // or default "sqs"
partition: self.arn.partition.clone(), // or default "rustfs"
};
if !arn_list.contains(&effective_arn.to_arn_string()) {
return Err(ParseConfigError::ArnNotFound(effective_arn.to_arn_string()));
}
Ok(())
}
/// Sets the region if it's not already set in the ARN.
pub fn set_region_if_empty(&mut self, region: &str) {
if self.arn.region.is_empty() {
self.arn.region = region.to_string();
}
}
}
/// Corresponding to the `lambda` structure in the Go code.
/// Used to parse <CloudFunction> ARN from inside the <CloudFunctionConfiguration> tag.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Default)]
pub struct LambdaConfigDetail {
#[serde(rename = "CloudFunction")]
pub arn: String,
// According to AWS S3 documentation, <CloudFunctionConfiguration> usually also contains Id, Event, Filter
// But in order to strictly correspond to the Go `lambda` structure provided, only ARN is included here.
// If full support is required, additional fields can be added.
// For example:
// #[serde(rename = "Id", skip_serializing_if = "Option::is_none")]
// pub id: Option<String>,
// #[serde(rename = "Event", default, skip_serializing_if = "Vec::is_empty")]
// pub events: Vec<EventName>,
// #[serde(rename = "Filter", default, skip_serializing_if = "S3KeyFilterIsEmpty")]
// pub filter: S3KeyFilter,
}
/// Corresponding to the `topic` structure in the Go code.
/// Used to parse <Topic> ARN from inside the <TopicConfiguration> tag.
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Default)]
pub struct TopicConfigDetail {
#[serde(rename = "Topic")]
pub arn: String,
// Similar to LambdaConfigDetail, it can be extended to include fields such as Id, Event, Filter, etc.
}
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
#[serde(rename = "NotificationConfiguration")]
pub struct NotificationConfiguration {
#[serde(rename = "xmlns", skip_serializing_if = "Option::is_none")]
pub xmlns: Option<String>,
#[serde(rename = "QueueConfiguration", default, skip_serializing_if = "Vec::is_empty")]
pub queue_list: Vec<QueueConfig>,
#[serde(
rename = "CloudFunctionConfiguration", // Tags for each lambda configuration item in XML
default,
skip_serializing_if = "Vec::is_empty"
)]
pub lambda_list: Vec<LambdaConfigDetail>, // Modify: Use a new structure
#[serde(
rename = "TopicConfiguration", // Tags for each topic configuration item in XML
default,
skip_serializing_if = "Vec::is_empty"
)]
pub topic_list: Vec<TopicConfigDetail>, // Modify: Use a new structure
}
impl NotificationConfiguration {
pub fn from_reader<R: Read + std::io::BufRead>(reader: R) -> Result<Self, ParseConfigError> {
let config: NotificationConfiguration = quick_xml::de::from_reader(reader)?;
Ok(config)
}
pub fn validate(&self, current_region: &str, arn_list: &[String]) -> Result<(), ParseConfigError> {
// Verification logic remains the same: if lambda_list or topic_list is not empty, it is considered an unsupported configuration
if !self.lambda_list.is_empty() || !self.topic_list.is_empty() {
return Err(ParseConfigError::UnsupportedConfiguration);
}
let mut unique_queues = HashSet::new();
for queue_config in &self.queue_list {
queue_config.validate(current_region, arn_list)?;
let queue_key = (
queue_config.id.clone(),
queue_config.arn.to_arn_string(), // Assuming that the ARN structure implements Display or ToString
);
if !unique_queues.insert(queue_key.clone()) {
return Err(ParseConfigError::DuplicateQueueConfiguration(queue_key.0, queue_key.1));
}
}
Ok(())
}
pub fn set_defaults(&mut self, region: &str) {
for queue_config in &mut self.queue_list {
queue_config.set_region_if_empty(region);
}
if self.xmlns.is_none() {
self.xmlns = Some("http://s3.amazonaws.com/doc/2006-03-01/".to_string());
}
// Note: If LambdaConfigDetail and TopicConfigDetail contain information such as regions in the future,
// You may also need to set the default value here. But according to the current definition, they only contain ARN strings.
}
}
crates/notify/src/store.rs
+498
View File
@@ -0,0 +1,498 @@
use crate::error::StoreError;
use serde::{de::DeserializeOwned, Serialize};
use snap::raw::{Decoder, Encoder};
use std::sync::{Arc, RwLock};
use std::{
collections::HashMap,
marker::PhantomData,
path::PathBuf,
time::{SystemTime, UNIX_EPOCH},
};
use tracing::{debug, warn};
use uuid::Uuid;
pub const DEFAULT_LIMIT: u64 = 100000; // Default store limit
pub const DEFAULT_EXT: &str = ".unknown"; // Default file extension
pub const COMPRESS_EXT: &str = ".snappy"; // Extension for compressed files
/// STORE_EXTENSION - file extension of an event file in store
pub const STORE_EXTENSION: &str = ".event";
/// Represents a key for an entry in the store
#[derive(Debug, Clone)]
pub struct Key {
/// The name of the key (UUID)
pub name: String,
/// The file extension for the entry
pub extension: String,
/// The number of items in the entry (for batch storage)
pub item_count: usize,
/// Whether the entry is compressed
pub compress: bool,
}
impl Key {
/// Converts the key to a string (filename)
pub fn to_key_string(&self) -> String {
let name_part = if self.item_count > 1 {
format!("{}:{}", self.item_count, self.name)
} else {
self.name.clone()
};
let mut file_name = name_part;
if !self.extension.is_empty() {
file_name.push_str(&self.extension);
}
if self.compress {
file_name.push_str(COMPRESS_EXT);
}
file_name
}
}
impl std::fmt::Display for Key {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let name_part = if self.item_count > 1 {
format!("{}:{}", self.item_count, self.name)
} else {
self.name.clone()
};
let mut file_name = name_part;
if !self.extension.is_empty() {
file_name.push_str(&self.extension);
}
if self.compress {
file_name.push_str(COMPRESS_EXT);
}
write!(f, "{}", file_name)
}
}
/// Parses a string into a Key
pub fn parse_key(s: &str) -> Key {
debug!("Parsing key: {}", s);
let mut name = s.to_string();
let mut extension = String::new();
let mut item_count = 1;
let mut compress = false;
// Check for compressed suffixes
if name.ends_with(COMPRESS_EXT) {
compress = true;
name = name[..name.len() - COMPRESS_EXT.len()].to_string();
}
// Number of batch items parsed
if let Some(colon_pos) = name.find(':') {
if let Ok(count) = name[..colon_pos].parse::<usize>() {
item_count = count;
name = name[colon_pos + 1..].to_string();
}
}
// Resolve extension
if let Some(dot_pos) = name.rfind('.') {
extension = name[dot_pos..].to_string();
name = name[..dot_pos].to_string();
}
debug!(
"Parsed key - name: {}, extension: {}, item_count: {}, compress: {}",
name, extension, item_count, compress
);
Key {
name,
extension,
item_count,
compress,
}
}
/// Trait for a store that can store and retrieve items of type T
pub trait Store<T>: Send + Sync {
/// The error type for the store
type Error;
/// The key type for the store
type Key;
/// Opens the store
fn open(&self) -> Result<(), Self::Error>;
/// Stores a single item
fn put(&self, item: T) -> Result<Self::Key, Self::Error>;
/// Stores multiple items in a single batch
fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error>;
/// Retrieves a single item by key
fn get(&self, key: &Self::Key) -> Result<T, Self::Error>;
/// Retrieves multiple items by key
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error>;
/// Deletes an item by key
fn del(&self, key: &Self::Key) -> Result<(), Self::Error>;
/// Lists all keys in the store
fn list(&self) -> Vec<Self::Key>;
/// Returns the number of items in the store
fn len(&self) -> usize;
/// Returns true if the store is empty
fn is_empty(&self) -> bool;
/// Clones the store into a boxed trait object
fn boxed_clone(&self) -> Box<dyn Store<T, Error = Self::Error, Key = Self::Key> + Send + Sync>;
}
/// A store that uses the filesystem to persist events in a queue
pub struct QueueStore<T> {
entry_limit: u64,
directory: PathBuf,
file_ext: String,
entries: Arc<RwLock<HashMap<String, i64>>>, // key -> modtime as unix nano
_phantom: PhantomData<T>,
}
impl<T> Clone for QueueStore<T> {
fn clone(&self) -> Self {
QueueStore {
entry_limit: self.entry_limit,
directory: self.directory.clone(),
file_ext: self.file_ext.clone(),
entries: Arc::clone(&self.entries),
_phantom: PhantomData,
}
}
}
impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
/// Creates a new QueueStore
pub fn new(directory: impl Into<PathBuf>, limit: u64, ext: &str) -> Self {
let file_ext = if ext.is_empty() { DEFAULT_EXT } else { ext };
QueueStore {
directory: directory.into(),
entry_limit: if limit == 0 { DEFAULT_LIMIT } else { limit },
file_ext: file_ext.to_string(),
entries: Arc::new(RwLock::new(HashMap::with_capacity(limit as usize))),
_phantom: PhantomData,
}
}
/// Returns the full path for a key
fn file_path(&self, key: &Key) -> PathBuf {
self.directory.join(key.to_string())
}
/// Reads a file for the given key
fn read_file(&self, key: &Key) -> Result<Vec<u8>, StoreError> {
let path = self.file_path(key);
debug!(
"Reading file for key: {},path: {}",
key.to_string(),
path.display()
);
let data = std::fs::read(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
StoreError::NotFound
} else {
StoreError::Io(e)
}
})?;
if data.is_empty() {
return Err(StoreError::NotFound);
}
if key.compress {
let mut decoder = Decoder::new();
decoder
.decompress_vec(&data)
.map_err(|e| StoreError::Compression(e.to_string()))
} else {
Ok(data)
}
}
/// Writes data to a file for the given key
fn write_file(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> {
let path = self.file_path(key);
// Create directory if it doesn't exist
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(StoreError::Io)?;
}
let data = if key.compress {
let mut encoder = Encoder::new();
encoder
.compress_vec(data)
.map_err(|e| StoreError::Compression(e.to_string()))?
} else {
data.to_vec()
};
std::fs::write(&path, &data).map_err(StoreError::Io)?;
let modified = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as i64;
let mut entries = self.entries.write().map_err(|_| {
StoreError::Internal("Failed to acquire write lock on entries".to_string())
})?;
entries.insert(key.to_string(), modified);
debug!("Wrote event to store: {}", key.to_string());
Ok(())
}
}
impl<T> Store<T> for QueueStore<T>
where
T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
{
type Error = StoreError;
type Key = Key;
fn open(&self) -> Result<(), Self::Error> {
std::fs::create_dir_all(&self.directory).map_err(StoreError::Io)?;
let entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?;
// Get the write lock to update the internal state
let mut entries_map = self.entries.write().map_err(|_| {
StoreError::Internal("Failed to acquire write lock on entries".to_string())
})?;
for entry in entries {
let entry = entry.map_err(StoreError::Io)?;
let metadata = entry.metadata().map_err(StoreError::Io)?;
if metadata.is_file() {
let modified = metadata.modified().map_err(StoreError::Io)?;
let unix_nano = modified
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as i64;
let file_name = entry.file_name().to_string_lossy().to_string();
entries_map.insert(file_name, unix_nano);
}
}
debug!("Opened store at: {:?}", self.directory);
Ok(())
}
fn put(&self, item: T) -> Result<Self::Key, Self::Error> {
// Check storage limits
{
let entries = self.entries.read().map_err(|_| {
StoreError::Internal("Failed to acquire read lock on entries".to_string())
})?;
if entries.len() as u64 >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
}
let uuid = Uuid::new_v4();
let key = Key {
name: uuid.to_string(),
extension: self.file_ext.clone(),
item_count: 1,
compress: true,
};
let data =
serde_json::to_vec(&item).map_err(|e| StoreError::Serialization(e.to_string()))?;
self.write_file(&key, &data)?;
Ok(key)
}
fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error> {
// Check storage limits
{
let entries = self.entries.read().map_err(|_| {
StoreError::Internal("Failed to acquire read lock on entries".to_string())
})?;
if entries.len() as u64 >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
}
if items.is_empty() {
// Or return an error, or a special key?
return Err(StoreError::Internal(
"Cannot put_multiple with empty items list".to_string(),
));
}
let uuid = Uuid::new_v4();
let key = Key {
name: uuid.to_string(),
extension: self.file_ext.clone(),
item_count: items.len(),
compress: true,
};
// Serialize all items into a single Vec<u8>
// This current approach for get_multiple/put_multiple assumes items are concatenated JSON objects.
// This might be problematic for deserialization if not handled carefully.
// A better approach for multiple items might be to store them as a JSON array `Vec<T>`.
// For now, sticking to current logic of concatenating.
let mut buffer = Vec::new();
for item in items {
// If items are Vec<Event>, and Event is large, this could be inefficient.
// The current get_multiple deserializes one by one.
let item_data =
serde_json::to_vec(&item).map_err(|e| StoreError::Serialization(e.to_string()))?;
buffer.extend_from_slice(&item_data);
// If using JSON array: buffer = serde_json::to_vec(&items)?
}
self.write_file(&key, &buffer)?;
Ok(key)
}
fn get(&self, key: &Self::Key) -> Result<T, Self::Error> {
if key.item_count != 1 {
return Err(StoreError::Internal(format!(
"get() called on a batch key ({} items), use get_multiple()",
key.item_count
)));
}
let items = self.get_multiple(key)?;
items.into_iter().next().ok_or(StoreError::NotFound)
}
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
debug!("Reading items from store for key: {}", key.to_string());
let data = self.read_file(key)?;
if data.is_empty() {
return Err(StoreError::Deserialization(
"Cannot deserialize empty data".to_string(),
));
}
let mut items = Vec::with_capacity(key.item_count);
// let mut deserializer = serde_json::Deserializer::from_slice(&data);
// while let Ok(item) = serde::Deserialize::deserialize(&mut deserializer) {
// items.push(item);
// }
// This deserialization logic assumes multiple JSON objects are simply concatenated in the file.
// This is fragile. It's better to store a JSON array `[item1, item2, ...]`
// or use a streaming deserializer that can handle multiple top-level objects if that's the format.
// For now, assuming serde_json::Deserializer::from_slice can handle this if input is well-formed.
let mut deserializer = serde_json::Deserializer::from_slice(&data).into_iter::<T>();
for _ in 0..key.item_count {
match deserializer.next() {
Some(Ok(item)) => items.push(item),
Some(Err(e)) => {
return Err(StoreError::Deserialization(format!(
"Failed to deserialize item in batch: {}",
e
)));
}
None => {
// Reached end of stream sooner than item_count
if items.len() < key.item_count && !items.is_empty() {
// Partial read
warn!(
"Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.",
key.item_count,
key.to_string(),
items.len()
);
// Depending on strictness, this could be an error.
} else if items.is_empty() {
// No items at all, but file existed
return Err(StoreError::Deserialization(format!(
"No items deserialized for key {} though file existed.",
key
)));
}
break;
}
}
}
if items.is_empty() && key.item_count > 0 {
return Err(StoreError::Deserialization("No items found".to_string()));
}
Ok(items)
}
fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
let path = self.file_path(key);
std::fs::remove_file(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
// If file not found, still try to remove from entries map in case of inconsistency
warn!("File not found for key {} during del, but proceeding to remove from entries map.", key.to_string());
StoreError::NotFound
} else {
StoreError::Io(e)
}
})?;
// Get the write lock to update the internal state
let mut entries = self.entries.write().map_err(|_| {
StoreError::Internal("Failed to acquire write lock on entries".to_string())
})?;
if entries.remove(&key.to_string()).is_none() {
// Key was not in the map, could be an inconsistency or already deleted.
// This is not necessarily an error if the file deletion succeeded or was NotFound.
debug!(
"Key {} not found in entries map during del, might have been already removed.",
key
);
}
debug!("Deleted event from store: {}", key.to_string());
Ok(())
}
fn list(&self) -> Vec<Self::Key> {
// Get the read lock to read the internal state
let entries = match self.entries.read() {
Ok(entries) => entries,
Err(_) => {
debug!("Failed to acquire read lock on entries for listing");
return Vec::new();
}
};
let mut entries_vec: Vec<_> = entries.iter().collect();
// Sort by modtime (value in HashMap) to process oldest first
entries_vec.sort_by(|a, b| a.1.cmp(b.1)); // Oldest first
entries_vec.into_iter().map(|(k, _)| parse_key(k)).collect()
}
fn len(&self) -> usize {
// Get the read lock to read the internal state
match self.entries.read() {
Ok(entries) => entries.len(),
Err(_) => {
debug!("Failed to acquire read lock on entries for len");
0
}
}
}
fn is_empty(&self) -> bool {
self.len() == 0
}
fn boxed_clone(&self) -> Box<dyn Store<T, Error = Self::Error, Key = Self::Key> + Send + Sync> {
Box::new(self.clone())
as Box<dyn Store<T, Error = Self::Error, Key = Self::Key> + Send + Sync>
}
}
crates/notify/src/stream.rs
+295
View File
@@ -0,0 +1,295 @@
use crate::{
error::TargetError, integration::NotificationMetrics,
store::{Key, Store},
target::Target,
Event,
StoreError,
};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{mpsc, Semaphore};
use tokio::time::sleep;
use tracing::{debug, error, info, warn};
/// Streams events from the store to the target
pub async fn stream_events(
store: &mut (dyn Store<Event, Error = StoreError, Key = Key> + Send),
target: &dyn Target,
mut cancel_rx: mpsc::Receiver<()>,
) {
info!("Starting event stream for target: {}", target.name());
// Retry configuration
const MAX_RETRIES: usize = 5;
const RETRY_DELAY: Duration = Duration::from_secs(5);
loop {
// Check for cancellation signal
if cancel_rx.try_recv().is_ok() {
info!("Cancellation received for target: {}", target.name());
return;
}
// Get list of events in the store
let keys = store.list();
if keys.is_empty() {
// No events, wait before checking again
sleep(Duration::from_secs(1)).await;
continue;
}
// Process each event
for key in keys {
// Check for cancellation before processing each event
if cancel_rx.try_recv().is_ok() {
info!("Cancellation received during processing for target: {}", target.name());
return;
}
let mut retry_count = 0;
let mut success = false;
// Retry logic
while retry_count < MAX_RETRIES && !success {
match target.send_from_store(key.clone()).await {
Ok(_) => {
info!("Successfully sent event for target: {}", target.name());
success = true;
}
Err(e) => {
// Handle specific errors
match &e {
TargetError::NotConnected => {
warn!("Target {} not connected, retrying...", target.name());
retry_count += 1;
sleep(RETRY_DELAY).await;
}
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
retry_count += 1;
sleep(Duration::from_secs((retry_count * 5) as u64)).await; // Exponential backoff
}
_ => {
// Permanent error, skip this event
error!("Permanent error for target {}: {}", target.name(), e);
break;
}
}
}
}
}
// Remove event from store if successfully sent
if retry_count >= MAX_RETRIES && !success {
warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
}
}
// Small delay before next iteration
sleep(Duration::from_millis(100)).await;
}
}
/// Starts the event streaming process for a target
pub fn start_event_stream(
mut store: Box<dyn Store<Event, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target + Send + Sync>,
) -> mpsc::Sender<()> {
let (cancel_tx, cancel_rx) = mpsc::channel(1);
tokio::spawn(async move {
stream_events(&mut *store, &*target, cancel_rx).await;
info!("Event stream stopped for target: {}", target.name());
});
cancel_tx
}
/// Start event stream with batch processing
pub fn start_event_stream_with_batching(
mut store: Box<dyn Store<Event, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target + Send + Sync>,
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
) -> mpsc::Sender<()> {
let (cancel_tx, cancel_rx) = mpsc::channel(1);
debug!("Starting event stream with batching for target: {}", target.name());
tokio::spawn(async move {
stream_events_with_batching(&mut *store, &*target, cancel_rx, metrics, semaphore).await;
info!("Event stream stopped for target: {}", target.name());
});
cancel_tx
}
/// Event stream processing with batch processing
pub async fn stream_events_with_batching(
store: &mut (dyn Store<Event, Error = StoreError, Key = Key> + Send),
target: &dyn Target,
mut cancel_rx: mpsc::Receiver<()>,
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
) {
info!("Starting event stream with batching for target: {}", target.name());
// Configuration parameters
const DEFAULT_BATCH_SIZE: usize = 1;
let batch_size = std::env::var("RUSTFS_EVENT_BATCH_SIZE")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(DEFAULT_BATCH_SIZE);
const BATCH_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_RETRIES: usize = 5;
const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
let mut batch = Vec::with_capacity(batch_size);
let mut batch_keys = Vec::with_capacity(batch_size);
let mut last_flush = Instant::now();
loop {
// Check the cancel signal
if cancel_rx.try_recv().is_ok() {
info!("Cancellation received for target: {}", target.name());
return;
}
// Get a list of events in storage
let keys = store.list();
debug!("Found {} keys in store for target: {}", keys.len(), target.name());
if keys.is_empty() {
// If there is data in the batch and timeout, refresh the batch
if !batch.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
last_flush = Instant::now();
}
// No event, wait before checking
tokio::time::sleep(Duration::from_millis(500)).await;
continue;
}
// Handle each event
for key in keys {
// Check the cancel signal again
if cancel_rx.try_recv().is_ok() {
info!("Cancellation received during processing for target: {}", target.name());
// Processing collected batches before exiting
if !batch.is_empty() {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
}
return;
}
// Try to get events from storage
match store.get(&key) {
Ok(event) => {
// Add to batch
batch.push(event);
batch_keys.push(key);
metrics.increment_processing();
// If the batch is full or enough time has passed since the last refresh, the batch will be processed
if batch.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore)
.await;
last_flush = Instant::now();
}
}
Err(e) => {
error!("Failed to target: {}, get event {} from store: {}", target.name(), key.to_string(), e);
// Consider deleting unreadable events to prevent infinite loops from trying to read
match store.del(&key) {
Ok(_) => {
info!("Deleted corrupted event {} from store", key.to_string());
}
Err(del_err) => {
error!("Failed to delete corrupted event {}: {}", key.to_string(), del_err);
}
}
metrics.increment_failed();
}
}
}
// A small delay will be conducted to check the next round
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
/// Processing event batches
async fn process_batch(
batch: &mut Vec<Event>,
batch_keys: &mut Vec<Key>,
target: &dyn Target,
max_retries: usize,
base_delay: Duration,
metrics: &Arc<NotificationMetrics>,
semaphore: &Arc<Semaphore>,
) {
debug!("Processing batch of {} events for target: {}", batch.len(), target.name());
if batch.is_empty() {
return;
}
// Obtain semaphore permission to limit concurrency
let permit = match semaphore.clone().acquire_owned().await {
Ok(permit) => permit,
Err(e) => {
error!("Failed to acquire semaphore permit: {}", e);
return;
}
};
// Handle every event in the batch
for (_event, key) in batch.iter().zip(batch_keys.iter()) {
let mut retry_count = 0;
let mut success = false;
// Retry logic
while retry_count < max_retries && !success {
match target.send_from_store(key.clone()).await {
Ok(_) => {
info!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
success = true;
metrics.increment_processed();
}
Err(e) => {
// Different retry strategies are adopted according to the error type
match &e {
TargetError::NotConnected => {
warn!("Target {} not connected, retrying...", target.name());
retry_count += 1;
tokio::time::sleep(base_delay * (1 << retry_count)).await; // Exponential backoff
}
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
retry_count += 1;
tokio::time::sleep(base_delay * (1 << retry_count)).await;
}
_ => {
// Permanent error, skip this event
error!("Permanent error for target {}: {}", target.name(), e);
metrics.increment_failed();
break;
}
}
}
}
}
// Handle the situation where the maximum number of retry exhaustion is exhausted
if retry_count >= max_retries && !success {
warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
metrics.increment_failed();
}
}
// Clear processed batches
batch.clear();
batch_keys.clear();
// Release semaphore permission (via drop)
drop(permit);
}
crates/notify/src/target/mod.rs
+96
View File
@@ -0,0 +1,96 @@
use crate::arn::TargetID;
use crate::store::{Key, Store};
use crate::{Event, StoreError, TargetError};
use async_trait::async_trait;
pub mod mqtt;
pub mod webhook;
/// Trait for notification targets
#[async_trait]
pub trait Target: Send + Sync + 'static {
/// Returns the ID of the target
fn id(&self) -> TargetID;
/// Returns the name of the target
fn name(&self) -> String {
self.id().to_string()
}
/// Checks if the target is active and reachable
async fn is_active(&self) -> Result<bool, TargetError>;
/// Saves an event (either sends it immediately or stores it for later)
async fn save(&self, event: Event) -> Result<(), TargetError>;
/// Sends an event from the store
async fn send_from_store(&self, key: Key) -> Result<(), TargetError>;
/// Closes the target and releases resources
async fn close(&self) -> Result<(), TargetError>;
/// Returns the store associated with the target (if any)
fn store(&self) -> Option<&(dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync)>;
/// Returns the type of the target
fn clone_dyn(&self) -> Box<dyn Target + Send + Sync>;
/// Initialize the target, such as establishing a connection, etc.
async fn init(&self) -> Result<(), TargetError> {
// The default implementation is empty
Ok(())
}
/// Check if the target is enabled
fn is_enabled(&self) -> bool;
}
/// The `ChannelTargetType` enum represents the different types of channel Target
/// used in the notification system.
///
/// It includes:
/// - `Webhook`: Represents a webhook target for sending notifications via HTTP requests.
/// - `Kafka`: Represents a Kafka target for sending notifications to a Kafka topic.
/// - `Mqtt`: Represents an MQTT target for sending notifications via MQTT protocol.
///
/// Each variant has an associated string representation that can be used for serialization
/// or logging purposes.
/// The `as_str` method returns the string representation of the target type,
/// and the `Display` implementation allows for easy formatting of the target type as a string.
///
/// example usage:
/// ```rust
/// use rustfs_notify::target::ChannelTargetType;
///
/// let target_type = ChannelTargetType::Webhook;
/// assert_eq!(target_type.as_str(), "webhook");
/// println!("Target type: {}", target_type);
/// ```
///
/// example output:
/// Target type: webhook
pub enum ChannelTargetType {
Webhook,
Kafka,
Mqtt,
}
impl ChannelTargetType {
pub fn as_str(&self) -> &'static str {
match self {
ChannelTargetType::Webhook => "webhook",
ChannelTargetType::Kafka => "kafka",
ChannelTargetType::Mqtt => "mqtt",
}
}
}
impl std::fmt::Display for ChannelTargetType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ChannelTargetType::Webhook => write!(f, "webhook"),
ChannelTargetType::Kafka => write!(f, "kafka"),
ChannelTargetType::Mqtt => write!(f, "mqtt"),
}
}
}
crates/notify/src/target/mqtt.rs
+671
View File
@@ -0,0 +1,671 @@
use crate::store::{Key, STORE_EXTENSION};
use crate::target::ChannelTargetType;
use crate::{
arn::TargetID, error::TargetError,
event::{Event, EventLog},
store::Store,
StoreError,
Target,
};
use async_trait::async_trait;
use rumqttc::{mqttbytes::Error as MqttBytesError, ConnectionError};
use rumqttc::{AsyncClient, EventLoop, MqttOptions, Outgoing, Packet, QoS};
use std::sync::Arc;
use std::{
path::PathBuf,
sync::atomic::{AtomicBool, Ordering},
time::Duration,
};
use tokio::sync::{mpsc, Mutex, OnceCell};
use tracing::{debug, error, info, instrument, trace, warn};
use url::Url;
use urlencoding;
const DEFAULT_CONNECTION_TIMEOUT: Duration = Duration::from_secs(15);
const EVENT_LOOP_POLL_TIMEOUT: Duration = Duration::from_secs(10); // For initial connection check in task
/// Arguments for configuring an MQTT target
#[derive(Debug, Clone)]
pub struct MQTTArgs {
/// Whether the target is enabled
pub enable: bool,
/// The broker URL
pub broker: Url,
/// The topic to publish to
pub topic: String,
/// The quality of service level
pub qos: QoS,
/// The username for the broker
pub username: String,
/// The password for the broker
pub password: String,
/// The maximum interval for reconnection attempts (Note: rumqttc has internal strategy)
pub max_reconnect_interval: Duration,
/// The keep alive interval
pub keep_alive: Duration,
/// The directory to store events in case of failure
pub queue_dir: String,
/// The maximum number of events to store
pub queue_limit: u64,
}
impl MQTTArgs {
pub fn validate(&self) -> Result<(), TargetError> {
if !self.enable {
return Ok(());
}
match self.broker.scheme() {
"ws" | "wss" | "tcp" | "ssl" | "tls" | "tcps" | "mqtt" | "mqtts" => {}
_ => {
return Err(TargetError::Configuration(
"unknown protocol in broker address".to_string(),
));
}
}
if !self.queue_dir.is_empty() {
let path = std::path::Path::new(&self.queue_dir);
if !path.is_absolute() {
return Err(TargetError::Configuration(
"mqtt queueDir path should be absolute".to_string(),
));
}
if self.qos == QoS::AtMostOnce {
return Err(TargetError::Configuration(
"QoS should be AtLeastOnce (1) or ExactlyOnce (2) if queueDir is set"
.to_string(),
));
}
}
Ok(())
}
}
struct BgTaskManager {
init_cell: OnceCell<tokio::task::JoinHandle<()>>,
cancel_tx: mpsc::Sender<()>,
initial_cancel_rx: Mutex<Option<mpsc::Receiver<()>>>,
}
/// A target that sends events to an MQTT broker
pub struct MQTTTarget {
id: TargetID,
args: MQTTArgs,
client: Arc<Mutex<Option<AsyncClient>>>,
store: Option<Box<dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync>>,
connected: Arc<AtomicBool>,
bg_task_manager: Arc<BgTaskManager>,
}
impl MQTTTarget {
/// Creates a new MQTTTarget
#[instrument(skip(args), fields(target_id_as_string = %id))]
pub fn new(id: String, args: MQTTArgs) -> Result<Self, TargetError> {
args.validate()?;
let target_id = TargetID::new(id.clone(), ChannelTargetType::Mqtt.as_str().to_string());
let queue_store = if !args.queue_dir.is_empty() {
let base_path = PathBuf::from(&args.queue_dir);
let unique_dir_name = format!(
"rustfs-{}-{}-{}",
ChannelTargetType::Mqtt.as_str(),
target_id.name,
target_id.id
)
.replace(":", "_");
// Ensure the directory name is valid for filesystem
let specific_queue_path = base_path.join(unique_dir_name);
debug!(target_id = %target_id, path = %specific_queue_path.display(), "Initializing queue store for MQTT target");
let store = crate::store::QueueStore::<Event>::new(
specific_queue_path,
args.queue_limit,
STORE_EXTENSION,
);
if let Err(e) = store.open() {
error!(
target_id = %target_id,
error = %e,
"Failed to open store for MQTT target"
);
return Err(TargetError::Storage(format!("{}", e)));
}
Some(Box::new(store)
as Box<
dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync,
>)
} else {
None
};
let (cancel_tx, cancel_rx) = mpsc::channel(1);
let bg_task_manager = Arc::new(BgTaskManager {
init_cell: OnceCell::new(),
cancel_tx,
initial_cancel_rx: Mutex::new(Some(cancel_rx)),
});
info!(target_id = %target_id, "MQTT target created");
Ok(MQTTTarget {
id: target_id,
args,
client: Arc::new(Mutex::new(None)),
store: queue_store,
connected: Arc::new(AtomicBool::new(false)),
bg_task_manager,
})
}
#[instrument(skip(self), fields(target_id = %self.id))]
async fn init(&self) -> Result<(), TargetError> {
if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "Already connected.");
return Ok(());
}
let bg_task_manager = Arc::clone(&self.bg_task_manager);
let client_arc = Arc::clone(&self.client);
let connected_arc = Arc::clone(&self.connected);
let target_id_clone = self.id.clone();
let args_clone = self.args.clone();
let _ = bg_task_manager
.init_cell
.get_or_try_init(|| async {
debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
let host = args_clone.broker.host_str().unwrap_or("localhost");
let port = args_clone.broker.port().unwrap_or(1883);
let mut mqtt_options = MqttOptions::new(
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
host,
port,
);
mqtt_options
.set_keep_alive(args_clone.keep_alive)
.set_max_packet_size(100 * 1024 * 1024, 100 * 1024 * 1024); // 100MB
if !args_clone.username.is_empty() {
mqtt_options
.set_credentials(args_clone.username.clone(), args_clone.password.clone());
}
let (new_client, eventloop) = AsyncClient::new(mqtt_options, 10);
if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await {
error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init");
return Err(TargetError::Network(format!("MQTT subscribe failed: {}", e)));
}
let mut rx_guard = bg_task_manager.initial_cancel_rx.lock().await;
let cancel_rx = rx_guard.take().ok_or_else(|| {
error!(target_id = %target_id_clone, "MQTT cancel receiver already taken for task.");
TargetError::Configuration("MQTT cancel receiver already taken for task".to_string())
})?;
drop(rx_guard);
*client_arc.lock().await = Some(new_client.clone());
info!(target_id = %target_id_clone, "Spawning MQTT event loop task.");
let task_handle = tokio::spawn(run_mqtt_event_loop(
eventloop,
connected_arc.clone(),
target_id_clone.clone(),
cancel_rx,
));
Ok(task_handle)
})
.await
.map_err(|e: TargetError| {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT background task");
e
})?;
debug!(target_id = %self.id, "MQTT background task initialized successfully.");
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
while !self.connected.load(Ordering::SeqCst) {
if let Some(handle) = self.bg_task_manager.init_cell.get() {
if handle.is_finished() && !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established.");
return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
}
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
debug!(target_id = %self.id, "MQTT target connected successfully.");
Ok(())
}).await {
Ok(Ok(_)) => {
info!(target_id = %self.id, "MQTT target initialized and connected.");
Ok(())
}
Ok(Err(e)) => Err(e),
Err(_) => {
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn.");
Err(TargetError::Network(
"Timeout waiting for MQTT connection".to_string(),
))
}
}
}
#[instrument(skip(self, event), fields(target_id = %self.id))]
async fn send(&self, event: &Event) -> Result<(), TargetError> {
let client_guard = self.client.lock().await;
let client = client_guard
.as_ref()
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
let object_name = urlencoding::decode(&event.s3.object.key)
.map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {}", e)))?;
let key = format!("{}/{}", event.s3.bucket.name, object_name);
let log = EventLog {
event_name: event.event_name,
key,
records: vec![event.clone()],
};
let data = serde_json::to_vec(&log)
.map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {}", e)))?;
// Vec<u8> Convert to String, only for printing logs
let data_string = String::from_utf8(data.clone()).map_err(|e| {
TargetError::Encoding(format!("Failed to convert event data to UTF-8: {}", e))
})?;
debug!(
"Sending event to mqtt target: {}, event log: {}",
self.id, data_string
);
client
.publish(&self.args.topic, self.args.qos, false, data)
.await
.map_err(|e| {
if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
self.connected.store(false, Ordering::SeqCst);
warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected.");
TargetError::NotConnected
} else {
TargetError::Request(format!("Failed to publish message: {}", e))
}
})?;
debug!(target_id = %self.id, topic = %self.args.topic, "Event published to MQTT topic");
Ok(())
}
pub fn clone_target(&self) -> Box<dyn Target + Send + Sync> {
Box::new(MQTTTarget {
id: self.id.clone(),
args: self.args.clone(),
client: self.client.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: self.connected.clone(),
bg_task_manager: self.bg_task_manager.clone(),
})
}
}
async fn run_mqtt_event_loop(
mut eventloop: EventLoop,
connected_status: Arc<AtomicBool>,
target_id: TargetID,
mut cancel_rx: mpsc::Receiver<()>,
) {
info!(target_id = %target_id, "MQTT event loop task started.");
let mut initial_connection_established = false;
loop {
tokio::select! {
biased;
_ = cancel_rx.recv() => {
info!(target_id = %target_id, "MQTT event loop task received cancellation signal. Shutting down.");
break;
}
polled_event_result = async {
if !initial_connection_established || !connected_status.load(Ordering::SeqCst) {
match tokio::time::timeout(EVENT_LOOP_POLL_TIMEOUT, eventloop.poll()).await {
Ok(Ok(event)) => Ok(event),
Ok(Err(e)) => Err(e),
Err(_) => {
debug!(target_id = %target_id, "MQTT poll timed out (EVENT_LOOP_POLL_TIMEOUT) while not connected or status pending.");
Err(rumqttc::ConnectionError::NetworkTimeout)
}
}
} else {
eventloop.poll().await
}
} => {
match polled_event_result {
Ok(notification) => {
trace!(target_id = %target_id, event = ?notification, "Received MQTT event");
match notification {
rumqttc::Event::Incoming(Packet::ConnAck(_conn_ack)) => {
info!(target_id = %target_id, "MQTT connected (ConnAck).");
connected_status.store(true, Ordering::SeqCst);
initial_connection_established = true;
}
rumqttc::Event::Incoming(Packet::Publish(publish)) => {
debug!(target_id = %target_id, topic = %publish.topic, payload_len = publish.payload.len(), "Received message on subscribed topic.");
}
rumqttc::Event::Incoming(Packet::Disconnect) => {
info!(target_id = %target_id, "Received Disconnect packet from broker. MQTT connection lost.");
connected_status.store(false, Ordering::SeqCst);
}
rumqttc::Event::Incoming(Packet::PingResp) => {
trace!(target_id = %target_id, "Received PingResp from broker. Connection is alive.");
}
rumqttc::Event::Incoming(Packet::SubAck(suback)) => {
trace!(target_id = %target_id, "Received SubAck for pkid: {}", suback.pkid);
}
rumqttc::Event::Incoming(Packet::PubAck(puback)) => {
trace!(target_id = %target_id, "Received PubAck for pkid: {}", puback.pkid);
}
// Process other incoming packet types as needed (PubRec, PubRel, PubComp, UnsubAck)
rumqttc::Event::Outgoing(Outgoing::Disconnect) => {
info!(target_id = %target_id, "MQTT outgoing disconnect initiated by client.");
connected_status.store(false, Ordering::SeqCst);
}
rumqttc::Event::Outgoing(Outgoing::PingReq) => {
trace!(target_id = %target_id, "Client sent PingReq to broker.");
}
// Other Outgoing events (Subscribe, Unsubscribe, Publish) usually do not need to handle connection status here,
// Because they are actions initiated by the client.
_ => {
// Log other unspecified MQTT events that are not handled, which helps debug
trace!(target_id = %target_id, "Unhandled or generic MQTT event: {:?}", notification);
}
}
}
Err(e) => {
connected_status.store(false, Ordering::SeqCst);
error!(target_id = %target_id, error = %e, "Error from MQTT event loop poll");
if matches!(e, rumqttc::ConnectionError::NetworkTimeout) && (!initial_connection_established || !connected_status.load(Ordering::SeqCst)) {
warn!(target_id = %target_id, "Timeout during initial poll or pending state, will retry.");
continue;
}
if matches!(e,
ConnectionError::Io(_) |
ConnectionError::NetworkTimeout |
ConnectionError::ConnectionRefused(_) |
ConnectionError::Tls(_)
) {
warn!(target_id = %target_id, error = %e, "MQTT connection error. Relying on rumqttc for reconnection if applicable.");
}
// Here you can decide whether to break loops based on the error type.
// For example, for some unrecoverable errors.
if is_fatal_mqtt_error(&e) {
error!(target_id = %target_id, error = %e, "Fatal MQTT error, terminating event loop.");
break;
}
// rumqttc's eventloop.poll() may return Err and terminate after some errors,
// Or it will handle reconnection internally. The continue here will make select! wait again.
// If the error is temporary and rumqttc is handling reconnection, poll() should eventually succeed or return a different error again.
// Sleep briefly to avoid busy cycles in case of rapid failure.
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
}
}
}
connected_status.store(false, Ordering::SeqCst);
info!(target_id = %target_id, "MQTT event loop task finished.");
}
/// Check whether the given MQTT connection error should be considered a fatal error,
/// For fatal errors, the event loop should terminate.
fn is_fatal_mqtt_error(err: &ConnectionError) -> bool {
match err {
// If the client request has been processed all (for example, AsyncClient is dropped), the event loop can end.
ConnectionError::RequestsDone => true,
// Check for the underlying MQTT status error
ConnectionError::MqttState(state_err) => {
// The type of state_err is &rumqttc::StateError
match state_err {
// If StateError is caused by deserialization issues, check the underlying MqttBytesError
rumqttc::StateError::Deserialization(mqtt_bytes_err) => { // The type of mqtt_bytes_err is &rumqttc::mqttbytes::Error
matches!(
mqtt_bytes_err,
MqttBytesError::InvalidProtocol // Invalid agreement
| MqttBytesError::InvalidProtocolLevel(_) // Invalid protocol level
| MqttBytesError::IncorrectPacketFormat // Package format is incorrect
| MqttBytesError::InvalidPacketType(_) // Invalid package type
| MqttBytesError::MalformedPacket // Package format error
| MqttBytesError::PayloadTooLong // Too long load
| MqttBytesError::PayloadSizeLimitExceeded(_) // Load size limit exceeded
| MqttBytesError::TopicNotUtf8 // Topic Non-UTF-8 (Serious Agreement Violation)
)
}
// Others that are fatal StateError variants
rumqttc::StateError::InvalidState // The internal state machine is in invalid state
| rumqttc::StateError::WrongPacket // Agreement Violation: Unexpected Data Packet Received
| rumqttc::StateError::Unsolicited(_) // Agreement Violation: Unsolicited ACK Received
| rumqttc::StateError::OutgoingPacketTooLarge { .. } // Try to send too large packets
| rumqttc::StateError::EmptySubscription // Agreement violation (if this stage occurs)
=> true,
// Other StateErrors (such as Io, AwaitPingResp, CollisionTimeout) are not considered deadly here.
// They may be processed internally by rumqttc or upgraded to other ConnectionError types.
_ => false,
}
}
// Other types of ConnectionErrors (such as Io, Tls, NetworkTimeout, ConnectionRefused, NotConnAck, etc.)
// It is usually considered temporary, or the reconnect logic inside rumqttc will be processed.
_ => false,
}
}
#[async_trait]
impl Target for MQTTTarget {
fn id(&self) -> TargetID {
self.id.clone()
}
#[instrument(skip(self), fields(target_id = %self.id))]
async fn is_active(&self) -> Result<bool, TargetError> {
debug!(target_id = %self.id, "Checking if MQTT target is active.");
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
// Check if the background task is running and has not panicked
if let Some(handle) = self.bg_task_manager.init_cell.get() {
if handle.is_finished() {
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active.");
return Err(TargetError::Network(
"MQTT background task terminated".to_string(),
));
}
}
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected.");
return Err(TargetError::Configuration(
"MQTT client not available or not initialized/connected".to_string(),
));
}
if self.connected.load(Ordering::SeqCst) {
debug!(target_id = %self.id, "MQTT target is active (connected flag is true).");
Ok(true)
} else {
debug!(target_id = %self.id, "MQTT target is not connected (connected flag is false).");
Err(TargetError::NotConnected)
}
}
#[instrument(skip(self, event), fields(target_id = %self.id))]
async fn save(&self, event: Event) -> Result<(), TargetError> {
if let Some(store) = &self.store {
debug!(target_id = %self.id, "Event saved to store start");
// If store is configured, ONLY put the event into the store.
// Do NOT send it directly here.
match store.put(event.clone()) {
Ok(_) => {
debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
Ok(())
}
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to save event to store");
return Err(TargetError::Storage(format!(
"Failed to save event to store: {}",
e
)));
}
}
} else {
if !self.is_enabled() {
return Err(TargetError::Disabled);
}
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init.");
// Call the struct's init method, not the trait's default
match MQTTTarget::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
return Err(TargetError::NotConnected);
}
}
if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot save (send directly) as target is not active after init attempt.");
return Err(TargetError::NotConnected);
}
}
self.send(&event).await
}
}
#[instrument(skip(self), fields(target_id = %self.id))]
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send event from store with key.");
if !self.is_enabled() {
return Err(TargetError::Disabled);
}
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Not connected; trying to init before sending from store.");
match MQTTTarget::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
return Err(TargetError::NotConnected);
}
}
if !self.connected.load(Ordering::SeqCst) {
error!(target_id = %self.id, "Cannot send from store as target is not active after init attempt.");
return Err(TargetError::NotConnected);
}
}
let store = self
.store
.as_ref()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
let event = match store.get(&key) {
Ok(event) => {
debug!(target_id = %self.id, ?key, "Retrieved event from store for sending.");
event
}
Err(StoreError::NotFound) => {
// Assuming NotFound takes the key
debug!(target_id = %self.id, ?key, "Event not found in store for sending.");
return Ok(());
}
Err(e) => {
error!(
target_id = %self.id,
error = %e,
"Failed to get event from store"
);
return Err(TargetError::Storage(format!(
"Failed to get event from store: {}",
e
)));
}
};
debug!(target_id = %self.id, ?key, "Sending event from store.");
if let Err(e) = self.send(&event).await {
if matches!(e, TargetError::NotConnected) {
warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store.");
return Err(TargetError::NotConnected);
}
error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error.");
return Err(e);
}
debug!(target_id = %self.id, ?key, "Event sent from store successfully. deleting from store. ");
match store.del(&key) {
Ok(_) => {
debug!(target_id = %self.id, ?key, "Event deleted from store after successful send.")
}
Err(StoreError::NotFound) => {
debug!(target_id = %self.id, ?key, "Event already deleted from store.");
}
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to delete event from store after send.");
return Err(TargetError::Storage(format!(
"Failed to delete event from store: {}",
e
)));
}
}
debug!(target_id = %self.id, ?key, "Event deleted from store.");
Ok(())
}
async fn close(&self) -> Result<(), TargetError> {
info!(target_id = %self.id, "Attempting to close MQTT target.");
if let Err(e) = self.bg_task_manager.cancel_tx.send(()).await {
warn!(target_id = %self.id, error = %e, "Failed to send cancel signal to MQTT background task. It might have already exited.");
}
// Wait for the task to finish if it was initialized
if let Some(_task_handle) = self.bg_task_manager.init_cell.get() {
debug!(target_id = %self.id, "Waiting for MQTT background task to complete...");
// It's tricky to await here if close is called from a sync context or Drop
// For async close, this is fine. Consider a timeout.
// let _ = tokio::time::timeout(Duration::from_secs(5), task_handle.await).await;
// If task_handle.await is directly used, ensure it's not awaited multiple times if close can be called multiple times.
// For now, we rely on the signal and the task's self-termination.
}
if let Some(client_instance) = self.client.lock().await.take() {
info!(target_id = %self.id, "Disconnecting MQTT client.");
if let Err(e) = client_instance.disconnect().await {
warn!(target_id = %self.id, error = %e, "Error during MQTT client disconnect.");
}
}
self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "MQTT target close method finished.");
Ok(())
}
fn store(&self) -> Option<&(dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync)> {
self.store.as_deref()
}
fn clone_dyn(&self) -> Box<dyn Target + Send + Sync> {
self.clone_target()
}
async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() {
debug!(target_id = %self.id, "Target is disabled, skipping init.");
return Ok(());
}
// Call the internal init logic
MQTTTarget::init(self).await
}
fn is_enabled(&self) -> bool {
self.args.enable
}
}
crates/notify/src/target/webhook.rs
+398
View File
@@ -0,0 +1,398 @@
use crate::store::STORE_EXTENSION;
use crate::target::ChannelTargetType;
use crate::{
arn::TargetID, error::TargetError,
event::{Event, EventLog},
store::{Key, Store},
StoreError,
Target,
};
use async_trait::async_trait;
use reqwest::{Client, StatusCode, Url};
use std::{
path::PathBuf,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
time::Duration,
};
use tokio::net::lookup_host;
use tokio::sync::mpsc;
use tracing::{debug, error, info, instrument};
use urlencoding;
/// Arguments for configuring a Webhook target
#[derive(Debug, Clone)]
pub struct WebhookArgs {
/// Whether the target is enabled
pub enable: bool,
/// The endpoint URL to send events to
pub endpoint: Url,
/// The authorization token for the endpoint
pub auth_token: String,
/// The directory to store events in case of failure
pub queue_dir: String,
/// The maximum number of events to store
pub queue_limit: u64,
/// The client certificate for TLS (PEM format)
pub client_cert: String,
/// The client key for TLS (PEM format)
pub client_key: String,
}
// WebhookArgs 的验证方法
impl WebhookArgs {
pub fn validate(&self) -> Result<(), TargetError> {
if !self.enable {
return Ok(());
}
if self.endpoint.as_str().is_empty() {
return Err(TargetError::Configuration("endpoint empty".to_string()));
}
if !self.queue_dir.is_empty() {
let path = std::path::Path::new(&self.queue_dir);
if !path.is_absolute() {
return Err(TargetError::Configuration("webhook queueDir path should be absolute".to_string()));
}
}
if !self.client_cert.is_empty() && self.client_key.is_empty()
|| self.client_cert.is_empty() && !self.client_key.is_empty()
{
return Err(TargetError::Configuration("cert and key must be specified as a pair".to_string()));
}
Ok(())
}
}
/// A target that sends events to a webhook
pub struct WebhookTarget {
id: TargetID,
args: WebhookArgs,
http_client: Arc<Client>,
// Add Send + Sync constraints to ensure thread safety
store: Option<Box<dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync>>,
initialized: AtomicBool,
addr: String,
cancel_sender: mpsc::Sender<()>,
}
impl WebhookTarget {
/// Clones the WebhookTarget, creating a new instance with the same configuration
pub fn clone_box(&self) -> Box<dyn Target + Send + Sync> {
Box::new(WebhookTarget {
id: self.id.clone(),
args: self.args.clone(),
http_client: Arc::clone(&self.http_client),
store: self.store.as_ref().map(|s| s.boxed_clone()),
initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
addr: self.addr.clone(),
cancel_sender: self.cancel_sender.clone(),
})
}
/// Creates a new WebhookTarget
#[instrument(skip(args), fields(target_id = %id))]
pub fn new(id: String, args: WebhookArgs) -> Result<Self, TargetError> {
// First verify the parameters
args.validate()?;
// Create a TargetID
let target_id = TargetID::new(id, ChannelTargetType::Webhook.as_str().to_string());
// Build HTTP client
let mut client_builder = Client::builder()
.timeout(Duration::from_secs(30))
.user_agent(rustfs_utils::sys::get_user_agent(rustfs_utils::sys::ServiceType::Basis));
// Supplementary certificate processing logic
if !args.client_cert.is_empty() && !args.client_key.is_empty() {
// Add client certificate
let cert = std::fs::read(&args.client_cert)
.map_err(|e| TargetError::Configuration(format!("Failed to read client cert: {}", e)))?;
let key = std::fs::read(&args.client_key)
.map_err(|e| TargetError::Configuration(format!("Failed to read client key: {}", e)))?;
let identity = reqwest::Identity::from_pem(&[cert, key].concat())
.map_err(|e| TargetError::Configuration(format!("Failed to create identity: {}", e)))?;
client_builder = client_builder.identity(identity);
}
let http_client = Arc::new(
client_builder
.build()
.map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {}", e)))?,
);
// Build storage
let queue_store = if !args.queue_dir.is_empty() {
let queue_dir = PathBuf::from(&args.queue_dir).join(format!(
"rustfs-{}-{}-{}",
ChannelTargetType::Webhook.as_str(),
target_id.name,
target_id.id
));
let store = super::super::store::QueueStore::<Event>::new(queue_dir, args.queue_limit, STORE_EXTENSION);
if let Err(e) = store.open() {
error!("Failed to open store for Webhook target {}: {}", target_id.id, e);
return Err(TargetError::Storage(format!("{}", e)));
}
// Make sure that the Store trait implemented by QueueStore matches the expected error type
Some(Box::new(store) as Box<dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
// resolved address
let addr = {
let host = args.endpoint.host_str().unwrap_or("localhost");
let port = args
.endpoint
.port()
.unwrap_or_else(|| if args.endpoint.scheme() == "https" { 443 } else { 80 });
format!("{}:{}", host, port)
};
// Create a cancel channel
let (cancel_sender, _) = mpsc::channel(1);
info!(target_id = %target_id.id, "Webhook target created");
Ok(WebhookTarget {
id: target_id,
args,
http_client,
store: queue_store,
initialized: AtomicBool::new(false),
addr,
cancel_sender,
})
}
async fn init(&self) -> Result<(), TargetError> {
// 使用 CAS 操作确保线程安全初始化
if !self.initialized.load(Ordering::SeqCst) {
// 检查连接
match self.is_active().await {
Ok(true) => {
info!("Webhook target {} is active", self.id);
}
Ok(false) => {
return Err(TargetError::NotConnected);
}
Err(e) => {
error!("Failed to check if Webhook target {} is active: {}", self.id, e);
return Err(e);
}
}
self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id);
}
Ok(())
}
async fn send(&self, event: &Event) -> Result<(), TargetError> {
info!("Webhook Sending event to webhook target: {}", self.id);
let object_name = urlencoding::decode(&event.s3.object.key)
.map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {}", e)))?;
let key = format!("{}/{}", event.s3.bucket.name, object_name);
let log = EventLog {
event_name: event.event_name,
key,
records: vec![event.clone()],
};
let data =
serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {}", e)))?;
// Vec<u8> 转换为 String
let data_string = String::from_utf8(data.clone())
.map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {}", e)))?;
debug!("Sending event to webhook target: {}, event log: {}", self.id, data_string);
// 构建请求
let mut req_builder = self
.http_client
.post(self.args.endpoint.as_str())
.header("Content-Type", "application/json");
if !self.args.auth_token.is_empty() {
// 分割 auth_token 字符串，检查是否已包含认证类型
let tokens: Vec<&str> = self.args.auth_token.split_whitespace().collect();
match tokens.len() {
2 => {
// 已经包含认证类型和令牌，如 "Bearer token123"
req_builder = req_builder.header("Authorization", &self.args.auth_token);
}
1 => {
// 只有令牌，需要添加 "Bearer" 前缀
req_builder = req_builder.header("Authorization", format!("Bearer {}", self.args.auth_token));
}
_ => {
// 空字符串或其他情况，不添加认证头
}
}
}
// 发送请求
let resp = req_builder.body(data).send().await.map_err(|e| {
if e.is_timeout() || e.is_connect() {
TargetError::NotConnected
} else {
TargetError::Request(format!("Failed to send request: {}", e))
}
})?;
let status = resp.status();
if status.is_success() {
debug!("Event sent to webhook target: {}", self.id);
Ok(())
} else if status == StatusCode::FORBIDDEN {
Err(TargetError::Authentication(format!(
"{} returned '{}', please check if your auth token is correctly set",
self.args.endpoint, status
)))
} else {
Err(TargetError::Request(format!(
"{} returned '{}', please check your endpoint configuration",
self.args.endpoint, status
)))
}
}
}
#[async_trait]
impl Target for WebhookTarget {
fn id(&self) -> TargetID {
self.id.clone()
}
// 确保 Future 是 Send
async fn is_active(&self) -> Result<bool, TargetError> {
let socket_addr = lookup_host(&self.addr)
.await
.map_err(|e| TargetError::Network(format!("Failed to resolve host: {}", e)))?
.next()
.ok_or_else(|| TargetError::Network("No address found".to_string()))?;
debug!("is_active socket addr: {},target id:{}", socket_addr, self.id.id);
match tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await {
Ok(Ok(_)) => {
debug!("Connection to {} is active", self.addr);
Ok(true)
}
Ok(Err(e)) => {
debug!("Connection to {} failed: {}", self.addr, e);
if e.kind() == std::io::ErrorKind::ConnectionRefused {
Err(TargetError::NotConnected)
} else {
Err(TargetError::Network(format!("Connection failed: {}", e)))
}
}
Err(_) => Err(TargetError::Timeout("Connection timed out".to_string())),
}
}
async fn save(&self, event: Event) -> Result<(), TargetError> {
if let Some(store) = &self.store {
// Call the store method directly, no longer need to acquire the lock
store
.put(event)
.map_err(|e| TargetError::Storage(format!("Failed to save event to store: {}", e)))?;
debug!("Event saved to store for target: {}", self.id);
Ok(())
} else {
match self.init().await {
Ok(_) => (),
Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
return Err(TargetError::NotConnected);
}
}
self.send(&event).await
}
}
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
debug!("Sending event from store for target: {}", self.id);
match self.init().await {
Ok(_) => {
debug!("Event sent to store for target: {}", self.name());
}
Err(e) => {
error!("Failed to initialize Webhook target {}: {}", self.id.id, e);
return Err(TargetError::NotConnected);
}
}
let store = self
.store
.as_ref()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
// Get events directly from the store, no longer need to acquire locks
let event = match store.get(&key) {
Ok(event) => event,
Err(StoreError::NotFound) => return Ok(()),
Err(e) => {
return Err(TargetError::Storage(format!("Failed to get event from store: {}", e)));
}
};
if let Err(e) = self.send(&event).await {
if let TargetError::NotConnected = e {
return Err(TargetError::NotConnected);
}
return Err(e);
}
// Use the immutable reference of the store to delete the event content corresponding to the key
debug!("Deleting event from store for target: {}, key:{}, start", self.id, key.to_string());
match store.del(&key) {
Ok(_) => debug!("Event deleted from store for target: {}, key:{}, end", self.id, key.to_string()),
Err(e) => {
error!("Failed to delete event from store: {}", e);
return Err(TargetError::Storage(format!("Failed to delete event from store: {}", e)));
}
}
debug!("Event sent from store and deleted for target: {}", self.id);
Ok(())
}
async fn close(&self) -> Result<(), TargetError> {
// Send cancel signal to background tasks
let _ = self.cancel_sender.try_send(());
info!("Webhook target closed: {}", self.id);
Ok(())
}
fn store(&self) -> Option<&(dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync)> {
// Returns the reference to the internal store
self.store.as_deref()
}
fn clone_dyn(&self) -> Box<dyn Target + Send + Sync> {
self.clone_box()
}
// The existing init method can meet the needs well, but we need to make sure it complies with the Target trait
// We can use the existing init method, but adjust the return value to match the trait requirement
async fn init(&self) -> Result<(), TargetError> {
// If the target is disabled, return to success directly
if !self.is_enabled() {
debug!("Webhook target {} is disabled, skipping initialization", self.id);
return Ok(());
}
// Use existing initialization logic
WebhookTarget::init(self).await
}
fn is_enabled(&self) -> bool {
self.args.enable
}
}
crates/obs/Cargo.toml
+2 -1
View File
@@ -17,10 +17,11 @@ webhook = ["dep:reqwest"]
kafka = ["dep:rdkafka"]
[dependencies]
rustfs-config = { workspace = true }
rustfs-config = { workspace = true, features = ["constants"] }
async-trait = { workspace = true }
chrono = { workspace = true }
flexi_logger = { workspace = true, features = ["trc", "kv"] }
lazy_static = { workspace = true }
nu-ansi-term = { workspace = true }
nvml-wrapper = { workspace = true, optional = true }
opentelemetry = { workspace = true }
crates/obs/examples/config.toml
+3 -2
View File
@@ -3,11 +3,12 @@ endpoint = "http://localhost:4317" # Default is "http://localhost:4317" if not s
use_stdout = false # Output with stdout, true output, false no output
sample_ratio = 1
meter_interval = 30
service_name = "rustfs_obs"
service_name = "rustfs"
service_version = "0.1.0"
environments = "develop"
logger_level = "debug"
local_logging_enabled = true
local_logging_enabled = true # Default is false if not specified
#[[sinks]]
#type = "Kafka"
crates/obs/src/lib.rs
+1
View File
@@ -34,6 +34,7 @@ mod config;
mod entry;
mod global;
mod logger;
mod metrics;
mod sinks;
mod system;
mod telemetry;
crates/obs/src/metrics/audit.rs
+32
View File
@@ -0,0 +1,32 @@
/// audit related metric descriptors
///
/// This module contains the metric descriptors for the audit subsystem.
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
const TARGET_ID: &str = "target_id";
lazy_static::lazy_static! {
pub static ref AUDIT_FAILED_MESSAGES_MD: MetricDescriptor =
new_counter_md(
MetricName::AuditFailedMessages,
"Total number of messages that failed to send since start",
&[TARGET_ID],
subsystems::AUDIT
);
pub static ref AUDIT_TARGET_QUEUE_LENGTH_MD: MetricDescriptor =
new_gauge_md(
MetricName::AuditTargetQueueLength,
"Number of unsent messages in queue for target",
&[TARGET_ID],
subsystems::AUDIT
);
pub static ref AUDIT_TOTAL_MESSAGES_MD: MetricDescriptor =
new_counter_md(
MetricName::AuditTotalMessages,
"Total number of messages sent since start",
&[TARGET_ID],
subsystems::AUDIT
);
}
crates/obs/src/metrics/bucket.rs
+68
View File
@@ -0,0 +1,68 @@
/// bucket level s3 metric descriptor
use crate::metrics::{new_counter_md, new_gauge_md, new_histogram_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref BUCKET_API_TRAFFIC_SENT_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiTrafficSentBytes,
"Total number of bytes received for a bucket",
&["bucket", "type"],
subsystems::BUCKET_API
);
pub static ref BUCKET_API_TRAFFIC_RECV_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiTrafficRecvBytes,
"Total number of bytes sent for a bucket",
&["bucket", "type"],
subsystems::BUCKET_API
);
pub static ref BUCKET_API_REQUESTS_IN_FLIGHT_MD: MetricDescriptor =
new_gauge_md(
MetricName::ApiRequestsInFlightTotal,
"Total number of requests currently in flight for a bucket",
&["bucket", "name", "type"],
subsystems::BUCKET_API
);
pub static ref BUCKET_API_REQUESTS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequestsTotal,
"Total number of requests for a bucket",
&["bucket", "name", "type"],
subsystems::BUCKET_API
);
pub static ref BUCKET_API_REQUESTS_CANCELED_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequestsCanceledTotal,
"Total number of requests canceled by the client for a bucket",
&["bucket", "name", "type"],
subsystems::BUCKET_API
);
pub static ref BUCKET_API_REQUESTS_4XX_ERRORS_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequests4xxErrorsTotal,
"Total number of requests with 4xx errors for a bucket",
&["bucket", "name", "type"],
subsystems::BUCKET_API
);
pub static ref BUCKET_API_REQUESTS_5XX_ERRORS_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequests5xxErrorsTotal,
"Total number of requests with 5xx errors for a bucket",
&["bucket", "name", "type"],
subsystems::BUCKET_API
);
pub static ref BUCKET_API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: MetricDescriptor =
new_histogram_md(
MetricName::ApiRequestsTTFBSecondsDistribution,
"Distribution of time to first byte across API calls for a bucket",
&["bucket", "name", "le", "type"],
subsystems::BUCKET_API
);
}
crates/obs/src/metrics/bucket_replication.rs
+168
View File
@@ -0,0 +1,168 @@
/// Bucket copy metric descriptor
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
/// Bucket level replication metric descriptor
pub const BUCKET_L: &str = "bucket";
/// Replication operation
pub const OPERATION_L: &str = "operation";
/// Replication target ARN
pub const TARGET_ARN_L: &str = "targetArn";
/// Replication range
pub const RANGE_L: &str = "range";
lazy_static::lazy_static! {
pub static ref BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::LastHourFailedBytes,
"Total number of bytes failed at least once to replicate in the last hour on a bucket",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::LastHourFailedCount,
"Total number of objects which failed replication in the last hour on a bucket",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::LastMinFailedBytes,
"Total number of bytes failed at least once to replicate in the last full minute on a bucket",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::LastMinFailedCount,
"Total number of objects which failed replication in the last full minute on a bucket",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_LATENCY_MS_MD: MetricDescriptor =
new_gauge_md(
MetricName::LatencyMilliSec,
"Replication latency on a bucket in milliseconds",
&[BUCKET_L, OPERATION_L, RANGE_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedDeleteTaggingRequestsTotal,
"Number of DELETE tagging requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedGetRequestsFailures,
"Number of failures in GET requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedGetRequestsTotal,
"Number of GET requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
// TODO - add a metric for the number of PUT requests proxied to replication target
pub static ref BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedGetTaggingRequestFailures,
"Number of failures in GET tagging requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedGetTaggingRequestsTotal,
"Number of GET tagging requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedHeadRequestsFailures,
"Number of failures in HEAD requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedHeadRequestsTotal,
"Number of HEAD requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
// TODO - add a metric for the number of PUT requests proxied to replication target
pub static ref BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedPutTaggingRequestFailures,
"Number of failures in PUT tagging requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedPutTaggingRequestsTotal,
"Number of PUT tagging requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_SENT_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::SentBytes,
"Total number of bytes replicated to the target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_SENT_COUNT_MD: MetricDescriptor =
new_counter_md(
MetricName::SentCount,
"Total number of objects replicated to the target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_TOTAL_FAILED_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::TotalFailedBytes,
"Total number of bytes failed at least once to replicate since server start",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
pub static ref BUCKET_REPL_TOTAL_FAILED_COUNT_MD: MetricDescriptor =
new_counter_md(
MetricName::TotalFailedCount,
"Total number of objects which failed replication since server start",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
// TODO - add a metric for the number of DELETE requests proxied to replication target
pub static ref BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProxiedDeleteTaggingRequestFailures,
"Number of failures in DELETE tagging requests proxied to replication target",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION
);
}
crates/obs/src/metrics/cluster_config.rs
+20
View File
@@ -0,0 +1,20 @@
/// Metric descriptors related to cluster configuration
use crate::metrics::{new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref CONFIG_RRS_PARITY_MD: MetricDescriptor =
new_gauge_md(
MetricName::ConfigRRSParity,
"Reduced redundancy storage class parity",
&[],
subsystems::CLUSTER_CONFIG
);
pub static ref CONFIG_STANDARD_PARITY_MD: MetricDescriptor =
new_gauge_md(
MetricName::ConfigStandardParity,
"Standard storage class parity",
&[],
subsystems::CLUSTER_CONFIG
);
}
crates/obs/src/metrics/cluster_erasure_set.rs
+97
View File
@@ -0,0 +1,97 @@
/// Erasure code set related metric descriptors
use crate::metrics::{new_gauge_md, subsystems, MetricDescriptor, MetricName};
/// The label for the pool ID
pub const POOL_ID_L: &str = "pool_id";
/// The label for the pool ID
pub const SET_ID_L: &str = "set_id";
lazy_static::lazy_static! {
pub static ref ERASURE_SET_OVERALL_WRITE_QUORUM_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetOverallWriteQuorum,
"Overall write quorum across pools and sets",
&[],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_OVERALL_HEALTH_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetOverallHealth,
"Overall health across pools and sets (1=healthy, 0=unhealthy)",
&[],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_READ_QUORUM_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetReadQuorum,
"Read quorum for the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_WRITE_QUORUM_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetWriteQuorum,
"Write quorum for the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_ONLINE_DRIVES_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetOnlineDrivesCount,
"Count of online drives in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_HEALING_DRIVES_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetHealingDrivesCount,
"Count of healing drives in the erasure set in a pool",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_HEALTH_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetHealth,
"Health of the erasure set in a pool (1=healthy, 0=unhealthy)",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_READ_TOLERANCE_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetReadTolerance,
"No of drive failures that can be tolerated without disrupting read operations",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_WRITE_TOLERANCE_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetWriteTolerance,
"No of drive failures that can be tolerated without disrupting write operations",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_READ_HEALTH_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetReadHealth,
"Health of the erasure set in a pool for read operations (1=healthy, 0=unhealthy)",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
pub static ref ERASURE_SET_WRITE_HEALTH_MD: MetricDescriptor =
new_gauge_md(
MetricName::ErasureSetWriteHealth,
"Health of the erasure set in a pool for write operations (1=healthy, 0=unhealthy)",
&[POOL_ID_L, SET_ID_L],
subsystems::CLUSTER_ERASURE_SET
);
}
crates/obs/src/metrics/cluster_health.rs
+28
View File
@@ -0,0 +1,28 @@
/// Cluster health-related metric descriptors
use crate::metrics::{new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref HEALTH_DRIVES_OFFLINE_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::HealthDrivesOfflineCount,
"Count of offline drives in the cluster",
&[],
subsystems::CLUSTER_HEALTH
);
pub static ref HEALTH_DRIVES_ONLINE_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::HealthDrivesOnlineCount,
"Count of online drives in the cluster",
&[],
subsystems::CLUSTER_HEALTH
);
pub static ref HEALTH_DRIVES_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::HealthDrivesCount,
"Count of all drives in the cluster",
&[],
subsystems::CLUSTER_HEALTH
);
}
crates/obs/src/metrics/cluster_iam.rs
+84
View File
@@ -0,0 +1,84 @@
/// IAM related metric descriptors
use crate::metrics::{new_counter_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref LAST_SYNC_DURATION_MILLIS_MD: MetricDescriptor =
new_counter_md(
MetricName::LastSyncDurationMillis,
"Last successful IAM data sync duration in milliseconds",
&[],
subsystems::CLUSTER_IAM
);
pub static ref PLUGIN_AUTHN_SERVICE_FAILED_REQUESTS_MINUTE_MD: MetricDescriptor =
new_counter_md(
MetricName::PluginAuthnServiceFailedRequestsMinute,
"When plugin authentication is configured, returns failed requests count in the last full minute",
&[],
subsystems::CLUSTER_IAM
);
pub static ref PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD: MetricDescriptor =
new_counter_md(
MetricName::PluginAuthnServiceLastFailSeconds,
"When plugin authentication is configured, returns time (in seconds) since the last failed request to the service",
&[],
subsystems::CLUSTER_IAM
);
pub static ref PLUGIN_AUTHN_SERVICE_LAST_SUCC_SECONDS_MD: MetricDescriptor =
new_counter_md(
MetricName::PluginAuthnServiceLastSuccSeconds,
"When plugin authentication is configured, returns time (in seconds) since the last successful request to the service",
&[],
subsystems::CLUSTER_IAM
);
pub static ref PLUGIN_AUTHN_SERVICE_SUCC_AVG_RTT_MS_MINUTE_MD: MetricDescriptor =
new_counter_md(
MetricName::PluginAuthnServiceSuccAvgRttMsMinute,
"When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute",
&[],
subsystems::CLUSTER_IAM
);
pub static ref PLUGIN_AUTHN_SERVICE_SUCC_MAX_RTT_MS_MINUTE_MD: MetricDescriptor =
new_counter_md(
MetricName::PluginAuthnServiceSuccMaxRttMsMinute,
"When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute",
&[],
subsystems::CLUSTER_IAM
);
pub static ref PLUGIN_AUTHN_SERVICE_TOTAL_REQUESTS_MINUTE_MD: MetricDescriptor =
new_counter_md(
MetricName::PluginAuthnServiceTotalRequestsMinute,
"When plugin authentication is configured, returns total requests count in the last full minute",
&[],
subsystems::CLUSTER_IAM
);
pub static ref SINCE_LAST_SYNC_MILLIS_MD: MetricDescriptor =
new_counter_md(
MetricName::SinceLastSyncMillis,
"Time (in milliseconds) since last successful IAM data sync.",
&[],
subsystems::CLUSTER_IAM
);
pub static ref SYNC_FAILURES_MD: MetricDescriptor =
new_counter_md(
MetricName::SyncFailures,
"Number of failed IAM data syncs since server start.",
&[],
subsystems::CLUSTER_IAM
);
pub static ref SYNC_SUCCESSES_MD: MetricDescriptor =
new_counter_md(
MetricName::SyncSuccesses,
"Number of successful IAM data syncs since server start.",
&[],
subsystems::CLUSTER_IAM
);
}
crates/obs/src/metrics/cluster_notification.rs
+36
View File
@@ -0,0 +1,36 @@
/// Notify the relevant metric descriptor
use crate::metrics::{new_counter_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref NOTIFICATION_CURRENT_SEND_IN_PROGRESS_MD: MetricDescriptor =
new_counter_md(
MetricName::NotificationCurrentSendInProgress,
"Number of concurrent async Send calls active to all targets",
&[],
subsystems::NOTIFICATION
);
pub static ref NOTIFICATION_EVENTS_ERRORS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::NotificationEventsErrorsTotal,
"Events that were failed to be sent to the targets",
&[],
subsystems::NOTIFICATION
);
pub static ref NOTIFICATION_EVENTS_SENT_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::NotificationEventsSentTotal,
"Total number of events sent to the targets",
&[],
subsystems::NOTIFICATION
);
pub static ref NOTIFICATION_EVENTS_SKIPPED_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::NotificationEventsSkippedTotal,
"Events that were skipped to be sent to the targets due to the in-memory queue being full",
&[],
subsystems::NOTIFICATION
);
}
crates/obs/src/metrics/cluster_usage.rs
+131
View File
@@ -0,0 +1,131 @@
/// Descriptors of metrics related to cluster object and bucket usage
use crate::metrics::{new_gauge_md, subsystems, MetricDescriptor, MetricName};
/// Bucket labels
pub const BUCKET_LABEL: &str = "bucket";
/// Range labels
pub const RANGE_LABEL: &str = "range";
lazy_static::lazy_static! {
pub static ref USAGE_SINCE_LAST_UPDATE_SECONDS_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageSinceLastUpdateSeconds,
"Time since last update of usage metrics in seconds",
&[],
subsystems::CLUSTER_USAGE_OBJECTS
);
pub static ref USAGE_TOTAL_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageTotalBytes,
"Total cluster usage in bytes",
&[],
subsystems::CLUSTER_USAGE_OBJECTS
);
pub static ref USAGE_OBJECTS_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageObjectsCount,
"Total cluster objects count",
&[],
subsystems::CLUSTER_USAGE_OBJECTS
);
pub static ref USAGE_VERSIONS_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageVersionsCount,
"Total cluster object versions (including delete markers) count",
&[],
subsystems::CLUSTER_USAGE_OBJECTS
);
pub static ref USAGE_DELETE_MARKERS_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageDeleteMarkersCount,
"Total cluster delete markers count",
&[],
subsystems::CLUSTER_USAGE_OBJECTS
);
pub static ref USAGE_BUCKETS_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketsCount,
"Total cluster buckets count",
&[],
subsystems::CLUSTER_USAGE_OBJECTS
);
pub static ref USAGE_OBJECTS_DISTRIBUTION_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageSizeDistribution,
"Cluster object size distribution",
&[RANGE_LABEL],
subsystems::CLUSTER_USAGE_OBJECTS
);
pub static ref USAGE_VERSIONS_DISTRIBUTION_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageVersionCountDistribution,
"Cluster object version count distribution",
&[RANGE_LABEL],
subsystems::CLUSTER_USAGE_OBJECTS
);
}
lazy_static::lazy_static! {
pub static ref USAGE_BUCKET_TOTAL_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketTotalBytes,
"Total bucket size in bytes",
&[BUCKET_LABEL],
subsystems::CLUSTER_USAGE_BUCKETS
);
pub static ref USAGE_BUCKET_OBJECTS_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketObjectsCount,
"Total objects count in bucket",
&[BUCKET_LABEL],
subsystems::CLUSTER_USAGE_BUCKETS
);
pub static ref USAGE_BUCKET_VERSIONS_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketVersionsCount,
"Total object versions (including delete markers) count in bucket",
&[BUCKET_LABEL],
subsystems::CLUSTER_USAGE_BUCKETS
);
pub static ref USAGE_BUCKET_DELETE_MARKERS_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketDeleteMarkersCount,
"Total delete markers count in bucket",
&[BUCKET_LABEL],
subsystems::CLUSTER_USAGE_BUCKETS
);
pub static ref USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketQuotaTotalBytes,
"Total bucket quota in bytes",
&[BUCKET_LABEL],
subsystems::CLUSTER_USAGE_BUCKETS
);
pub static ref USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketObjectSizeDistribution,
"Bucket object size distribution",
&[RANGE_LABEL, BUCKET_LABEL],
subsystems::CLUSTER_USAGE_BUCKETS
);
pub static ref USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD: MetricDescriptor =
new_gauge_md(
MetricName::UsageBucketObjectVersionCountDistribution,
"Bucket object version count distribution",
&[RANGE_LABEL, BUCKET_LABEL],
subsystems::CLUSTER_USAGE_BUCKETS
);
}
crates/obs/src/metrics/entry/descriptor.rs
+67
View File
@@ -0,0 +1,67 @@
use crate::metrics::{MetricName, MetricNamespace, MetricSubsystem, MetricType};
use std::collections::HashSet;
/// MetricDescriptor - Metric descriptors
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct MetricDescriptor {
pub name: MetricName,
pub metric_type: MetricType,
pub help: String,
pub variable_labels: Vec<String>,
pub namespace: MetricNamespace,
pub subsystem: MetricSubsystem,
// Internal management values
label_set: Option<HashSet<String>>,
}
impl MetricDescriptor {
/// Create a new metric descriptor
pub fn new(
name: MetricName,
metric_type: MetricType,
help: String,
variable_labels: Vec<String>,
namespace: MetricNamespace,
subsystem: impl Into<MetricSubsystem>, // Modify the parameter type
) -> Self {
Self {
name,
metric_type,
help,
variable_labels,
namespace,
subsystem: subsystem.into(),
label_set: None,
}
}
/// Get the full metric name, including the prefix and formatting path
#[allow(dead_code)]
pub fn get_full_metric_name(&self) -> String {
let prefix = self.metric_type.as_prom();
let namespace = self.namespace.as_str();
let formatted_subsystem = self.subsystem.as_str();
format!("{}{}_{}_{}", prefix, namespace, formatted_subsystem, self.name.as_str())
}
/// check whether the label is in the label set
#[allow(dead_code)]
pub fn has_label(&mut self, label: &str) -> bool {
self.get_label_set().contains(label)
}
/// Gets a collection of tags and creates them if they don't exist
pub fn get_label_set(&mut self) -> &HashSet<String> {
if self.label_set.is_none() {
let mut set = HashSet::with_capacity(self.variable_labels.len());
for label in &self.variable_labels {
set.insert(label.clone());
}
self.label_set = Some(set);
}
self.label_set.as_ref().unwrap()
}
}
crates/obs/src/metrics/entry/metric_name.rs
+666
View File
@@ -0,0 +1,666 @@
/// The metric name is the individual name of the metric
#[allow(dead_code)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MetricName {
// The generic metric name
AuthTotal,
CanceledTotal,
ErrorsTotal,
HeaderTotal,
HealTotal,
HitsTotal,
InflightTotal,
InvalidTotal,
LimitTotal,
MissedTotal,
WaitingTotal,
IncomingTotal,
ObjectTotal,
VersionTotal,
DeleteMarkerTotal,
OfflineTotal,
OnlineTotal,
OpenTotal,
ReadTotal,
TimestampTotal,
WriteTotal,
Total,
FreeInodes,
// Failure statistical metrics
LastMinFailedCount,
LastMinFailedBytes,
LastHourFailedCount,
LastHourFailedBytes,
TotalFailedCount,
TotalFailedBytes,
// Worker metrics
CurrActiveWorkers,
AvgActiveWorkers,
MaxActiveWorkers,
RecentBacklogCount,
CurrInQueueCount,
CurrInQueueBytes,
ReceivedCount,
SentCount,
CurrTransferRate,
AvgTransferRate,
MaxTransferRate,
CredentialErrors,
// Link latency metrics
CurrLinkLatency,
AvgLinkLatency,
MaxLinkLatency,
// Link status metrics
LinkOnline,
LinkOfflineDuration,
LinkDowntimeTotalDuration,
// Queue metrics
AvgInQueueCount,
AvgInQueueBytes,
MaxInQueueCount,
MaxInQueueBytes,
// Proxy request metrics
ProxiedGetRequestsTotal,
ProxiedHeadRequestsTotal,
ProxiedPutTaggingRequestsTotal,
ProxiedGetTaggingRequestsTotal,
ProxiedDeleteTaggingRequestsTotal,
ProxiedGetRequestsFailures,
ProxiedHeadRequestsFailures,
ProxiedPutTaggingRequestFailures,
ProxiedGetTaggingRequestFailures,
ProxiedDeleteTaggingRequestFailures,
// Byte-related metrics
FreeBytes,
ReadBytes,
RcharBytes,
ReceivedBytes,
LatencyMilliSec,
SentBytes,
TotalBytes,
UsedBytes,
WriteBytes,
WcharBytes,
// Latency metrics
LatencyMicroSec,
LatencyNanoSec,
// Information metrics
CommitInfo,
UsageInfo,
VersionInfo,
// Distribution metrics
SizeDistribution,
VersionDistribution,
TtfbDistribution,
TtlbDistribution,
// Time metrics
LastActivityTime,
StartTime,
UpTime,
Memory,
Vmemory,
Cpu,
// Expiration and conversion metrics
ExpiryMissedTasks,
ExpiryMissedFreeVersions,
ExpiryMissedTierJournalTasks,
ExpiryNumWorkers,
TransitionMissedTasks,
TransitionedBytes,
TransitionedObjects,
TransitionedVersions,
//Tier request metrics
TierRequestsSuccess,
TierRequestsFailure,
// KMS metrics
KmsOnline,
KmsRequestsSuccess,
KmsRequestsError,
KmsRequestsFail,
KmsUptime,
// Webhook metrics
WebhookOnline,
// API 拒绝指标
ApiRejectedAuthTotal,
ApiRejectedHeaderTotal,
ApiRejectedTimestampTotal,
ApiRejectedInvalidTotal,
//API request metrics
ApiRequestsWaitingTotal,
ApiRequestsIncomingTotal,
ApiRequestsInFlightTotal,
ApiRequestsTotal,
ApiRequestsErrorsTotal,
ApiRequests5xxErrorsTotal,
ApiRequests4xxErrorsTotal,
ApiRequestsCanceledTotal,
// API distribution metrics
ApiRequestsTTFBSecondsDistribution,
// API traffic metrics
ApiTrafficSentBytes,
ApiTrafficRecvBytes,
// Audit metrics
AuditFailedMessages,
AuditTargetQueueLength,
AuditTotalMessages,
// Metrics related to cluster configurations
ConfigRRSParity,
ConfigStandardParity,
// Erasure coding set related metrics
ErasureSetOverallWriteQuorum,
ErasureSetOverallHealth,
ErasureSetReadQuorum,
ErasureSetWriteQuorum,
ErasureSetOnlineDrivesCount,
ErasureSetHealingDrivesCount,
ErasureSetHealth,
ErasureSetReadTolerance,
ErasureSetWriteTolerance,
ErasureSetReadHealth,
ErasureSetWriteHealth,
// Cluster health-related metrics
HealthDrivesOfflineCount,
HealthDrivesOnlineCount,
HealthDrivesCount,
// IAM-related metrics
LastSyncDurationMillis,
PluginAuthnServiceFailedRequestsMinute,
PluginAuthnServiceLastFailSeconds,
PluginAuthnServiceLastSuccSeconds,
PluginAuthnServiceSuccAvgRttMsMinute,
PluginAuthnServiceSuccMaxRttMsMinute,
PluginAuthnServiceTotalRequestsMinute,
SinceLastSyncMillis,
SyncFailures,
SyncSuccesses,
// Notify relevant metrics
NotificationCurrentSendInProgress,
NotificationEventsErrorsTotal,
NotificationEventsSentTotal,
NotificationEventsSkippedTotal,
// Metrics related to the usage of cluster objects
UsageSinceLastUpdateSeconds,
UsageTotalBytes,
UsageObjectsCount,
UsageVersionsCount,
UsageDeleteMarkersCount,
UsageBucketsCount,
UsageSizeDistribution,
UsageVersionCountDistribution,
// Metrics related to bucket usage
UsageBucketQuotaTotalBytes,
UsageBucketTotalBytes,
UsageBucketObjectsCount,
UsageBucketVersionsCount,
UsageBucketDeleteMarkersCount,
UsageBucketObjectSizeDistribution,
UsageBucketObjectVersionCountDistribution,
// ILM-related metrics
IlmExpiryPendingTasks,
IlmTransitionActiveTasks,
IlmTransitionPendingTasks,
IlmTransitionMissedImmediateTasks,
IlmVersionsScanned,
// Webhook logs
WebhookQueueLength,
WebhookTotalMessages,
WebhookFailedMessages,
// Copy the relevant metrics
ReplicationAverageActiveWorkers,
ReplicationAverageQueuedBytes,
ReplicationAverageQueuedCount,
ReplicationAverageDataTransferRate,
ReplicationCurrentActiveWorkers,
ReplicationCurrentDataTransferRate,
ReplicationLastMinuteQueuedBytes,
ReplicationLastMinuteQueuedCount,
ReplicationMaxActiveWorkers,
ReplicationMaxQueuedBytes,
ReplicationMaxQueuedCount,
ReplicationMaxDataTransferRate,
ReplicationRecentBacklogCount,
// Scanner-related metrics
ScannerBucketScansFinished,
ScannerBucketScansStarted,
ScannerDirectoriesScanned,
ScannerObjectsScanned,
ScannerVersionsScanned,
ScannerLastActivitySeconds,
// CPU system-related metrics
SysCPUAvgIdle,
SysCPUAvgIOWait,
SysCPULoad,
SysCPULoadPerc,
SysCPUNice,
SysCPUSteal,
SysCPUSystem,
SysCPUUser,
// Drive-related metrics
DriveUsedBytes,
DriveFreeBytes,
DriveTotalBytes,
DriveUsedInodes,
DriveFreeInodes,
DriveTotalInodes,
DriveTimeoutErrorsTotal,
DriveIOErrorsTotal,
DriveAvailabilityErrorsTotal,
DriveWaitingIO,
DriveAPILatencyMicros,
DriveHealth,
DriveOfflineCount,
DriveOnlineCount,
DriveCount,
// iostat related metrics
DriveReadsPerSec,
DriveReadsKBPerSec,
DriveReadsAwait,
DriveWritesPerSec,
DriveWritesKBPerSec,
DriveWritesAwait,
DrivePercUtil,
// Memory-related metrics
MemTotal,
MemUsed,
MemUsedPerc,
MemFree,
MemBuffers,
MemCache,
MemShared,
MemAvailable,
// Network-related metrics
InternodeErrorsTotal,
InternodeDialErrorsTotal,
InternodeDialAvgTimeNanos,
InternodeSentBytesTotal,
InternodeRecvBytesTotal,
// Process-related metrics
ProcessLocksReadTotal,
ProcessLocksWriteTotal,
ProcessCPUTotalSeconds,
ProcessGoRoutineTotal,
ProcessIORCharBytes,
ProcessIOReadBytes,
ProcessIOWCharBytes,
ProcessIOWriteBytes,
ProcessStartTimeSeconds,
ProcessUptimeSeconds,
ProcessFileDescriptorLimitTotal,
ProcessFileDescriptorOpenTotal,
ProcessSyscallReadTotal,
ProcessSyscallWriteTotal,
ProcessResidentMemoryBytes,
ProcessVirtualMemoryBytes,
ProcessVirtualMemoryMaxBytes,
// Custom metrics
Custom(String),
}
impl MetricName {
#[allow(dead_code)]
pub fn as_str(&self) -> String {
match self {
Self::AuthTotal => "auth_total".to_string(),
Self::CanceledTotal => "canceled_total".to_string(),
Self::ErrorsTotal => "errors_total".to_string(),
Self::HeaderTotal => "header_total".to_string(),
Self::HealTotal => "heal_total".to_string(),
Self::HitsTotal => "hits_total".to_string(),
Self::InflightTotal => "inflight_total".to_string(),
Self::InvalidTotal => "invalid_total".to_string(),
Self::LimitTotal => "limit_total".to_string(),
Self::MissedTotal => "missed_total".to_string(),
Self::WaitingTotal => "waiting_total".to_string(),
Self::IncomingTotal => "incoming_total".to_string(),
Self::ObjectTotal => "object_total".to_string(),
Self::VersionTotal => "version_total".to_string(),
Self::DeleteMarkerTotal => "deletemarker_total".to_string(),
Self::OfflineTotal => "offline_total".to_string(),
Self::OnlineTotal => "online_total".to_string(),
Self::OpenTotal => "open_total".to_string(),
Self::ReadTotal => "read_total".to_string(),
Self::TimestampTotal => "timestamp_total".to_string(),
Self::WriteTotal => "write_total".to_string(),
Self::Total => "total".to_string(),
Self::FreeInodes => "free_inodes".to_string(),
Self::LastMinFailedCount => "last_minute_failed_count".to_string(),
Self::LastMinFailedBytes => "last_minute_failed_bytes".to_string(),
Self::LastHourFailedCount => "last_hour_failed_count".to_string(),
Self::LastHourFailedBytes => "last_hour_failed_bytes".to_string(),
Self::TotalFailedCount => "total_failed_count".to_string(),
Self::TotalFailedBytes => "total_failed_bytes".to_string(),
Self::CurrActiveWorkers => "current_active_workers".to_string(),
Self::AvgActiveWorkers => "average_active_workers".to_string(),
Self::MaxActiveWorkers => "max_active_workers".to_string(),
Self::RecentBacklogCount => "recent_backlog_count".to_string(),
Self::CurrInQueueCount => "last_minute_queued_count".to_string(),
Self::CurrInQueueBytes => "last_minute_queued_bytes".to_string(),
Self::ReceivedCount => "received_count".to_string(),
Self::SentCount => "sent_count".to_string(),
Self::CurrTransferRate => "current_transfer_rate".to_string(),
Self::AvgTransferRate => "average_transfer_rate".to_string(),
Self::MaxTransferRate => "max_transfer_rate".to_string(),
Self::CredentialErrors => "credential_errors".to_string(),
Self::CurrLinkLatency => "current_link_latency_ms".to_string(),
Self::AvgLinkLatency => "average_link_latency_ms".to_string(),
Self::MaxLinkLatency => "max_link_latency_ms".to_string(),
Self::LinkOnline => "link_online".to_string(),
Self::LinkOfflineDuration => "link_offline_duration_seconds".to_string(),
Self::LinkDowntimeTotalDuration => "link_downtime_duration_seconds".to_string(),
Self::AvgInQueueCount => "average_queued_count".to_string(),
Self::AvgInQueueBytes => "average_queued_bytes".to_string(),
Self::MaxInQueueCount => "max_queued_count".to_string(),
Self::MaxInQueueBytes => "max_queued_bytes".to_string(),
Self::ProxiedGetRequestsTotal => "proxied_get_requests_total".to_string(),
Self::ProxiedHeadRequestsTotal => "proxied_head_requests_total".to_string(),
Self::ProxiedPutTaggingRequestsTotal => "proxied_put_tagging_requests_total".to_string(),
Self::ProxiedGetTaggingRequestsTotal => "proxied_get_tagging_requests_total".to_string(),
Self::ProxiedDeleteTaggingRequestsTotal => "proxied_delete_tagging_requests_total".to_string(),
Self::ProxiedGetRequestsFailures => "proxied_get_requests_failures".to_string(),
Self::ProxiedHeadRequestsFailures => "proxied_head_requests_failures".to_string(),
Self::ProxiedPutTaggingRequestFailures => "proxied_put_tagging_requests_failures".to_string(),
Self::ProxiedGetTaggingRequestFailures => "proxied_get_tagging_requests_failures".to_string(),
Self::ProxiedDeleteTaggingRequestFailures => "proxied_delete_tagging_requests_failures".to_string(),
Self::FreeBytes => "free_bytes".to_string(),
Self::ReadBytes => "read_bytes".to_string(),
Self::RcharBytes => "rchar_bytes".to_string(),
Self::ReceivedBytes => "received_bytes".to_string(),
Self::LatencyMilliSec => "latency_ms".to_string(),
Self::SentBytes => "sent_bytes".to_string(),
Self::TotalBytes => "total_bytes".to_string(),
Self::UsedBytes => "used_bytes".to_string(),
Self::WriteBytes => "write_bytes".to_string(),
Self::WcharBytes => "wchar_bytes".to_string(),
Self::LatencyMicroSec => "latency_us".to_string(),
Self::LatencyNanoSec => "latency_ns".to_string(),
Self::CommitInfo => "commit_info".to_string(),
Self::UsageInfo => "usage_info".to_string(),
Self::VersionInfo => "version_info".to_string(),
Self::SizeDistribution => "size_distribution".to_string(),
Self::VersionDistribution => "version_distribution".to_string(),
Self::TtfbDistribution => "seconds_distribution".to_string(),
Self::TtlbDistribution => "ttlb_seconds_distribution".to_string(),
Self::LastActivityTime => "last_activity_nano_seconds".to_string(),
Self::StartTime => "starttime_seconds".to_string(),
Self::UpTime => "uptime_seconds".to_string(),
Self::Memory => "resident_memory_bytes".to_string(),
Self::Vmemory => "virtual_memory_bytes".to_string(),
Self::Cpu => "cpu_total_seconds".to_string(),
Self::ExpiryMissedTasks => "expiry_missed_tasks".to_string(),
Self::ExpiryMissedFreeVersions => "expiry_missed_freeversions".to_string(),
Self::ExpiryMissedTierJournalTasks => "expiry_missed_tierjournal_tasks".to_string(),
Self::ExpiryNumWorkers => "expiry_num_workers".to_string(),
Self::TransitionMissedTasks => "transition_missed_immediate_tasks".to_string(),
Self::TransitionedBytes => "transitioned_bytes".to_string(),
Self::TransitionedObjects => "transitioned_objects".to_string(),
Self::TransitionedVersions => "transitioned_versions".to_string(),
Self::TierRequestsSuccess => "requests_success".to_string(),
Self::TierRequestsFailure => "requests_failure".to_string(),
Self::KmsOnline => "online".to_string(),
Self::KmsRequestsSuccess => "request_success".to_string(),
Self::KmsRequestsError => "request_error".to_string(),
Self::KmsRequestsFail => "request_failure".to_string(),
Self::KmsUptime => "uptime".to_string(),
Self::WebhookOnline => "online".to_string(),
Self::ApiRejectedAuthTotal => "rejected_auth_total".to_string(),
Self::ApiRejectedHeaderTotal => "rejected_header_total".to_string(),
Self::ApiRejectedTimestampTotal => "rejected_timestamp_total".to_string(),
Self::ApiRejectedInvalidTotal => "rejected_invalid_total".to_string(),
Self::ApiRequestsWaitingTotal => "waiting_total".to_string(),
Self::ApiRequestsIncomingTotal => "incoming_total".to_string(),
Self::ApiRequestsInFlightTotal => "inflight_total".to_string(),
Self::ApiRequestsTotal => "total".to_string(),
Self::ApiRequestsErrorsTotal => "errors_total".to_string(),
Self::ApiRequests5xxErrorsTotal => "5xx_errors_total".to_string(),
Self::ApiRequests4xxErrorsTotal => "4xx_errors_total".to_string(),
Self::ApiRequestsCanceledTotal => "canceled_total".to_string(),
Self::ApiRequestsTTFBSecondsDistribution => "ttfb_seconds_distribution".to_string(),
Self::ApiTrafficSentBytes => "traffic_sent_bytes".to_string(),
Self::ApiTrafficRecvBytes => "traffic_received_bytes".to_string(),
Self::AuditFailedMessages => "failed_messages".to_string(),
Self::AuditTargetQueueLength => "target_queue_length".to_string(),
Self::AuditTotalMessages => "total_messages".to_string(),
// metrics related to cluster configurations
Self::ConfigRRSParity => "rrs_parity".to_string(),
Self::ConfigStandardParity => "standard_parity".to_string(),
// Erasure coding set related metrics
Self::ErasureSetOverallWriteQuorum => "overall_write_quorum".to_string(),
Self::ErasureSetOverallHealth => "overall_health".to_string(),
Self::ErasureSetReadQuorum => "read_quorum".to_string(),
Self::ErasureSetWriteQuorum => "write_quorum".to_string(),
Self::ErasureSetOnlineDrivesCount => "online_drives_count".to_string(),
Self::ErasureSetHealingDrivesCount => "healing_drives_count".to_string(),
Self::ErasureSetHealth => "health".to_string(),
Self::ErasureSetReadTolerance => "read_tolerance".to_string(),
Self::ErasureSetWriteTolerance => "write_tolerance".to_string(),
Self::ErasureSetReadHealth => "read_health".to_string(),
Self::ErasureSetWriteHealth => "write_health".to_string(),
// Cluster health-related metrics
Self::HealthDrivesOfflineCount => "drives_offline_count".to_string(),
Self::HealthDrivesOnlineCount => "drives_online_count".to_string(),
Self::HealthDrivesCount => "drives_count".to_string(),
// IAM-related metrics
Self::LastSyncDurationMillis => "last_sync_duration_millis".to_string(),
Self::PluginAuthnServiceFailedRequestsMinute => "plugin_authn_service_failed_requests_minute".to_string(),
Self::PluginAuthnServiceLastFailSeconds => "plugin_authn_service_last_fail_seconds".to_string(),
Self::PluginAuthnServiceLastSuccSeconds => "plugin_authn_service_last_succ_seconds".to_string(),
Self::PluginAuthnServiceSuccAvgRttMsMinute => "plugin_authn_service_succ_avg_rtt_ms_minute".to_string(),
Self::PluginAuthnServiceSuccMaxRttMsMinute => "plugin_authn_service_succ_max_rtt_ms_minute".to_string(),
Self::PluginAuthnServiceTotalRequestsMinute => "plugin_authn_service_total_requests_minute".to_string(),
Self::SinceLastSyncMillis => "since_last_sync_millis".to_string(),
Self::SyncFailures => "sync_failures".to_string(),
Self::SyncSuccesses => "sync_successes".to_string(),
// Notify relevant metrics
Self::NotificationCurrentSendInProgress => "current_send_in_progress".to_string(),
Self::NotificationEventsErrorsTotal => "events_errors_total".to_string(),
Self::NotificationEventsSentTotal => "events_sent_total".to_string(),
Self::NotificationEventsSkippedTotal => "events_skipped_total".to_string(),
// Metrics related to the usage of cluster objects
Self::UsageSinceLastUpdateSeconds => "since_last_update_seconds".to_string(),
Self::UsageTotalBytes => "total_bytes".to_string(),
Self::UsageObjectsCount => "count".to_string(),
Self::UsageVersionsCount => "versions_count".to_string(),
Self::UsageDeleteMarkersCount => "delete_markers_count".to_string(),
Self::UsageBucketsCount => "buckets_count".to_string(),
Self::UsageSizeDistribution => "size_distribution".to_string(),
Self::UsageVersionCountDistribution => "version_count_distribution".to_string(),
// Metrics related to bucket usage
Self::UsageBucketQuotaTotalBytes => "quota_total_bytes".to_string(),
Self::UsageBucketTotalBytes => "total_bytes".to_string(),
Self::UsageBucketObjectsCount => "objects_count".to_string(),
Self::UsageBucketVersionsCount => "versions_count".to_string(),
Self::UsageBucketDeleteMarkersCount => "delete_markers_count".to_string(),
Self::UsageBucketObjectSizeDistribution => "object_size_distribution".to_string(),
Self::UsageBucketObjectVersionCountDistribution => "object_version_count_distribution".to_string(),
// ILM-related metrics
Self::IlmExpiryPendingTasks => "expiry_pending_tasks".to_string(),
Self::IlmTransitionActiveTasks => "transition_active_tasks".to_string(),
Self::IlmTransitionPendingTasks => "transition_pending_tasks".to_string(),
Self::IlmTransitionMissedImmediateTasks => "transition_missed_immediate_tasks".to_string(),
Self::IlmVersionsScanned => "versions_scanned".to_string(),
// Webhook logs
Self::WebhookQueueLength => "queue_length".to_string(),
Self::WebhookTotalMessages => "total_messages".to_string(),
Self::WebhookFailedMessages => "failed_messages".to_string(),
// Copy the relevant metrics
Self::ReplicationAverageActiveWorkers => "average_active_workers".to_string(),
Self::ReplicationAverageQueuedBytes => "average_queued_bytes".to_string(),
Self::ReplicationAverageQueuedCount => "average_queued_count".to_string(),
Self::ReplicationAverageDataTransferRate => "average_data_transfer_rate".to_string(),
Self::ReplicationCurrentActiveWorkers => "current_active_workers".to_string(),
Self::ReplicationCurrentDataTransferRate => "current_data_transfer_rate".to_string(),
Self::ReplicationLastMinuteQueuedBytes => "last_minute_queued_bytes".to_string(),
Self::ReplicationLastMinuteQueuedCount => "last_minute_queued_count".to_string(),
Self::ReplicationMaxActiveWorkers => "max_active_workers".to_string(),
Self::ReplicationMaxQueuedBytes => "max_queued_bytes".to_string(),
Self::ReplicationMaxQueuedCount => "max_queued_count".to_string(),
Self::ReplicationMaxDataTransferRate => "max_data_transfer_rate".to_string(),
Self::ReplicationRecentBacklogCount => "recent_backlog_count".to_string(),
// Scanner-related metrics
Self::ScannerBucketScansFinished => "bucket_scans_finished".to_string(),
Self::ScannerBucketScansStarted => "bucket_scans_started".to_string(),
Self::ScannerDirectoriesScanned => "directories_scanned".to_string(),
Self::ScannerObjectsScanned => "objects_scanned".to_string(),
Self::ScannerVersionsScanned => "versions_scanned".to_string(),
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
// CPU system-related metrics
Self::SysCPUAvgIdle => "avg_idle".to_string(),
Self::SysCPUAvgIOWait => "avg_iowait".to_string(),
Self::SysCPULoad => "load".to_string(),
Self::SysCPULoadPerc => "load_perc".to_string(),
Self::SysCPUNice => "nice".to_string(),
Self::SysCPUSteal => "steal".to_string(),
Self::SysCPUSystem => "system".to_string(),
Self::SysCPUUser => "user".to_string(),
// Drive-related metrics
Self::DriveUsedBytes => "used_bytes".to_string(),
Self::DriveFreeBytes => "free_bytes".to_string(),
Self::DriveTotalBytes => "total_bytes".to_string(),
Self::DriveUsedInodes => "used_inodes".to_string(),
Self::DriveFreeInodes => "free_inodes".to_string(),
Self::DriveTotalInodes => "total_inodes".to_string(),
Self::DriveTimeoutErrorsTotal => "timeout_errors_total".to_string(),
Self::DriveIOErrorsTotal => "io_errors_total".to_string(),
Self::DriveAvailabilityErrorsTotal => "availability_errors_total".to_string(),
Self::DriveWaitingIO => "waiting_io".to_string(),
Self::DriveAPILatencyMicros => "api_latency_micros".to_string(),
Self::DriveHealth => "health".to_string(),
Self::DriveOfflineCount => "offline_count".to_string(),
Self::DriveOnlineCount => "online_count".to_string(),
Self::DriveCount => "count".to_string(),
// iostat related metrics
Self::DriveReadsPerSec => "reads_per_sec".to_string(),
Self::DriveReadsKBPerSec => "reads_kb_per_sec".to_string(),
Self::DriveReadsAwait => "reads_await".to_string(),
Self::DriveWritesPerSec => "writes_per_sec".to_string(),
Self::DriveWritesKBPerSec => "writes_kb_per_sec".to_string(),
Self::DriveWritesAwait => "writes_await".to_string(),
Self::DrivePercUtil => "perc_util".to_string(),
// Memory-related metrics
Self::MemTotal => "total".to_string(),
Self::MemUsed => "used".to_string(),
Self::MemUsedPerc => "used_perc".to_string(),
Self::MemFree => "free".to_string(),
Self::MemBuffers => "buffers".to_string(),
Self::MemCache => "cache".to_string(),
Self::MemShared => "shared".to_string(),
Self::MemAvailable => "available".to_string(),
// Network-related metrics
Self::InternodeErrorsTotal => "errors_total".to_string(),
Self::InternodeDialErrorsTotal => "dial_errors_total".to_string(),
Self::InternodeDialAvgTimeNanos => "dial_avg_time_nanos".to_string(),
Self::InternodeSentBytesTotal => "sent_bytes_total".to_string(),
Self::InternodeRecvBytesTotal => "recv_bytes_total".to_string(),
// Process-related metrics
Self::ProcessLocksReadTotal => "locks_read_total".to_string(),
Self::ProcessLocksWriteTotal => "locks_write_total".to_string(),
Self::ProcessCPUTotalSeconds => "cpu_total_seconds".to_string(),
Self::ProcessGoRoutineTotal => "go_routine_total".to_string(),
Self::ProcessIORCharBytes => "io_rchar_bytes".to_string(),
Self::ProcessIOReadBytes => "io_read_bytes".to_string(),
Self::ProcessIOWCharBytes => "io_wchar_bytes".to_string(),
Self::ProcessIOWriteBytes => "io_write_bytes".to_string(),
Self::ProcessStartTimeSeconds => "start_time_seconds".to_string(),
Self::ProcessUptimeSeconds => "uptime_seconds".to_string(),
Self::ProcessFileDescriptorLimitTotal => "file_descriptor_limit_total".to_string(),
Self::ProcessFileDescriptorOpenTotal => "file_descriptor_open_total".to_string(),
Self::ProcessSyscallReadTotal => "syscall_read_total".to_string(),
Self::ProcessSyscallWriteTotal => "syscall_write_total".to_string(),
Self::ProcessResidentMemoryBytes => "resident_memory_bytes".to_string(),
Self::ProcessVirtualMemoryBytes => "virtual_memory_bytes".to_string(),
Self::ProcessVirtualMemoryMaxBytes => "virtual_memory_max_bytes".to_string(),
Self::Custom(name) => name.clone(),
}
}
}
impl From<String> for MetricName {
fn from(s: String) -> Self {
Self::Custom(s)
}
}
impl From<&str> for MetricName {
fn from(s: &str) -> Self {
Self::Custom(s.to_string())
}
}
crates/obs/src/metrics/entry/metric_type.rs
+31
View File
@@ -0,0 +1,31 @@
/// MetricType - Indicates the type of indicator
#[allow(dead_code)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MetricType {
Counter,
Gauge,
Histogram,
}
impl MetricType {
/// convert the metric type to a string representation
#[allow(dead_code)]
pub fn as_str(&self) -> &'static str {
match self {
Self::Counter => "counter",
Self::Gauge => "gauge",
Self::Histogram => "histogram",
}
}
/// Convert the metric type to the Prometheus value type
/// In a Rust implementation, this might return the corresponding Prometheus Rust client type
#[allow(dead_code)]
pub fn as_prom(&self) -> &'static str {
match self {
Self::Counter => "counter.",
Self::Gauge => "gauge.",
Self::Histogram => "histogram.", // Histograms still use the counter value in Prometheus
}
}
}
crates/obs/src/metrics/entry/mod.rs
+115
View File
@@ -0,0 +1,115 @@
use crate::metrics::{MetricDescriptor, MetricName, MetricNamespace, MetricSubsystem, MetricType};
pub(crate) mod descriptor;
pub(crate) mod metric_name;
pub(crate) mod metric_type;
pub(crate) mod namespace;
mod path_utils;
pub(crate) mod subsystem;
/// Create a new counter metric descriptor
pub fn new_counter_md(
name: impl Into<MetricName>,
help: impl Into<String>,
labels: &[&str],
subsystem: impl Into<MetricSubsystem>,
) -> MetricDescriptor {
MetricDescriptor::new(
name.into(),
MetricType::Counter,
help.into(),
labels.iter().map(|&s| s.to_string()).collect(),
MetricNamespace::RustFS,
subsystem,
)
}
/// create a new dashboard metric descriptor
pub fn new_gauge_md(
name: impl Into<MetricName>,
help: impl Into<String>,
labels: &[&str],
subsystem: impl Into<MetricSubsystem>,
) -> MetricDescriptor {
MetricDescriptor::new(
name.into(),
MetricType::Gauge,
help.into(),
labels.iter().map(|&s| s.to_string()).collect(),
MetricNamespace::RustFS,
subsystem,
)
}
/// create a new histogram indicator descriptor
#[allow(dead_code)]
pub fn new_histogram_md(
name: impl Into<MetricName>,
help: impl Into<String>,
labels: &[&str],
subsystem: impl Into<MetricSubsystem>,
) -> MetricDescriptor {
MetricDescriptor::new(
name.into(),
MetricType::Histogram,
help.into(),
labels.iter().map(|&s| s.to_string()).collect(),
MetricNamespace::RustFS,
subsystem,
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::metrics::subsystems;
#[test]
fn test_new_histogram_md() {
// create a histogram indicator descriptor
let histogram_md = new_histogram_md(
MetricName::TtfbDistribution,
"test the response time distribution",
&["api", "method", "le"],
subsystems::API_REQUESTS,
);
// verify that the metric type is correct
assert_eq!(histogram_md.metric_type, MetricType::Histogram);
// verify that the metric name is correct
assert_eq!(histogram_md.name.as_str(), "seconds_distribution");
// verify that the help information is correct
assert_eq!(histogram_md.help, "test the response time distribution");
// Verify that the label is correct
assert_eq!(histogram_md.variable_labels.len(), 3);
assert!(histogram_md.variable_labels.contains(&"api".to_string()));
assert!(histogram_md.variable_labels.contains(&"method".to_string()));
assert!(histogram_md.variable_labels.contains(&"le".to_string()));
// Verify that the namespace is correct
assert_eq!(histogram_md.namespace, MetricNamespace::RustFS);
// Verify that the subsystem is correct
assert_eq!(histogram_md.subsystem, MetricSubsystem::ApiRequests);
// Verify that the full metric name generated is formatted correctly
assert_eq!(histogram_md.get_full_metric_name(), "histogram.rustfs_api_requests_seconds_distribution");
// Tests use custom subsystems
let custom_histogram_md = new_histogram_md(
"custom_latency_distribution",
"custom latency distribution",
&["endpoint", "le"],
MetricSubsystem::new("/custom/path-metrics"),
);
// Verify the custom name and subsystem
assert_eq!(
custom_histogram_md.get_full_metric_name(),
"histogram.rustfs_custom_path_metrics_custom_latency_distribution"
);
}
}
crates/obs/src/metrics/entry/namespace.rs
+14
View File
@@ -0,0 +1,14 @@
/// The metric namespace, which represents the top-level grouping of the metric
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MetricNamespace {
RustFS,
}
impl MetricNamespace {
#[allow(dead_code)]
pub fn as_str(&self) -> &'static str {
match self {
Self::RustFS => "rustfs",
}
}
}
crates/obs/src/metrics/entry/path_utils.rs
+19
View File
@@ -0,0 +1,19 @@
/// Format the path to the metric name format
/// Replace '/' and '-' with '_'
#[allow(dead_code)]
pub fn format_path_to_metric_name(path: &str) -> String {
path.trim_start_matches('/').replace(['/', '-'], "_")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_format_path_to_metric_name() {
assert_eq!(format_path_to_metric_name("/api/requests"), "api_requests");
assert_eq!(format_path_to_metric_name("/system/network/internode"), "system_network_internode");
assert_eq!(format_path_to_metric_name("/bucket-api"), "bucket_api");
assert_eq!(format_path_to_metric_name("cluster/health"), "cluster_health");
}
}
crates/obs/src/metrics/entry/subsystem.rs
+232
View File
@@ -0,0 +1,232 @@
use crate::metrics::entry::path_utils::format_path_to_metric_name;
/// The metrics subsystem is a subgroup of metrics within a namespace
/// The metrics subsystem, which represents a subgroup of metrics within a namespace
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum MetricSubsystem {
// API related subsystems
ApiRequests,
// bucket related subsystems
BucketApi,
BucketReplication,
// system related subsystems
SystemNetworkInternode,
SystemDrive,
SystemMemory,
SystemCpu,
SystemProcess,
// debug related subsystems
DebugGo,
// cluster related subsystems
ClusterHealth,
ClusterUsageObjects,
ClusterUsageBuckets,
ClusterErasureSet,
ClusterIam,
ClusterConfig,
// other service related subsystems
Ilm,
Audit,
LoggerWebhook,
Replication,
Notification,
Scanner,
// Custom paths
Custom(String),
}
impl MetricSubsystem {
/// Gets the original path string
pub fn path(&self) -> &str {
match self {
// api related subsystems
Self::ApiRequests => "/api/requests",
// bucket related subsystems
Self::BucketApi => "/bucket/api",
Self::BucketReplication => "/bucket/replication",
// system related subsystems
Self::SystemNetworkInternode => "/system/network/internode",
Self::SystemDrive => "/system/drive",
Self::SystemMemory => "/system/memory",
Self::SystemCpu => "/system/cpu",
Self::SystemProcess => "/system/process",
// debug related subsystems
Self::DebugGo => "/debug/go",
// cluster related subsystems
Self::ClusterHealth => "/cluster/health",
Self::ClusterUsageObjects => "/cluster/usage/objects",
Self::ClusterUsageBuckets => "/cluster/usage/buckets",
Self::ClusterErasureSet => "/cluster/erasure-set",
Self::ClusterIam => "/cluster/iam",
Self::ClusterConfig => "/cluster/config",
// other service related subsystems
Self::Ilm => "/ilm",
Self::Audit => "/audit",
Self::LoggerWebhook => "/logger/webhook",
Self::Replication => "/replication",
Self::Notification => "/notification",
Self::Scanner => "/scanner",
// Custom paths
Self::Custom(path) => path,
}
}
/// Get the formatted metric name format string
#[allow(dead_code)]
pub fn as_str(&self) -> String {
format_path_to_metric_name(self.path())
}
/// Create a subsystem enumeration from a path string
pub fn from_path(path: &str) -> Self {
match path {
// API-related subsystems
"/api/requests" => Self::ApiRequests,
// Bucket-related subsystems
"/bucket/api" => Self::BucketApi,
"/bucket/replication" => Self::BucketReplication,
// System-related subsystems
"/system/network/internode" => Self::SystemNetworkInternode,
"/system/drive" => Self::SystemDrive,
"/system/memory" => Self::SystemMemory,
"/system/cpu" => Self::SystemCpu,
"/system/process" => Self::SystemProcess,
// Debug related subsystems
"/debug/go" => Self::DebugGo,
// 集群相关子系统
"/cluster/health" => Self::ClusterHealth,
"/cluster/usage/objects" => Self::ClusterUsageObjects,
"/cluster/usage/buckets" => Self::ClusterUsageBuckets,
"/cluster/erasure-set" => Self::ClusterErasureSet,
"/cluster/iam" => Self::ClusterIam,
"/cluster/config" => Self::ClusterConfig,
// 其他服务相关子系统
"/ilm" => Self::Ilm,
"/audit" => Self::Audit,
"/logger/webhook" => Self::LoggerWebhook,
"/replication" => Self::Replication,
"/notification" => Self::Notification,
"/scanner" => Self::Scanner,
// 其他路径作为自定义处理
_ => Self::Custom(path.to_string()),
}
}
/// A convenient way to create custom subsystems directly
#[allow(dead_code)]
pub fn new(path: impl Into<String>) -> Self {
Self::Custom(path.into())
}
}
/// Implementations that facilitate conversion to and from strings
impl From<&str> for MetricSubsystem {
fn from(s: &str) -> Self {
Self::from_path(s)
}
}
impl From<String> for MetricSubsystem {
fn from(s: String) -> Self {
Self::from_path(&s)
}
}
impl std::fmt::Display for MetricSubsystem {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.path())
}
}
#[allow(dead_code)]
pub mod subsystems {
use super::MetricSubsystem;
// cluster base path constant
pub const CLUSTER_BASE_PATH: &str = "/cluster";
// Quick access to constants for each subsystem
pub const API_REQUESTS: MetricSubsystem = MetricSubsystem::ApiRequests;
pub const BUCKET_API: MetricSubsystem = MetricSubsystem::BucketApi;
pub const BUCKET_REPLICATION: MetricSubsystem = MetricSubsystem::BucketReplication;
pub const SYSTEM_NETWORK_INTERNODE: MetricSubsystem = MetricSubsystem::SystemNetworkInternode;
pub const SYSTEM_DRIVE: MetricSubsystem = MetricSubsystem::SystemDrive;
pub const SYSTEM_MEMORY: MetricSubsystem = MetricSubsystem::SystemMemory;
pub const SYSTEM_CPU: MetricSubsystem = MetricSubsystem::SystemCpu;
pub const SYSTEM_PROCESS: MetricSubsystem = MetricSubsystem::SystemProcess;
pub const DEBUG_GO: MetricSubsystem = MetricSubsystem::DebugGo;
pub const CLUSTER_HEALTH: MetricSubsystem = MetricSubsystem::ClusterHealth;
pub const CLUSTER_USAGE_OBJECTS: MetricSubsystem = MetricSubsystem::ClusterUsageObjects;
pub const CLUSTER_USAGE_BUCKETS: MetricSubsystem = MetricSubsystem::ClusterUsageBuckets;
pub const CLUSTER_ERASURE_SET: MetricSubsystem = MetricSubsystem::ClusterErasureSet;
pub const CLUSTER_IAM: MetricSubsystem = MetricSubsystem::ClusterIam;
pub const CLUSTER_CONFIG: MetricSubsystem = MetricSubsystem::ClusterConfig;
pub const ILM: MetricSubsystem = MetricSubsystem::Ilm;
pub const AUDIT: MetricSubsystem = MetricSubsystem::Audit;
pub const LOGGER_WEBHOOK: MetricSubsystem = MetricSubsystem::LoggerWebhook;
pub const REPLICATION: MetricSubsystem = MetricSubsystem::Replication;
pub const NOTIFICATION: MetricSubsystem = MetricSubsystem::Notification;
pub const SCANNER: MetricSubsystem = MetricSubsystem::Scanner;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::metrics::MetricType;
use crate::metrics::{MetricDescriptor, MetricName, MetricNamespace};
#[test]
fn test_metric_subsystem_formatting() {
assert_eq!(MetricSubsystem::ApiRequests.as_str(), "api_requests");
assert_eq!(MetricSubsystem::SystemNetworkInternode.as_str(), "system_network_internode");
assert_eq!(MetricSubsystem::BucketApi.as_str(), "bucket_api");
assert_eq!(MetricSubsystem::ClusterHealth.as_str(), "cluster_health");
// Test custom paths
let custom = MetricSubsystem::new("/custom/path-test");
assert_eq!(custom.as_str(), "custom_path_test");
}
#[test]
fn test_metric_descriptor_name_generation() {
let md = MetricDescriptor::new(
MetricName::ApiRequestsTotal,
MetricType::Counter,
"Test help".to_string(),
vec!["label1".to_string(), "label2".to_string()],
MetricNamespace::RustFS,
MetricSubsystem::ApiRequests,
);
assert_eq!(md.get_full_metric_name(), "counter.rustfs_api_requests_total");
let custom_md = MetricDescriptor::new(
MetricName::Custom("test_metric".to_string()),
MetricType::Gauge,
"Test help".to_string(),
vec!["label1".to_string()],
MetricNamespace::RustFS,
MetricSubsystem::new("/custom/path-with-dash"),
);
assert_eq!(custom_md.get_full_metric_name(), "gauge.rustfs_custom_path_with_dash_test_metric");
}
}
crates/obs/src/metrics/ilm.rs
+44
View File
@@ -0,0 +1,44 @@
/// ILM-related metric descriptors
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref ILM_EXPIRY_PENDING_TASKS_MD: MetricDescriptor =
new_gauge_md(
MetricName::IlmExpiryPendingTasks,
"Number of pending ILM expiry tasks in the queue",
&[],
subsystems::ILM
);
pub static ref ILM_TRANSITION_ACTIVE_TASKS_MD: MetricDescriptor =
new_gauge_md(
MetricName::IlmTransitionActiveTasks,
"Number of active ILM transition tasks",
&[],
subsystems::ILM
);
pub static ref ILM_TRANSITION_PENDING_TASKS_MD: MetricDescriptor =
new_gauge_md(
MetricName::IlmTransitionPendingTasks,
"Number of pending ILM transition tasks in the queue",
&[],
subsystems::ILM
);
pub static ref ILM_TRANSITION_MISSED_IMMEDIATE_TASKS_MD: MetricDescriptor =
new_counter_md(
MetricName::IlmTransitionMissedImmediateTasks,
"Number of missed immediate ILM transition tasks",
&[],
subsystems::ILM
);
pub static ref ILM_VERSIONS_SCANNED_MD: MetricDescriptor =
new_counter_md(
MetricName::IlmVersionsScanned,
"Total number of object versions checked for ILM actions since server start",
&[],
subsystems::ILM
);
}
crates/obs/src/metrics/logger_webhook.rs
+37
View File
@@ -0,0 +1,37 @@
/// A descriptor for metrics related to webhook logs
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
/// Define label constants for webhook metrics
/// name label
pub const NAME_LABEL: &str = "name";
/// endpoint label
pub const ENDPOINT_LABEL: &str = "endpoint";
lazy_static::lazy_static! {
// The label used by all webhook metrics
static ref ALL_WEBHOOK_LABELS: [&'static str; 2] = [NAME_LABEL, ENDPOINT_LABEL];
pub static ref WEBHOOK_FAILED_MESSAGES_MD: MetricDescriptor =
new_counter_md(
MetricName::WebhookFailedMessages,
"Number of messages that failed to send",
&ALL_WEBHOOK_LABELS[..],
subsystems::LOGGER_WEBHOOK
);
pub static ref WEBHOOK_QUEUE_LENGTH_MD: MetricDescriptor =
new_gauge_md(
MetricName::WebhookQueueLength,
"Webhook queue length",
&ALL_WEBHOOK_LABELS[..],
subsystems::LOGGER_WEBHOOK
);
pub static ref WEBHOOK_TOTAL_MESSAGES_MD: MetricDescriptor =
new_counter_md(
MetricName::WebhookTotalMessages,
"Total number of messages sent to this target",
&ALL_WEBHOOK_LABELS[..],
subsystems::LOGGER_WEBHOOK
);
}
crates/obs/src/metrics/mod.rs
+28
View File
@@ -0,0 +1,28 @@
pub(crate) mod audit;
pub(crate) mod bucket;
pub(crate) mod bucket_replication;
pub(crate) mod cluster_config;
pub(crate) mod cluster_erasure_set;
pub(crate) mod cluster_health;
pub(crate) mod cluster_iam;
pub(crate) mod cluster_notification;
pub(crate) mod cluster_usage;
pub(crate) mod entry;
pub(crate) mod ilm;
pub(crate) mod logger_webhook;
pub(crate) mod replication;
pub(crate) mod request;
pub(crate) mod scanner;
pub(crate) mod system_cpu;
pub(crate) mod system_drive;
pub(crate) mod system_memory;
pub(crate) mod system_network;
pub(crate) mod system_process;
pub use entry::descriptor::MetricDescriptor;
pub use entry::metric_name::MetricName;
pub use entry::metric_type::MetricType;
pub use entry::namespace::MetricNamespace;
pub use entry::subsystem::subsystems;
pub use entry::subsystem::MetricSubsystem;
pub use entry::{new_counter_md, new_gauge_md, new_histogram_md};
crates/obs/src/metrics/replication.rs
+108
View File
@@ -0,0 +1,108 @@
/// Copy the relevant metric descriptor
use crate::metrics::{new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationAverageActiveWorkers,
"Average number of active replication workers",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_AVERAGE_QUEUED_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationAverageQueuedBytes,
"Average number of bytes queued for replication since server start",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_AVERAGE_QUEUED_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationAverageQueuedCount,
"Average number of objects queued for replication since server start",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationAverageDataTransferRate,
"Average replication data transfer rate in bytes/sec",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_CURRENT_ACTIVE_WORKERS_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationCurrentActiveWorkers,
"Total number of active replication workers",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationCurrentDataTransferRate,
"Current replication data transfer rate in bytes/sec",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationLastMinuteQueuedBytes,
"Number of bytes queued for replication in the last full minute",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationLastMinuteQueuedCount,
"Number of objects queued for replication in the last full minute",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_MAX_ACTIVE_WORKERS_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationMaxActiveWorkers,
"Maximum number of active replication workers seen since server start",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_MAX_QUEUED_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationMaxQueuedBytes,
"Maximum number of bytes queued for replication since server start",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_MAX_QUEUED_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationMaxQueuedCount,
"Maximum number of objects queued for replication since server start",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_MAX_DATA_TRANSFER_RATE_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationMaxDataTransferRate,
"Maximum replication data transfer rate in bytes/sec seen since server start",
&[],
subsystems::REPLICATION
);
pub static ref REPLICATION_RECENT_BACKLOG_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::ReplicationRecentBacklogCount,
"Total number of objects seen in replication backlog in the last 5 minutes",
&[],
subsystems::REPLICATION
);
}
crates/obs/src/metrics/request.rs
+123
View File
@@ -0,0 +1,123 @@
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName, MetricSubsystem};
lazy_static::lazy_static! {
pub static ref API_REJECTED_AUTH_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRejectedAuthTotal,
"Total number of requests rejected for auth failure",
&["type"],
subsystems::API_REQUESTS
);
pub static ref API_REJECTED_HEADER_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRejectedHeaderTotal,
"Total number of requests rejected for invalid header",
&["type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REJECTED_TIMESTAMP_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRejectedTimestampTotal,
"Total number of requests rejected for invalid timestamp",
&["type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REJECTED_INVALID_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRejectedInvalidTotal,
"Total number of invalid requests",
&["type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_WAITING_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ApiRequestsWaitingTotal,
"Total number of requests in the waiting queue",
&["type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_INCOMING_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ApiRequestsIncomingTotal,
"Total number of incoming requests",
&["type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_IN_FLIGHT_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ApiRequestsInFlightTotal,
"Total number of requests currently in flight",
&["name", "type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequestsTotal,
"Total number of requests",
&["name", "type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_ERRORS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequestsErrorsTotal,
"Total number of requests with (4xx and 5xx) errors",
&["name", "type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_5XX_ERRORS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequests5xxErrorsTotal,
"Total number of requests with 5xx errors",
&["name", "type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_4XX_ERRORS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequests4xxErrorsTotal,
"Total number of requests with 4xx errors",
&["name", "type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_CANCELED_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequestsCanceledTotal,
"Total number of requests canceled by the client",
&["name", "type"],
MetricSubsystem::ApiRequests
);
pub static ref API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiRequestsTTFBSecondsDistribution,
"Distribution of time to first byte across API calls",
&["name", "type", "le"],
MetricSubsystem::ApiRequests
);
pub static ref API_TRAFFIC_SENT_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiTrafficSentBytes,
"Total number of bytes sent",
&["type"],
MetricSubsystem::ApiRequests
);
pub static ref API_TRAFFIC_RECV_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ApiTrafficRecvBytes,
"Total number of bytes received",
&["type"],
MetricSubsystem::ApiRequests
);
}
crates/obs/src/metrics/scanner.rs
+52
View File
@@ -0,0 +1,52 @@
/// Scanner-related metric descriptors
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref SCANNER_BUCKET_SCANS_FINISHED_MD: MetricDescriptor =
new_counter_md(
MetricName::ScannerBucketScansFinished,
"Total number of bucket scans finished since server start",
&[],
subsystems::SCANNER
);
pub static ref SCANNER_BUCKET_SCANS_STARTED_MD: MetricDescriptor =
new_counter_md(
MetricName::ScannerBucketScansStarted,
"Total number of bucket scans started since server start",
&[],
subsystems::SCANNER
);
pub static ref SCANNER_DIRECTORIES_SCANNED_MD: MetricDescriptor =
new_counter_md(
MetricName::ScannerDirectoriesScanned,
"Total number of directories scanned since server start",
&[],
subsystems::SCANNER
);
pub static ref SCANNER_OBJECTS_SCANNED_MD: MetricDescriptor =
new_counter_md(
MetricName::ScannerObjectsScanned,
"Total number of unique objects scanned since server start",
&[],
subsystems::SCANNER
);
pub static ref SCANNER_VERSIONS_SCANNED_MD: MetricDescriptor =
new_counter_md(
MetricName::ScannerVersionsScanned,
"Total number of object versions scanned since server start",
&[],
subsystems::SCANNER
);
pub static ref SCANNER_LAST_ACTIVITY_SECONDS_MD: MetricDescriptor =
new_gauge_md(
MetricName::ScannerLastActivitySeconds,
"Time elapsed (in seconds) since last scan activity.",
&[],
subsystems::SCANNER
);
}
crates/obs/src/metrics/system_cpu.rs
+68
View File
@@ -0,0 +1,68 @@
/// CPU system-related metric descriptors
use crate::metrics::{new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref SYS_CPU_AVG_IDLE_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPUAvgIdle,
"Average CPU idle time",
&[],
subsystems::SYSTEM_CPU
);
pub static ref SYS_CPU_AVG_IOWAIT_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPUAvgIOWait,
"Average CPU IOWait time",
&[],
subsystems::SYSTEM_CPU
);
pub static ref SYS_CPU_LOAD_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPULoad,
"CPU load average 1min",
&[],
subsystems::SYSTEM_CPU
);
pub static ref SYS_CPU_LOAD_PERC_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPULoadPerc,
"CPU load average 1min (percentage)",
&[],
subsystems::SYSTEM_CPU
);
pub static ref SYS_CPU_NICE_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPUNice,
"CPU nice time",
&[],
subsystems::SYSTEM_CPU
);
pub static ref SYS_CPU_STEAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPUSteal,
"CPU steal time",
&[],
subsystems::SYSTEM_CPU
);
pub static ref SYS_CPU_SYSTEM_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPUSystem,
"CPU system time",
&[],
subsystems::SYSTEM_CPU
);
pub static ref SYS_CPU_USER_MD: MetricDescriptor =
new_gauge_md(
MetricName::SysCPUUser,
"CPU user time",
&[],
subsystems::SYSTEM_CPU
);
}
crates/obs/src/metrics/system_drive.rs
+196
View File
@@ -0,0 +1,196 @@
/// Drive-related metric descriptors
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
/// drive related labels
pub const DRIVE_LABEL: &str = "drive";
/// pool index label
pub const POOL_INDEX_LABEL: &str = "pool_index";
/// set index label
pub const SET_INDEX_LABEL: &str = "set_index";
/// drive index label
pub const DRIVE_INDEX_LABEL: &str = "drive_index";
/// API label
pub const API_LABEL: &str = "api";
lazy_static::lazy_static! {
/// All drive-related labels
static ref ALL_DRIVE_LABELS: [&'static str; 4] = [DRIVE_LABEL, POOL_INDEX_LABEL, SET_INDEX_LABEL, DRIVE_INDEX_LABEL];
}
lazy_static::lazy_static! {
pub static ref DRIVE_USED_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveUsedBytes,
"Total storage used on a drive in bytes",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_FREE_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveFreeBytes,
"Total storage free on a drive in bytes",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_TOTAL_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveTotalBytes,
"Total storage available on a drive in bytes",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_USED_INODES_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveUsedInodes,
"Total used inodes on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_FREE_INODES_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveFreeInodes,
"Total free inodes on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_TOTAL_INODES_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveTotalInodes,
"Total inodes available on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_TIMEOUT_ERRORS_MD: MetricDescriptor =
new_counter_md(
MetricName::DriveTimeoutErrorsTotal,
"Total timeout errors on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_IO_ERRORS_MD: MetricDescriptor =
new_counter_md(
MetricName::DriveIOErrorsTotal,
"Total I/O errors on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_AVAILABILITY_ERRORS_MD: MetricDescriptor =
new_counter_md(
MetricName::DriveAvailabilityErrorsTotal,
"Total availability errors (I/O errors, timeouts) on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_WAITING_IO_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveWaitingIO,
"Total waiting I/O operations on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_API_LATENCY_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveAPILatencyMicros,
"Average last minute latency in µs for drive API storage operations",
&[&ALL_DRIVE_LABELS[..], &[API_LABEL]].concat(),
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_HEALTH_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveHealth,
"Drive health (0 = offline, 1 = healthy, 2 = healing)",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_OFFLINE_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveOfflineCount,
"Count of offline drives",
&[],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_ONLINE_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveOnlineCount,
"Count of online drives",
&[],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_COUNT_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveCount,
"Count of all drives",
&[],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_READS_PER_SEC_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveReadsPerSec,
"Reads per second on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_READS_KB_PER_SEC_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveReadsKBPerSec,
"Kilobytes read per second on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_READS_AWAIT_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveReadsAwait,
"Average time for read requests served on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_WRITES_PER_SEC_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveWritesPerSec,
"Writes per second on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_WRITES_KB_PER_SEC_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveWritesKBPerSec,
"Kilobytes written per second on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_WRITES_AWAIT_MD: MetricDescriptor =
new_gauge_md(
MetricName::DriveWritesAwait,
"Average time for write requests served on a drive",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
pub static ref DRIVE_PERC_UTIL_MD: MetricDescriptor =
new_gauge_md(
MetricName::DrivePercUtil,
"Percentage of time the disk was busy",
&ALL_DRIVE_LABELS[..],
subsystems::SYSTEM_DRIVE
);
}
crates/obs/src/metrics/system_memory.rs
+68
View File
@@ -0,0 +1,68 @@
/// Memory-related metric descriptors
use crate::metrics::{new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref MEM_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemTotal,
"Total memory on the node",
&[],
subsystems::SYSTEM_MEMORY
);
pub static ref MEM_USED_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemUsed,
"Used memory on the node",
&[],
subsystems::SYSTEM_MEMORY
);
pub static ref MEM_USED_PERC_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemUsedPerc,
"Used memory percentage on the node",
&[],
subsystems::SYSTEM_MEMORY
);
pub static ref MEM_FREE_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemFree,
"Free memory on the node",
&[],
subsystems::SYSTEM_MEMORY
);
pub static ref MEM_BUFFERS_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemBuffers,
"Buffers memory on the node",
&[],
subsystems::SYSTEM_MEMORY
);
pub static ref MEM_CACHE_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemCache,
"Cache memory on the node",
&[],
subsystems::SYSTEM_MEMORY
);
pub static ref MEM_SHARED_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemShared,
"Shared memory on the node",
&[],
subsystems::SYSTEM_MEMORY
);
pub static ref MEM_AVAILABLE_MD: MetricDescriptor =
new_gauge_md(
MetricName::MemAvailable,
"Available memory on the node",
&[],
subsystems::SYSTEM_MEMORY
);
}
crates/obs/src/metrics/system_network.rs
+44
View File
@@ -0,0 +1,44 @@
/// Network-related metric descriptors
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref INTERNODE_ERRORS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::InternodeErrorsTotal,
"Total number of failed internode calls",
&[],
subsystems::SYSTEM_NETWORK_INTERNODE
);
pub static ref INTERNODE_DIAL_ERRORS_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::InternodeDialErrorsTotal,
"Total number of internode TCP dial timeouts and errors",
&[],
subsystems::SYSTEM_NETWORK_INTERNODE
);
pub static ref INTERNODE_DIAL_AVG_TIME_NANOS_MD: MetricDescriptor =
new_gauge_md(
MetricName::InternodeDialAvgTimeNanos,
"Average dial time of internode TCP calls in nanoseconds",
&[],
subsystems::SYSTEM_NETWORK_INTERNODE
);
pub static ref INTERNODE_SENT_BYTES_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::InternodeSentBytesTotal,
"Total number of bytes sent to other peer nodes",
&[],
subsystems::SYSTEM_NETWORK_INTERNODE
);
pub static ref INTERNODE_RECV_BYTES_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::InternodeRecvBytesTotal,
"Total number of bytes received from other peer nodes",
&[],
subsystems::SYSTEM_NETWORK_INTERNODE
);
}
crates/obs/src/metrics/system_process.rs
+140
View File
@@ -0,0 +1,140 @@
/// process related metric descriptors
use crate::metrics::{new_counter_md, new_gauge_md, subsystems, MetricDescriptor, MetricName};
lazy_static::lazy_static! {
pub static ref PROCESS_LOCKS_READ_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessLocksReadTotal,
"Number of current READ locks on this peer",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_LOCKS_WRITE_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessLocksWriteTotal,
"Number of current WRITE locks on this peer",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_CPU_TOTAL_SECONDS_MD: MetricDescriptor =
new_counter_md(
MetricName::ProcessCPUTotalSeconds,
"Total user and system CPU time spent in seconds",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_GO_ROUTINE_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessGoRoutineTotal,
"Total number of go routines running",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_IO_RCHAR_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProcessIORCharBytes,
"Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_IO_READ_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProcessIOReadBytes,
"Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_IO_WCHAR_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProcessIOWCharBytes,
"Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_IO_WRITE_BYTES_MD: MetricDescriptor =
new_counter_md(
MetricName::ProcessIOWriteBytes,
"Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_START_TIME_SECONDS_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessStartTimeSeconds,
"Start time for RustFS process in seconds since Unix epoc",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_UPTIME_SECONDS_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessUptimeSeconds,
"Uptime for RustFS process in seconds",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessFileDescriptorLimitTotal,
"Limit on total number of open file descriptors for the RustFS Server process",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessFileDescriptorOpenTotal,
"Total number of open file descriptors by the RustFS Server process",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_SYSCALL_READ_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ProcessSyscallReadTotal,
"Total read SysCalls to the kernel. /proc/[pid]/io syscr",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_SYSCALL_WRITE_TOTAL_MD: MetricDescriptor =
new_counter_md(
MetricName::ProcessSyscallWriteTotal,
"Total write SysCalls to the kernel. /proc/[pid]/io syscw",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_RESIDENT_MEMORY_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessResidentMemoryBytes,
"Resident memory size in bytes",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_VIRTUAL_MEMORY_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessVirtualMemoryBytes,
"Virtual memory size in bytes",
&[],
subsystems::SYSTEM_PROCESS
);
pub static ref PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD: MetricDescriptor =
new_gauge_md(
MetricName::ProcessVirtualMemoryMaxBytes,
"Maximum virtual memory size in bytes",
&[],
subsystems::SYSTEM_PROCESS
);
}
crates/obs/src/telemetry.rs
+7 -5
View File
@@ -1,19 +1,19 @@
use crate::OtelConfig;
use flexi_logger::{Age, Cleanup, Criterion, DeferredNow, FileSpec, LogSpecification, Naming, Record, WriteMode, style};
use flexi_logger::{style, Age, Cleanup, Criterion, DeferredNow, FileSpec, LogSpecification, Naming, Record, WriteMode};
use nu_ansi_term::Color;
use opentelemetry::trace::TracerProvider;
use opentelemetry::{KeyValue, global};
use opentelemetry::{global, KeyValue};
use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::logs::SdkLoggerProvider;
use opentelemetry_sdk::{
Resource,
metrics::{MeterProviderBuilder, PeriodicReader, SdkMeterProvider},
trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
Resource,
};
use opentelemetry_semantic_conventions::{
SCHEMA_URL,
attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
SCHEMA_URL,
};
use rustfs_config::{
APP_NAME, DEFAULT_LOG_DIR, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO,
@@ -27,7 +27,8 @@ use tracing::info;
use tracing_error::ErrorLayer;
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::{EnvFilter, Layer, layer::SubscriberExt, util::SubscriberInitExt};
use tracing_subscriber::fmt::time::LocalTime;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
/// A guard object that manages the lifecycle of OpenTelemetry components.
///
@@ -224,6 +225,7 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
let fmt_layer = {
let enable_color = std::io::stdout().is_terminal();
let mut layer = tracing_subscriber::fmt::layer()
.with_timer(LocalTime::rfc_3339())
.with_target(true)
.with_ansi(enable_color)
.with_thread_names(true)
crates/utils/Cargo.toml
+23 -21
View File
@@ -7,18 +7,18 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
base64-simd= { workspace = true , optional = true}
base64-simd = { workspace = true, optional = true }
blake3 = { workspace = true, optional = true }
crc32fast.workspace = true
hex-simd= { workspace = true , optional = true}
hex-simd = { workspace = true, optional = true }
highway = { workspace = true, optional = true }
lazy_static= { workspace = true , optional = true}
lazy_static = { workspace = true, optional = true }
local-ip-address = { workspace = true, optional = true }
md-5 = { workspace = true, optional = true }
netif= { workspace = true , optional = true}
netif = { workspace = true, optional = true }
nix = { workspace = true, optional = true }
regex= { workspace = true, optional = true }
rustfs-config = { workspace = true }
regex = { workspace = true, optional = true }
rustfs-config = { workspace = true, features = ["constants"] }
rustls = { workspace = true, optional = true }
rustls-pemfile = { workspace = true, optional = true }
rustls-pki-types = { workspace = true, optional = true }
@@ -28,20 +28,21 @@ siphasher = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
tokio = { workspace = true, optional = true, features = ["io-util", "macros"] }
tracing = { workspace = true }
url = { workspace = true , optional = true}
flate2 = { workspace = true , optional = true}
brotli = { workspace = true , optional = true}
zstd = { workspace = true , optional = true}
snap = { workspace = true , optional = true}
lz4 = { workspace = true , optional = true}
url = { workspace = true, optional = true }
flate2 = { workspace = true, optional = true }
brotli = { workspace = true, optional = true }
zstd = { workspace = true, optional = true }
snap = { workspace = true, optional = true }
lz4 = { workspace = true, optional = true }
rand = { workspace = true, optional = true }
futures= { workspace = true, optional = true }
transform-stream= { workspace = true, optional = true }
bytes= { workspace = true, optional = true }
futures = { workspace = true, optional = true }
transform-stream = { workspace = true, optional = true }
bytes = { workspace = true, optional = true }
sysinfo = { workspace = true, optional = true }
[dev-dependencies]
tempfile = { workspace = true }
rand = {workspace = true}
rand = { workspace = true }
[target.'cfg(windows)'.dependencies]
winapi = { workspace = true, optional = true, features = ["std", "fileapi", "minwindef", "ntdef", "winnt"] }
@@ -53,13 +54,14 @@ workspace = true
default = ["ip"] # features that are enabled by default
ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies
net = ["ip","dep:url", "dep:netif", "dep:lazy_static", "dep:futures", "dep:transform-stream", "dep:bytes"] # empty network features
net = ["ip", "dep:url", "dep:netif", "dep:lazy_static", "dep:futures", "dep:transform-stream", "dep:bytes"] # empty network features
io = ["dep:tokio"]
path = []
compress =["dep:flate2","dep:brotli","dep:snap","dep:lz4","dep:zstd"]
string = ["dep:regex","dep:lazy_static","dep:rand"]
crypto = ["dep:base64-simd","dep:hex-simd"]
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
string = ["dep:regex", "dep:lazy_static", "dep:rand"]
crypto = ["dep:base64-simd", "dep:hex-simd"]
hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher"]
os = ["dep:nix", "dep:tempfile", "winapi"] # operating system utilities
integration = [] # integration test features
full = ["ip", "tls", "net", "io","hash", "os", "integration","path","crypto", "string","compress"] # all features
sys = ["dep:sysinfo"] # system information features
full = ["ip", "tls", "net", "io", "hash", "os", "integration", "path", "crypto", "string", "compress", "sys"] # all features

Some files were not shown because too many files have changed in this diff Show More