mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 08:38:58 +00:00
refactor(s3): consolidate semantic boundaries and remove s3-common (#3012)
* refactor(common): introduce rustfs-data-usage core crate * refactor(concurrency): migrate workers crate into concurrency * refactor(crypto): migrate appauth token APIs into crypto * fix docs urls * remove unused crate * refactor(data-usage): switch consumers to rustfs-data-usage * chore(fmt): apply cargo fmt and lockfile sync * refactor(common): remove data_usage compatibility re-export * refactor(capacity): move capacity_scope to object-capacity * refactor(io-metrics): relocate internode metrics from common * refactor(common): decouple scanner report from madmin * chore(fmt): normalize import ordering after pre-commit * refactor(s3): split s3 types and ops crates * refactor(s3): centralize event version and safe parsing * refactor(s3): add op-event compatibility guardrails * refactor(s3): add runtime op-event mismatch observability * refactor(s3): extract delete event mapping helper * refactor(s3): extract put event mapping helper * refactor(s3): consolidate remaining event semantic helpers * refactor(s3): add op-event coverage checks and observability alerts * refactor(s3-ops): consolidate op-event semantic mapping * refactor(scanner): remove last_minute wrapper module * refactor(scanner): consolidate duplicated data usage models
This commit is contained in:
houseme
@@ -9037,13 +9037,13 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{__name__=~\"rustfs_(notification|audit)_.*\",job=~\"$job\"}",
|
||||
"expr": "{__name__=~\"rustfs_(notification|audit|log_chain)_.*\",job=~\"$job\"}",
|
||||
"legendFormat": "{{__name__}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Notification / Audit (All)",
|
||||
"title": "Notification / Audit / LogChain (All)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -38,3 +38,16 @@ groups:
|
||||
expr: sum by (job) (rate(rustfs_scanner_buckets_scanned_total[5m]))
|
||||
- record: rustfs:scanner_cycles_success:rate5m
|
||||
expr: sum by (job) (rate(rustfs_scanner_cycles_total{result="success"}[5m]))
|
||||
|
||||
- record: rustfs:log_chain_op_event_mismatch:rate5m
|
||||
expr: sum by (job) (rate(rustfs_log_chain_op_event_mismatch_total[5m]))
|
||||
|
||||
- alert: RustFSLogChainOpEventMismatchDetected
|
||||
expr: rustfs:log_chain_op_event_mismatch:rate5m > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
component: s3-log-chain
|
||||
annotations:
|
||||
summary: "RustFS log-chain op/event mismatch detected"
|
||||
description: "job={{ $labels.job }} has non-zero rustfs_log_chain_op_event_mismatch_total rate for more than 10m. Check s3 op/event mapping changes."
|
||||
|
||||
Generated
+41
-33
@@ -3698,6 +3698,7 @@ dependencies = [
|
||||
"russh-sftp",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-lock",
|
||||
@@ -9315,13 +9316,13 @@ dependencies = [
|
||||
"rmp-serde",
|
||||
"rsa 0.10.0-rc.18",
|
||||
"rust-embed",
|
||||
"rustfs-appauth",
|
||||
"rustfs-audit",
|
||||
"rustfs-common",
|
||||
"rustfs-concurrency",
|
||||
"rustfs-config",
|
||||
"rustfs-credentials",
|
||||
"rustfs-crypto",
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-heal",
|
||||
@@ -9339,7 +9340,8 @@ dependencies = [
|
||||
"rustfs-protocols",
|
||||
"rustfs-protos",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-common",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-s3select-api",
|
||||
"rustfs-s3select-query",
|
||||
"rustfs-scanner",
|
||||
@@ -9381,17 +9383,6 @@ dependencies = [
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-appauth"
|
||||
version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"rand 0.10.1",
|
||||
"rsa 0.10.0-rc.18",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-beta.3"
|
||||
@@ -9404,7 +9395,8 @@ dependencies = [
|
||||
"metrics",
|
||||
"rustfs-config",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-s3-common",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-targets",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9433,13 +9425,9 @@ dependencies = [
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"metrics",
|
||||
"path-clean",
|
||||
"rmp-serde",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-madmin",
|
||||
"s3s",
|
||||
"serde",
|
||||
"tokio",
|
||||
@@ -9484,10 +9472,13 @@ version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"aes-gcm 0.11.0-rc.3",
|
||||
"argon2 0.6.0-rc.8",
|
||||
"base64-simd",
|
||||
"chacha20poly1305",
|
||||
"jsonwebtoken",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand 0.10.1",
|
||||
"rsa 0.10.0-rc.18",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"test-case",
|
||||
@@ -9495,6 +9486,17 @@ dependencies = [
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"path-clean",
|
||||
"rmp-serde",
|
||||
"rustfs-filemeta",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-beta.3"
|
||||
@@ -9553,20 +9555,23 @@ dependencies = [
|
||||
"rmp-serde",
|
||||
"rustfs-checksums",
|
||||
"rustfs-common",
|
||||
"rustfs-concurrency",
|
||||
"rustfs-config",
|
||||
"rustfs-credentials",
|
||||
"rustfs-data-usage",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-kms",
|
||||
"rustfs-lock",
|
||||
"rustfs-madmin",
|
||||
"rustfs-object-capacity",
|
||||
"rustfs-policy",
|
||||
"rustfs-protos",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3-common",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-signer",
|
||||
"rustfs-utils",
|
||||
"rustfs-workers",
|
||||
"rustix 1.1.4",
|
||||
"rustls",
|
||||
"s3s",
|
||||
@@ -9700,6 +9705,7 @@ dependencies = [
|
||||
"criterion",
|
||||
"metrics",
|
||||
"num_cpus",
|
||||
"rustfs-s3-ops",
|
||||
"sysinfo",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
@@ -9860,7 +9866,8 @@ dependencies = [
|
||||
"rustc-hash",
|
||||
"rustfs-config",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-s3-common",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-targets",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
@@ -9881,7 +9888,6 @@ version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
@@ -10034,6 +10040,7 @@ dependencies = [
|
||||
"prost 0.14.3",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
@@ -10061,6 +10068,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde",
|
||||
@@ -10075,10 +10083,16 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-common"
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"rustfs-s3-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"metrics",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
@@ -10141,6 +10155,7 @@ dependencies = [
|
||||
"rmp-serde",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-utils",
|
||||
@@ -10195,7 +10210,8 @@ dependencies = [
|
||||
"rustfs-config",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-kafka-async",
|
||||
"rustfs-s3-common",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
@@ -10284,14 +10300,6 @@ dependencies = [
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-workers"
|
||||
version = "1.0.0-beta.3"
|
||||
dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-beta.3"
|
||||
|
||||
+6
-7
@@ -15,10 +15,10 @@
|
||||
[workspace]
|
||||
members = [
|
||||
"rustfs", # Core file system implementation
|
||||
"crates/appauth", # Application authentication and authorization
|
||||
"crates/audit", # Audit target management system with multi-target fan-out
|
||||
"crates/checksums", # client checksums
|
||||
"crates/common", # Shared utilities and data structures
|
||||
"crates/data-usage", # Shared data usage models and algorithms
|
||||
"crates/config", # Configuration management
|
||||
"crates/credentials", # Credential management system
|
||||
"crates/crypto", # Cryptography and security features
|
||||
@@ -39,7 +39,8 @@ members = [
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-common", # Common utilities and data structures for S3 compatibility
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
"crates/s3-ops", # S3 operation definitions and mapping
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
"crates/s3select-query", # S3 Select query engine
|
||||
"crates/scanner", # Scanner for data integrity checks and health monitoring
|
||||
@@ -47,7 +48,6 @@ members = [
|
||||
"crates/targets", # Target-specific configurations and utilities
|
||||
"crates/trusted-proxies", # Trusted proxies management
|
||||
"crates/utils", # Utility functions and helpers
|
||||
"crates/workers", # Worker thread pools and task scheduling
|
||||
"crates/io-metrics", # Zero-copy metrics collection for performance analysis
|
||||
"crates/io-core", # Zero-copy core reader and writer implementations
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
@@ -78,10 +78,10 @@ redundant_clone = "warn"
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.3" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.3" }
|
||||
rustfs-appauth = { path = "crates/appauth", version = "1.0.0-beta.3" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.3" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.3" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.3" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.3" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.3" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.3" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.3" }
|
||||
@@ -102,7 +102,8 @@ rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.3" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.3" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.3" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.3" }
|
||||
rustfs-s3-common = { path = "crates/s3-common", version = "1.0.0-beta.3" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.3" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.3" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.3" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.3" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.3" }
|
||||
@@ -110,7 +111,6 @@ rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.3" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.3" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.3" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.3" }
|
||||
rustfs-workers = { path = "crates/workers", version = "1.0.0-beta.3" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.3" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
@@ -202,7 +202,6 @@ aws-credential-types = { version = "1.2.14" }
|
||||
aws-sdk-s3 = { version = "1.132.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
|
||||
aws-smithy-types = { version = "1.4.7" }
|
||||
backtrace = "0.3.76"
|
||||
base64 = "0.22.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.2"
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
[](https://rustfs.com)
|
||||
|
||||
# RustFS AppAuth - Application Authentication
|
||||
|
||||
<p align="center">
|
||||
<strong>Application-level authentication and authorization module for RustFS distributed object storage</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
|
||||
<a href="https://docs.rustfs.com/">📖 Documentation</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/issues">🐛 Bug Reports</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/discussions">💬 Discussions</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 📖 Overview
|
||||
|
||||
**RustFS AppAuth** provides application-level authentication and authorization capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
|
||||
|
||||
## ✨ Features
|
||||
|
||||
- JWT-based authentication with secure token management
|
||||
- RBAC (Role-Based Access Control) for fine-grained permissions
|
||||
- Multi-tenant application isolation and management
|
||||
- OAuth 2.0 and OpenID Connect integration
|
||||
- API key management and rotation
|
||||
- Session management with configurable expiration
|
||||
|
||||
## 📚 Documentation
|
||||
|
||||
For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
@@ -29,7 +29,8 @@ categories = ["web-programming", "development-tools", "asynchronous", "api-bindi
|
||||
rustfs-targets = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["audit", "constants"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-s3-common = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
const-str = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ homepage.workspace = true
|
||||
description = "Checksum calculation and verification callbacks for HTTP request and response bodies sent by service clients generated by RustFS, ensuring data integrity and authenticity."
|
||||
keywords = ["checksum-calculation", "verification", "integrity", "authenticity", "rustfs"]
|
||||
categories = ["web-programming", "development-tools", "network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-signer/latest/rustfs_checksum/"
|
||||
documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/"
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true }
|
||||
|
||||
@@ -33,12 +33,8 @@ tonic = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
rustfs-madmin = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
path-clean = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
|
||||
@@ -13,12 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod bucket_stats;
|
||||
pub mod capacity_scope;
|
||||
// pub mod error;
|
||||
pub mod data_usage;
|
||||
pub mod globals;
|
||||
pub mod heal_channel;
|
||||
pub mod internode_metrics;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
mod readiness;
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use crate::last_minute::{AccElem, LastMinuteLatency};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_madmin::metrics::{ScannerMetrics as M_ScannerMetrics, TimedAction};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
@@ -351,6 +350,32 @@ pub struct CurrentCycle {
|
||||
pub started: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerTimedAction {
|
||||
pub count: u64,
|
||||
pub acc_time: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerLastMinute {
|
||||
pub actions: HashMap<String, ScannerTimedAction>,
|
||||
pub ilm: HashMap<String, ScannerTimedAction>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerMetricsReport {
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub current_cycle: u64,
|
||||
pub current_started: DateTime<Utc>,
|
||||
pub cycles_completed_at: Vec<DateTime<Utc>>,
|
||||
pub ongoing_buckets: usize,
|
||||
pub life_time_ops: HashMap<String, u64>,
|
||||
pub life_time_ilm: HashMap<String, u64>,
|
||||
pub last_minute: ScannerLastMinute,
|
||||
pub active_paths: Vec<String>,
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
*self = rmp_serde::from_slice(buf)?;
|
||||
@@ -572,8 +597,8 @@ impl Metrics {
|
||||
}
|
||||
|
||||
/// Build a full metrics report snapshot.
|
||||
pub async fn report(&self) -> M_ScannerMetrics {
|
||||
let mut m = M_ScannerMetrics::default();
|
||||
pub async fn report(&self) -> ScannerMetricsReport {
|
||||
let mut m = ScannerMetricsReport::default();
|
||||
|
||||
if let Some(cycle) = self.get_cycle().await {
|
||||
m.current_cycle = cycle.current;
|
||||
@@ -606,7 +631,7 @@ impl Metrics {
|
||||
{
|
||||
m.last_minute.actions.insert(
|
||||
metric.as_str().to_string(),
|
||||
TimedAction {
|
||||
ScannerTimedAction {
|
||||
count: last_min.n,
|
||||
acc_time: last_min.total,
|
||||
bytes: last_min.size,
|
||||
@@ -633,7 +658,7 @@ impl Metrics {
|
||||
{
|
||||
m.last_minute.ilm.insert(
|
||||
action.as_str().to_string(),
|
||||
TimedAction {
|
||||
ScannerTimedAction {
|
||||
count: last_min.n,
|
||||
acc_time: last_min.total,
|
||||
bytes: last_min.size,
|
||||
|
||||
@@ -124,6 +124,8 @@ mod backpressure;
|
||||
#[cfg(feature = "scheduler")]
|
||||
mod scheduler;
|
||||
|
||||
pub mod workers;
|
||||
|
||||
// Public module exports with feature gates
|
||||
#[cfg(feature = "timeout")]
|
||||
pub use timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager};
|
||||
|
||||
@@ -12,10 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Worker slot limiter used by long-running background workflows.
|
||||
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
use tracing::info;
|
||||
|
||||
/// Cooperative worker-slot controller for async tasks.
|
||||
pub struct Workers {
|
||||
available: Mutex<usize>, // Available working slots
|
||||
notify: Notify, // Used to notify waiting tasks
|
||||
@@ -23,20 +26,20 @@ pub struct Workers {
|
||||
}
|
||||
|
||||
impl Workers {
|
||||
// Create a Workers object that allows up to n jobs to execute concurrently
|
||||
pub fn new(n: usize) -> Result<Arc<Workers>, &'static str> {
|
||||
/// Create a [`Workers`] object that allows up to `n` jobs to execute concurrently.
|
||||
pub fn new(n: usize) -> Result<Arc<Self>, &'static str> {
|
||||
if n == 0 {
|
||||
return Err("n must be > 0");
|
||||
}
|
||||
|
||||
Ok(Arc::new(Workers {
|
||||
Ok(Arc::new(Self {
|
||||
available: Mutex::new(n),
|
||||
notify: Notify::new(),
|
||||
limit: n,
|
||||
}))
|
||||
}
|
||||
|
||||
// Give a job a chance to be executed
|
||||
/// Acquire a worker slot, waiting until one becomes available.
|
||||
pub async fn take(&self) {
|
||||
loop {
|
||||
let mut available = self.available.lock().await;
|
||||
@@ -51,7 +54,7 @@ impl Workers {
|
||||
}
|
||||
}
|
||||
|
||||
// Release a job's slot
|
||||
/// Release a worker slot.
|
||||
pub async fn give(&self) {
|
||||
let mut available = self.available.lock().await;
|
||||
info!("worker give, {}", *available);
|
||||
@@ -59,7 +62,7 @@ impl Workers {
|
||||
self.notify.notify_one(); // Notify a waiting task
|
||||
}
|
||||
|
||||
// Wait for all concurrent jobs to complete
|
||||
/// Wait until all worker slots are released.
|
||||
pub async fn wait(&self) {
|
||||
loop {
|
||||
{
|
||||
@@ -74,6 +77,7 @@ impl Workers {
|
||||
info!("worker wait end");
|
||||
}
|
||||
|
||||
/// Return the current number of available worker slots.
|
||||
pub async fn available(&self) -> usize {
|
||||
*self.available.lock().await
|
||||
}
|
||||
@@ -33,8 +33,11 @@ aes-gcm = { workspace = true, optional = true }
|
||||
argon2 = { workspace = true, optional = true }
|
||||
chacha20poly1305 = { workspace = true, optional = true }
|
||||
jsonwebtoken = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, optional = true }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
mod encdec;
|
||||
mod error;
|
||||
mod jwt;
|
||||
pub mod license_token;
|
||||
|
||||
pub use encdec::decrypt::decrypt_data;
|
||||
pub use encdec::encrypt::encrypt_data;
|
||||
@@ -25,3 +26,4 @@ pub use encdec::stream_io::{decrypt_stream_io, encrypt_stream_io};
|
||||
pub use error::Error;
|
||||
pub use jwt::decode::decode as jwt_decode;
|
||||
pub use jwt::encode::encode as jwt_encode;
|
||||
pub use license_token::{Token, parse_license_with_public_key, parse_signed_license_token, sign_license_token};
|
||||
|
||||
@@ -115,12 +115,20 @@ mod tests {
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key.to_pkcs8_pem(LineEnding::LF).unwrap();
|
||||
let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap();
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
let token = Token {
|
||||
name: "test_app".to_string(),
|
||||
expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600, // 1 hour from now
|
||||
expired: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock before unix epoch")
|
||||
.as_secs()
|
||||
+ 3600, // 1 hour from now
|
||||
};
|
||||
|
||||
let encoded = sign_license_token(&token, &private_key_pem).expect("Failed to encode token");
|
||||
@@ -139,12 +147,20 @@ mod tests {
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key.to_pkcs8_pem(LineEnding::LF).unwrap();
|
||||
let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap();
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
let token = Token {
|
||||
name: "test_app".to_string(),
|
||||
expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600,
|
||||
expired: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock before unix epoch")
|
||||
.as_secs()
|
||||
+ 3600,
|
||||
};
|
||||
|
||||
let encoded = gencode(&token, &public_key_pem).expect("Failed to encode token");
|
||||
@@ -159,11 +175,19 @@ mod tests {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key_pem = private_key.to_pkcs8_pem(LineEnding::LF).unwrap();
|
||||
let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap();
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
let token = Token {
|
||||
name: "test_app".to_string(),
|
||||
expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600,
|
||||
expired: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock before unix epoch")
|
||||
.as_secs()
|
||||
+ 3600,
|
||||
};
|
||||
|
||||
let encoded = sign_license_token(&token, &private_key_pem).expect("Failed to encode token");
|
||||
@@ -181,7 +205,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_source_does_not_embed_private_key() {
|
||||
let source = include_str!("token.rs");
|
||||
let source = include_str!("license_token.rs");
|
||||
let forbidden = ["BEGIN", "PRIVATE KEY"].join(" ");
|
||||
|
||||
assert!(!source.contains(&forbidden));
|
||||
@@ -192,7 +216,9 @@ mod tests {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap();
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
let invalid_token = "invalid_base64_token";
|
||||
let result = parse_signed_license_token(invalid_token, &public_key_pem);
|
||||
@@ -204,7 +230,11 @@ mod tests {
|
||||
fn test_sign_license_token_with_invalid_signing_key() {
|
||||
let token = Token {
|
||||
name: "test_app".to_string(),
|
||||
expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600, // 1 hour from now
|
||||
expired: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock before unix epoch")
|
||||
.as_secs()
|
||||
+ 3600, // 1 hour from now
|
||||
};
|
||||
|
||||
let invalid_key = "invalid_private_key";
|
||||
@@ -13,24 +13,26 @@
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-workers"
|
||||
name = "rustfs-data-usage"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "Workers for RustFS, providing background processing capabilities for tasks such as data synchronization, indexing, and more."
|
||||
keywords = ["workers", "tasks", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools"]
|
||||
documentation = "https://docs.rs/rustfs-workers/latest/rustfs_workers/"
|
||||
description = "Shared data usage models and algorithms for RustFS"
|
||||
keywords = ["rustfs", "data-usage", "cache", "histogram"]
|
||||
categories = ["data-structures", "filesystem"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true, features = ["sync", "time", "macros"] }
|
||||
tracing.workspace = true
|
||||
serde = { workspace = true }
|
||||
path-clean = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
@@ -312,6 +312,12 @@ impl SizeHistogram {
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub fn merge_from(&mut self, other: &Self) {
|
||||
for (dst, src) in self.0.iter_mut().zip(other.0.iter()) {
|
||||
*dst += src;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Versions histogram for version count distribution
|
||||
@@ -361,6 +367,12 @@ impl VersionsHistogram {
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub fn merge_from(&mut self, other: &Self) {
|
||||
for (dst, src) in self.0.iter_mut().zip(other.0.iter()) {
|
||||
*dst += src;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication statistics for a single target
|
||||
@@ -419,6 +431,9 @@ pub struct DataUsageEntry {
|
||||
pub obj_versions: VersionsHistogram,
|
||||
pub replication_stats: Option<ReplicationAllStats>,
|
||||
pub compacted: bool,
|
||||
/// Number of objects that failed to scan (e.g., IO errors)
|
||||
#[serde(default)]
|
||||
pub failed_objects: usize,
|
||||
}
|
||||
|
||||
impl DataUsageEntry {
|
||||
@@ -456,6 +471,7 @@ impl DataUsageEntry {
|
||||
self.versions += other.versions;
|
||||
self.delete_markers += other.delete_markers;
|
||||
self.size += other.size;
|
||||
self.failed_objects += other.failed_objects;
|
||||
|
||||
if let Some(o_rep) = &other.replication_stats {
|
||||
let s_rep = self.replication_stats.get_or_insert_with(ReplicationAllStats::default);
|
||||
@@ -490,9 +506,11 @@ impl DataUsageEntry {
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct DataUsageCacheInfo {
|
||||
pub name: String,
|
||||
pub next_cycle: u32,
|
||||
pub next_cycle: u64,
|
||||
pub last_update: Option<SystemTime>,
|
||||
pub skip_healing: bool,
|
||||
#[serde(default)]
|
||||
pub failed_objects: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
/// Data usage cache
|
||||
@@ -12,4 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod workers;
|
||||
pub mod data_usage;
|
||||
|
||||
pub use data_usage::*;
|
||||
@@ -32,6 +32,7 @@ sftp = []
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-ecstore.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-rio.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
futures.workspace = true
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use rustfs_common::data_usage::DataUsageInfo;
|
||||
use rustfs_data_usage::DataUsageInfo;
|
||||
use serial_test::serial;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
|
||||
|
||||
@@ -45,7 +45,10 @@ rustfs-common.workspace = true
|
||||
rustfs-policy.workspace = true
|
||||
rustfs-protos.workspace = true
|
||||
rustfs-kms.workspace = true
|
||||
rustfs-s3-common = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
byteorder = { workspace = true }
|
||||
@@ -103,7 +106,7 @@ memmap2 = { workspace = true }
|
||||
libc.workspace = true
|
||||
rustix = { workspace = true }
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-workers.workspace = true
|
||||
rustfs-concurrency.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
aes-gcm.workspace = true
|
||||
aws-sdk-s3 = { workspace = true }
|
||||
|
||||
@@ -44,7 +44,6 @@ use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
|
||||
use futures::Future;
|
||||
use http::HeaderMap;
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_common::data_usage::TierStats;
|
||||
use rustfs_common::heal_channel::rep_has_active_rules;
|
||||
use rustfs_common::metrics::{IlmAction, Metrics};
|
||||
use rustfs_config::{
|
||||
@@ -52,11 +51,12 @@ use rustfs_config::{
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, ENV_TRANSITION_QUEUE_CAPACITY,
|
||||
ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
};
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::{
|
||||
FileInfo, FileInfoOpts, NULL_VERSION_ID, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicationState, RestoreStatusOps,
|
||||
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use rustfs_common::data_usage::TierStats;
|
||||
use rustfs_data_usage::TierStats;
|
||||
use sha2::Sha256;
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Sub;
|
||||
|
||||
@@ -56,7 +56,7 @@ use rustfs_filemeta::{
|
||||
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
|
||||
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
|
||||
@@ -27,7 +27,7 @@ pub use local_snapshot::{
|
||||
data_usage_dir, data_usage_state_dir, ensure_data_usage_layout, read_snapshot as read_local_snapshot, snapshot_file_name,
|
||||
snapshot_object_path, snapshot_path, write_snapshot as write_local_snapshot,
|
||||
};
|
||||
use rustfs_common::data_usage::{
|
||||
use rustfs_data_usage::{
|
||||
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
|
||||
};
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
@@ -686,7 +686,7 @@ pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate:
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_common::data_usage::BucketUsageInfo;
|
||||
use rustfs_data_usage::BucketUsageInfo;
|
||||
|
||||
fn aggregate_for_test(
|
||||
inputs: Vec<(DiskUsageStatus, Result<Option<LocalUsageSnapshot>, Error>)>,
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Compatibility re-export for the legacy `rustfs_ecstore::event::name::EventName` path.
|
||||
//! The canonical event definition now lives in `rustfs_s3_common::EventName`.
|
||||
//! The canonical event definition now lives in `rustfs_s3_types::EventName`.
|
||||
|
||||
pub use rustfs_s3_common::EventName;
|
||||
pub use rustfs_s3_types::EventName;
|
||||
|
||||
@@ -14,11 +14,12 @@
|
||||
|
||||
use crate::{admin_server_info::get_local_server_property, new_object_layer_fn, store_api::StorageAPI};
|
||||
use chrono::Utc;
|
||||
use rustfs_common::{
|
||||
GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, internode_metrics::global_internode_metrics,
|
||||
metrics::global_metrics,
|
||||
use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, metrics::global_metrics};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_madmin::metrics::{
|
||||
DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics,
|
||||
ScannerMetrics as MadminScannerMetrics, TimedAction as MadminTimedAction,
|
||||
};
|
||||
use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics};
|
||||
use rustfs_utils::os::get_drive_stats;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -56,6 +57,51 @@ impl MetricType {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
|
||||
MadminScannerMetrics {
|
||||
collected_at: metrics.collected_at,
|
||||
current_cycle: metrics.current_cycle,
|
||||
current_started: metrics.current_started,
|
||||
cycles_completed_at: metrics.cycles_completed_at,
|
||||
ongoing_buckets: metrics.ongoing_buckets,
|
||||
life_time_ops: metrics.life_time_ops,
|
||||
life_time_ilm: metrics.life_time_ilm,
|
||||
last_minute: MadminLastMinute {
|
||||
actions: metrics
|
||||
.last_minute
|
||||
.actions
|
||||
.into_iter()
|
||||
.map(|(key, value)| {
|
||||
(
|
||||
key,
|
||||
MadminTimedAction {
|
||||
count: value.count,
|
||||
acc_time: value.acc_time,
|
||||
bytes: value.bytes,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
ilm: metrics
|
||||
.last_minute
|
||||
.ilm
|
||||
.into_iter()
|
||||
.map(|(key, value)| {
|
||||
(
|
||||
key,
|
||||
MadminTimedAction {
|
||||
count: value.count,
|
||||
acc_time: value.acc_time,
|
||||
bytes: value.bytes,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
active_paths: metrics.active_paths,
|
||||
}
|
||||
}
|
||||
|
||||
impl MetricType {
|
||||
fn contains(&self, x: &MetricType) -> bool {
|
||||
(self.0 & x.0) == x.0
|
||||
@@ -114,7 +160,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
if let Some(init_time) = rustfs_common::get_global_init_time().await {
|
||||
metrics.current_started = init_time;
|
||||
}
|
||||
real_time_metrics.aggregated.scanner = Some(metrics);
|
||||
real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
|
||||
}
|
||||
|
||||
// if types.contains(&MetricType::OS) {}
|
||||
@@ -252,7 +298,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use rustfs_common::internode_metrics::global_internode_metrics;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -51,9 +51,9 @@ use rmp_serde::Deserializer;
|
||||
use rmp_serde::Serializer;
|
||||
use rustfs_common::defer;
|
||||
use rustfs_common::heal_channel::HealOpts;
|
||||
use rustfs_concurrency::workers::Workers;
|
||||
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
|
||||
use rustfs_workers::workers::Workers;
|
||||
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
@@ -36,10 +36,10 @@ use bytes::Bytes;
|
||||
use futures::lock::Mutex;
|
||||
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
|
||||
use metrics::counter;
|
||||
use rustfs_common::internode_metrics::{
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, global_internode_metrics,
|
||||
};
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
|
||||
@@ -68,7 +68,6 @@ use http::HeaderMap;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::{Rng, seq::SliceRandom};
|
||||
use regex::Regex;
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope};
|
||||
use rustfs_common::heal_channel::{DriveState, HealChannelPriority, HealItemType, HealOpts, HealScanMode, send_heal_disk};
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::{
|
||||
@@ -80,8 +79,11 @@ use rustfs_lock::fast_lock::types::LockResult;
|
||||
use rustfs_lock::local_lock::LocalLock;
|
||||
use rustfs_lock::{FastLockGuard, LockManager, NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper, ObjectKey};
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
|
||||
use rustfs_object_capacity::capacity_scope::{
|
||||
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
|
||||
};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||
use rustfs_utils::http::headers::{
|
||||
@@ -97,7 +99,6 @@ use rustfs_utils::{
|
||||
crypto::hex,
|
||||
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
|
||||
};
|
||||
use rustfs_workers::workers::Workers;
|
||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::hash::Hash;
|
||||
|
||||
@@ -30,6 +30,7 @@ harness = false
|
||||
|
||||
[dependencies]
|
||||
metrics = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
num_cpus = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync","rt"] }
|
||||
|
||||
@@ -60,10 +60,12 @@ pub mod capacity_metrics;
|
||||
pub mod collector;
|
||||
pub mod config;
|
||||
pub mod deadlock_metrics;
|
||||
pub mod internode_metrics;
|
||||
pub mod io_metrics;
|
||||
pub mod lock_metrics;
|
||||
pub mod performance;
|
||||
pub mod process_lock_metrics;
|
||||
pub mod s3_api_metrics;
|
||||
pub mod sampler;
|
||||
pub mod system_path_metrics;
|
||||
pub mod timeout_metrics;
|
||||
@@ -118,6 +120,7 @@ pub use process_lock_metrics::{
|
||||
record_write_lock_held_acquire, record_write_lock_held_release, snapshot_process_lock_counts,
|
||||
snapshot_process_platform_stats,
|
||||
};
|
||||
pub use s3_api_metrics::{init_s3_metrics, record_s3_op};
|
||||
pub use sampler::{
|
||||
ProcessResourceSnapshot, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_platform, snapshot_process_resource,
|
||||
snapshot_process_resource_and_system, snapshot_process_system,
|
||||
|
||||
@@ -12,29 +12,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::S3Operation;
|
||||
use metrics::{counter, describe_counter};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const S3_OPS_METRIC: &str = "rustfs_s3_operations_total";
|
||||
|
||||
/// Records an S3 operation in the metrics system.
|
||||
/// This function should be called whenever an S3 API operation is handled, allowing us to track the usage of different S3 operations across buckets.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `op` - The S3 operation being recorded.
|
||||
/// * `bucket` - The name of the bucket associated with the operation, used as a label for more granular metrics analysis.
|
||||
///
|
||||
/// Example usage:
|
||||
/// ```ignore
|
||||
/// record_s3_op(S3Operation::GetObject, "my-bucket");
|
||||
/// ```
|
||||
pub fn record_s3_op(op: S3Operation, bucket: &str) {
|
||||
counter!(S3_OPS_METRIC, "op" => op.as_str(), "bucket" => bucket.to_owned()).increment(1);
|
||||
}
|
||||
|
||||
/// One-time registration of indicator meta information
|
||||
/// This function ensures that metric descriptors are registered only once.
|
||||
pub fn init_s3_metrics() {
|
||||
static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new();
|
||||
METRICS_DESC_INIT.get_or_init(|| {
|
||||
@@ -28,7 +28,8 @@ documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify", "constants"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-s3-common = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
rustfs-targets = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
|
||||
@@ -6,7 +6,7 @@ use tokio::runtime::Runtime;
|
||||
|
||||
fn build_rule_map(target: &TargetID) -> RulesMap {
|
||||
let mut rules_map = RulesMap::new();
|
||||
rules_map.add_rule_config(&[rustfs_s3_common::EventName::ObjectCreatedPut], "*".to_string(), target.clone());
|
||||
rules_map.add_rule_config(&[rustfs_s3_types::EventName::ObjectCreatedPut], "*".to_string(), target.clone());
|
||||
rules_map
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ use rustfs_config::{
|
||||
use rustfs_ecstore::config::{Config, KV, KVS};
|
||||
use rustfs_notify::{BucketNotificationConfig, Event, NotificationError};
|
||||
use rustfs_notify::{initialize, notification_system};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -24,7 +24,7 @@ use rustfs_config::{
|
||||
use rustfs_ecstore::config::{Config, KV, KVS};
|
||||
use rustfs_notify::{BucketNotificationConfig, Event, NotificationError};
|
||||
use rustfs_notify::{initialize, notification_system};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::{
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
rules::ParseConfigError,
|
||||
};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -91,7 +91,7 @@ mod tests {
|
||||
BucketNotificationConfig, integration::NotificationMetrics,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
|
||||
use chrono::{DateTime, SecondsFormat, Utc};
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_ops::is_object_removed_event;
|
||||
use rustfs_s3_types::{EventName, event_schema_version};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use url::form_urlencoded;
|
||||
|
||||
@@ -135,25 +136,6 @@ pub struct Event {
|
||||
}
|
||||
|
||||
impl Event {
|
||||
fn event_version_for(event_name: EventName) -> &'static str {
|
||||
match event_name {
|
||||
EventName::ObjectReplicationFailed
|
||||
| EventName::ObjectReplicationComplete
|
||||
| EventName::ObjectReplicationMissedThreshold
|
||||
| EventName::ObjectReplicationReplicatedAfterThreshold
|
||||
| EventName::ObjectReplicationNotTracked => "2.2",
|
||||
EventName::ObjectRestoreCompleted
|
||||
| EventName::ObjectAclPut
|
||||
| EventName::ObjectTaggingPut
|
||||
| EventName::ObjectTaggingDelete
|
||||
| EventName::LifecycleExpirationDelete
|
||||
| EventName::LifecycleExpirationDeleteMarkerCreated
|
||||
| EventName::LifecycleTransition
|
||||
| EventName::IntelligentTiering => "2.3",
|
||||
_ => "2.1",
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a test event for a given bucket and object
|
||||
pub fn new_test_event(bucket: &str, key: &str, event_name: EventName) -> Self {
|
||||
let mut user_metadata = HashMap::new();
|
||||
@@ -176,7 +158,7 @@ impl Event {
|
||||
user_metadata.insert("x-request-time".to_string(), Utc::now().to_rfc3339());
|
||||
|
||||
Event {
|
||||
event_version: Self::event_version_for(event_name).to_string(),
|
||||
event_version: event_schema_version(event_name).to_string(),
|
||||
event_source: "rustfs:s3".to_string(),
|
||||
aws_region: "us-east-1".to_string(),
|
||||
event_time: Utc::now(),
|
||||
@@ -256,10 +238,7 @@ impl Event {
|
||||
},
|
||||
};
|
||||
|
||||
let is_removed_event = matches!(
|
||||
args.event_name,
|
||||
EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated
|
||||
);
|
||||
let is_removed_event = is_object_removed_event(args.event_name);
|
||||
|
||||
if !is_removed_event {
|
||||
s3_metadata.object.size = Some(args.object.size);
|
||||
@@ -293,7 +272,7 @@ impl Event {
|
||||
};
|
||||
|
||||
Self {
|
||||
event_version: Self::event_version_for(args.event_name).to_string(),
|
||||
event_version: event_schema_version(args.event_name).to_string(),
|
||||
event_source: "rustfs:s3".to_string(),
|
||||
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
|
||||
event_time: event_time.and_utc(),
|
||||
@@ -542,7 +521,7 @@ mod event_args_tests {
|
||||
use super::EventArgs;
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_ecstore::store_api::ObjectInfo;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
fn args_with_headers(pairs: &[(&str, &str)]) -> EventArgs {
|
||||
let mut req_params = HashMap::new();
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::{
|
||||
NotificationSystem, NotificationTargetMetricSnapshot,
|
||||
};
|
||||
use rustfs_ecstore::config::Config;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tracing::error;
|
||||
|
||||
@@ -23,7 +23,7 @@ use hashbrown::HashMap;
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget};
|
||||
use std::sync::Arc;
|
||||
@@ -402,7 +402,7 @@ pub async fn load_config_from_file(path: &str, system: &NotificationSystem) -> R
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
#[test]
|
||||
fn live_event_history_snapshots_from_sequence() {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::BucketNotificationConfig;
|
||||
use crate::rules::{BucketRulesSnapshot, DynRulesContainer, SubscriberIndex};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
/// NotificationSystemSubscriberView - Provides an interface to manage and query
|
||||
/// the subscription status of buckets in the notification system.
|
||||
|
||||
@@ -308,7 +308,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::StoreError;
|
||||
use rustfs_targets::{
|
||||
TargetError,
|
||||
|
||||
@@ -105,7 +105,7 @@ pub type NotifyEventBridge = NotifyPipeline;
|
||||
mod tests {
|
||||
use super::{LiveEventHistory, NotifyPipeline};
|
||||
use crate::{Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{RwLock, broadcast};
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::rules::{RulesMap, TargetIdSet};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
|
||||
use std::sync::Arc;
|
||||
@@ -99,7 +99,7 @@ impl Default for NotifyRuleEngine {
|
||||
mod tests {
|
||||
use super::NotifyRuleEngine;
|
||||
use crate::rules::RulesMap;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::rules_map::RulesMap;
|
||||
use super::xml_config::ParseConfigError as BucketNotificationConfigError;
|
||||
use crate::rules::NotificationConfiguration;
|
||||
use crate::rules::subscriber_snapshot::{BucketRulesSnapshot, DynRulesContainer, RuleEvents, RulesContainer};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::io::Read;
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
//! to event matching, including filter rules with prefix and suffix.
|
||||
|
||||
use super::*;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::{ARN, TargetID};
|
||||
use std::io::Cursor;
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
//! configuration to event matching.
|
||||
|
||||
use super::*;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::rules::{PatternRules, TargetIdSet};
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::rules::{BucketRulesSnapshot, BucketSnapshotRef, DynRulesContainer};
|
||||
use arc_swap::ArcSwap;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use starshard::{DEFAULT_SHARDS, ShardedHashMap};
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Let the rules structure provide "what events it is subscribed to".
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::rules::pattern;
|
||||
use hashbrown::HashSet;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::{ARN, ArnError, TargetIDError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::io::Read;
|
||||
|
||||
@@ -35,7 +35,6 @@ harness = false
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
|
||||
use super::scan::refresh_capacity_with_scope;
|
||||
use super::types::CapacityDiskRef;
|
||||
use crate::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope};
|
||||
use futures::FutureExt;
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope};
|
||||
use rustfs_config::{
|
||||
DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH,
|
||||
DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS,
|
||||
@@ -1017,7 +1017,7 @@ pub async fn start_background_task(disks: Vec<CapacityDiskRef>) {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope};
|
||||
use crate::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope};
|
||||
use rustfs_config::{
|
||||
ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_METRICS_INTERVAL,
|
||||
ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod capacity_manager;
|
||||
pub mod capacity_scope;
|
||||
pub mod scan;
|
||||
pub mod types;
|
||||
|
||||
|
||||
@@ -18,8 +18,8 @@ use super::capacity_manager::{
|
||||
get_stat_timeout,
|
||||
};
|
||||
use super::types::{CapacityDiskRef, CapacityScanResult, CapacityScanSummary};
|
||||
use crate::capacity_scope::CapacityScopeDisk;
|
||||
use futures::{StreamExt, stream};
|
||||
use rustfs_common::capacity_scope::CapacityScopeDisk;
|
||||
use rustfs_io_metrics::capacity_metrics::{
|
||||
record_capacity_dynamic_timeout, record_capacity_scan_disk, record_capacity_scan_mode, record_capacity_scan_sampling,
|
||||
record_capacity_stall_detected, record_capacity_symlink, record_capacity_timeout_fallback,
|
||||
@@ -647,7 +647,7 @@ async fn get_dir_size_async(path: &Path) -> Result<CapacityScanResult, std::io::
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::capacity_manager::{DataSource, HybridStrategyConfig, create_isolated_manager};
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk};
|
||||
use crate::capacity_scope::{CapacityScope, CapacityScopeDisk};
|
||||
#[cfg(unix)]
|
||||
use rustfs_config::ENV_CAPACITY_FOLLOW_SYMLINKS;
|
||||
use serial_test::serial;
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::metrics::collectors::{
|
||||
ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
|
||||
};
|
||||
use chrono::Utc;
|
||||
use rustfs_common::{internode_metrics::global_internode_metrics, metrics::global_metrics};
|
||||
use rustfs_common::metrics::global_metrics;
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
|
||||
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
|
||||
use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
|
||||
@@ -37,6 +37,7 @@ use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity
|
||||
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
|
||||
use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
|
||||
use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -34,6 +34,7 @@ path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
rustfs-common.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
rustfs-utils.workspace = true
|
||||
flatbuffers = { workspace = true }
|
||||
|
||||
@@ -18,9 +18,8 @@
|
||||
mod generated;
|
||||
|
||||
use proto_gen::node_service::node_service_client::NodeServiceClient;
|
||||
use rustfs_common::{
|
||||
GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection, internode_metrics::global_internode_metrics,
|
||||
};
|
||||
use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use std::{
|
||||
error::Error,
|
||||
time::{Duration, Instant},
|
||||
|
||||
@@ -43,6 +43,7 @@ faster-hex.workspace = true
|
||||
futures.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-common.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["io", "hash", "compress", "tls"] }
|
||||
serde_json.workspace = true
|
||||
md-5 = { workspace = true }
|
||||
|
||||
@@ -18,7 +18,7 @@ use futures::{Stream, TryStreamExt as _};
|
||||
use http::HeaderMap;
|
||||
use pin_project_lite::pin_project;
|
||||
use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
|
||||
use rustfs_common::internode_metrics::{
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
|
||||
global_internode_metrics,
|
||||
};
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
# RustFS S3 Common
|
||||
|
||||
`rustfs-s3-common` provides shared types, utilities, and definitions for S3-compatible operations within the RustFS
|
||||
ecosystem. It serves as a foundational crate for handling S3 event notifications, metrics, and operation definitions.
|
||||
|
||||
## Features
|
||||
|
||||
- **Event Definitions**: Comprehensive `EventName` enum covering standard AWS S3 event notifications (e.g.,
|
||||
`s3:ObjectCreated:Put`, `s3:ObjectRemoved:Delete`) and RustFS-specific extensions.
|
||||
- **S3 Operations**: `S3Operation` enum defining supported S3 API actions, used for metrics tracking and audit logging.
|
||||
- **Metrics Integration**: Utilities for recording S3 operation metrics (`record_s3_op`) using the `metrics` crate.
|
||||
- **Type Mapping**: robust mapping between `EventName` and `S3Operation` to bridge the gap between API calls and event
|
||||
notifications.
|
||||
|
||||
## Usage
|
||||
|
||||
Add this crate to your `Cargo.toml`:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
rustfs-s3-common = { path = "../s3-common" }
|
||||
```
|
||||
|
||||
### Event Names and Operations
|
||||
|
||||
```rust
|
||||
use rustfs_s3_common::{EventName, S3Operation};
|
||||
|
||||
// Parse an event string
|
||||
let event = EventName::parse("s3:ObjectCreated:Put").unwrap();
|
||||
assert_eq!(event, EventName::ObjectCreatedPut);
|
||||
|
||||
// Map event to S3 operation
|
||||
let op = event.to_s3_operation();
|
||||
assert_eq!(op, Some(S3Operation::PutObject));
|
||||
|
||||
// Get string representation
|
||||
assert_eq!(S3Operation::PutObject.as_str(), "s3:PutObject");
|
||||
```
|
||||
|
||||
### Metrics
|
||||
|
||||
Initialize and record metrics for S3 operations:
|
||||
|
||||
```rust
|
||||
use rustfs_s3_common::{init_s3_metrics, record_s3_op};
|
||||
use rustfs_s3_common::S3Operation;
|
||||
|
||||
// Initialize metrics (call once)
|
||||
init_s3_metrics();
|
||||
|
||||
// Record an operation
|
||||
record_s3_op(S3Operation::GetObject, "my-bucket");
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the [Apache-2.0 License](../../LICENSE).
|
||||
@@ -1,19 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod event_name;
|
||||
mod s3_metrics;
|
||||
|
||||
pub use event_name::{EventName, ParseEventNameError, S3Operation};
|
||||
pub use s3_metrics::{init_s3_metrics, record_s3_op};
|
||||
@@ -13,26 +13,22 @@
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-appauth"
|
||||
name = "rustfs-s3-ops"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "Application authentication and authorization for RustFS, providing secure access control and user management."
|
||||
keywords = ["authentication", "authorization", "security", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "authentication"]
|
||||
|
||||
[dependencies]
|
||||
base64-simd = { workspace = true }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
rand.workspace = true
|
||||
description = "S3 operation enum and event mapping for RustFS."
|
||||
keywords = ["s3", "operations", "rustfs"]
|
||||
categories = ["data-structures"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-s3-types = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
@@ -0,0 +1,445 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum S3Operation {
|
||||
AbortMultipartUpload,
|
||||
CompleteMultipartUpload,
|
||||
CopyObject,
|
||||
CreateBucket,
|
||||
CreateMultipartUpload,
|
||||
DeleteBucket,
|
||||
DeleteBucketCors,
|
||||
DeleteBucketEncryption,
|
||||
DeleteBucketLifecycle,
|
||||
DeleteBucketPolicy,
|
||||
DeleteBucketReplication,
|
||||
DeleteBucketTagging,
|
||||
DeleteObject,
|
||||
DeleteObjectTagging,
|
||||
DeleteObjects,
|
||||
DeletePublicAccessBlock,
|
||||
GetBucketAcl,
|
||||
GetBucketCors,
|
||||
GetBucketEncryption,
|
||||
GetBucketLifecycleConfiguration,
|
||||
GetBucketLocation,
|
||||
GetBucketLogging,
|
||||
GetBucketNotificationConfiguration,
|
||||
GetBucketPolicy,
|
||||
GetBucketPolicyStatus,
|
||||
GetBucketReplication,
|
||||
GetBucketTagging,
|
||||
GetBucketVersioning,
|
||||
GetObject,
|
||||
GetObjectAcl,
|
||||
GetObjectAttributes,
|
||||
GetObjectLegalHold,
|
||||
GetObjectLockConfiguration,
|
||||
GetObjectRetention,
|
||||
GetObjectTagging,
|
||||
GetObjectTorrent,
|
||||
GetPublicAccessBlock,
|
||||
HeadBucket,
|
||||
HeadObject,
|
||||
ListBuckets,
|
||||
ListMultipartUploads,
|
||||
ListObjectVersions,
|
||||
ListObjects,
|
||||
ListObjectsV2,
|
||||
ListParts,
|
||||
PutBucketAcl,
|
||||
PutBucketCors,
|
||||
PutBucketEncryption,
|
||||
PutBucketLifecycleConfiguration,
|
||||
PutBucketLogging,
|
||||
PutBucketNotificationConfiguration,
|
||||
PutBucketPolicy,
|
||||
PutBucketReplication,
|
||||
PutBucketTagging,
|
||||
PutBucketVersioning,
|
||||
PutObject,
|
||||
PutObjectAcl,
|
||||
PutObjectLegalHold,
|
||||
PutObjectLockConfiguration,
|
||||
PutObjectRetention,
|
||||
PutObjectTagging,
|
||||
PutPublicAccessBlock,
|
||||
RestoreObject,
|
||||
SelectObjectContent,
|
||||
UploadPart,
|
||||
UploadPartCopy,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum EventMapping {
|
||||
None,
|
||||
Single(EventName),
|
||||
PutObject,
|
||||
DeleteObject,
|
||||
DeleteObjects,
|
||||
}
|
||||
|
||||
impl EventMapping {
|
||||
#[inline]
|
||||
fn primary_event(self) -> Option<EventName> {
|
||||
match self {
|
||||
Self::None => None,
|
||||
Self::Single(event_name) => Some(event_name),
|
||||
Self::PutObject => Some(EventName::ObjectCreatedPut),
|
||||
Self::DeleteObject => Some(EventName::ObjectRemovedDelete),
|
||||
Self::DeleteObjects => Some(EventName::ObjectRemovedDeleteObjects),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn matches_event(self, event_name: EventName) -> bool {
|
||||
match self {
|
||||
Self::None => false,
|
||||
Self::Single(mapped_event_name) => mapped_event_name == event_name,
|
||||
Self::PutObject => matches!(event_name, EventName::ObjectCreatedPut | EventName::ObjectCreatedPost),
|
||||
Self::DeleteObject => matches!(
|
||||
event_name,
|
||||
EventName::ObjectRemovedDelete
|
||||
| EventName::ObjectRemovedDeleteMarkerCreated
|
||||
| EventName::ObjectRemovedDeleteAllVersions
|
||||
),
|
||||
Self::DeleteObjects => {
|
||||
matches!(event_name, EventName::ObjectRemovedDeleteObjects | EventName::ObjectRemovedDelete)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Operation {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::AbortMultipartUpload => "s3:AbortMultipartUpload",
|
||||
Self::CompleteMultipartUpload => "s3:CompleteMultipartUpload",
|
||||
Self::CopyObject => "s3:CopyObject",
|
||||
Self::CreateBucket => "s3:CreateBucket",
|
||||
Self::CreateMultipartUpload => "s3:CreateMultipartUpload",
|
||||
Self::DeleteBucket => "s3:DeleteBucket",
|
||||
Self::DeleteBucketCors => "s3:DeleteBucketCors",
|
||||
Self::DeleteBucketEncryption => "s3:DeleteBucketEncryption",
|
||||
Self::DeleteBucketLifecycle => "s3:DeleteBucketLifecycle",
|
||||
Self::DeleteBucketPolicy => "s3:DeleteBucketPolicy",
|
||||
Self::DeleteBucketReplication => "s3:DeleteBucketReplication",
|
||||
Self::DeleteBucketTagging => "s3:DeleteBucketTagging",
|
||||
Self::DeleteObject => "s3:DeleteObject",
|
||||
Self::DeleteObjectTagging => "s3:DeleteObjectTagging",
|
||||
Self::DeleteObjects => "s3:DeleteObjects",
|
||||
Self::DeletePublicAccessBlock => "s3:DeletePublicAccessBlock",
|
||||
Self::GetBucketAcl => "s3:GetBucketAcl",
|
||||
Self::GetBucketCors => "s3:GetBucketCors",
|
||||
Self::GetBucketEncryption => "s3:GetBucketEncryption",
|
||||
Self::GetBucketLifecycleConfiguration => "s3:GetBucketLifecycleConfiguration",
|
||||
Self::GetBucketLocation => "s3:GetBucketLocation",
|
||||
Self::GetBucketLogging => "s3:GetBucketLogging",
|
||||
Self::GetBucketNotificationConfiguration => "s3:GetBucketNotificationConfiguration",
|
||||
Self::GetBucketPolicy => "s3:GetBucketPolicy",
|
||||
Self::GetBucketPolicyStatus => "s3:GetBucketPolicyStatus",
|
||||
Self::GetBucketReplication => "s3:GetBucketReplication",
|
||||
Self::GetBucketTagging => "s3:GetBucketTagging",
|
||||
Self::GetBucketVersioning => "s3:GetBucketVersioning",
|
||||
Self::GetObject => "s3:GetObject",
|
||||
Self::GetObjectAcl => "s3:GetObjectAcl",
|
||||
Self::GetObjectAttributes => "s3:GetObjectAttributes",
|
||||
Self::GetObjectLegalHold => "s3:GetObjectLegalHold",
|
||||
Self::GetObjectLockConfiguration => "s3:GetObjectLockConfiguration",
|
||||
Self::GetObjectRetention => "s3:GetObjectRetention",
|
||||
Self::GetObjectTagging => "s3:GetObjectTagging",
|
||||
Self::GetObjectTorrent => "s3:GetObjectTorrent",
|
||||
Self::GetPublicAccessBlock => "s3:GetPublicAccessBlock",
|
||||
Self::HeadBucket => "s3:HeadBucket",
|
||||
Self::HeadObject => "s3:HeadObject",
|
||||
Self::ListBuckets => "s3:ListBuckets",
|
||||
Self::ListMultipartUploads => "s3:ListMultipartUploads",
|
||||
Self::ListObjectVersions => "s3:ListObjectVersions",
|
||||
Self::ListObjects => "s3:ListObjects",
|
||||
Self::ListObjectsV2 => "s3:ListObjectsV2",
|
||||
Self::ListParts => "s3:ListParts",
|
||||
Self::PutBucketAcl => "s3:PutBucketAcl",
|
||||
Self::PutBucketCors => "s3:PutBucketCors",
|
||||
Self::PutBucketEncryption => "s3:PutBucketEncryption",
|
||||
Self::PutBucketLifecycleConfiguration => "s3:PutBucketLifecycleConfiguration",
|
||||
Self::PutBucketLogging => "s3:PutBucketLogging",
|
||||
Self::PutBucketNotificationConfiguration => "s3:PutBucketNotificationConfiguration",
|
||||
Self::PutBucketPolicy => "s3:PutBucketPolicy",
|
||||
Self::PutBucketReplication => "s3:PutBucketReplication",
|
||||
Self::PutBucketTagging => "s3:PutBucketTagging",
|
||||
Self::PutBucketVersioning => "s3:PutBucketVersioning",
|
||||
Self::PutObject => "s3:PutObject",
|
||||
Self::PutObjectAcl => "s3:PutObjectAcl",
|
||||
Self::PutObjectLegalHold => "s3:PutObjectLegalHold",
|
||||
Self::PutObjectLockConfiguration => "s3:PutObjectLockConfiguration",
|
||||
Self::PutObjectRetention => "s3:PutObjectRetention",
|
||||
Self::PutObjectTagging => "s3:PutObjectTagging",
|
||||
Self::PutPublicAccessBlock => "s3:PutPublicAccessBlock",
|
||||
Self::RestoreObject => "s3:RestoreObject",
|
||||
Self::SelectObjectContent => "s3:SelectObjectContent",
|
||||
Self::UploadPart => "s3:UploadPart",
|
||||
Self::UploadPartCopy => "s3:UploadPartCopy",
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn event_mapping(self) -> EventMapping {
|
||||
match self {
|
||||
Self::AbortMultipartUpload => EventMapping::Single(EventName::ObjectRemovedAbortMultipartUpload),
|
||||
Self::CompleteMultipartUpload => EventMapping::Single(EventName::ObjectCreatedCompleteMultipartUpload),
|
||||
Self::CopyObject => EventMapping::Single(EventName::ObjectCreatedCopy),
|
||||
Self::CreateBucket => EventMapping::Single(EventName::BucketCreated),
|
||||
Self::CreateMultipartUpload => EventMapping::Single(EventName::ObjectCreatedCreateMultipartUpload),
|
||||
Self::DeleteBucket => EventMapping::Single(EventName::BucketRemoved),
|
||||
Self::DeleteObjectTagging => EventMapping::Single(EventName::ObjectTaggingDelete),
|
||||
Self::DeleteObject => EventMapping::DeleteObject,
|
||||
Self::DeleteObjects => EventMapping::DeleteObjects,
|
||||
Self::GetObject => EventMapping::Single(EventName::ObjectAccessedGet),
|
||||
Self::GetObjectAttributes => EventMapping::Single(EventName::ObjectAccessedAttributes),
|
||||
Self::GetObjectLegalHold => EventMapping::Single(EventName::ObjectAccessedGetLegalHold),
|
||||
Self::GetObjectRetention => EventMapping::Single(EventName::ObjectAccessedGetRetention),
|
||||
Self::HeadObject => EventMapping::Single(EventName::ObjectAccessedHead),
|
||||
Self::PutObject => EventMapping::PutObject,
|
||||
Self::PutObjectAcl => EventMapping::Single(EventName::ObjectAclPut),
|
||||
Self::PutObjectLegalHold => EventMapping::Single(EventName::ObjectCreatedPutLegalHold),
|
||||
Self::PutObjectRetention => EventMapping::Single(EventName::ObjectCreatedPutRetention),
|
||||
Self::PutObjectTagging => EventMapping::Single(EventName::ObjectTaggingPut),
|
||||
Self::RestoreObject => EventMapping::Single(EventName::ObjectRestorePost),
|
||||
Self::SelectObjectContent => EventMapping::Single(EventName::ObjectAccessedGet),
|
||||
Self::DeleteBucketCors
|
||||
| Self::DeleteBucketEncryption
|
||||
| Self::DeleteBucketLifecycle
|
||||
| Self::DeleteBucketPolicy
|
||||
| Self::DeleteBucketReplication
|
||||
| Self::DeleteBucketTagging
|
||||
| Self::DeletePublicAccessBlock
|
||||
| Self::GetBucketAcl
|
||||
| Self::GetBucketCors
|
||||
| Self::GetBucketEncryption
|
||||
| Self::GetBucketLifecycleConfiguration
|
||||
| Self::GetBucketLocation
|
||||
| Self::GetBucketLogging
|
||||
| Self::GetBucketNotificationConfiguration
|
||||
| Self::GetBucketPolicy
|
||||
| Self::GetBucketPolicyStatus
|
||||
| Self::GetBucketReplication
|
||||
| Self::GetBucketTagging
|
||||
| Self::GetBucketVersioning
|
||||
| Self::GetObjectAcl
|
||||
| Self::GetObjectLockConfiguration
|
||||
| Self::GetObjectTagging
|
||||
| Self::GetObjectTorrent
|
||||
| Self::GetPublicAccessBlock
|
||||
| Self::HeadBucket
|
||||
| Self::ListBuckets
|
||||
| Self::ListMultipartUploads
|
||||
| Self::ListObjectVersions
|
||||
| Self::ListObjects
|
||||
| Self::ListObjectsV2
|
||||
| Self::ListParts
|
||||
| Self::PutBucketAcl
|
||||
| Self::PutBucketCors
|
||||
| Self::PutBucketEncryption
|
||||
| Self::PutBucketLifecycleConfiguration
|
||||
| Self::PutBucketLogging
|
||||
| Self::PutBucketNotificationConfiguration
|
||||
| Self::PutBucketPolicy
|
||||
| Self::PutBucketReplication
|
||||
| Self::PutBucketTagging
|
||||
| Self::PutBucketVersioning
|
||||
| Self::PutObjectLockConfiguration
|
||||
| Self::PutPublicAccessBlock
|
||||
| Self::UploadPart
|
||||
| Self::UploadPartCopy => EventMapping::None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_event_name(self) -> Option<EventName> {
|
||||
self.event_mapping().primary_event()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn event_name_to_s3_operation(event_name: EventName) -> Option<S3Operation> {
|
||||
match event_name {
|
||||
EventName::BucketCreated => Some(S3Operation::CreateBucket),
|
||||
EventName::BucketRemoved => Some(S3Operation::DeleteBucket),
|
||||
EventName::ObjectAccessedGet => Some(S3Operation::GetObject),
|
||||
EventName::ObjectAccessedGetRetention => Some(S3Operation::GetObjectRetention),
|
||||
EventName::ObjectAccessedGetLegalHold => Some(S3Operation::GetObjectLegalHold),
|
||||
EventName::ObjectAccessedHead => Some(S3Operation::HeadObject),
|
||||
EventName::ObjectAccessedAttributes => Some(S3Operation::GetObjectAttributes),
|
||||
EventName::ObjectCreatedCompleteMultipartUpload => Some(S3Operation::CompleteMultipartUpload),
|
||||
EventName::ObjectCreatedCopy => Some(S3Operation::CopyObject),
|
||||
EventName::ObjectCreatedPost => Some(S3Operation::PutObject),
|
||||
EventName::ObjectCreatedPut => Some(S3Operation::PutObject),
|
||||
EventName::ObjectCreatedPutRetention => Some(S3Operation::PutObjectRetention),
|
||||
EventName::ObjectCreatedPutLegalHold => Some(S3Operation::PutObjectLegalHold),
|
||||
EventName::ObjectTaggingPut => Some(S3Operation::PutObjectTagging),
|
||||
EventName::ObjectTaggingDelete => Some(S3Operation::DeleteObjectTagging),
|
||||
EventName::ObjectAclPut => Some(S3Operation::PutObjectAcl),
|
||||
EventName::ObjectRemovedDelete => Some(S3Operation::DeleteObject),
|
||||
EventName::ObjectRemovedDeleteMarkerCreated => Some(S3Operation::DeleteObject),
|
||||
EventName::ObjectRemovedDeleteAllVersions => Some(S3Operation::DeleteObject),
|
||||
EventName::ObjectRestorePost => Some(S3Operation::RestoreObject),
|
||||
EventName::ObjectRemovedAbortMultipartUpload => Some(S3Operation::AbortMultipartUpload),
|
||||
EventName::ObjectCreatedCreateMultipartUpload => Some(S3Operation::CreateMultipartUpload),
|
||||
EventName::ObjectRemovedDeleteObjects => Some(S3Operation::DeleteObjects),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether an S3 operation is semantically compatible with an event name.
|
||||
///
|
||||
/// Some S3 operations intentionally map to multiple event variants:
|
||||
/// - `PutObject` can emit both `ObjectCreatedPut` and `ObjectCreatedPost`.
|
||||
/// - `DeleteObject` can emit delete/delete-marker/all-versions variants.
|
||||
/// - `DeleteObjects` can emit per-object delete events in addition to the
|
||||
/// internal batch-delete event.
|
||||
pub fn operation_matches_event_name(op: S3Operation, event_name: EventName) -> bool {
|
||||
op.event_mapping().matches_event(event_name)
|
||||
}
|
||||
|
||||
/// Resolves the object-delete notification event name from delete-marker state.
|
||||
#[inline]
|
||||
pub fn delete_event_name_for_marker(delete_marker: bool) -> EventName {
|
||||
if delete_marker {
|
||||
EventName::ObjectRemovedDeleteMarkerCreated
|
||||
} else {
|
||||
EventName::ObjectRemovedDelete
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the object-create notification event name from POST-object mode.
|
||||
#[inline]
|
||||
pub fn put_event_name_for_post_object(is_post_object: bool) -> EventName {
|
||||
if is_post_object {
|
||||
EventName::ObjectCreatedPost
|
||||
} else {
|
||||
EventName::ObjectCreatedPut
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` when the event is one of object-remove notification variants
|
||||
/// that should omit size/etag metadata.
|
||||
#[inline]
|
||||
pub fn is_object_removed_event(event_name: EventName) -> bool {
|
||||
matches!(event_name, EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated)
|
||||
}
|
||||
|
||||
/// Returns the event mask that matches both PUT and POST object-created events.
|
||||
#[inline]
|
||||
pub fn put_object_created_event_mask() -> u64 {
|
||||
EventName::ObjectCreatedPut.mask() | EventName::ObjectCreatedPost.mask()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_s3_operation_to_event_name() {
|
||||
assert_eq!(S3Operation::PutObject.to_event_name(), Some(EventName::ObjectCreatedPut));
|
||||
assert_eq!(S3Operation::PutObjectAcl.to_event_name(), Some(EventName::ObjectAclPut));
|
||||
assert_eq!(S3Operation::PutObjectTagging.to_event_name(), Some(EventName::ObjectTaggingPut));
|
||||
assert_eq!(S3Operation::DeleteObjectTagging.to_event_name(), Some(EventName::ObjectTaggingDelete));
|
||||
assert_eq!(S3Operation::GetObject.to_event_name(), Some(EventName::ObjectAccessedGet));
|
||||
assert_eq!(S3Operation::ListBuckets.to_event_name(), None);
|
||||
assert_eq!(S3Operation::RestoreObject.to_event_name(), Some(EventName::ObjectRestorePost));
|
||||
assert_eq!(S3Operation::SelectObjectContent.to_event_name(), Some(EventName::ObjectAccessedGet));
|
||||
assert_eq!(
|
||||
S3Operation::AbortMultipartUpload.to_event_name(),
|
||||
Some(EventName::ObjectRemovedAbortMultipartUpload)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_event_name_to_s3_operation() {
|
||||
assert_eq!(event_name_to_s3_operation(EventName::ObjectCreatedPut), Some(S3Operation::PutObject));
|
||||
assert_eq!(event_name_to_s3_operation(EventName::ObjectAclPut), Some(S3Operation::PutObjectAcl));
|
||||
assert_eq!(
|
||||
event_name_to_s3_operation(EventName::ObjectTaggingPut),
|
||||
Some(S3Operation::PutObjectTagging)
|
||||
);
|
||||
assert_eq!(
|
||||
event_name_to_s3_operation(EventName::ObjectTaggingDelete),
|
||||
Some(S3Operation::DeleteObjectTagging)
|
||||
);
|
||||
assert_eq!(event_name_to_s3_operation(EventName::ObjectAccessedGet), Some(S3Operation::GetObject));
|
||||
assert_eq!(event_name_to_s3_operation(EventName::BucketCreated), Some(S3Operation::CreateBucket));
|
||||
assert_eq!(event_name_to_s3_operation(EventName::Everything), None);
|
||||
assert_eq!(event_name_to_s3_operation(EventName::ObjectRestorePost), Some(S3Operation::RestoreObject));
|
||||
assert_eq!(event_name_to_s3_operation(EventName::ObjectCreatedPost), Some(S3Operation::PutObject));
|
||||
assert_eq!(
|
||||
event_name_to_s3_operation(EventName::ObjectRemovedAbortMultipartUpload),
|
||||
Some(S3Operation::AbortMultipartUpload)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operation_matches_event_name() {
|
||||
assert!(operation_matches_event_name(S3Operation::PutObject, EventName::ObjectCreatedPut));
|
||||
assert!(operation_matches_event_name(S3Operation::PutObject, EventName::ObjectCreatedPost));
|
||||
assert!(operation_matches_event_name(
|
||||
S3Operation::DeleteObject,
|
||||
EventName::ObjectRemovedDeleteMarkerCreated
|
||||
));
|
||||
assert!(operation_matches_event_name(S3Operation::DeleteObjects, EventName::ObjectRemovedDelete));
|
||||
|
||||
assert!(!operation_matches_event_name(S3Operation::GetObject, EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_event_name_for_marker() {
|
||||
assert_eq!(delete_event_name_for_marker(true), EventName::ObjectRemovedDeleteMarkerCreated);
|
||||
assert_eq!(delete_event_name_for_marker(false), EventName::ObjectRemovedDelete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_event_name_for_post_object() {
|
||||
assert_eq!(put_event_name_for_post_object(true), EventName::ObjectCreatedPost);
|
||||
assert_eq!(put_event_name_for_post_object(false), EventName::ObjectCreatedPut);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_object_removed_event() {
|
||||
assert!(is_object_removed_event(EventName::ObjectRemovedDelete));
|
||||
assert!(is_object_removed_event(EventName::ObjectRemovedDeleteMarkerCreated));
|
||||
assert!(!is_object_removed_event(EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_object_created_event_mask() {
|
||||
let mask = put_object_created_event_mask();
|
||||
assert_ne!(mask & EventName::ObjectCreatedPut.mask(), 0);
|
||||
assert_ne!(mask & EventName::ObjectCreatedPost.mask(), 0);
|
||||
assert_eq!(mask & EventName::ObjectRemovedDelete.mask(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operations_without_event_mapping_remain_unmapped() {
|
||||
let unmapped = [
|
||||
S3Operation::GetBucketAcl,
|
||||
S3Operation::ListObjectsV2,
|
||||
S3Operation::PutBucketNotificationConfiguration,
|
||||
S3Operation::UploadPart,
|
||||
];
|
||||
for op in unmapped {
|
||||
assert_eq!(op.to_event_name(), None);
|
||||
assert!(!operation_matches_event_name(op, EventName::ObjectCreatedPut));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,22 +13,21 @@
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-s3-common"
|
||||
name = "rustfs-s3-types"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "Common S3 definitions and metrics for RustFS."
|
||||
keywords = ["s3", "common", "rustfs"]
|
||||
description = "S3 event type definitions for RustFS."
|
||||
keywords = ["s3", "event", "types", "rustfs"]
|
||||
categories = ["data-structures"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
@@ -190,6 +190,12 @@ impl EventName {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses an event string into an EventName with explicit error handling.
|
||||
#[inline]
|
||||
pub fn try_from_event_str(s: &str) -> Result<Self, ParseEventNameError> {
|
||||
Self::parse(s)
|
||||
}
|
||||
|
||||
/// Returns a string representation of the event type.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
@@ -315,35 +321,26 @@ impl EventName {
|
||||
mask
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the corresponding S3Operation if the event triggers a notification event.
|
||||
pub fn to_s3_operation(&self) -> Option<S3Operation> {
|
||||
match self {
|
||||
EventName::BucketCreated => Some(S3Operation::CreateBucket),
|
||||
EventName::BucketRemoved => Some(S3Operation::DeleteBucket),
|
||||
EventName::ObjectAccessedGet => Some(S3Operation::GetObject),
|
||||
EventName::ObjectAccessedGetRetention => Some(S3Operation::GetObjectRetention),
|
||||
EventName::ObjectAccessedGetLegalHold => Some(S3Operation::GetObjectLegalHold),
|
||||
EventName::ObjectAccessedHead => Some(S3Operation::HeadObject),
|
||||
EventName::ObjectAccessedAttributes => Some(S3Operation::GetObjectAttributes),
|
||||
EventName::ObjectCreatedCompleteMultipartUpload => Some(S3Operation::CompleteMultipartUpload),
|
||||
EventName::ObjectCreatedCopy => Some(S3Operation::CopyObject),
|
||||
EventName::ObjectCreatedPost => Some(S3Operation::PutObject),
|
||||
EventName::ObjectCreatedPut => Some(S3Operation::PutObject),
|
||||
EventName::ObjectCreatedPutRetention => Some(S3Operation::PutObjectRetention),
|
||||
EventName::ObjectCreatedPutLegalHold => Some(S3Operation::PutObjectLegalHold),
|
||||
EventName::ObjectTaggingPut => Some(S3Operation::PutObjectTagging),
|
||||
EventName::ObjectTaggingDelete => Some(S3Operation::DeleteObjectTagging),
|
||||
EventName::ObjectAclPut => Some(S3Operation::PutObjectAcl),
|
||||
EventName::ObjectRemovedDelete => Some(S3Operation::DeleteObject),
|
||||
EventName::ObjectRemovedDeleteMarkerCreated => Some(S3Operation::DeleteObject),
|
||||
EventName::ObjectRemovedDeleteAllVersions => Some(S3Operation::DeleteObject),
|
||||
EventName::ObjectRestorePost => Some(S3Operation::RestoreObject),
|
||||
EventName::ObjectRemovedAbortMultipartUpload => Some(S3Operation::AbortMultipartUpload),
|
||||
EventName::ObjectCreatedCreateMultipartUpload => Some(S3Operation::CreateMultipartUpload),
|
||||
EventName::ObjectRemovedDeleteObjects => Some(S3Operation::DeleteObjects),
|
||||
_ => None,
|
||||
}
|
||||
/// Returns the S3 notification event schema version for a given event.
|
||||
#[inline]
|
||||
pub fn event_schema_version(event_name: EventName) -> &'static str {
|
||||
match event_name {
|
||||
EventName::ObjectReplicationFailed
|
||||
| EventName::ObjectReplicationComplete
|
||||
| EventName::ObjectReplicationMissedThreshold
|
||||
| EventName::ObjectReplicationReplicatedAfterThreshold
|
||||
| EventName::ObjectReplicationNotTracked => "2.2",
|
||||
EventName::ObjectRestoreCompleted
|
||||
| EventName::ObjectAclPut
|
||||
| EventName::ObjectTaggingPut
|
||||
| EventName::ObjectTaggingDelete
|
||||
| EventName::LifecycleExpirationDelete
|
||||
| EventName::LifecycleExpirationDeleteMarkerCreated
|
||||
| EventName::LifecycleTransition
|
||||
| EventName::IntelligentTiering => "2.3",
|
||||
_ => "2.1",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -380,177 +377,6 @@ impl<'de> serde::de::Deserialize<'de> for EventName {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum S3Operation {
|
||||
AbortMultipartUpload,
|
||||
CompleteMultipartUpload,
|
||||
CopyObject,
|
||||
CreateBucket,
|
||||
CreateMultipartUpload,
|
||||
DeleteBucket,
|
||||
DeleteBucketCors,
|
||||
DeleteBucketEncryption,
|
||||
DeleteBucketLifecycle,
|
||||
DeleteBucketPolicy,
|
||||
DeleteBucketReplication,
|
||||
DeleteBucketTagging,
|
||||
DeleteObject,
|
||||
DeleteObjectTagging,
|
||||
DeleteObjects,
|
||||
DeletePublicAccessBlock,
|
||||
GetBucketAcl,
|
||||
GetBucketCors,
|
||||
GetBucketEncryption,
|
||||
GetBucketLifecycleConfiguration,
|
||||
GetBucketLocation,
|
||||
GetBucketLogging,
|
||||
GetBucketNotificationConfiguration,
|
||||
GetBucketPolicy,
|
||||
GetBucketPolicyStatus,
|
||||
GetBucketReplication,
|
||||
GetBucketTagging,
|
||||
GetBucketVersioning,
|
||||
GetObject,
|
||||
GetObjectAcl,
|
||||
GetObjectAttributes,
|
||||
GetObjectLegalHold,
|
||||
GetObjectLockConfiguration,
|
||||
GetObjectRetention,
|
||||
GetObjectTagging,
|
||||
GetObjectTorrent,
|
||||
GetPublicAccessBlock,
|
||||
HeadBucket,
|
||||
HeadObject,
|
||||
ListBuckets,
|
||||
ListMultipartUploads,
|
||||
ListObjectVersions,
|
||||
ListObjects,
|
||||
ListObjectsV2,
|
||||
ListParts,
|
||||
PutBucketAcl,
|
||||
PutBucketCors,
|
||||
PutBucketEncryption,
|
||||
PutBucketLifecycleConfiguration,
|
||||
PutBucketLogging,
|
||||
PutBucketNotificationConfiguration,
|
||||
PutBucketPolicy,
|
||||
PutBucketReplication,
|
||||
PutBucketTagging,
|
||||
PutBucketVersioning,
|
||||
PutObject,
|
||||
PutObjectAcl,
|
||||
PutObjectLegalHold,
|
||||
PutObjectLockConfiguration,
|
||||
PutObjectRetention,
|
||||
PutObjectTagging,
|
||||
PutPublicAccessBlock,
|
||||
RestoreObject,
|
||||
SelectObjectContent,
|
||||
UploadPart,
|
||||
UploadPartCopy,
|
||||
}
|
||||
|
||||
impl S3Operation {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::AbortMultipartUpload => "s3:AbortMultipartUpload",
|
||||
Self::CompleteMultipartUpload => "s3:CompleteMultipartUpload",
|
||||
Self::CopyObject => "s3:CopyObject",
|
||||
Self::CreateBucket => "s3:CreateBucket",
|
||||
Self::CreateMultipartUpload => "s3:CreateMultipartUpload",
|
||||
Self::DeleteBucket => "s3:DeleteBucket",
|
||||
Self::DeleteBucketCors => "s3:DeleteBucketCors",
|
||||
Self::DeleteBucketEncryption => "s3:DeleteBucketEncryption",
|
||||
Self::DeleteBucketLifecycle => "s3:DeleteBucketLifecycle",
|
||||
Self::DeleteBucketPolicy => "s3:DeleteBucketPolicy",
|
||||
Self::DeleteBucketReplication => "s3:DeleteBucketReplication",
|
||||
Self::DeleteBucketTagging => "s3:DeleteBucketTagging",
|
||||
Self::DeleteObject => "s3:DeleteObject",
|
||||
Self::DeleteObjectTagging => "s3:DeleteObjectTagging",
|
||||
Self::DeleteObjects => "s3:DeleteObjects",
|
||||
Self::DeletePublicAccessBlock => "s3:DeletePublicAccessBlock",
|
||||
Self::GetBucketAcl => "s3:GetBucketAcl",
|
||||
Self::GetBucketCors => "s3:GetBucketCors",
|
||||
Self::GetBucketEncryption => "s3:GetBucketEncryption",
|
||||
Self::GetBucketLifecycleConfiguration => "s3:GetBucketLifecycleConfiguration",
|
||||
Self::GetBucketLocation => "s3:GetBucketLocation",
|
||||
Self::GetBucketLogging => "s3:GetBucketLogging",
|
||||
Self::GetBucketNotificationConfiguration => "s3:GetBucketNotificationConfiguration",
|
||||
Self::GetBucketPolicy => "s3:GetBucketPolicy",
|
||||
Self::GetBucketPolicyStatus => "s3:GetBucketPolicyStatus",
|
||||
Self::GetBucketReplication => "s3:GetBucketReplication",
|
||||
Self::GetBucketTagging => "s3:GetBucketTagging",
|
||||
Self::GetBucketVersioning => "s3:GetBucketVersioning",
|
||||
Self::GetObject => "s3:GetObject",
|
||||
Self::GetObjectAcl => "s3:GetObjectAcl",
|
||||
Self::GetObjectAttributes => "s3:GetObjectAttributes",
|
||||
Self::GetObjectLegalHold => "s3:GetObjectLegalHold",
|
||||
Self::GetObjectLockConfiguration => "s3:GetObjectLockConfiguration",
|
||||
Self::GetObjectRetention => "s3:GetObjectRetention",
|
||||
Self::GetObjectTagging => "s3:GetObjectTagging",
|
||||
Self::GetObjectTorrent => "s3:GetObjectTorrent",
|
||||
Self::GetPublicAccessBlock => "s3:GetPublicAccessBlock",
|
||||
Self::HeadBucket => "s3:HeadBucket",
|
||||
Self::HeadObject => "s3:HeadObject",
|
||||
Self::ListBuckets => "s3:ListBuckets",
|
||||
Self::ListMultipartUploads => "s3:ListMultipartUploads",
|
||||
Self::ListObjectVersions => "s3:ListObjectVersions",
|
||||
Self::ListObjects => "s3:ListObjects",
|
||||
Self::ListObjectsV2 => "s3:ListObjectsV2",
|
||||
Self::ListParts => "s3:ListParts",
|
||||
Self::PutBucketAcl => "s3:PutBucketAcl",
|
||||
Self::PutBucketCors => "s3:PutBucketCors",
|
||||
Self::PutBucketEncryption => "s3:PutBucketEncryption",
|
||||
Self::PutBucketLifecycleConfiguration => "s3:PutBucketLifecycleConfiguration",
|
||||
Self::PutBucketLogging => "s3:PutBucketLogging",
|
||||
Self::PutBucketNotificationConfiguration => "s3:PutBucketNotificationConfiguration",
|
||||
Self::PutBucketPolicy => "s3:PutBucketPolicy",
|
||||
Self::PutBucketReplication => "s3:PutBucketReplication",
|
||||
Self::PutBucketTagging => "s3:PutBucketTagging",
|
||||
Self::PutBucketVersioning => "s3:PutBucketVersioning",
|
||||
Self::PutObject => "s3:PutObject",
|
||||
Self::PutObjectAcl => "s3:PutObjectAcl",
|
||||
Self::PutObjectLegalHold => "s3:PutObjectLegalHold",
|
||||
Self::PutObjectLockConfiguration => "s3:PutObjectLockConfiguration",
|
||||
Self::PutObjectRetention => "s3:PutObjectRetention",
|
||||
Self::PutObjectTagging => "s3:PutObjectTagging",
|
||||
Self::PutPublicAccessBlock => "s3:PutPublicAccessBlock",
|
||||
Self::RestoreObject => "s3:RestoreObject",
|
||||
Self::SelectObjectContent => "s3:SelectObjectContent",
|
||||
Self::UploadPart => "s3:UploadPart",
|
||||
Self::UploadPartCopy => "s3:UploadPartCopy",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the corresponding EventName if the operation triggers a notification event.
|
||||
pub fn to_event_name(self) -> Option<EventName> {
|
||||
match self {
|
||||
Self::CompleteMultipartUpload => Some(EventName::ObjectCreatedCompleteMultipartUpload),
|
||||
Self::CopyObject => Some(EventName::ObjectCreatedCopy),
|
||||
Self::CreateBucket => Some(EventName::BucketCreated),
|
||||
Self::DeleteBucket => Some(EventName::BucketRemoved),
|
||||
Self::DeleteObject => Some(EventName::ObjectRemovedDelete),
|
||||
Self::DeleteObjects => Some(EventName::ObjectRemovedDeleteObjects),
|
||||
Self::DeleteObjectTagging => Some(EventName::ObjectTaggingDelete),
|
||||
Self::GetObject => Some(EventName::ObjectAccessedGet),
|
||||
Self::GetObjectAttributes => Some(EventName::ObjectAccessedAttributes),
|
||||
Self::GetObjectLegalHold => Some(EventName::ObjectAccessedGetLegalHold),
|
||||
Self::GetObjectRetention => Some(EventName::ObjectAccessedGetRetention),
|
||||
Self::HeadObject => Some(EventName::ObjectAccessedHead),
|
||||
Self::PutObject => Some(EventName::ObjectCreatedPut),
|
||||
Self::PutObjectAcl => Some(EventName::ObjectAclPut),
|
||||
Self::PutObjectLegalHold => Some(EventName::ObjectCreatedPutLegalHold),
|
||||
Self::PutObjectRetention => Some(EventName::ObjectCreatedPutRetention),
|
||||
Self::PutObjectTagging => Some(EventName::ObjectTaggingPut),
|
||||
Self::RestoreObject => Some(EventName::ObjectRestorePost),
|
||||
Self::SelectObjectContent => Some(EventName::ObjectAccessedGet),
|
||||
Self::AbortMultipartUpload => Some(EventName::ObjectRemovedAbortMultipartUpload),
|
||||
Self::CreateMultipartUpload => Some(EventName::ObjectCreatedCreateMultipartUpload),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -610,39 +436,6 @@ mod tests {
|
||||
assert!(deserialized.is_err(), "Deserialization should fail for empty string");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_s3_operation_to_event_name() {
|
||||
assert_eq!(S3Operation::PutObject.to_event_name(), Some(EventName::ObjectCreatedPut));
|
||||
assert_eq!(S3Operation::PutObjectAcl.to_event_name(), Some(EventName::ObjectAclPut));
|
||||
assert_eq!(S3Operation::PutObjectTagging.to_event_name(), Some(EventName::ObjectTaggingPut));
|
||||
assert_eq!(S3Operation::DeleteObjectTagging.to_event_name(), Some(EventName::ObjectTaggingDelete));
|
||||
assert_eq!(S3Operation::GetObject.to_event_name(), Some(EventName::ObjectAccessedGet));
|
||||
assert_eq!(S3Operation::ListBuckets.to_event_name(), None);
|
||||
assert_eq!(S3Operation::RestoreObject.to_event_name(), Some(EventName::ObjectRestorePost));
|
||||
assert_eq!(S3Operation::SelectObjectContent.to_event_name(), Some(EventName::ObjectAccessedGet));
|
||||
assert_eq!(
|
||||
S3Operation::AbortMultipartUpload.to_event_name(),
|
||||
Some(EventName::ObjectRemovedAbortMultipartUpload)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_event_name_to_s3_operation() {
|
||||
assert_eq!(EventName::ObjectCreatedPut.to_s3_operation(), Some(S3Operation::PutObject));
|
||||
assert_eq!(EventName::ObjectAclPut.to_s3_operation(), Some(S3Operation::PutObjectAcl));
|
||||
assert_eq!(EventName::ObjectTaggingPut.to_s3_operation(), Some(S3Operation::PutObjectTagging));
|
||||
assert_eq!(EventName::ObjectTaggingDelete.to_s3_operation(), Some(S3Operation::DeleteObjectTagging));
|
||||
assert_eq!(EventName::ObjectAccessedGet.to_s3_operation(), Some(S3Operation::GetObject));
|
||||
assert_eq!(EventName::BucketCreated.to_s3_operation(), Some(S3Operation::CreateBucket));
|
||||
assert_eq!(EventName::Everything.to_s3_operation(), None);
|
||||
assert_eq!(EventName::ObjectRestorePost.to_s3_operation(), Some(S3Operation::RestoreObject));
|
||||
assert_eq!(EventName::ObjectCreatedPost.to_s3_operation(), Some(S3Operation::PutObject));
|
||||
assert_eq!(
|
||||
EventName::ObjectRemovedAbortMultipartUpload.to_s3_operation(),
|
||||
Some(S3Operation::AbortMultipartUpload)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_event_name_aliases_parse_to_aws_compatible_variants() {
|
||||
assert_eq!(EventName::parse("s3:ObjectCreated:PutTagging").unwrap(), EventName::ObjectTaggingPut);
|
||||
@@ -673,4 +466,18 @@ mod tests {
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_event_schema_version_mapping() {
|
||||
assert_eq!(event_schema_version(EventName::ObjectCreatedPut), "2.1");
|
||||
assert_eq!(event_schema_version(EventName::ObjectReplicationFailed), "2.2");
|
||||
assert_eq!(event_schema_version(EventName::LifecycleTransition), "2.3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_try_from_event_str_matches_parse() {
|
||||
let parsed = EventName::try_from_event_str("s3:ObjectCreated:Put").unwrap();
|
||||
assert_eq!(parsed, EventName::ObjectCreatedPut);
|
||||
assert!(EventName::try_from_event_str("s3:Invalid").is_err());
|
||||
}
|
||||
}
|
||||
@@ -12,4 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod token;
|
||||
mod event_name;
|
||||
|
||||
pub use event_name::{EventName, ParseEventNameError, event_schema_version};
|
||||
@@ -51,6 +51,7 @@ http = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
rustfs-data-usage = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,886 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
pub struct TimedAction {
|
||||
count: u64,
|
||||
acc_time: u64,
|
||||
min_time: Option<u64>,
|
||||
max_time: Option<u64>,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TimedAction {
|
||||
// Avg returns the average time spent on the action.
|
||||
pub fn avg(&self) -> Option<Duration> {
|
||||
if self.count == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(Duration::from_nanos(self.acc_time / self.count))
|
||||
}
|
||||
|
||||
// AvgBytes returns the average bytes processed.
|
||||
pub fn avg_bytes(&self) -> u64 {
|
||||
if self.count == 0 {
|
||||
return 0;
|
||||
}
|
||||
self.bytes / self.count
|
||||
}
|
||||
|
||||
// Merge other into t.
|
||||
pub fn merge(&mut self, other: TimedAction) {
|
||||
self.count += other.count;
|
||||
self.acc_time += other.acc_time;
|
||||
self.bytes += other.bytes;
|
||||
|
||||
if self.count == 0 {
|
||||
self.min_time = other.min_time;
|
||||
}
|
||||
if let Some(other_min) = other.min_time {
|
||||
self.min_time = self.min_time.map_or(Some(other_min), |min| Some(min.min(other_min)));
|
||||
}
|
||||
|
||||
self.max_time = self
|
||||
.max_time
|
||||
.map_or(other.max_time, |max| Some(max.max(other.max_time.unwrap_or(0))));
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
enum SizeCategory {
|
||||
SizeLessThan1KiB = 0,
|
||||
SizeLessThan1MiB,
|
||||
SizeLessThan10MiB,
|
||||
SizeLessThan100MiB,
|
||||
SizeLessThan1GiB,
|
||||
SizeGreaterThan1GiB,
|
||||
// Add new entries here
|
||||
SizeLastElemMarker,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SizeCategory {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let s = match *self {
|
||||
SizeCategory::SizeLessThan1KiB => "SizeLessThan1KiB",
|
||||
SizeCategory::SizeLessThan1MiB => "SizeLessThan1MiB",
|
||||
SizeCategory::SizeLessThan10MiB => "SizeLessThan10MiB",
|
||||
SizeCategory::SizeLessThan100MiB => "SizeLessThan100MiB",
|
||||
SizeCategory::SizeLessThan1GiB => "SizeLessThan1GiB",
|
||||
SizeCategory::SizeGreaterThan1GiB => "SizeGreaterThan1GiB",
|
||||
SizeCategory::SizeLastElemMarker => "SizeLastElemMarker",
|
||||
};
|
||||
write!(f, "{s}")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Copy)]
|
||||
pub struct AccElem {
|
||||
pub total: u64,
|
||||
pub size: u64,
|
||||
pub n: u64,
|
||||
}
|
||||
|
||||
impl AccElem {
|
||||
pub fn add(&mut self, dur: &Duration) {
|
||||
let dur = dur.as_secs();
|
||||
self.total = self.total.wrapping_add(dur);
|
||||
self.n = self.n.wrapping_add(1);
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, b: &AccElem) {
|
||||
self.n = self.n.wrapping_add(b.n);
|
||||
self.total = self.total.wrapping_add(b.total);
|
||||
self.size = self.size.wrapping_add(b.size);
|
||||
}
|
||||
|
||||
pub fn avg(&self) -> Duration {
|
||||
if self.n >= 1 && self.total > 0 {
|
||||
return Duration::from_secs(self.total / self.n);
|
||||
}
|
||||
Duration::from_secs(0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LastMinuteLatency {
|
||||
pub totals: Vec<AccElem>,
|
||||
pub last_sec: u64,
|
||||
}
|
||||
|
||||
impl Default for LastMinuteLatency {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
totals: vec![AccElem::default(); 60],
|
||||
last_sec: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LastMinuteLatency {
|
||||
pub fn merge(&mut self, o: &LastMinuteLatency) -> LastMinuteLatency {
|
||||
let mut merged = LastMinuteLatency::default();
|
||||
let mut x = o.clone();
|
||||
if self.last_sec > o.last_sec {
|
||||
x.forward_to(self.last_sec);
|
||||
merged.last_sec = self.last_sec;
|
||||
} else {
|
||||
self.forward_to(o.last_sec);
|
||||
merged.last_sec = o.last_sec;
|
||||
}
|
||||
|
||||
for i in 0..merged.totals.len() {
|
||||
merged.totals[i] = AccElem {
|
||||
total: self.totals[i].total + o.totals[i].total,
|
||||
n: self.totals[i].n + o.totals[i].n,
|
||||
size: self.totals[i].size + o.totals[i].size,
|
||||
}
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
pub fn add(&mut self, t: &Duration) {
|
||||
let sec = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time went backwards")
|
||||
.as_secs();
|
||||
self.forward_to(sec);
|
||||
let win_idx = sec % 60;
|
||||
self.totals[win_idx as usize].add(t);
|
||||
self.last_sec = sec;
|
||||
}
|
||||
|
||||
pub fn add_all(&mut self, sec: u64, a: &AccElem) {
|
||||
self.forward_to(sec);
|
||||
let win_idx = sec % 60;
|
||||
self.totals[win_idx as usize].merge(a);
|
||||
self.last_sec = sec;
|
||||
}
|
||||
|
||||
pub fn get_total(&mut self) -> AccElem {
|
||||
let mut res = AccElem::default();
|
||||
let sec = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time went backwards")
|
||||
.as_secs();
|
||||
self.forward_to(sec);
|
||||
for elem in self.totals.iter() {
|
||||
res.merge(elem);
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub fn forward_to(&mut self, t: u64) {
|
||||
if self.last_sec >= t {
|
||||
return;
|
||||
}
|
||||
if t - self.last_sec >= 60 {
|
||||
self.totals = vec![AccElem::default(); 60];
|
||||
self.last_sec = t;
|
||||
return;
|
||||
}
|
||||
while self.last_sec != t {
|
||||
let idx = (self.last_sec + 1) % 60;
|
||||
self.totals[idx as usize] = AccElem::default();
|
||||
self.last_sec += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_default() {
|
||||
let elem = AccElem::default();
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.size, 0);
|
||||
assert_eq!(elem.n, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_add_single_duration() {
|
||||
let mut elem = AccElem::default();
|
||||
let duration = Duration::from_secs(5);
|
||||
|
||||
elem.add(&duration);
|
||||
|
||||
assert_eq!(elem.total, 5);
|
||||
assert_eq!(elem.n, 1);
|
||||
assert_eq!(elem.size, 0); // size is not modified by add
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_add_multiple_durations() {
|
||||
let mut elem = AccElem::default();
|
||||
|
||||
elem.add(&Duration::from_secs(3));
|
||||
elem.add(&Duration::from_secs(7));
|
||||
elem.add(&Duration::from_secs(2));
|
||||
|
||||
assert_eq!(elem.total, 12);
|
||||
assert_eq!(elem.n, 3);
|
||||
assert_eq!(elem.size, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_add_zero_duration() {
|
||||
let mut elem = AccElem::default();
|
||||
let duration = Duration::from_secs(0);
|
||||
|
||||
elem.add(&duration);
|
||||
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.n, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_add_subsecond_duration() {
|
||||
let mut elem = AccElem::default();
|
||||
// Duration less than 1 second should be truncated to 0
|
||||
let duration = Duration::from_millis(500);
|
||||
|
||||
elem.add(&duration);
|
||||
|
||||
assert_eq!(elem.total, 0); // as_secs() truncates subsecond values
|
||||
assert_eq!(elem.n, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_merge_empty_elements() {
|
||||
let mut elem1 = AccElem::default();
|
||||
let elem2 = AccElem::default();
|
||||
|
||||
elem1.merge(&elem2);
|
||||
|
||||
assert_eq!(elem1.total, 0);
|
||||
assert_eq!(elem1.size, 0);
|
||||
assert_eq!(elem1.n, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_merge_with_data() {
|
||||
let mut elem1 = AccElem {
|
||||
total: 10,
|
||||
size: 100,
|
||||
n: 2,
|
||||
};
|
||||
let elem2 = AccElem {
|
||||
total: 15,
|
||||
size: 200,
|
||||
n: 3,
|
||||
};
|
||||
|
||||
elem1.merge(&elem2);
|
||||
|
||||
assert_eq!(elem1.total, 25);
|
||||
assert_eq!(elem1.size, 300);
|
||||
assert_eq!(elem1.n, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_merge_one_empty() {
|
||||
let mut elem1 = AccElem {
|
||||
total: 10,
|
||||
size: 100,
|
||||
n: 2,
|
||||
};
|
||||
let elem2 = AccElem::default();
|
||||
|
||||
elem1.merge(&elem2);
|
||||
|
||||
assert_eq!(elem1.total, 10);
|
||||
assert_eq!(elem1.size, 100);
|
||||
assert_eq!(elem1.n, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_avg_with_data() {
|
||||
let elem = AccElem {
|
||||
total: 15,
|
||||
size: 0,
|
||||
n: 3,
|
||||
};
|
||||
|
||||
let avg = elem.avg();
|
||||
assert_eq!(avg, Duration::from_secs(5)); // 15 / 3 = 5
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_avg_zero_count() {
|
||||
let elem = AccElem {
|
||||
total: 10,
|
||||
size: 0,
|
||||
n: 0,
|
||||
};
|
||||
|
||||
let avg = elem.avg();
|
||||
assert_eq!(avg, Duration::from_secs(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_avg_zero_total() {
|
||||
let elem = AccElem { total: 0, size: 0, n: 5 };
|
||||
|
||||
let avg = elem.avg();
|
||||
assert_eq!(avg, Duration::from_secs(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_avg_rounding() {
|
||||
let elem = AccElem {
|
||||
total: 10,
|
||||
size: 0,
|
||||
n: 3,
|
||||
};
|
||||
|
||||
let avg = elem.avg();
|
||||
assert_eq!(avg, Duration::from_secs(3)); // 10 / 3 = 3 (integer division)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_default() {
|
||||
let latency = LastMinuteLatency::default();
|
||||
|
||||
assert_eq!(latency.totals.len(), 60);
|
||||
assert_eq!(latency.last_sec, 0);
|
||||
|
||||
// All elements should be default (empty)
|
||||
for elem in &latency.totals {
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.size, 0);
|
||||
assert_eq!(elem.n, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_forward_to_same_time() {
|
||||
let mut latency = LastMinuteLatency {
|
||||
last_sec: 100,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Add some data to verify it's not cleared
|
||||
latency.totals[0].total = 10;
|
||||
latency.totals[0].n = 1;
|
||||
|
||||
latency.forward_to(100); // Same time
|
||||
|
||||
assert_eq!(latency.last_sec, 100);
|
||||
assert_eq!(latency.totals[0].total, 10); // Data should remain
|
||||
assert_eq!(latency.totals[0].n, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_forward_to_past_time() {
|
||||
let mut latency = LastMinuteLatency {
|
||||
last_sec: 100,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Add some data to verify it's not cleared
|
||||
latency.totals[0].total = 10;
|
||||
latency.totals[0].n = 1;
|
||||
|
||||
latency.forward_to(50); // Past time
|
||||
|
||||
assert_eq!(latency.last_sec, 100); // Should not change
|
||||
assert_eq!(latency.totals[0].total, 10); // Data should remain
|
||||
assert_eq!(latency.totals[0].n, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_forward_to_large_gap() {
|
||||
let mut latency = LastMinuteLatency {
|
||||
last_sec: 100,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Add some data to verify it's cleared
|
||||
latency.totals[0].total = 10;
|
||||
latency.totals[0].n = 1;
|
||||
|
||||
latency.forward_to(200); // Gap >= 60 seconds
|
||||
|
||||
assert_eq!(latency.last_sec, 200); // last_sec should be updated to target time
|
||||
|
||||
// All data should be cleared
|
||||
for elem in &latency.totals {
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.size, 0);
|
||||
assert_eq!(elem.n, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_forward_to_small_gap() {
|
||||
let mut latency = LastMinuteLatency {
|
||||
last_sec: 100,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Add data at specific indices
|
||||
latency.totals[41].total = 10; // (100 + 1) % 60 = 41
|
||||
latency.totals[42].total = 20; // (100 + 2) % 60 = 42
|
||||
|
||||
latency.forward_to(102); // Forward by 2 seconds
|
||||
|
||||
assert_eq!(latency.last_sec, 102);
|
||||
|
||||
// The slots that were advanced should be cleared
|
||||
assert_eq!(latency.totals[41].total, 0); // Cleared during forward
|
||||
assert_eq!(latency.totals[42].total, 0); // Cleared during forward
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_add_all() {
|
||||
let mut latency = LastMinuteLatency::default();
|
||||
let acc_elem = AccElem {
|
||||
total: 15,
|
||||
size: 100,
|
||||
n: 3,
|
||||
};
|
||||
|
||||
latency.add_all(1000, &acc_elem);
|
||||
|
||||
assert_eq!(latency.last_sec, 1000);
|
||||
let idx = 1000 % 60; // Should be 40
|
||||
assert_eq!(latency.totals[idx as usize].total, 15);
|
||||
assert_eq!(latency.totals[idx as usize].size, 100);
|
||||
assert_eq!(latency.totals[idx as usize].n, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_add_all_multiple() {
|
||||
let mut latency = LastMinuteLatency::default();
|
||||
|
||||
let acc_elem1 = AccElem {
|
||||
total: 10,
|
||||
size: 50,
|
||||
n: 2,
|
||||
};
|
||||
let acc_elem2 = AccElem {
|
||||
total: 20,
|
||||
size: 100,
|
||||
n: 4,
|
||||
};
|
||||
|
||||
latency.add_all(1000, &acc_elem1);
|
||||
latency.add_all(1000, &acc_elem2); // Same second
|
||||
|
||||
let idx = 1000 % 60;
|
||||
assert_eq!(latency.totals[idx as usize].total, 30); // 10 + 20
|
||||
assert_eq!(latency.totals[idx as usize].size, 150); // 50 + 100
|
||||
assert_eq!(latency.totals[idx as usize].n, 6); // 2 + 4
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_merge_same_time() {
|
||||
let mut latency1 = LastMinuteLatency::default();
|
||||
let mut latency2 = LastMinuteLatency::default();
|
||||
|
||||
latency1.last_sec = 1000;
|
||||
latency2.last_sec = 1000;
|
||||
|
||||
// Add data to both
|
||||
latency1.totals[0].total = 10;
|
||||
latency1.totals[0].n = 2;
|
||||
latency2.totals[0].total = 20;
|
||||
latency2.totals[0].n = 3;
|
||||
|
||||
let merged = latency1.merge(&latency2);
|
||||
|
||||
assert_eq!(merged.last_sec, 1000);
|
||||
assert_eq!(merged.totals[0].total, 30); // 10 + 20
|
||||
assert_eq!(merged.totals[0].n, 5); // 2 + 3
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_merge_different_times() {
|
||||
let mut latency1 = LastMinuteLatency::default();
|
||||
let mut latency2 = LastMinuteLatency::default();
|
||||
|
||||
latency1.last_sec = 1000;
|
||||
latency2.last_sec = 1010; // 10 seconds later
|
||||
|
||||
// Add data to both
|
||||
latency1.totals[0].total = 10;
|
||||
latency2.totals[0].total = 20;
|
||||
|
||||
let merged = latency1.merge(&latency2);
|
||||
|
||||
assert_eq!(merged.last_sec, 1010); // Should use the later time
|
||||
assert_eq!(merged.totals[0].total, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_merge_empty() {
|
||||
let mut latency1 = LastMinuteLatency::default();
|
||||
let latency2 = LastMinuteLatency::default();
|
||||
|
||||
let merged = latency1.merge(&latency2);
|
||||
|
||||
assert_eq!(merged.last_sec, 0);
|
||||
for elem in &merged.totals {
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.size, 0);
|
||||
assert_eq!(elem.n, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_window_wraparound() {
|
||||
let mut latency = LastMinuteLatency::default();
|
||||
|
||||
// Test that indices wrap around correctly
|
||||
for sec in 0..120 {
|
||||
// Test for 2 minutes
|
||||
let acc_elem = AccElem {
|
||||
total: sec,
|
||||
size: 0,
|
||||
n: 1,
|
||||
};
|
||||
latency.add_all(sec, &acc_elem);
|
||||
|
||||
let expected_idx = sec % 60;
|
||||
assert_eq!(latency.totals[expected_idx as usize].total, sec);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_time_progression() {
|
||||
let mut latency = LastMinuteLatency::default();
|
||||
|
||||
// Add data at time 1000
|
||||
latency.add_all(
|
||||
1000,
|
||||
&AccElem {
|
||||
total: 10,
|
||||
size: 0,
|
||||
n: 1,
|
||||
},
|
||||
);
|
||||
|
||||
// Forward to time 1030 (30 seconds later)
|
||||
latency.forward_to(1030);
|
||||
|
||||
// Original data should still be there
|
||||
let idx_1000 = 1000 % 60;
|
||||
assert_eq!(latency.totals[idx_1000 as usize].total, 10);
|
||||
|
||||
// Forward to time 1070 (70 seconds from original, > 60 seconds)
|
||||
latency.forward_to(1070);
|
||||
|
||||
// All data should be cleared due to large gap
|
||||
for elem in &latency.totals {
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.n, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_realistic_scenario() {
|
||||
let mut latency = LastMinuteLatency::default();
|
||||
let base_time = 1000u64;
|
||||
|
||||
// Add data for exactly 60 seconds to fill the window
|
||||
for i in 0..60 {
|
||||
let current_time = base_time + i;
|
||||
let duration_secs = i % 10 + 1; // Varying durations 1-10 seconds
|
||||
let acc_elem = AccElem {
|
||||
total: duration_secs,
|
||||
size: 1024 * (i % 5 + 1), // Varying sizes
|
||||
n: 1,
|
||||
};
|
||||
|
||||
latency.add_all(current_time, &acc_elem);
|
||||
}
|
||||
|
||||
// Count non-empty slots after filling the window
|
||||
let mut non_empty_count = 0;
|
||||
let mut total_n = 0;
|
||||
let mut total_sum = 0;
|
||||
|
||||
for elem in &latency.totals {
|
||||
if elem.n > 0 {
|
||||
non_empty_count += 1;
|
||||
total_n += elem.n;
|
||||
total_sum += elem.total;
|
||||
}
|
||||
}
|
||||
|
||||
// We should have exactly 60 non-empty slots (one for each second in the window)
|
||||
assert_eq!(non_empty_count, 60);
|
||||
assert_eq!(total_n, 60); // 60 data points total
|
||||
assert!(total_sum > 0);
|
||||
|
||||
// Test manual total calculation (get_total uses system time which interferes with test)
|
||||
let mut manual_total = AccElem::default();
|
||||
for elem in &latency.totals {
|
||||
manual_total.merge(elem);
|
||||
}
|
||||
assert_eq!(manual_total.n, 60);
|
||||
assert_eq!(manual_total.total, total_sum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_clone_and_debug() {
|
||||
let elem = AccElem {
|
||||
total: 100,
|
||||
size: 200,
|
||||
n: 5,
|
||||
};
|
||||
|
||||
let cloned = elem;
|
||||
assert_eq!(elem.total, cloned.total);
|
||||
assert_eq!(elem.size, cloned.size);
|
||||
assert_eq!(elem.n, cloned.n);
|
||||
|
||||
// Test Debug trait
|
||||
let debug_str = format!("{elem:?}");
|
||||
assert!(debug_str.contains("100"));
|
||||
assert!(debug_str.contains("200"));
|
||||
assert!(debug_str.contains("5"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_minute_latency_clone() {
|
||||
let mut latency = LastMinuteLatency {
|
||||
last_sec: 1000,
|
||||
..Default::default()
|
||||
};
|
||||
latency.totals[0].total = 100;
|
||||
latency.totals[0].n = 5;
|
||||
|
||||
let cloned = latency.clone();
|
||||
assert_eq!(latency.last_sec, cloned.last_sec);
|
||||
assert_eq!(latency.totals[0].total, cloned.totals[0].total);
|
||||
assert_eq!(latency.totals[0].n, cloned.totals[0].n);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_edge_case_max_values() {
|
||||
let mut elem = AccElem {
|
||||
total: u64::MAX - 50,
|
||||
size: u64::MAX - 50,
|
||||
n: u64::MAX - 50,
|
||||
};
|
||||
|
||||
let other = AccElem {
|
||||
total: 100,
|
||||
size: 100,
|
||||
n: 100,
|
||||
};
|
||||
|
||||
// This should not panic due to overflow, values will wrap around
|
||||
elem.merge(&other);
|
||||
|
||||
// Values should wrap around due to overflow (wrapping_add behavior)
|
||||
assert_eq!(elem.total, 49); // (u64::MAX - 50) + 100 wraps to 49
|
||||
assert_eq!(elem.size, 49);
|
||||
assert_eq!(elem.n, 49);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_forward_to_boundary_conditions() {
|
||||
let mut latency = LastMinuteLatency {
|
||||
last_sec: 59,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Add data at the last slot
|
||||
latency.totals[59].total = 100;
|
||||
latency.totals[59].n = 1;
|
||||
|
||||
// Forward exactly 60 seconds (boundary case)
|
||||
latency.forward_to(119);
|
||||
|
||||
// All data should be cleared
|
||||
for elem in &latency.totals {
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.n, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_total_with_data() {
|
||||
let mut latency = LastMinuteLatency::default();
|
||||
|
||||
// Set a recent timestamp to avoid forward_to clearing data
|
||||
let current_time = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time went backwards")
|
||||
.as_secs();
|
||||
latency.last_sec = current_time;
|
||||
|
||||
// Add data to multiple slots
|
||||
latency.totals[0] = AccElem {
|
||||
total: 10,
|
||||
size: 100,
|
||||
n: 1,
|
||||
};
|
||||
latency.totals[1] = AccElem {
|
||||
total: 20,
|
||||
size: 200,
|
||||
n: 2,
|
||||
};
|
||||
latency.totals[59] = AccElem {
|
||||
total: 30,
|
||||
size: 300,
|
||||
n: 3,
|
||||
};
|
||||
|
||||
let total = latency.get_total();
|
||||
|
||||
assert_eq!(total.total, 60);
|
||||
assert_eq!(total.size, 600);
|
||||
assert_eq!(total.n, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_window_index_calculation() {
|
||||
// Test that window index calculation works correctly
|
||||
let _latency = LastMinuteLatency::default();
|
||||
|
||||
let acc_elem = AccElem { total: 1, size: 1, n: 1 };
|
||||
|
||||
// Test various timestamps
|
||||
let test_cases = [(0, 0), (1, 1), (59, 59), (60, 0), (61, 1), (119, 59), (120, 0)];
|
||||
|
||||
for (timestamp, expected_idx) in test_cases {
|
||||
let mut test_latency = LastMinuteLatency::default();
|
||||
test_latency.add_all(timestamp, &acc_elem);
|
||||
|
||||
assert_eq!(
|
||||
test_latency.totals[expected_idx].n, 1,
|
||||
"Failed for timestamp {timestamp} (expected index {expected_idx})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_safety_simulation() {
|
||||
// Simulate concurrent access patterns
|
||||
let mut latency = LastMinuteLatency::default();
|
||||
|
||||
// Use current time to ensure data doesn't get cleared by get_total
|
||||
let current_time = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time went backwards")
|
||||
.as_secs();
|
||||
|
||||
// Simulate rapid additions within a 60-second window
|
||||
for i in 0..1000 {
|
||||
let acc_elem = AccElem {
|
||||
total: (i % 10) + 1, // Ensure non-zero values
|
||||
size: (i % 100) + 1,
|
||||
n: 1,
|
||||
};
|
||||
// Keep all timestamps within the current minute window
|
||||
latency.add_all(current_time - (i % 60), &acc_elem);
|
||||
}
|
||||
|
||||
let total = latency.get_total();
|
||||
assert!(total.n > 0, "Total count should be greater than 0");
|
||||
assert!(total.total > 0, "Total time should be greater than 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_acc_elem_debug_format() {
|
||||
let elem = AccElem {
|
||||
total: 123,
|
||||
size: 456,
|
||||
n: 789,
|
||||
};
|
||||
|
||||
let debug_str = format!("{elem:?}");
|
||||
assert!(debug_str.contains("123"));
|
||||
assert!(debug_str.contains("456"));
|
||||
assert!(debug_str.contains("789"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_values() {
|
||||
let mut elem = AccElem::default();
|
||||
|
||||
// Test with large duration values
|
||||
let large_duration = Duration::from_secs(u64::MAX / 2);
|
||||
elem.add(&large_duration);
|
||||
|
||||
assert_eq!(elem.total, u64::MAX / 2);
|
||||
assert_eq!(elem.n, 1);
|
||||
|
||||
// Test average calculation with large values
|
||||
let avg = elem.avg();
|
||||
assert_eq!(avg, Duration::from_secs(u64::MAX / 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_zero_duration_handling() {
|
||||
let mut elem = AccElem::default();
|
||||
|
||||
let zero_duration = Duration::from_secs(0);
|
||||
elem.add(&zero_duration);
|
||||
|
||||
assert_eq!(elem.total, 0);
|
||||
assert_eq!(elem.n, 1);
|
||||
assert_eq!(elem.avg(), Duration::from_secs(0));
|
||||
}
|
||||
}
|
||||
|
||||
const SIZE_LAST_ELEM_MARKER: usize = 10; // Assumed marker size is 10, modify according to actual situation
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
pub struct LastMinuteHistogram {
|
||||
histogram: Vec<LastMinuteLatency>,
|
||||
size: u32,
|
||||
}
|
||||
|
||||
impl LastMinuteHistogram {
|
||||
pub fn merge(&mut self, other: &LastMinuteHistogram) {
|
||||
for i in 0..self.histogram.len() {
|
||||
self.histogram[i].merge(&other.histogram[i]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add(&mut self, size: i64, t: Duration) {
|
||||
let index = size_to_tag(size);
|
||||
self.histogram[index].add(&t);
|
||||
}
|
||||
|
||||
pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] {
|
||||
let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER];
|
||||
for (i, elem) in self.histogram.iter_mut().enumerate() {
|
||||
res[i] = elem.get_total();
|
||||
}
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
fn size_to_tag(size: i64) -> usize {
|
||||
match size {
|
||||
_ if size < 1024 => 0, // sizeLessThan1KiB
|
||||
_ if size < 1024 * 1024 => 1, // sizeLessThan1MiB
|
||||
_ if size < 10 * 1024 * 1024 => 2, // sizeLessThan10MiB
|
||||
_ if size < 100 * 1024 * 1024 => 3, // sizeLessThan100MiB
|
||||
_ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB
|
||||
_ => 5, // sizeGreaterThan1GiB
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
pub mod data_usage_define;
|
||||
pub mod error;
|
||||
pub mod last_minute;
|
||||
pub mod scanner;
|
||||
pub mod scanner_folder;
|
||||
pub mod scanner_io;
|
||||
@@ -30,6 +29,7 @@ pub mod sleeper;
|
||||
|
||||
pub use data_usage_define::*;
|
||||
pub use error::ScannerError;
|
||||
pub use rustfs_common::last_minute;
|
||||
pub use scanner::init_data_scanner;
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
@@ -115,6 +115,32 @@ fn warn_inline_heal_compat_requested() {
|
||||
});
|
||||
}
|
||||
|
||||
fn non_negative_i64_to_u64(value: i64) -> u64 {
|
||||
value.max(0) as u64
|
||||
}
|
||||
|
||||
fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) {
|
||||
into.size += summary.total_size;
|
||||
into.versions += summary.versions;
|
||||
into.delete_markers += summary.delete_markers;
|
||||
into.obj_sizes.add(summary.total_size as u64);
|
||||
into.obj_versions.add(summary.versions as u64);
|
||||
|
||||
let replication_stats = into.replication_stats.get_or_insert_with(Default::default);
|
||||
replication_stats.replica_size += non_negative_i64_to_u64(summary.replica_size);
|
||||
replication_stats.replica_count += summary.replica_count as u64;
|
||||
|
||||
for (arn, st) in &summary.repl_target_stats {
|
||||
let tgt_stat = replication_stats.targets.entry(arn.clone()).or_default();
|
||||
tgt_stat.pending_size += non_negative_i64_to_u64(st.pending_size);
|
||||
tgt_stat.failed_size += non_negative_i64_to_u64(st.failed_size);
|
||||
tgt_stat.replicated_size += non_negative_i64_to_u64(st.replicated_size);
|
||||
tgt_stat.replicated_count += st.replicated_count as u64;
|
||||
tgt_stat.failed_count += st.failed_count as u64;
|
||||
tgt_stat.pending_count += st.pending_count as u64;
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached folder information for scanning
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CachedFolder {
|
||||
@@ -963,7 +989,7 @@ impl FolderScanner {
|
||||
|
||||
abandoned_children.remove(&path_join_buf(&[&item.bucket, &item.object_path()]));
|
||||
|
||||
into.add_sizes(&sz);
|
||||
apply_scanner_size_summary(into, &sz);
|
||||
into.objects += 1;
|
||||
object_count += 1;
|
||||
|
||||
|
||||
@@ -9,13 +9,14 @@ homepage.workspace = true
|
||||
description = "Notification target abstraction and implementations for RustFS"
|
||||
keywords = ["file-system", "notification", "target", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-target/latest/rustfs_target/"
|
||||
documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["notify", "tls"] }
|
||||
rustfs-s3-common = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
async-nats = { workspace = true }
|
||||
deadpool-postgres = { workspace = true }
|
||||
|
||||
@@ -61,7 +61,7 @@ pub use runtime::{
|
||||
sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability},
|
||||
start_replay_worker,
|
||||
};
|
||||
pub use rustfs_s3_common::EventName;
|
||||
pub use rustfs_s3_types::EventName;
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub use sys::user_agent::*;
|
||||
pub use target::{Target, TargetDeliverySnapshot};
|
||||
|
||||
@@ -501,7 +501,7 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use serde_json::json;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::arn::TargetID;
|
||||
use crate::store::{Key, QueueStore, Store};
|
||||
use crate::{StoreError, TargetError, TargetLog};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt::Formatter;
|
||||
|
||||
@@ -1015,7 +1015,7 @@ mod tests {
|
||||
let entity = EntityTarget {
|
||||
object_name: "bucket%2Fobj.txt".to_string(),
|
||||
bucket_name: "testbucket".to_string(),
|
||||
event_name: rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
event_name: rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
data: serde_json::json!({"eventTime": "2026-05-03T10:00:00Z"}),
|
||||
};
|
||||
|
||||
@@ -1061,7 +1061,7 @@ mod tests {
|
||||
|
||||
let body = br#"{"Records":[]}"#.to_vec();
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"testbucket".to_string(),
|
||||
"obj.txt".to_string(),
|
||||
"application/json",
|
||||
@@ -1120,7 +1120,7 @@ mod tests {
|
||||
br#"{"EventName":"s3:ObjectCreated:Put","Key":"bucket/obj.txt","Records":[{"eventTime":"2026-05-03T10:00:00Z"}]}"#
|
||||
.to_vec();
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"testbucket".to_string(),
|
||||
"obj.txt".to_string(),
|
||||
"application/json",
|
||||
|
||||
@@ -1042,7 +1042,7 @@ mod tests {
|
||||
"Records": []
|
||||
});
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"bucket-a".to_string(),
|
||||
"fallback%2Fvalue.txt".to_string(),
|
||||
"application/json",
|
||||
@@ -1059,7 +1059,7 @@ mod tests {
|
||||
"Records": []
|
||||
});
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"bucket-a".to_string(),
|
||||
"hello+world%2Ftest.txt".to_string(),
|
||||
"application/json",
|
||||
|
||||
@@ -969,7 +969,7 @@ mod tests {
|
||||
let payload = build_queued_payload(&EntityTarget {
|
||||
object_name: "greeting+file+%282%29.csv".to_string(),
|
||||
bucket_name: "bucket".to_string(),
|
||||
event_name: rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
event_name: rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
data: "payload-data".to_string(),
|
||||
})
|
||||
.expect("payload should build");
|
||||
@@ -1071,7 +1071,7 @@ mod tests {
|
||||
|
||||
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"bucket".to_string(),
|
||||
"object".to_string(),
|
||||
"application/json",
|
||||
@@ -1110,7 +1110,7 @@ mod tests {
|
||||
|
||||
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"bucket".to_string(),
|
||||
"object".to_string(),
|
||||
"application/json",
|
||||
@@ -1149,7 +1149,7 @@ mod tests {
|
||||
|
||||
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
rustfs_s3_common::EventName::ObjectCreatedPut,
|
||||
rustfs_s3_types::EventName::ObjectCreatedPut,
|
||||
"bucket".to_string(),
|
||||
"object".to_string(),
|
||||
"application/json",
|
||||
|
||||
@@ -29,7 +29,7 @@ use lapin::{
|
||||
options::{BasicAckOptions, BasicGetOptions, QueueBindOptions, QueueDeclareOptions, QueueDeleteOptions},
|
||||
types::FieldTable,
|
||||
};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::check_amqp_broker_available;
|
||||
use rustfs_targets::target::EntityTarget;
|
||||
|
||||
@@ -70,7 +70,7 @@ fn make_args(dsn: &str, table: &str, queue_dir: &str) -> MySqlArgs {
|
||||
}
|
||||
}
|
||||
|
||||
fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_common::EventName) -> EntityTarget<serde_json::Value> {
|
||||
fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_types::EventName) -> EntityTarget<serde_json::Value> {
|
||||
EntityTarget {
|
||||
object_name: object.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
@@ -117,7 +117,7 @@ async fn direct_write_and_read() {
|
||||
|
||||
target.init().await.expect("init");
|
||||
|
||||
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
|
||||
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
|
||||
target.save(Arc::new(entity)).await.expect("save");
|
||||
|
||||
let pool = build_test_pool(&dsn);
|
||||
@@ -141,10 +141,10 @@ async fn delete_appends_row_does_not_remove_old() {
|
||||
|
||||
target.init().await.expect("init");
|
||||
|
||||
let put = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
|
||||
let put = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
|
||||
target.save(Arc::new(put)).await.expect("save put");
|
||||
|
||||
let delete = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectRemovedDelete);
|
||||
let delete = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectRemovedDelete);
|
||||
target.save(Arc::new(delete)).await.expect("save delete");
|
||||
|
||||
let pool = build_test_pool(&dsn);
|
||||
@@ -166,7 +166,7 @@ async fn queue_store_saves_entry_and_replays() {
|
||||
let target: MySqlTarget<serde_json::Value> =
|
||||
MySqlTarget::new("queue".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target");
|
||||
|
||||
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
|
||||
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
|
||||
target.save(Arc::new(entity)).await.expect("save to queue");
|
||||
|
||||
let store = target.store().expect("store should exist");
|
||||
@@ -206,7 +206,7 @@ async fn duplicate_replay_produces_duplicate_rows() {
|
||||
let target: MySqlTarget<serde_json::Value> =
|
||||
MySqlTarget::new("dupe".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target");
|
||||
|
||||
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
|
||||
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
|
||||
target.save(Arc::new(entity)).await.expect("save to queue");
|
||||
|
||||
target.init().await.expect("init");
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
//! podman rm -f rustfs-pg-test
|
||||
//! ```
|
||||
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::check_postgres_server_available;
|
||||
use rustfs_targets::target::EntityTarget;
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
[](https://rustfs.com)
|
||||
|
||||
# RustFS Workers - Background Job System
|
||||
|
||||
<p align="center">
|
||||
<strong>Distributed background job processing system for RustFS object storage</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/rustfs/rustfs/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/rustfs/rustfs/actions/workflows/ci.yml/badge.svg" /></a>
|
||||
<a href="https://docs.rustfs.com/">📖 Documentation</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/issues">🐛 Bug Reports</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/discussions">💬 Discussions</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 📖 Overview
|
||||
|
||||
**RustFS Workers** provides distributed background job processing capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
|
||||
|
||||
## ✨ Features
|
||||
|
||||
- Distributed job execution across cluster nodes
|
||||
- Priority-based job scheduling and queue management
|
||||
- Built-in workers for replication, cleanup, healing, and indexing
|
||||
- Automatic retry logic with exponential backoff
|
||||
- Horizontal scaling with load balancing
|
||||
- Real-time job monitoring and administrative interface
|
||||
|
||||
## 📚 Documentation
|
||||
|
||||
For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
+3
-2
@@ -61,7 +61,6 @@ workspace = true
|
||||
[dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs-heal = { workspace = true }
|
||||
rustfs-appauth = { workspace = true }
|
||||
rustfs-audit = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants", "notify"] }
|
||||
@@ -80,7 +79,9 @@ rustfs-policy = { workspace = true }
|
||||
rustfs-protocols = { workspace = true }
|
||||
rustfs-protos = { workspace = true }
|
||||
rustfs-rio.workspace = true
|
||||
rustfs-s3-common = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
rustfs-data-usage = { workspace = true }
|
||||
rustfs-s3select-api = { workspace = true }
|
||||
rustfs-s3select-query = { workspace = true }
|
||||
rustfs-targets = { workspace = true }
|
||||
|
||||
@@ -27,8 +27,8 @@ use http::{HeaderMap, StatusCode};
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_common::data_usage::TierStats;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
|
||||
use rustfs_ecstore::{
|
||||
bucket::lifecycle::tier_last_day_stats::DailyAllTierStats,
|
||||
|
||||
@@ -65,7 +65,7 @@ use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
|
||||
use rustfs_madmin::utils::parse_duration;
|
||||
use rustfs_notify::{Event as NotificationEvent, notification_system};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_signer::pre_sign_v4;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST,
|
||||
@@ -3803,7 +3803,7 @@ mod tests {
|
||||
fn event_matches_listen_notification_respects_bucket_event_and_object_filters() {
|
||||
let filter = ListenNotificationFilter {
|
||||
bucket: Some("demo-bucket".to_string()),
|
||||
event_mask: EventName::ObjectCreatedPut.mask() | EventName::ObjectCreatedPost.mask(),
|
||||
event_mask: rustfs_s3_ops::put_object_created_event_mask(),
|
||||
prefix: Some("logs/".to_string()),
|
||||
suffix: Some(".json".to_string()),
|
||||
};
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::app::context::{AppContext, get_global_app_context};
|
||||
use crate::capacity::resolve_admin_used_capacity;
|
||||
use crate::error::ApiError;
|
||||
use rustfs_common::data_usage::DataUsageInfo;
|
||||
use rustfs_data_usage::DataUsageInfo;
|
||||
use rustfs_ecstore::admin_server_info::get_server_info;
|
||||
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
|
||||
use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
|
||||
@@ -64,7 +64,7 @@ use rustfs_policy::policy::{
|
||||
action::{Action, S3Action},
|
||||
{BucketPolicy, BucketPolicyArgs, Effect, Validator},
|
||||
};
|
||||
use rustfs_s3_common::S3Operation;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_targets::{
|
||||
EventName,
|
||||
arn::{ARN, TargetIDError},
|
||||
|
||||
@@ -52,7 +52,7 @@ use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
|
||||
use rustfs_rio::{CompressReader, EncryptReader, HashReader};
|
||||
#[cfg(test)]
|
||||
use rustfs_rio::{DecryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||
use rustfs_s3_common::S3Operation;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use rustfs_utils::http::{
|
||||
|
||||
@@ -86,7 +86,7 @@ use rustfs_io_metrics;
|
||||
use rustfs_notify::EventArgsBuilder;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_rio::{CompressReader, DynReader, EncryptReader, HashReader, wrap_reader};
|
||||
use rustfs_s3_common::S3Operation;
|
||||
use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object};
|
||||
use rustfs_s3select_api::{
|
||||
object_store::bytes_stream,
|
||||
query::{Context, Query},
|
||||
@@ -1625,9 +1625,9 @@ impl DefaultObjectUsecase {
|
||||
|
||||
fn put_object_execution_context(req: &S3Request<PutObjectInput>) -> (EventName, QuotaOperation, &'static str) {
|
||||
if req.extensions.get::<PostObjectRequestMarker>().is_some() {
|
||||
(EventName::ObjectCreatedPost, QuotaOperation::PostObject, "POST")
|
||||
(put_event_name_for_post_object(true), QuotaOperation::PostObject, "POST")
|
||||
} else {
|
||||
(EventName::ObjectCreatedPut, QuotaOperation::PutObject, "PUT")
|
||||
(put_event_name_for_post_object(false), QuotaOperation::PutObject, "PUT")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3141,11 +3141,7 @@ impl DefaultObjectUsecase {
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify);
|
||||
for res in delete_results {
|
||||
if let Some(dobj) = res.delete_object {
|
||||
let event_name = if dobj.delete_marker {
|
||||
EventName::ObjectRemovedDeleteMarkerCreated
|
||||
} else {
|
||||
EventName::ObjectRemovedDelete
|
||||
};
|
||||
let event_name = delete_event_name_for_marker(dobj.delete_marker);
|
||||
let event_args = EventArgsBuilder::new(
|
||||
event_name,
|
||||
bucket.clone(),
|
||||
@@ -3310,7 +3306,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
// Prefix/force-delete returns empty ObjectInfo; still emit bucket notification so webhooks match S3 DELETE.
|
||||
helper = helper
|
||||
.event_name(EventName::ObjectRemovedDelete)
|
||||
.event_name(delete_event_name_for_marker(false))
|
||||
.object(ObjectInfo {
|
||||
name: key.clone(),
|
||||
bucket: bucket.clone(),
|
||||
@@ -3378,11 +3374,7 @@ impl DefaultObjectUsecase {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let event_name = if delete_marker {
|
||||
EventName::ObjectRemovedDeleteMarkerCreated
|
||||
} else {
|
||||
EventName::ObjectRemovedDelete
|
||||
};
|
||||
let event_name = delete_event_name_for_marker(delete_marker);
|
||||
|
||||
helper = helper.event_name(event_name);
|
||||
helper = helper
|
||||
@@ -4341,7 +4333,7 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
let event_args = rustfs_notify::EventArgs {
|
||||
event_name: EventName::ObjectCreatedPut,
|
||||
event_name: put_event_name_for_post_object(false),
|
||||
bucket_name: bucket.clone(),
|
||||
object: obj_info.clone(),
|
||||
req_params: req_params.clone(),
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_appauth::token::{Token, parse_license_with_public_key};
|
||||
use rustfs_crypto::{Token, parse_license_with_public_key};
|
||||
use std::fmt;
|
||||
use std::io::{Error, ErrorKind, Result};
|
||||
use std::sync::Arc;
|
||||
@@ -328,7 +328,7 @@ mod tests {
|
||||
RsaPrivateKey, RsaPublicKey,
|
||||
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||
};
|
||||
use rustfs_appauth::token::sign_license_token;
|
||||
use rustfs_crypto::sign_license_token;
|
||||
use serial_test::serial;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::{module_switch::resolve_notify_module_state, refresh_persisted_module
|
||||
use crate::app::context::resolve_server_config;
|
||||
use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
|
||||
use rustfs_notify::EventArgs as NotifyEventArgs;
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::spawn;
|
||||
@@ -38,14 +38,26 @@ pub fn is_notify_module_enabled() -> bool {
|
||||
NOTIFY_MODULE_ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs {
|
||||
fn convert_ecstore_event_args(args: EcstoreEventArgs) -> Option<NotifyEventArgs> {
|
||||
let version_id = args.object.version_id.map(|v| v.to_string()).unwrap_or_default();
|
||||
let (host, port) = parse_host_and_port(args.host);
|
||||
let req_params = args.req_params.into_iter().collect();
|
||||
let resp_elements = args.resp_elements.into_iter().collect();
|
||||
let event_name = match EventName::try_from_event_str(args.event_name.as_str()) {
|
||||
Ok(event_name) => event_name,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event_name = args.event_name,
|
||||
bucket = args.bucket_name,
|
||||
error = %err,
|
||||
"dropping ecstore event with invalid event name"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
NotifyEventArgs {
|
||||
event_name: EventName::from(args.event_name.as_str()),
|
||||
Some(NotifyEventArgs {
|
||||
event_name,
|
||||
bucket_name: args.bucket_name,
|
||||
object: args.object,
|
||||
req_params,
|
||||
@@ -54,7 +66,7 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs {
|
||||
host,
|
||||
port,
|
||||
user_agent: args.user_agent,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_host_and_port(host: String) -> (String, u16) {
|
||||
@@ -77,7 +89,9 @@ fn parse_host_and_port(host: String) -> (String, u16) {
|
||||
|
||||
fn install_ecstore_event_dispatch_hook() {
|
||||
let installed = register_event_dispatch_hook(|args| {
|
||||
let notify_args = convert_ecstore_event_args(args);
|
||||
let Some(notify_args) = convert_ecstore_event_args(args) else {
|
||||
return;
|
||||
};
|
||||
spawn(async move {
|
||||
rustfs_notify::notifier_global::notify(notify_args).await;
|
||||
});
|
||||
|
||||
@@ -41,7 +41,8 @@ use rustfs_ecstore::{
|
||||
new_object_layer_fn,
|
||||
store_api::{BucketOperations, BucketOptions, ObjectLockRetentionOptions, ObjectOperations, ObjectOptions},
|
||||
};
|
||||
use rustfs_s3_common::{S3Operation, record_s3_op};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::http::headers::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
@@ -71,7 +72,7 @@ impl Default for FS {
|
||||
|
||||
impl FS {
|
||||
pub fn new() -> Self {
|
||||
rustfs_s3_common::init_s3_metrics();
|
||||
rustfs_io_metrics::init_s3_metrics();
|
||||
Self {}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,9 +23,10 @@ use rustfs_audit::{
|
||||
global::AuditLogger,
|
||||
};
|
||||
use rustfs_ecstore::store_api::ObjectInfo;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_notify::{EventArgsBuilder, notifier_global};
|
||||
use rustfs_s3_common::record_s3_op;
|
||||
use rustfs_s3_common::{EventName, S3Operation};
|
||||
use rustfs_s3_ops::{S3Operation, operation_matches_event_name};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::{
|
||||
extract_params_header, extract_req_params, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
|
||||
http::headers::AMZ_REQUEST_ID,
|
||||
@@ -34,7 +35,7 @@ use s3s::{S3Request, S3Response, S3Result};
|
||||
use serde_json::Value;
|
||||
use std::future::Future;
|
||||
use tokio::runtime::{Builder, Handle};
|
||||
use tracing::{Instrument, info_span};
|
||||
use tracing::{Instrument, info_span, warn};
|
||||
|
||||
/// Schedules an asynchronous task on the current runtime;
|
||||
/// if there is no runtime, creates a minimal runtime execution on a new thread.
|
||||
@@ -89,6 +90,22 @@ pub struct EnabledOperationHelper {
|
||||
impl OperationHelper {
|
||||
/// Create a new OperationHelper for S3 requests.
|
||||
pub fn new(req: &S3Request<impl Send + Sync>, event: EventName, op: S3Operation) -> Self {
|
||||
let op_event_matches = operation_matches_event_name(op, event);
|
||||
debug_assert!(op_event_matches, "operation/event mismatch: op={} event={}", op.as_str(), event.as_str());
|
||||
if !op_event_matches {
|
||||
counter!(
|
||||
"rustfs_log_chain_op_event_mismatch_total",
|
||||
"op" => op.as_str(),
|
||||
"event" => event.as_str().to_string()
|
||||
)
|
||||
.increment(1);
|
||||
warn!(
|
||||
op = op.as_str(),
|
||||
event = event.as_str(),
|
||||
"operation/event mismatch detected; check S3 semantic mapping"
|
||||
);
|
||||
}
|
||||
|
||||
let audit_enabled = is_audit_module_enabled();
|
||||
let notify_enabled = should_build_notification_event(is_notify_module_enabled());
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use rustfs_common::internode_metrics::{
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, global_internode_metrics,
|
||||
};
|
||||
use serde::de::DeserializeOwned;
|
||||
|
||||
@@ -18,15 +18,15 @@ use futures_util::TryStreamExt;
|
||||
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
|
||||
use http_body_util::{BodyExt, Limited};
|
||||
use hyper::body::Incoming;
|
||||
use rustfs_common::internode_metrics::{
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_ecstore::disk::{DiskAPI, WalkDirOptions};
|
||||
use rustfs_ecstore::rpc::verify_rpc_signature;
|
||||
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
use rustfs_ecstore::store::find_local_disk_by_ref;
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_utils::net::bytes_stream;
|
||||
use s3s::Body;
|
||||
use s3s::dto::StreamingBlob;
|
||||
|
||||
Reference in New Issue
Block a user