diff --git a/.docker/observability/grafana/dashboards/rustfs.json b/.docker/observability/grafana/dashboards/rustfs.json index 315de3c69..86f595180 100644 --- a/.docker/observability/grafana/dashboards/rustfs.json +++ b/.docker/observability/grafana/dashboards/rustfs.json @@ -9037,13 +9037,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "{__name__=~\"rustfs_(notification|audit)_.*\",job=~\"$job\"}", + "expr": "{__name__=~\"rustfs_(notification|audit|log_chain)_.*\",job=~\"$job\"}", "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], - "title": "Notification / Audit (All)", + "title": "Notification / Audit / LogChain (All)", "type": "timeseries" }, { diff --git a/.docker/observability/prometheus-rules/rustfs-dashboard.yml b/.docker/observability/prometheus-rules/rustfs-dashboard.yml index 898b82b2a..328df017e 100644 --- a/.docker/observability/prometheus-rules/rustfs-dashboard.yml +++ b/.docker/observability/prometheus-rules/rustfs-dashboard.yml @@ -38,3 +38,16 @@ groups: expr: sum by (job) (rate(rustfs_scanner_buckets_scanned_total[5m])) - record: rustfs:scanner_cycles_success:rate5m expr: sum by (job) (rate(rustfs_scanner_cycles_total{result="success"}[5m])) + + - record: rustfs:log_chain_op_event_mismatch:rate5m + expr: sum by (job) (rate(rustfs_log_chain_op_event_mismatch_total[5m])) + + - alert: RustFSLogChainOpEventMismatchDetected + expr: rustfs:log_chain_op_event_mismatch:rate5m > 0 + for: 10m + labels: + severity: warning + component: s3-log-chain + annotations: + summary: "RustFS log-chain op/event mismatch detected" + description: "job={{ $labels.job }} has non-zero rustfs_log_chain_op_event_mismatch_total rate for more than 10m. Check s3 op/event mapping changes." diff --git a/Cargo.lock b/Cargo.lock index 1c4331f5b..b5e4de12d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3698,6 +3698,7 @@ dependencies = [ "russh-sftp", "rustfs-common", "rustfs-config", + "rustfs-data-usage", "rustfs-ecstore", "rustfs-filemeta", "rustfs-lock", @@ -9315,13 +9316,13 @@ dependencies = [ "rmp-serde", "rsa 0.10.0-rc.18", "rust-embed", - "rustfs-appauth", "rustfs-audit", "rustfs-common", "rustfs-concurrency", "rustfs-config", "rustfs-credentials", "rustfs-crypto", + "rustfs-data-usage", "rustfs-ecstore", "rustfs-filemeta", "rustfs-heal", @@ -9339,7 +9340,8 @@ dependencies = [ "rustfs-protocols", "rustfs-protos", "rustfs-rio", - "rustfs-s3-common", + "rustfs-s3-ops", + "rustfs-s3-types", "rustfs-s3select-api", "rustfs-s3select-query", "rustfs-scanner", @@ -9381,17 +9383,6 @@ dependencies = [ "zip", ] -[[package]] -name = "rustfs-appauth" -version = "1.0.0-beta.3" -dependencies = [ - "base64-simd", - "rand 0.10.1", - "rsa 0.10.0-rc.18", - "serde", - "serde_json", -] - [[package]] name = "rustfs-audit" version = "1.0.0-beta.3" @@ -9404,7 +9395,8 @@ dependencies = [ "metrics", "rustfs-config", "rustfs-ecstore", - "rustfs-s3-common", + "rustfs-s3-ops", + "rustfs-s3-types", "rustfs-targets", "serde", "serde_json", @@ -9433,13 +9425,9 @@ dependencies = [ name = "rustfs-common" version = "1.0.0-beta.3" dependencies = [ - "async-trait", "chrono", "metrics", - "path-clean", "rmp-serde", - "rustfs-filemeta", - "rustfs-madmin", "s3s", "serde", "tokio", @@ -9484,10 +9472,13 @@ version = "1.0.0-beta.3" dependencies = [ "aes-gcm 0.11.0-rc.3", "argon2 0.6.0-rc.8", + "base64-simd", "chacha20poly1305", "jsonwebtoken", "pbkdf2 0.13.0", "rand 0.10.1", + "rsa 0.10.0-rc.18", + "serde", "serde_json", "sha2 0.11.0", "test-case", @@ -9495,6 +9486,17 @@ dependencies = [ "time", ] +[[package]] +name = "rustfs-data-usage" +version = "1.0.0-beta.3" +dependencies = [ + "async-trait", + "path-clean", + "rmp-serde", + "rustfs-filemeta", + "serde", +] + [[package]] name = "rustfs-ecstore" version = "1.0.0-beta.3" @@ -9553,20 +9555,23 @@ dependencies = [ "rmp-serde", "rustfs-checksums", "rustfs-common", + "rustfs-concurrency", "rustfs-config", "rustfs-credentials", + "rustfs-data-usage", "rustfs-filemeta", "rustfs-io-metrics", "rustfs-kms", "rustfs-lock", "rustfs-madmin", + "rustfs-object-capacity", "rustfs-policy", "rustfs-protos", "rustfs-rio", - "rustfs-s3-common", + "rustfs-s3-ops", + "rustfs-s3-types", "rustfs-signer", "rustfs-utils", - "rustfs-workers", "rustix 1.1.4", "rustls", "s3s", @@ -9700,6 +9705,7 @@ dependencies = [ "criterion", "metrics", "num_cpus", + "rustfs-s3-ops", "sysinfo", "thiserror 2.0.18", "tokio", @@ -9860,7 +9866,8 @@ dependencies = [ "rustc-hash", "rustfs-config", "rustfs-ecstore", - "rustfs-s3-common", + "rustfs-s3-ops", + "rustfs-s3-types", "rustfs-targets", "rustfs-utils", "serde", @@ -9881,7 +9888,6 @@ version = "1.0.0-beta.3" dependencies = [ "criterion", "futures", - "rustfs-common", "rustfs-config", "rustfs-io-metrics", "rustfs-utils", @@ -10034,6 +10040,7 @@ dependencies = [ "prost 0.14.3", "rustfs-common", "rustfs-config", + "rustfs-io-metrics", "rustfs-utils", "tonic", "tonic-prost", @@ -10061,6 +10068,7 @@ dependencies = [ "reqwest", "rustfs-common", "rustfs-config", + "rustfs-io-metrics", "rustfs-utils", "s3s", "serde", @@ -10075,10 +10083,16 @@ dependencies = [ ] [[package]] -name = "rustfs-s3-common" +name = "rustfs-s3-ops" +version = "1.0.0-beta.3" +dependencies = [ + "rustfs-s3-types", +] + +[[package]] +name = "rustfs-s3-types" version = "1.0.0-beta.3" dependencies = [ - "metrics", "serde", "serde_json", ] @@ -10141,6 +10155,7 @@ dependencies = [ "rmp-serde", "rustfs-common", "rustfs-config", + "rustfs-data-usage", "rustfs-ecstore", "rustfs-filemeta", "rustfs-utils", @@ -10195,7 +10210,8 @@ dependencies = [ "rustfs-config", "rustfs-ecstore", "rustfs-kafka-async", - "rustfs-s3-common", + "rustfs-s3-ops", + "rustfs-s3-types", "rustfs-utils", "rustls", "rustls-native-certs", @@ -10284,14 +10300,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "rustfs-workers" -version = "1.0.0-beta.3" -dependencies = [ - "tokio", - "tracing", -] - [[package]] name = "rustfs-zip" version = "1.0.0-beta.3" diff --git a/Cargo.toml b/Cargo.toml index d2c18d0c8..385136776 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,10 @@ [workspace] members = [ "rustfs", # Core file system implementation - "crates/appauth", # Application authentication and authorization "crates/audit", # Audit target management system with multi-target fan-out "crates/checksums", # client checksums "crates/common", # Shared utilities and data structures + "crates/data-usage", # Shared data usage models and algorithms "crates/config", # Configuration management "crates/credentials", # Credential management system "crates/crypto", # Cryptography and security features @@ -39,7 +39,8 @@ members = [ "crates/protos", # Protocol buffer definitions "crates/rio", # Rust I/O utilities and abstractions "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling - "crates/s3-common", # Common utilities and data structures for S3 compatibility + "crates/s3-types", # S3 event type definitions + "crates/s3-ops", # S3 operation definitions and mapping "crates/s3select-api", # S3 Select API interface "crates/s3select-query", # S3 Select query engine "crates/scanner", # Scanner for data integrity checks and health monitoring @@ -47,7 +48,6 @@ members = [ "crates/targets", # Target-specific configurations and utilities "crates/trusted-proxies", # Trusted proxies management "crates/utils", # Utility functions and helpers - "crates/workers", # Worker thread pools and task scheduling "crates/io-metrics", # Zero-copy metrics collection for performance analysis "crates/io-core", # Zero-copy core reader and writer implementations "crates/zip", # ZIP file handling and compression @@ -78,10 +78,10 @@ redundant_clone = "warn" # RustFS Internal Crates rustfs = { path = "./rustfs", version = "1.0.0-beta.3" } rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.3" } -rustfs-appauth = { path = "crates/appauth", version = "1.0.0-beta.3" } rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.3" } rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.3" } rustfs-common = { path = "crates/common", version = "1.0.0-beta.3" } +rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.3" } rustfs-config = { path = "./crates/config", version = "1.0.0-beta.3" } rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.3" } rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.3" } @@ -102,7 +102,8 @@ rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.3" } rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.3" } rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.3" } rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.3" } -rustfs-s3-common = { path = "crates/s3-common", version = "1.0.0-beta.3" } +rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.3" } +rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.3" } rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.3" } rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.3" } rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.3" } @@ -110,7 +111,6 @@ rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.3" } rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.3" } rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.3" } rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.3" } -rustfs-workers = { path = "crates/workers", version = "1.0.0-beta.3" } rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.3" } # Async Runtime and Networking @@ -202,7 +202,6 @@ aws-credential-types = { version = "1.2.14" } aws-sdk-s3 = { version = "1.132.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] } aws-smithy-types = { version = "1.4.7" } -backtrace = "0.3.76" base64 = "0.22.1" base64-simd = "0.8.0" brotli = "8.0.2" diff --git a/crates/appauth/README.md b/crates/appauth/README.md deleted file mode 100644 index 43f4b9654..000000000 --- a/crates/appauth/README.md +++ /dev/null @@ -1,37 +0,0 @@ -[![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com) - -# RustFS AppAuth - Application Authentication - -

- Application-level authentication and authorization module for RustFS distributed object storage -

- -

- CI - ๐Ÿ“– Documentation - ยท ๐Ÿ› Bug Reports - ยท ๐Ÿ’ฌ Discussions -

- ---- - -## ๐Ÿ“– Overview - -**RustFS AppAuth** provides application-level authentication and authorization capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs). - -## โœจ Features - -- JWT-based authentication with secure token management -- RBAC (Role-Based Access Control) for fine-grained permissions -- Multi-tenant application isolation and management -- OAuth 2.0 and OpenID Connect integration -- API key management and rotation -- Session management with configurable expiration - -## ๐Ÿ“š Documentation - -For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs). - -## ๐Ÿ“„ License - -This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details. diff --git a/crates/audit/Cargo.toml b/crates/audit/Cargo.toml index 04ce7cda6..f58ae10ef 100644 --- a/crates/audit/Cargo.toml +++ b/crates/audit/Cargo.toml @@ -29,7 +29,8 @@ categories = ["web-programming", "development-tools", "asynchronous", "api-bindi rustfs-targets = { workspace = true } rustfs-config = { workspace = true, features = ["audit", "constants"] } rustfs-ecstore = { workspace = true } -rustfs-s3-common = { workspace = true } +rustfs-s3-types = { workspace = true } +rustfs-s3-ops = { workspace = true } chrono = { workspace = true } const-str = { workspace = true } futures = { workspace = true } diff --git a/crates/audit/src/entity.rs b/crates/audit/src/entity.rs index 3e2375f5e..39931ecfb 100644 --- a/crates/audit/src/entity.rs +++ b/crates/audit/src/entity.rs @@ -14,7 +14,7 @@ use chrono::{DateTime, Utc}; use hashbrown::HashMap; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use serde::{Deserialize, Serialize}; use serde_json::Value; diff --git a/crates/checksums/Cargo.toml b/crates/checksums/Cargo.toml index 8d6fe9dc1..6ac09669b 100644 --- a/crates/checksums/Cargo.toml +++ b/crates/checksums/Cargo.toml @@ -23,7 +23,7 @@ homepage.workspace = true description = "Checksum calculation and verification callbacks for HTTP request and response bodies sent by service clients generated by RustFS, ensuring data integrity and authenticity." keywords = ["checksum-calculation", "verification", "integrity", "authenticity", "rustfs"] categories = ["web-programming", "development-tools", "network-programming"] -documentation = "https://docs.rs/rustfs-signer/latest/rustfs_checksum/" +documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/" [dependencies] bytes = { workspace = true } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 4886ce906..c86ba3ea3 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -33,12 +33,8 @@ tonic = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } metrics = { workspace = true } -rustfs-madmin = { workspace = true } -rustfs-filemeta = { workspace = true } serde = { workspace = true } -path-clean = { workspace = true } rmp-serde = { workspace = true } -async-trait = { workspace = true } s3s = { workspace = true } tracing = { workspace = true } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index df06c5966..bab6931e8 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -13,12 +13,9 @@ // limitations under the License. pub mod bucket_stats; -pub mod capacity_scope; // pub mod error; -pub mod data_usage; pub mod globals; pub mod heal_channel; -pub mod internode_metrics; pub mod last_minute; pub mod metrics; mod readiness; diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 3b860ab63..744435ccd 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -14,7 +14,6 @@ use crate::last_minute::{AccElem, LastMinuteLatency}; use chrono::{DateTime, Utc}; -use rustfs_madmin::metrics::{ScannerMetrics as M_ScannerMetrics, TimedAction}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -351,6 +350,32 @@ pub struct CurrentCycle { pub started: DateTime, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScannerTimedAction { + pub count: u64, + pub acc_time: u64, + pub bytes: u64, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ScannerLastMinute { + pub actions: HashMap, + pub ilm: HashMap, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ScannerMetricsReport { + pub collected_at: DateTime, + pub current_cycle: u64, + pub current_started: DateTime, + pub cycles_completed_at: Vec>, + pub ongoing_buckets: usize, + pub life_time_ops: HashMap, + pub life_time_ilm: HashMap, + pub last_minute: ScannerLastMinute, + pub active_paths: Vec, +} + impl CurrentCycle { pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box> { *self = rmp_serde::from_slice(buf)?; @@ -572,8 +597,8 @@ impl Metrics { } /// Build a full metrics report snapshot. - pub async fn report(&self) -> M_ScannerMetrics { - let mut m = M_ScannerMetrics::default(); + pub async fn report(&self) -> ScannerMetricsReport { + let mut m = ScannerMetricsReport::default(); if let Some(cycle) = self.get_cycle().await { m.current_cycle = cycle.current; @@ -606,7 +631,7 @@ impl Metrics { { m.last_minute.actions.insert( metric.as_str().to_string(), - TimedAction { + ScannerTimedAction { count: last_min.n, acc_time: last_min.total, bytes: last_min.size, @@ -633,7 +658,7 @@ impl Metrics { { m.last_minute.ilm.insert( action.as_str().to_string(), - TimedAction { + ScannerTimedAction { count: last_min.n, acc_time: last_min.total, bytes: last_min.size, diff --git a/crates/concurrency/src/lib.rs b/crates/concurrency/src/lib.rs index d7dbb0344..6d0b5ff8e 100644 --- a/crates/concurrency/src/lib.rs +++ b/crates/concurrency/src/lib.rs @@ -124,6 +124,8 @@ mod backpressure; #[cfg(feature = "scheduler")] mod scheduler; +pub mod workers; + // Public module exports with feature gates #[cfg(feature = "timeout")] pub use timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager}; diff --git a/crates/workers/src/workers.rs b/crates/concurrency/src/workers.rs similarity index 86% rename from crates/workers/src/workers.rs rename to crates/concurrency/src/workers.rs index 3b56444c0..0c116a9bd 100644 --- a/crates/workers/src/workers.rs +++ b/crates/concurrency/src/workers.rs @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Worker slot limiter used by long-running background workflows. + use std::sync::Arc; use tokio::sync::{Mutex, Notify}; use tracing::info; +/// Cooperative worker-slot controller for async tasks. pub struct Workers { available: Mutex, // Available working slots notify: Notify, // Used to notify waiting tasks @@ -23,20 +26,20 @@ pub struct Workers { } impl Workers { - // Create a Workers object that allows up to n jobs to execute concurrently - pub fn new(n: usize) -> Result, &'static str> { + /// Create a [`Workers`] object that allows up to `n` jobs to execute concurrently. + pub fn new(n: usize) -> Result, &'static str> { if n == 0 { return Err("n must be > 0"); } - Ok(Arc::new(Workers { + Ok(Arc::new(Self { available: Mutex::new(n), notify: Notify::new(), limit: n, })) } - // Give a job a chance to be executed + /// Acquire a worker slot, waiting until one becomes available. pub async fn take(&self) { loop { let mut available = self.available.lock().await; @@ -51,7 +54,7 @@ impl Workers { } } - // Release a job's slot + /// Release a worker slot. pub async fn give(&self) { let mut available = self.available.lock().await; info!("worker give, {}", *available); @@ -59,7 +62,7 @@ impl Workers { self.notify.notify_one(); // Notify a waiting task } - // Wait for all concurrent jobs to complete + /// Wait until all worker slots are released. pub async fn wait(&self) { loop { { @@ -74,6 +77,7 @@ impl Workers { info!("worker wait end"); } + /// Return the current number of available worker slots. pub async fn available(&self) -> usize { *self.available.lock().await } diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index f1e2b4772..651066731 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -33,8 +33,11 @@ aes-gcm = { workspace = true, optional = true } argon2 = { workspace = true, optional = true } chacha20poly1305 = { workspace = true, optional = true } jsonwebtoken = { workspace = true } +base64-simd = { workspace = true } pbkdf2 = { workspace = true, optional = true } rand = { workspace = true, optional = true } +rsa = { workspace = true, features = ["sha2"] } +serde = { workspace = true, features = ["derive"] } sha2 = { workspace = true, optional = true } thiserror.workspace = true serde_json.workspace = true diff --git a/crates/crypto/src/lib.rs b/crates/crypto/src/lib.rs index 8bed5bfd8..e66c7484c 100644 --- a/crates/crypto/src/lib.rs +++ b/crates/crypto/src/lib.rs @@ -16,6 +16,7 @@ mod encdec; mod error; mod jwt; +pub mod license_token; pub use encdec::decrypt::decrypt_data; pub use encdec::encrypt::encrypt_data; @@ -25,3 +26,4 @@ pub use encdec::stream_io::{decrypt_stream_io, encrypt_stream_io}; pub use error::Error; pub use jwt::decode::decode as jwt_decode; pub use jwt::encode::encode as jwt_encode; +pub use license_token::{Token, parse_license_with_public_key, parse_signed_license_token, sign_license_token}; diff --git a/crates/appauth/src/token.rs b/crates/crypto/src/license_token.rs similarity index 81% rename from crates/appauth/src/token.rs rename to crates/crypto/src/license_token.rs index 22952a7d2..8128605ec 100644 --- a/crates/appauth/src/token.rs +++ b/crates/crypto/src/license_token.rs @@ -115,12 +115,20 @@ mod tests { let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key"); let public_key = RsaPublicKey::from(&private_key); - let private_key_pem = private_key.to_pkcs8_pem(LineEnding::LF).unwrap(); - let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap(); + let private_key_pem = private_key + .to_pkcs8_pem(LineEnding::LF) + .expect("failed to encode private key pem"); + let public_key_pem = public_key + .to_public_key_pem(LineEnding::LF) + .expect("failed to encode public key pem"); let token = Token { name: "test_app".to_string(), - expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600, // 1 hour from now + expired: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before unix epoch") + .as_secs() + + 3600, // 1 hour from now }; let encoded = sign_license_token(&token, &private_key_pem).expect("Failed to encode token"); @@ -139,12 +147,20 @@ mod tests { let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key"); let public_key = RsaPublicKey::from(&private_key); - let private_key_pem = private_key.to_pkcs8_pem(LineEnding::LF).unwrap(); - let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap(); + let private_key_pem = private_key + .to_pkcs8_pem(LineEnding::LF) + .expect("failed to encode private key pem"); + let public_key_pem = public_key + .to_public_key_pem(LineEnding::LF) + .expect("failed to encode public key pem"); let token = Token { name: "test_app".to_string(), - expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600, + expired: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before unix epoch") + .as_secs() + + 3600, }; let encoded = gencode(&token, &public_key_pem).expect("Failed to encode token"); @@ -159,11 +175,19 @@ mod tests { let mut rng = rand::rng(); let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key"); let public_key = RsaPublicKey::from(&private_key); - let private_key_pem = private_key.to_pkcs8_pem(LineEnding::LF).unwrap(); - let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap(); + let private_key_pem = private_key + .to_pkcs8_pem(LineEnding::LF) + .expect("failed to encode private key pem"); + let public_key_pem = public_key + .to_public_key_pem(LineEnding::LF) + .expect("failed to encode public key pem"); let token = Token { name: "test_app".to_string(), - expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600, + expired: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before unix epoch") + .as_secs() + + 3600, }; let encoded = sign_license_token(&token, &private_key_pem).expect("Failed to encode token"); @@ -181,7 +205,7 @@ mod tests { #[test] fn test_source_does_not_embed_private_key() { - let source = include_str!("token.rs"); + let source = include_str!("license_token.rs"); let forbidden = ["BEGIN", "PRIVATE KEY"].join(" "); assert!(!source.contains(&forbidden)); @@ -192,7 +216,9 @@ mod tests { let mut rng = rand::rng(); let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key"); let public_key = RsaPublicKey::from(&private_key); - let public_key_pem = public_key.to_public_key_pem(LineEnding::LF).unwrap(); + let public_key_pem = public_key + .to_public_key_pem(LineEnding::LF) + .expect("failed to encode public key pem"); let invalid_token = "invalid_base64_token"; let result = parse_signed_license_token(invalid_token, &public_key_pem); @@ -204,7 +230,11 @@ mod tests { fn test_sign_license_token_with_invalid_signing_key() { let token = Token { name: "test_app".to_string(), - expired: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + 3600, // 1 hour from now + expired: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before unix epoch") + .as_secs() + + 3600, // 1 hour from now }; let invalid_key = "invalid_private_key"; diff --git a/crates/workers/Cargo.toml b/crates/data-usage/Cargo.toml similarity index 65% rename from crates/workers/Cargo.toml rename to crates/data-usage/Cargo.toml index 70a360ef9..d6cd797ae 100644 --- a/crates/workers/Cargo.toml +++ b/crates/data-usage/Cargo.toml @@ -13,24 +13,26 @@ # limitations under the License. [package] -name = "rustfs-workers" +name = "rustfs-data-usage" +version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true -version.workspace = true homepage.workspace = true -description = "Workers for RustFS, providing background processing capabilities for tasks such as data synchronization, indexing, and more." -keywords = ["workers", "tasks", "rustfs", "Minio"] -categories = ["web-programming", "development-tools"] -documentation = "https://docs.rs/rustfs-workers/latest/rustfs_workers/" +description = "Shared data usage models and algorithms for RustFS" +keywords = ["rustfs", "data-usage", "cache", "histogram"] +categories = ["data-structures", "filesystem"] [lints] workspace = true [dependencies] -tokio = { workspace = true, features = ["sync", "time", "macros"] } -tracing.workspace = true +serde = { workspace = true } +path-clean = { workspace = true } +rmp-serde = { workspace = true } +async-trait = { workspace = true } +rustfs-filemeta = { workspace = true } [lib] doctest = false diff --git a/crates/common/src/data_usage.rs b/crates/data-usage/src/data_usage.rs similarity index 98% rename from crates/common/src/data_usage.rs rename to crates/data-usage/src/data_usage.rs index 21a8f9932..9071ff81e 100644 --- a/crates/common/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -312,6 +312,12 @@ impl SizeHistogram { } res } + + pub fn merge_from(&mut self, other: &Self) { + for (dst, src) in self.0.iter_mut().zip(other.0.iter()) { + *dst += src; + } + } } /// Versions histogram for version count distribution @@ -361,6 +367,12 @@ impl VersionsHistogram { } res } + + pub fn merge_from(&mut self, other: &Self) { + for (dst, src) in self.0.iter_mut().zip(other.0.iter()) { + *dst += src; + } + } } /// Replication statistics for a single target @@ -419,6 +431,9 @@ pub struct DataUsageEntry { pub obj_versions: VersionsHistogram, pub replication_stats: Option, pub compacted: bool, + /// Number of objects that failed to scan (e.g., IO errors) + #[serde(default)] + pub failed_objects: usize, } impl DataUsageEntry { @@ -456,6 +471,7 @@ impl DataUsageEntry { self.versions += other.versions; self.delete_markers += other.delete_markers; self.size += other.size; + self.failed_objects += other.failed_objects; if let Some(o_rep) = &other.replication_stats { let s_rep = self.replication_stats.get_or_insert_with(ReplicationAllStats::default); @@ -490,9 +506,11 @@ impl DataUsageEntry { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct DataUsageCacheInfo { pub name: String, - pub next_cycle: u32, + pub next_cycle: u64, pub last_update: Option, pub skip_healing: bool, + #[serde(default)] + pub failed_objects: HashMap, } /// Data usage cache diff --git a/crates/workers/src/lib.rs b/crates/data-usage/src/lib.rs similarity index 93% rename from crates/workers/src/lib.rs rename to crates/data-usage/src/lib.rs index 1512b3655..6a67b6f88 100644 --- a/crates/workers/src/lib.rs +++ b/crates/data-usage/src/lib.rs @@ -12,4 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod workers; +pub mod data_usage; + +pub use data_usage::*; diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index e978e0a95..c410a0009 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -32,6 +32,7 @@ sftp = [] rustfs-config = { workspace = true, features = ["constants"] } rustfs-ecstore.workspace = true rustfs-common.workspace = true +rustfs-data-usage.workspace = true rustfs-rio.workspace = true flatbuffers.workspace = true futures.workspace = true diff --git a/crates/e2e_test/src/data_usage_test.rs b/crates/e2e_test/src/data_usage_test.rs index 1121b366c..a64168397 100644 --- a/crates/e2e_test/src/data_usage_test.rs +++ b/crates/e2e_test/src/data_usage_test.rs @@ -13,7 +13,7 @@ // limitations under the License. use aws_sdk_s3::primitives::ByteStream; -use rustfs_common::data_usage::DataUsageInfo; +use rustfs_data_usage::DataUsageInfo; use serial_test::serial; use crate::common::{RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging}; diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index c91fa821d..7e8ec1312 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -45,7 +45,10 @@ rustfs-common.workspace = true rustfs-policy.workspace = true rustfs-protos.workspace = true rustfs-kms.workspace = true -rustfs-s3-common = { workspace = true } +rustfs-s3-types = { workspace = true } +rustfs-s3-ops = { workspace = true } +rustfs-data-usage.workspace = true +rustfs-object-capacity.workspace = true async-trait.workspace = true bytes.workspace = true byteorder = { workspace = true } @@ -103,7 +106,7 @@ memmap2 = { workspace = true } libc.workspace = true rustix = { workspace = true } rustfs-madmin.workspace = true -rustfs-workers.workspace = true +rustfs-concurrency.workspace = true reqwest = { workspace = true } aes-gcm.workspace = true aws-sdk-s3 = { workspace = true } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 8b41f25ed..9ea0947d9 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -44,7 +44,6 @@ use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; use futures::Future; use http::HeaderMap; use lazy_static::lazy_static; -use rustfs_common::data_usage::TierStats; use rustfs_common::heal_channel::rep_has_active_rules; use rustfs_common::metrics::{IlmAction, Metrics}; use rustfs_config::{ @@ -52,11 +51,12 @@ use rustfs_config::{ DEFAULT_TRANSITION_WORKERS_CAP, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX, }; +use rustfs_data_usage::TierStats; use rustfs_filemeta::{ FileInfo, FileInfoOpts, NULL_VERSION_ID, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicationState, RestoreStatusOps, VersionPurgeStatusType, get_file_info, is_restored_object_on_disk, }; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold}; use s3s::dto::{ BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest, diff --git a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs index cd2fcb36d..b32fb32f5 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -18,7 +18,7 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use rustfs_common::data_usage::TierStats; +use rustfs_data_usage::TierStats; use sha2::Sha256; use std::collections::HashMap; use std::ops::Sub; diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 093b3c372..37b9ba469 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -56,7 +56,7 @@ use rustfs_filemeta::{ ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType, get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map, }; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs index defb7b20e..ed80acff8 100644 --- a/crates/ecstore/src/data_usage.rs +++ b/crates/ecstore/src/data_usage.rs @@ -27,7 +27,7 @@ pub use local_snapshot::{ data_usage_dir, data_usage_state_dir, ensure_data_usage_layout, read_snapshot as read_local_snapshot, snapshot_file_name, snapshot_object_path, snapshot_path, write_snapshot as write_local_snapshot, }; -use rustfs_common::data_usage::{ +use rustfs_data_usage::{ BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary, }; use rustfs_io_metrics::record_system_path_failure; @@ -686,7 +686,7 @@ pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate: #[cfg(test)] mod tests { use super::*; - use rustfs_common::data_usage::BucketUsageInfo; + use rustfs_data_usage::BucketUsageInfo; fn aggregate_for_test( inputs: Vec<(DiskUsageStatus, Result, Error>)>, diff --git a/crates/ecstore/src/event/name.rs b/crates/ecstore/src/event/name.rs index 43da8bc75..82781925f 100644 --- a/crates/ecstore/src/event/name.rs +++ b/crates/ecstore/src/event/name.rs @@ -13,6 +13,6 @@ // limitations under the License. //! Compatibility re-export for the legacy `rustfs_ecstore::event::name::EventName` path. -//! The canonical event definition now lives in `rustfs_s3_common::EventName`. +//! The canonical event definition now lives in `rustfs_s3_types::EventName`. -pub use rustfs_s3_common::EventName; +pub use rustfs_s3_types::EventName; diff --git a/crates/ecstore/src/metrics_realtime.rs b/crates/ecstore/src/metrics_realtime.rs index 0d1fa07a8..1944eb8ad 100644 --- a/crates/ecstore/src/metrics_realtime.rs +++ b/crates/ecstore/src/metrics_realtime.rs @@ -14,11 +14,12 @@ use crate::{admin_server_info::get_local_server_property, new_object_layer_fn, store_api::StorageAPI}; use chrono::Utc; -use rustfs_common::{ - GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, internode_metrics::global_internode_metrics, - metrics::global_metrics, +use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, metrics::global_metrics}; +use rustfs_io_metrics::internode_metrics::global_internode_metrics; +use rustfs_madmin::metrics::{ + DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics, + ScannerMetrics as MadminScannerMetrics, TimedAction as MadminTimedAction, }; -use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics}; use rustfs_utils::os::get_drive_stats; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; @@ -56,6 +57,51 @@ impl MetricType { } } +fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsReport) -> MadminScannerMetrics { + MadminScannerMetrics { + collected_at: metrics.collected_at, + current_cycle: metrics.current_cycle, + current_started: metrics.current_started, + cycles_completed_at: metrics.cycles_completed_at, + ongoing_buckets: metrics.ongoing_buckets, + life_time_ops: metrics.life_time_ops, + life_time_ilm: metrics.life_time_ilm, + last_minute: MadminLastMinute { + actions: metrics + .last_minute + .actions + .into_iter() + .map(|(key, value)| { + ( + key, + MadminTimedAction { + count: value.count, + acc_time: value.acc_time, + bytes: value.bytes, + }, + ) + }) + .collect(), + ilm: metrics + .last_minute + .ilm + .into_iter() + .map(|(key, value)| { + ( + key, + MadminTimedAction { + count: value.count, + acc_time: value.acc_time, + bytes: value.bytes, + }, + ) + }) + .collect(), + }, + active_paths: metrics.active_paths, + } +} + impl MetricType { fn contains(&self, x: &MetricType) -> bool { (self.0 & x.0) == x.0 @@ -114,7 +160,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) if let Some(init_time) = rustfs_common::get_global_init_time().await { metrics.current_started = init_time; } - real_time_metrics.aggregated.scanner = Some(metrics); + real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics)); } // if types.contains(&MetricType::OS) {} @@ -252,7 +298,7 @@ async fn collect_local_disks_metrics(disks: &HashSet) -> HashMap op.as_str(), "bucket" => bucket.to_owned()).increment(1); } -/// One-time registration of indicator meta information -/// This function ensures that metric descriptors are registered only once. pub fn init_s3_metrics() { static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new(); METRICS_DESC_INIT.get_or_init(|| { diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index 85b86f6a9..aa54cea2e 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -28,7 +28,8 @@ documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/" [dependencies] rustfs-config = { workspace = true, features = ["notify", "constants"] } rustfs-ecstore = { workspace = true } -rustfs-s3-common = { workspace = true } +rustfs-s3-types = { workspace = true } +rustfs-s3-ops = { workspace = true } rustfs-targets = { workspace = true } rustfs-utils = { workspace = true } arc-swap = { workspace = true } diff --git a/crates/notify/benches/snapshot_mode_scan.rs b/crates/notify/benches/snapshot_mode_scan.rs index 22907f537..7cc4fe09c 100644 --- a/crates/notify/benches/snapshot_mode_scan.rs +++ b/crates/notify/benches/snapshot_mode_scan.rs @@ -6,7 +6,7 @@ use tokio::runtime::Runtime; fn build_rule_map(target: &TargetID) -> RulesMap { let mut rules_map = RulesMap::new(); - rules_map.add_rule_config(&[rustfs_s3_common::EventName::ObjectCreatedPut], "*".to_string(), target.clone()); + rules_map.add_rule_config(&[rustfs_s3_types::EventName::ObjectCreatedPut], "*".to_string(), target.clone()); rules_map } diff --git a/crates/notify/examples/full_demo.rs b/crates/notify/examples/full_demo.rs index d778871f8..facda2737 100644 --- a/crates/notify/examples/full_demo.rs +++ b/crates/notify/examples/full_demo.rs @@ -24,7 +24,7 @@ use rustfs_config::{ use rustfs_ecstore::config::{Config, KV, KVS}; use rustfs_notify::{BucketNotificationConfig, Event, NotificationError}; use rustfs_notify::{initialize, notification_system}; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use std::sync::Arc; use std::time::Duration; diff --git a/crates/notify/examples/full_demo_one.rs b/crates/notify/examples/full_demo_one.rs index 3e45463cc..afd4578a8 100644 --- a/crates/notify/examples/full_demo_one.rs +++ b/crates/notify/examples/full_demo_one.rs @@ -24,7 +24,7 @@ use rustfs_config::{ use rustfs_ecstore::config::{Config, KV, KVS}; use rustfs_notify::{BucketNotificationConfig, Event, NotificationError}; use rustfs_notify::{initialize, notification_system}; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use std::sync::Arc; use std::time::Duration; diff --git a/crates/notify/src/bucket_config_manager.rs b/crates/notify/src/bucket_config_manager.rs index 8e041b52d..bbfe1ebd9 100644 --- a/crates/notify/src/bucket_config_manager.rs +++ b/crates/notify/src/bucket_config_manager.rs @@ -17,7 +17,7 @@ use crate::{ notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine, rules::ParseConfigError, }; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use std::sync::Arc; use tracing::{debug, info, warn}; @@ -91,7 +91,7 @@ mod tests { BucketNotificationConfig, integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine, }; - use rustfs_s3_common::EventName; + use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use std::sync::Arc; diff --git a/crates/notify/src/event.rs b/crates/notify/src/event.rs index ee803bd87..2369cceb5 100644 --- a/crates/notify/src/event.rs +++ b/crates/notify/src/event.rs @@ -14,7 +14,8 @@ use chrono::{DateTime, SecondsFormat, Utc}; use hashbrown::HashMap; -use rustfs_s3_common::EventName; +use rustfs_s3_ops::is_object_removed_event; +use rustfs_s3_types::{EventName, event_schema_version}; use serde::{Deserialize, Serialize}; use url::form_urlencoded; @@ -135,25 +136,6 @@ pub struct Event { } impl Event { - fn event_version_for(event_name: EventName) -> &'static str { - match event_name { - EventName::ObjectReplicationFailed - | EventName::ObjectReplicationComplete - | EventName::ObjectReplicationMissedThreshold - | EventName::ObjectReplicationReplicatedAfterThreshold - | EventName::ObjectReplicationNotTracked => "2.2", - EventName::ObjectRestoreCompleted - | EventName::ObjectAclPut - | EventName::ObjectTaggingPut - | EventName::ObjectTaggingDelete - | EventName::LifecycleExpirationDelete - | EventName::LifecycleExpirationDeleteMarkerCreated - | EventName::LifecycleTransition - | EventName::IntelligentTiering => "2.3", - _ => "2.1", - } - } - /// Creates a test event for a given bucket and object pub fn new_test_event(bucket: &str, key: &str, event_name: EventName) -> Self { let mut user_metadata = HashMap::new(); @@ -176,7 +158,7 @@ impl Event { user_metadata.insert("x-request-time".to_string(), Utc::now().to_rfc3339()); Event { - event_version: Self::event_version_for(event_name).to_string(), + event_version: event_schema_version(event_name).to_string(), event_source: "rustfs:s3".to_string(), aws_region: "us-east-1".to_string(), event_time: Utc::now(), @@ -256,10 +238,7 @@ impl Event { }, }; - let is_removed_event = matches!( - args.event_name, - EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated - ); + let is_removed_event = is_object_removed_event(args.event_name); if !is_removed_event { s3_metadata.object.size = Some(args.object.size); @@ -293,7 +272,7 @@ impl Event { }; Self { - event_version: Self::event_version_for(args.event_name).to_string(), + event_version: event_schema_version(args.event_name).to_string(), event_source: "rustfs:s3".to_string(), aws_region: args.req_params.get("region").cloned().unwrap_or_default(), event_time: event_time.and_utc(), @@ -542,7 +521,7 @@ mod event_args_tests { use super::EventArgs; use hashbrown::HashMap; use rustfs_ecstore::store_api::ObjectInfo; - use rustfs_s3_common::EventName; + use rustfs_s3_types::EventName; fn args_with_headers(pairs: &[(&str, &str)]) -> EventArgs { let mut req_params = HashMap::new(); diff --git a/crates/notify/src/global.rs b/crates/notify/src/global.rs index 767ce8e76..0d61ee591 100644 --- a/crates/notify/src/global.rs +++ b/crates/notify/src/global.rs @@ -17,7 +17,7 @@ use crate::{ NotificationSystem, NotificationTargetMetricSnapshot, }; use rustfs_ecstore::config::Config; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use std::sync::{Arc, OnceLock}; use tracing::error; diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index dd8f5dc16..70da7f8ea 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -23,7 +23,7 @@ use hashbrown::HashMap; use metrics::{counter, gauge}; use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY}; use rustfs_ecstore::config::{Config, KVS}; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget}; use std::sync::Arc; @@ -402,7 +402,7 @@ pub async fn load_config_from_file(path: &str, system: &NotificationSystem) -> R #[cfg(test)] mod tests { use super::*; - use rustfs_s3_common::EventName; + use rustfs_s3_types::EventName; #[test] fn live_event_history_snapshots_from_sequence() { diff --git a/crates/notify/src/notification_system_subscriber.rs b/crates/notify/src/notification_system_subscriber.rs index a7ca8735a..9f56f30b4 100644 --- a/crates/notify/src/notification_system_subscriber.rs +++ b/crates/notify/src/notification_system_subscriber.rs @@ -14,7 +14,7 @@ use crate::BucketNotificationConfig; use crate::rules::{BucketRulesSnapshot, DynRulesContainer, SubscriberIndex}; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; /// NotificationSystemSubscriberView - Provides an interface to manage and query /// the subscription status of buckets in the notification system. diff --git a/crates/notify/src/notifier.rs b/crates/notify/src/notifier.rs index 0196fc68d..edba03854 100644 --- a/crates/notify/src/notifier.rs +++ b/crates/notify/src/notifier.rs @@ -308,7 +308,7 @@ mod tests { use super::*; use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap}; use async_trait::async_trait; - use rustfs_s3_common::EventName; + use rustfs_s3_types::EventName; use rustfs_targets::StoreError; use rustfs_targets::{ TargetError, diff --git a/crates/notify/src/pipeline.rs b/crates/notify/src/pipeline.rs index 6e611d82e..41d87cb13 100644 --- a/crates/notify/src/pipeline.rs +++ b/crates/notify/src/pipeline.rs @@ -105,7 +105,7 @@ pub type NotifyEventBridge = NotifyPipeline; mod tests { use super::{LiveEventHistory, NotifyPipeline}; use crate::{Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine}; - use rustfs_s3_common::EventName; + use rustfs_s3_types::EventName; use std::sync::Arc; use tokio::sync::{RwLock, broadcast}; diff --git a/crates/notify/src/rule_engine.rs b/crates/notify/src/rule_engine.rs index 42bcacf7a..0c5d1072d 100644 --- a/crates/notify/src/rule_engine.rs +++ b/crates/notify/src/rule_engine.rs @@ -14,7 +14,7 @@ use crate::rules::{RulesMap, TargetIdSet}; use percent_encoding::percent_decode_str; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode}; use std::sync::Arc; @@ -99,7 +99,7 @@ impl Default for NotifyRuleEngine { mod tests { use super::NotifyRuleEngine; use crate::rules::RulesMap; - use rustfs_s3_common::EventName; + use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; #[tokio::test] diff --git a/crates/notify/src/rules/config.rs b/crates/notify/src/rules/config.rs index 8890399b0..42ca136fa 100644 --- a/crates/notify/src/rules/config.rs +++ b/crates/notify/src/rules/config.rs @@ -16,7 +16,7 @@ use super::rules_map::RulesMap; use super::xml_config::ParseConfigError as BucketNotificationConfigError; use crate::rules::NotificationConfiguration; use crate::rules::subscriber_snapshot::{BucketRulesSnapshot, DynRulesContainer, RuleEvents, RulesContainer}; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use serde::{Deserialize, Serialize}; use std::io::Read; diff --git a/crates/notify/src/rules/config_test.rs b/crates/notify/src/rules/config_test.rs index 34a6b57dd..7f1d0d06d 100644 --- a/crates/notify/src/rules/config_test.rs +++ b/crates/notify/src/rules/config_test.rs @@ -18,7 +18,7 @@ //! to event matching, including filter rules with prefix and suffix. use super::*; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::{ARN, TargetID}; use std::io::Cursor; diff --git a/crates/notify/src/rules/pattern_rules_test.rs b/crates/notify/src/rules/pattern_rules_test.rs index 3672828f9..03dc4c4b0 100644 --- a/crates/notify/src/rules/pattern_rules_test.rs +++ b/crates/notify/src/rules/pattern_rules_test.rs @@ -19,7 +19,7 @@ //! configuration to event matching. use super::*; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; #[cfg(test)] diff --git a/crates/notify/src/rules/rules_map.rs b/crates/notify/src/rules/rules_map.rs index 00ed317d9..9ae40ad85 100644 --- a/crates/notify/src/rules/rules_map.rs +++ b/crates/notify/src/rules/rules_map.rs @@ -14,7 +14,7 @@ use crate::rules::{PatternRules, TargetIdSet}; use hashbrown::HashMap; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; use serde::{Deserialize, Serialize}; diff --git a/crates/notify/src/rules/subscriber_index.rs b/crates/notify/src/rules/subscriber_index.rs index 3b57bc8f3..853aba330 100644 --- a/crates/notify/src/rules/subscriber_index.rs +++ b/crates/notify/src/rules/subscriber_index.rs @@ -14,7 +14,7 @@ use crate::rules::{BucketRulesSnapshot, BucketSnapshotRef, DynRulesContainer}; use arc_swap::ArcSwap; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use starshard::{DEFAULT_SHARDS, ShardedHashMap}; use std::fmt; use std::sync::Arc; diff --git a/crates/notify/src/rules/subscriber_snapshot.rs b/crates/notify/src/rules/subscriber_snapshot.rs index a8c951eb8..3747a6082 100644 --- a/crates/notify/src/rules/subscriber_snapshot.rs +++ b/crates/notify/src/rules/subscriber_snapshot.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use std::sync::Arc; /// Let the rules structure provide "what events it is subscribed to". diff --git a/crates/notify/src/rules/xml_config.rs b/crates/notify/src/rules/xml_config.rs index 51b5a1b5f..fd6a344e4 100644 --- a/crates/notify/src/rules/xml_config.rs +++ b/crates/notify/src/rules/xml_config.rs @@ -14,7 +14,7 @@ use crate::rules::pattern; use hashbrown::HashSet; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::arn::{ARN, ArnError, TargetIDError}; use serde::{Deserialize, Serialize}; use std::io::Read; diff --git a/crates/object-capacity/Cargo.toml b/crates/object-capacity/Cargo.toml index 08cdda9de..04413ec34 100644 --- a/crates/object-capacity/Cargo.toml +++ b/crates/object-capacity/Cargo.toml @@ -35,7 +35,6 @@ harness = false workspace = true [dependencies] -rustfs-common = { workspace = true } rustfs-config = { workspace = true, features = ["constants"] } rustfs-io-metrics = { workspace = true } rustfs-utils = { workspace = true } diff --git a/crates/object-capacity/src/capacity_manager.rs b/crates/object-capacity/src/capacity_manager.rs index 92edb6ba1..f67d74d18 100644 --- a/crates/object-capacity/src/capacity_manager.rs +++ b/crates/object-capacity/src/capacity_manager.rs @@ -16,8 +16,8 @@ use super::scan::refresh_capacity_with_scope; use super::types::CapacityDiskRef; +use crate::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope}; use futures::FutureExt; -use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope}; use rustfs_config::{ DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, @@ -1017,7 +1017,7 @@ pub async fn start_background_task(disks: Vec) { #[cfg(test)] mod tests { use super::*; - use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope}; + use crate::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope}; use rustfs_config::{ ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_METRICS_INTERVAL, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, diff --git a/crates/common/src/capacity_scope.rs b/crates/object-capacity/src/capacity_scope.rs similarity index 100% rename from crates/common/src/capacity_scope.rs rename to crates/object-capacity/src/capacity_scope.rs diff --git a/crates/object-capacity/src/lib.rs b/crates/object-capacity/src/lib.rs index c5730cdbf..19ff9dbf2 100644 --- a/crates/object-capacity/src/lib.rs +++ b/crates/object-capacity/src/lib.rs @@ -13,6 +13,7 @@ // limitations under the License. pub mod capacity_manager; +pub mod capacity_scope; pub mod scan; pub mod types; diff --git a/crates/object-capacity/src/scan.rs b/crates/object-capacity/src/scan.rs index 7074bd467..5b4b4355e 100644 --- a/crates/object-capacity/src/scan.rs +++ b/crates/object-capacity/src/scan.rs @@ -18,8 +18,8 @@ use super::capacity_manager::{ get_stat_timeout, }; use super::types::{CapacityDiskRef, CapacityScanResult, CapacityScanSummary}; +use crate::capacity_scope::CapacityScopeDisk; use futures::{StreamExt, stream}; -use rustfs_common::capacity_scope::CapacityScopeDisk; use rustfs_io_metrics::capacity_metrics::{ record_capacity_dynamic_timeout, record_capacity_scan_disk, record_capacity_scan_mode, record_capacity_scan_sampling, record_capacity_stall_detected, record_capacity_symlink, record_capacity_timeout_fallback, @@ -647,7 +647,7 @@ async fn get_dir_size_async(path: &Path) -> Result Option { + match self { + Self::None => None, + Self::Single(event_name) => Some(event_name), + Self::PutObject => Some(EventName::ObjectCreatedPut), + Self::DeleteObject => Some(EventName::ObjectRemovedDelete), + Self::DeleteObjects => Some(EventName::ObjectRemovedDeleteObjects), + } + } + + #[inline] + fn matches_event(self, event_name: EventName) -> bool { + match self { + Self::None => false, + Self::Single(mapped_event_name) => mapped_event_name == event_name, + Self::PutObject => matches!(event_name, EventName::ObjectCreatedPut | EventName::ObjectCreatedPost), + Self::DeleteObject => matches!( + event_name, + EventName::ObjectRemovedDelete + | EventName::ObjectRemovedDeleteMarkerCreated + | EventName::ObjectRemovedDeleteAllVersions + ), + Self::DeleteObjects => { + matches!(event_name, EventName::ObjectRemovedDeleteObjects | EventName::ObjectRemovedDelete) + } + } + } +} + +impl S3Operation { + pub fn as_str(self) -> &'static str { + match self { + Self::AbortMultipartUpload => "s3:AbortMultipartUpload", + Self::CompleteMultipartUpload => "s3:CompleteMultipartUpload", + Self::CopyObject => "s3:CopyObject", + Self::CreateBucket => "s3:CreateBucket", + Self::CreateMultipartUpload => "s3:CreateMultipartUpload", + Self::DeleteBucket => "s3:DeleteBucket", + Self::DeleteBucketCors => "s3:DeleteBucketCors", + Self::DeleteBucketEncryption => "s3:DeleteBucketEncryption", + Self::DeleteBucketLifecycle => "s3:DeleteBucketLifecycle", + Self::DeleteBucketPolicy => "s3:DeleteBucketPolicy", + Self::DeleteBucketReplication => "s3:DeleteBucketReplication", + Self::DeleteBucketTagging => "s3:DeleteBucketTagging", + Self::DeleteObject => "s3:DeleteObject", + Self::DeleteObjectTagging => "s3:DeleteObjectTagging", + Self::DeleteObjects => "s3:DeleteObjects", + Self::DeletePublicAccessBlock => "s3:DeletePublicAccessBlock", + Self::GetBucketAcl => "s3:GetBucketAcl", + Self::GetBucketCors => "s3:GetBucketCors", + Self::GetBucketEncryption => "s3:GetBucketEncryption", + Self::GetBucketLifecycleConfiguration => "s3:GetBucketLifecycleConfiguration", + Self::GetBucketLocation => "s3:GetBucketLocation", + Self::GetBucketLogging => "s3:GetBucketLogging", + Self::GetBucketNotificationConfiguration => "s3:GetBucketNotificationConfiguration", + Self::GetBucketPolicy => "s3:GetBucketPolicy", + Self::GetBucketPolicyStatus => "s3:GetBucketPolicyStatus", + Self::GetBucketReplication => "s3:GetBucketReplication", + Self::GetBucketTagging => "s3:GetBucketTagging", + Self::GetBucketVersioning => "s3:GetBucketVersioning", + Self::GetObject => "s3:GetObject", + Self::GetObjectAcl => "s3:GetObjectAcl", + Self::GetObjectAttributes => "s3:GetObjectAttributes", + Self::GetObjectLegalHold => "s3:GetObjectLegalHold", + Self::GetObjectLockConfiguration => "s3:GetObjectLockConfiguration", + Self::GetObjectRetention => "s3:GetObjectRetention", + Self::GetObjectTagging => "s3:GetObjectTagging", + Self::GetObjectTorrent => "s3:GetObjectTorrent", + Self::GetPublicAccessBlock => "s3:GetPublicAccessBlock", + Self::HeadBucket => "s3:HeadBucket", + Self::HeadObject => "s3:HeadObject", + Self::ListBuckets => "s3:ListBuckets", + Self::ListMultipartUploads => "s3:ListMultipartUploads", + Self::ListObjectVersions => "s3:ListObjectVersions", + Self::ListObjects => "s3:ListObjects", + Self::ListObjectsV2 => "s3:ListObjectsV2", + Self::ListParts => "s3:ListParts", + Self::PutBucketAcl => "s3:PutBucketAcl", + Self::PutBucketCors => "s3:PutBucketCors", + Self::PutBucketEncryption => "s3:PutBucketEncryption", + Self::PutBucketLifecycleConfiguration => "s3:PutBucketLifecycleConfiguration", + Self::PutBucketLogging => "s3:PutBucketLogging", + Self::PutBucketNotificationConfiguration => "s3:PutBucketNotificationConfiguration", + Self::PutBucketPolicy => "s3:PutBucketPolicy", + Self::PutBucketReplication => "s3:PutBucketReplication", + Self::PutBucketTagging => "s3:PutBucketTagging", + Self::PutBucketVersioning => "s3:PutBucketVersioning", + Self::PutObject => "s3:PutObject", + Self::PutObjectAcl => "s3:PutObjectAcl", + Self::PutObjectLegalHold => "s3:PutObjectLegalHold", + Self::PutObjectLockConfiguration => "s3:PutObjectLockConfiguration", + Self::PutObjectRetention => "s3:PutObjectRetention", + Self::PutObjectTagging => "s3:PutObjectTagging", + Self::PutPublicAccessBlock => "s3:PutPublicAccessBlock", + Self::RestoreObject => "s3:RestoreObject", + Self::SelectObjectContent => "s3:SelectObjectContent", + Self::UploadPart => "s3:UploadPart", + Self::UploadPartCopy => "s3:UploadPartCopy", + } + } + + #[inline] + fn event_mapping(self) -> EventMapping { + match self { + Self::AbortMultipartUpload => EventMapping::Single(EventName::ObjectRemovedAbortMultipartUpload), + Self::CompleteMultipartUpload => EventMapping::Single(EventName::ObjectCreatedCompleteMultipartUpload), + Self::CopyObject => EventMapping::Single(EventName::ObjectCreatedCopy), + Self::CreateBucket => EventMapping::Single(EventName::BucketCreated), + Self::CreateMultipartUpload => EventMapping::Single(EventName::ObjectCreatedCreateMultipartUpload), + Self::DeleteBucket => EventMapping::Single(EventName::BucketRemoved), + Self::DeleteObjectTagging => EventMapping::Single(EventName::ObjectTaggingDelete), + Self::DeleteObject => EventMapping::DeleteObject, + Self::DeleteObjects => EventMapping::DeleteObjects, + Self::GetObject => EventMapping::Single(EventName::ObjectAccessedGet), + Self::GetObjectAttributes => EventMapping::Single(EventName::ObjectAccessedAttributes), + Self::GetObjectLegalHold => EventMapping::Single(EventName::ObjectAccessedGetLegalHold), + Self::GetObjectRetention => EventMapping::Single(EventName::ObjectAccessedGetRetention), + Self::HeadObject => EventMapping::Single(EventName::ObjectAccessedHead), + Self::PutObject => EventMapping::PutObject, + Self::PutObjectAcl => EventMapping::Single(EventName::ObjectAclPut), + Self::PutObjectLegalHold => EventMapping::Single(EventName::ObjectCreatedPutLegalHold), + Self::PutObjectRetention => EventMapping::Single(EventName::ObjectCreatedPutRetention), + Self::PutObjectTagging => EventMapping::Single(EventName::ObjectTaggingPut), + Self::RestoreObject => EventMapping::Single(EventName::ObjectRestorePost), + Self::SelectObjectContent => EventMapping::Single(EventName::ObjectAccessedGet), + Self::DeleteBucketCors + | Self::DeleteBucketEncryption + | Self::DeleteBucketLifecycle + | Self::DeleteBucketPolicy + | Self::DeleteBucketReplication + | Self::DeleteBucketTagging + | Self::DeletePublicAccessBlock + | Self::GetBucketAcl + | Self::GetBucketCors + | Self::GetBucketEncryption + | Self::GetBucketLifecycleConfiguration + | Self::GetBucketLocation + | Self::GetBucketLogging + | Self::GetBucketNotificationConfiguration + | Self::GetBucketPolicy + | Self::GetBucketPolicyStatus + | Self::GetBucketReplication + | Self::GetBucketTagging + | Self::GetBucketVersioning + | Self::GetObjectAcl + | Self::GetObjectLockConfiguration + | Self::GetObjectTagging + | Self::GetObjectTorrent + | Self::GetPublicAccessBlock + | Self::HeadBucket + | Self::ListBuckets + | Self::ListMultipartUploads + | Self::ListObjectVersions + | Self::ListObjects + | Self::ListObjectsV2 + | Self::ListParts + | Self::PutBucketAcl + | Self::PutBucketCors + | Self::PutBucketEncryption + | Self::PutBucketLifecycleConfiguration + | Self::PutBucketLogging + | Self::PutBucketNotificationConfiguration + | Self::PutBucketPolicy + | Self::PutBucketReplication + | Self::PutBucketTagging + | Self::PutBucketVersioning + | Self::PutObjectLockConfiguration + | Self::PutPublicAccessBlock + | Self::UploadPart + | Self::UploadPartCopy => EventMapping::None, + } + } + + pub fn to_event_name(self) -> Option { + self.event_mapping().primary_event() + } +} + +pub fn event_name_to_s3_operation(event_name: EventName) -> Option { + match event_name { + EventName::BucketCreated => Some(S3Operation::CreateBucket), + EventName::BucketRemoved => Some(S3Operation::DeleteBucket), + EventName::ObjectAccessedGet => Some(S3Operation::GetObject), + EventName::ObjectAccessedGetRetention => Some(S3Operation::GetObjectRetention), + EventName::ObjectAccessedGetLegalHold => Some(S3Operation::GetObjectLegalHold), + EventName::ObjectAccessedHead => Some(S3Operation::HeadObject), + EventName::ObjectAccessedAttributes => Some(S3Operation::GetObjectAttributes), + EventName::ObjectCreatedCompleteMultipartUpload => Some(S3Operation::CompleteMultipartUpload), + EventName::ObjectCreatedCopy => Some(S3Operation::CopyObject), + EventName::ObjectCreatedPost => Some(S3Operation::PutObject), + EventName::ObjectCreatedPut => Some(S3Operation::PutObject), + EventName::ObjectCreatedPutRetention => Some(S3Operation::PutObjectRetention), + EventName::ObjectCreatedPutLegalHold => Some(S3Operation::PutObjectLegalHold), + EventName::ObjectTaggingPut => Some(S3Operation::PutObjectTagging), + EventName::ObjectTaggingDelete => Some(S3Operation::DeleteObjectTagging), + EventName::ObjectAclPut => Some(S3Operation::PutObjectAcl), + EventName::ObjectRemovedDelete => Some(S3Operation::DeleteObject), + EventName::ObjectRemovedDeleteMarkerCreated => Some(S3Operation::DeleteObject), + EventName::ObjectRemovedDeleteAllVersions => Some(S3Operation::DeleteObject), + EventName::ObjectRestorePost => Some(S3Operation::RestoreObject), + EventName::ObjectRemovedAbortMultipartUpload => Some(S3Operation::AbortMultipartUpload), + EventName::ObjectCreatedCreateMultipartUpload => Some(S3Operation::CreateMultipartUpload), + EventName::ObjectRemovedDeleteObjects => Some(S3Operation::DeleteObjects), + _ => None, + } +} + +/// Returns whether an S3 operation is semantically compatible with an event name. +/// +/// Some S3 operations intentionally map to multiple event variants: +/// - `PutObject` can emit both `ObjectCreatedPut` and `ObjectCreatedPost`. +/// - `DeleteObject` can emit delete/delete-marker/all-versions variants. +/// - `DeleteObjects` can emit per-object delete events in addition to the +/// internal batch-delete event. +pub fn operation_matches_event_name(op: S3Operation, event_name: EventName) -> bool { + op.event_mapping().matches_event(event_name) +} + +/// Resolves the object-delete notification event name from delete-marker state. +#[inline] +pub fn delete_event_name_for_marker(delete_marker: bool) -> EventName { + if delete_marker { + EventName::ObjectRemovedDeleteMarkerCreated + } else { + EventName::ObjectRemovedDelete + } +} + +/// Resolves the object-create notification event name from POST-object mode. +#[inline] +pub fn put_event_name_for_post_object(is_post_object: bool) -> EventName { + if is_post_object { + EventName::ObjectCreatedPost + } else { + EventName::ObjectCreatedPut + } +} + +/// Returns `true` when the event is one of object-remove notification variants +/// that should omit size/etag metadata. +#[inline] +pub fn is_object_removed_event(event_name: EventName) -> bool { + matches!(event_name, EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated) +} + +/// Returns the event mask that matches both PUT and POST object-created events. +#[inline] +pub fn put_object_created_event_mask() -> u64 { + EventName::ObjectCreatedPut.mask() | EventName::ObjectCreatedPost.mask() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_s3_operation_to_event_name() { + assert_eq!(S3Operation::PutObject.to_event_name(), Some(EventName::ObjectCreatedPut)); + assert_eq!(S3Operation::PutObjectAcl.to_event_name(), Some(EventName::ObjectAclPut)); + assert_eq!(S3Operation::PutObjectTagging.to_event_name(), Some(EventName::ObjectTaggingPut)); + assert_eq!(S3Operation::DeleteObjectTagging.to_event_name(), Some(EventName::ObjectTaggingDelete)); + assert_eq!(S3Operation::GetObject.to_event_name(), Some(EventName::ObjectAccessedGet)); + assert_eq!(S3Operation::ListBuckets.to_event_name(), None); + assert_eq!(S3Operation::RestoreObject.to_event_name(), Some(EventName::ObjectRestorePost)); + assert_eq!(S3Operation::SelectObjectContent.to_event_name(), Some(EventName::ObjectAccessedGet)); + assert_eq!( + S3Operation::AbortMultipartUpload.to_event_name(), + Some(EventName::ObjectRemovedAbortMultipartUpload) + ); + } + + #[test] + fn test_event_name_to_s3_operation() { + assert_eq!(event_name_to_s3_operation(EventName::ObjectCreatedPut), Some(S3Operation::PutObject)); + assert_eq!(event_name_to_s3_operation(EventName::ObjectAclPut), Some(S3Operation::PutObjectAcl)); + assert_eq!( + event_name_to_s3_operation(EventName::ObjectTaggingPut), + Some(S3Operation::PutObjectTagging) + ); + assert_eq!( + event_name_to_s3_operation(EventName::ObjectTaggingDelete), + Some(S3Operation::DeleteObjectTagging) + ); + assert_eq!(event_name_to_s3_operation(EventName::ObjectAccessedGet), Some(S3Operation::GetObject)); + assert_eq!(event_name_to_s3_operation(EventName::BucketCreated), Some(S3Operation::CreateBucket)); + assert_eq!(event_name_to_s3_operation(EventName::Everything), None); + assert_eq!(event_name_to_s3_operation(EventName::ObjectRestorePost), Some(S3Operation::RestoreObject)); + assert_eq!(event_name_to_s3_operation(EventName::ObjectCreatedPost), Some(S3Operation::PutObject)); + assert_eq!( + event_name_to_s3_operation(EventName::ObjectRemovedAbortMultipartUpload), + Some(S3Operation::AbortMultipartUpload) + ); + } + + #[test] + fn test_operation_matches_event_name() { + assert!(operation_matches_event_name(S3Operation::PutObject, EventName::ObjectCreatedPut)); + assert!(operation_matches_event_name(S3Operation::PutObject, EventName::ObjectCreatedPost)); + assert!(operation_matches_event_name( + S3Operation::DeleteObject, + EventName::ObjectRemovedDeleteMarkerCreated + )); + assert!(operation_matches_event_name(S3Operation::DeleteObjects, EventName::ObjectRemovedDelete)); + + assert!(!operation_matches_event_name(S3Operation::GetObject, EventName::ObjectCreatedPut)); + } + + #[test] + fn test_delete_event_name_for_marker() { + assert_eq!(delete_event_name_for_marker(true), EventName::ObjectRemovedDeleteMarkerCreated); + assert_eq!(delete_event_name_for_marker(false), EventName::ObjectRemovedDelete); + } + + #[test] + fn test_put_event_name_for_post_object() { + assert_eq!(put_event_name_for_post_object(true), EventName::ObjectCreatedPost); + assert_eq!(put_event_name_for_post_object(false), EventName::ObjectCreatedPut); + } + + #[test] + fn test_is_object_removed_event() { + assert!(is_object_removed_event(EventName::ObjectRemovedDelete)); + assert!(is_object_removed_event(EventName::ObjectRemovedDeleteMarkerCreated)); + assert!(!is_object_removed_event(EventName::ObjectCreatedPut)); + } + + #[test] + fn test_put_object_created_event_mask() { + let mask = put_object_created_event_mask(); + assert_ne!(mask & EventName::ObjectCreatedPut.mask(), 0); + assert_ne!(mask & EventName::ObjectCreatedPost.mask(), 0); + assert_eq!(mask & EventName::ObjectRemovedDelete.mask(), 0); + } + + #[test] + fn test_operations_without_event_mapping_remain_unmapped() { + let unmapped = [ + S3Operation::GetBucketAcl, + S3Operation::ListObjectsV2, + S3Operation::PutBucketNotificationConfiguration, + S3Operation::UploadPart, + ]; + for op in unmapped { + assert_eq!(op.to_event_name(), None); + assert!(!operation_matches_event_name(op, EventName::ObjectCreatedPut)); + } + } +} diff --git a/crates/s3-common/Cargo.toml b/crates/s3-types/Cargo.toml similarity index 85% rename from crates/s3-common/Cargo.toml rename to crates/s3-types/Cargo.toml index 6b1ae5811..21a361739 100644 --- a/crates/s3-common/Cargo.toml +++ b/crates/s3-types/Cargo.toml @@ -13,22 +13,21 @@ # limitations under the License. [package] -name = "rustfs-s3-common" +name = "rustfs-s3-types" version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true homepage.workspace = true -description = "Common S3 definitions and metrics for RustFS." -keywords = ["s3", "common", "rustfs"] +description = "S3 event type definitions for RustFS." +keywords = ["s3", "event", "types", "rustfs"] categories = ["data-structures"] [lints] workspace = true [dependencies] -metrics = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/s3-common/src/event_name.rs b/crates/s3-types/src/event_name.rs similarity index 63% rename from crates/s3-common/src/event_name.rs rename to crates/s3-types/src/event_name.rs index 7776dc751..88344bc23 100644 --- a/crates/s3-common/src/event_name.rs +++ b/crates/s3-types/src/event_name.rs @@ -190,6 +190,12 @@ impl EventName { } } + /// Parses an event string into an EventName with explicit error handling. + #[inline] + pub fn try_from_event_str(s: &str) -> Result { + Self::parse(s) + } + /// Returns a string representation of the event type. pub fn as_str(&self) -> &'static str { match self { @@ -315,35 +321,26 @@ impl EventName { mask } } +} - /// Returns the corresponding S3Operation if the event triggers a notification event. - pub fn to_s3_operation(&self) -> Option { - match self { - EventName::BucketCreated => Some(S3Operation::CreateBucket), - EventName::BucketRemoved => Some(S3Operation::DeleteBucket), - EventName::ObjectAccessedGet => Some(S3Operation::GetObject), - EventName::ObjectAccessedGetRetention => Some(S3Operation::GetObjectRetention), - EventName::ObjectAccessedGetLegalHold => Some(S3Operation::GetObjectLegalHold), - EventName::ObjectAccessedHead => Some(S3Operation::HeadObject), - EventName::ObjectAccessedAttributes => Some(S3Operation::GetObjectAttributes), - EventName::ObjectCreatedCompleteMultipartUpload => Some(S3Operation::CompleteMultipartUpload), - EventName::ObjectCreatedCopy => Some(S3Operation::CopyObject), - EventName::ObjectCreatedPost => Some(S3Operation::PutObject), - EventName::ObjectCreatedPut => Some(S3Operation::PutObject), - EventName::ObjectCreatedPutRetention => Some(S3Operation::PutObjectRetention), - EventName::ObjectCreatedPutLegalHold => Some(S3Operation::PutObjectLegalHold), - EventName::ObjectTaggingPut => Some(S3Operation::PutObjectTagging), - EventName::ObjectTaggingDelete => Some(S3Operation::DeleteObjectTagging), - EventName::ObjectAclPut => Some(S3Operation::PutObjectAcl), - EventName::ObjectRemovedDelete => Some(S3Operation::DeleteObject), - EventName::ObjectRemovedDeleteMarkerCreated => Some(S3Operation::DeleteObject), - EventName::ObjectRemovedDeleteAllVersions => Some(S3Operation::DeleteObject), - EventName::ObjectRestorePost => Some(S3Operation::RestoreObject), - EventName::ObjectRemovedAbortMultipartUpload => Some(S3Operation::AbortMultipartUpload), - EventName::ObjectCreatedCreateMultipartUpload => Some(S3Operation::CreateMultipartUpload), - EventName::ObjectRemovedDeleteObjects => Some(S3Operation::DeleteObjects), - _ => None, - } +/// Returns the S3 notification event schema version for a given event. +#[inline] +pub fn event_schema_version(event_name: EventName) -> &'static str { + match event_name { + EventName::ObjectReplicationFailed + | EventName::ObjectReplicationComplete + | EventName::ObjectReplicationMissedThreshold + | EventName::ObjectReplicationReplicatedAfterThreshold + | EventName::ObjectReplicationNotTracked => "2.2", + EventName::ObjectRestoreCompleted + | EventName::ObjectAclPut + | EventName::ObjectTaggingPut + | EventName::ObjectTaggingDelete + | EventName::LifecycleExpirationDelete + | EventName::LifecycleExpirationDeleteMarkerCreated + | EventName::LifecycleTransition + | EventName::IntelligentTiering => "2.3", + _ => "2.1", } } @@ -380,177 +377,6 @@ impl<'de> serde::de::Deserialize<'de> for EventName { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum S3Operation { - AbortMultipartUpload, - CompleteMultipartUpload, - CopyObject, - CreateBucket, - CreateMultipartUpload, - DeleteBucket, - DeleteBucketCors, - DeleteBucketEncryption, - DeleteBucketLifecycle, - DeleteBucketPolicy, - DeleteBucketReplication, - DeleteBucketTagging, - DeleteObject, - DeleteObjectTagging, - DeleteObjects, - DeletePublicAccessBlock, - GetBucketAcl, - GetBucketCors, - GetBucketEncryption, - GetBucketLifecycleConfiguration, - GetBucketLocation, - GetBucketLogging, - GetBucketNotificationConfiguration, - GetBucketPolicy, - GetBucketPolicyStatus, - GetBucketReplication, - GetBucketTagging, - GetBucketVersioning, - GetObject, - GetObjectAcl, - GetObjectAttributes, - GetObjectLegalHold, - GetObjectLockConfiguration, - GetObjectRetention, - GetObjectTagging, - GetObjectTorrent, - GetPublicAccessBlock, - HeadBucket, - HeadObject, - ListBuckets, - ListMultipartUploads, - ListObjectVersions, - ListObjects, - ListObjectsV2, - ListParts, - PutBucketAcl, - PutBucketCors, - PutBucketEncryption, - PutBucketLifecycleConfiguration, - PutBucketLogging, - PutBucketNotificationConfiguration, - PutBucketPolicy, - PutBucketReplication, - PutBucketTagging, - PutBucketVersioning, - PutObject, - PutObjectAcl, - PutObjectLegalHold, - PutObjectLockConfiguration, - PutObjectRetention, - PutObjectTagging, - PutPublicAccessBlock, - RestoreObject, - SelectObjectContent, - UploadPart, - UploadPartCopy, -} - -impl S3Operation { - pub fn as_str(self) -> &'static str { - match self { - Self::AbortMultipartUpload => "s3:AbortMultipartUpload", - Self::CompleteMultipartUpload => "s3:CompleteMultipartUpload", - Self::CopyObject => "s3:CopyObject", - Self::CreateBucket => "s3:CreateBucket", - Self::CreateMultipartUpload => "s3:CreateMultipartUpload", - Self::DeleteBucket => "s3:DeleteBucket", - Self::DeleteBucketCors => "s3:DeleteBucketCors", - Self::DeleteBucketEncryption => "s3:DeleteBucketEncryption", - Self::DeleteBucketLifecycle => "s3:DeleteBucketLifecycle", - Self::DeleteBucketPolicy => "s3:DeleteBucketPolicy", - Self::DeleteBucketReplication => "s3:DeleteBucketReplication", - Self::DeleteBucketTagging => "s3:DeleteBucketTagging", - Self::DeleteObject => "s3:DeleteObject", - Self::DeleteObjectTagging => "s3:DeleteObjectTagging", - Self::DeleteObjects => "s3:DeleteObjects", - Self::DeletePublicAccessBlock => "s3:DeletePublicAccessBlock", - Self::GetBucketAcl => "s3:GetBucketAcl", - Self::GetBucketCors => "s3:GetBucketCors", - Self::GetBucketEncryption => "s3:GetBucketEncryption", - Self::GetBucketLifecycleConfiguration => "s3:GetBucketLifecycleConfiguration", - Self::GetBucketLocation => "s3:GetBucketLocation", - Self::GetBucketLogging => "s3:GetBucketLogging", - Self::GetBucketNotificationConfiguration => "s3:GetBucketNotificationConfiguration", - Self::GetBucketPolicy => "s3:GetBucketPolicy", - Self::GetBucketPolicyStatus => "s3:GetBucketPolicyStatus", - Self::GetBucketReplication => "s3:GetBucketReplication", - Self::GetBucketTagging => "s3:GetBucketTagging", - Self::GetBucketVersioning => "s3:GetBucketVersioning", - Self::GetObject => "s3:GetObject", - Self::GetObjectAcl => "s3:GetObjectAcl", - Self::GetObjectAttributes => "s3:GetObjectAttributes", - Self::GetObjectLegalHold => "s3:GetObjectLegalHold", - Self::GetObjectLockConfiguration => "s3:GetObjectLockConfiguration", - Self::GetObjectRetention => "s3:GetObjectRetention", - Self::GetObjectTagging => "s3:GetObjectTagging", - Self::GetObjectTorrent => "s3:GetObjectTorrent", - Self::GetPublicAccessBlock => "s3:GetPublicAccessBlock", - Self::HeadBucket => "s3:HeadBucket", - Self::HeadObject => "s3:HeadObject", - Self::ListBuckets => "s3:ListBuckets", - Self::ListMultipartUploads => "s3:ListMultipartUploads", - Self::ListObjectVersions => "s3:ListObjectVersions", - Self::ListObjects => "s3:ListObjects", - Self::ListObjectsV2 => "s3:ListObjectsV2", - Self::ListParts => "s3:ListParts", - Self::PutBucketAcl => "s3:PutBucketAcl", - Self::PutBucketCors => "s3:PutBucketCors", - Self::PutBucketEncryption => "s3:PutBucketEncryption", - Self::PutBucketLifecycleConfiguration => "s3:PutBucketLifecycleConfiguration", - Self::PutBucketLogging => "s3:PutBucketLogging", - Self::PutBucketNotificationConfiguration => "s3:PutBucketNotificationConfiguration", - Self::PutBucketPolicy => "s3:PutBucketPolicy", - Self::PutBucketReplication => "s3:PutBucketReplication", - Self::PutBucketTagging => "s3:PutBucketTagging", - Self::PutBucketVersioning => "s3:PutBucketVersioning", - Self::PutObject => "s3:PutObject", - Self::PutObjectAcl => "s3:PutObjectAcl", - Self::PutObjectLegalHold => "s3:PutObjectLegalHold", - Self::PutObjectLockConfiguration => "s3:PutObjectLockConfiguration", - Self::PutObjectRetention => "s3:PutObjectRetention", - Self::PutObjectTagging => "s3:PutObjectTagging", - Self::PutPublicAccessBlock => "s3:PutPublicAccessBlock", - Self::RestoreObject => "s3:RestoreObject", - Self::SelectObjectContent => "s3:SelectObjectContent", - Self::UploadPart => "s3:UploadPart", - Self::UploadPartCopy => "s3:UploadPartCopy", - } - } - - /// Returns the corresponding EventName if the operation triggers a notification event. - pub fn to_event_name(self) -> Option { - match self { - Self::CompleteMultipartUpload => Some(EventName::ObjectCreatedCompleteMultipartUpload), - Self::CopyObject => Some(EventName::ObjectCreatedCopy), - Self::CreateBucket => Some(EventName::BucketCreated), - Self::DeleteBucket => Some(EventName::BucketRemoved), - Self::DeleteObject => Some(EventName::ObjectRemovedDelete), - Self::DeleteObjects => Some(EventName::ObjectRemovedDeleteObjects), - Self::DeleteObjectTagging => Some(EventName::ObjectTaggingDelete), - Self::GetObject => Some(EventName::ObjectAccessedGet), - Self::GetObjectAttributes => Some(EventName::ObjectAccessedAttributes), - Self::GetObjectLegalHold => Some(EventName::ObjectAccessedGetLegalHold), - Self::GetObjectRetention => Some(EventName::ObjectAccessedGetRetention), - Self::HeadObject => Some(EventName::ObjectAccessedHead), - Self::PutObject => Some(EventName::ObjectCreatedPut), - Self::PutObjectAcl => Some(EventName::ObjectAclPut), - Self::PutObjectLegalHold => Some(EventName::ObjectCreatedPutLegalHold), - Self::PutObjectRetention => Some(EventName::ObjectCreatedPutRetention), - Self::PutObjectTagging => Some(EventName::ObjectTaggingPut), - Self::RestoreObject => Some(EventName::ObjectRestorePost), - Self::SelectObjectContent => Some(EventName::ObjectAccessedGet), - Self::AbortMultipartUpload => Some(EventName::ObjectRemovedAbortMultipartUpload), - Self::CreateMultipartUpload => Some(EventName::ObjectCreatedCreateMultipartUpload), - _ => None, - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -610,39 +436,6 @@ mod tests { assert!(deserialized.is_err(), "Deserialization should fail for empty string"); } - #[test] - fn test_s3_operation_to_event_name() { - assert_eq!(S3Operation::PutObject.to_event_name(), Some(EventName::ObjectCreatedPut)); - assert_eq!(S3Operation::PutObjectAcl.to_event_name(), Some(EventName::ObjectAclPut)); - assert_eq!(S3Operation::PutObjectTagging.to_event_name(), Some(EventName::ObjectTaggingPut)); - assert_eq!(S3Operation::DeleteObjectTagging.to_event_name(), Some(EventName::ObjectTaggingDelete)); - assert_eq!(S3Operation::GetObject.to_event_name(), Some(EventName::ObjectAccessedGet)); - assert_eq!(S3Operation::ListBuckets.to_event_name(), None); - assert_eq!(S3Operation::RestoreObject.to_event_name(), Some(EventName::ObjectRestorePost)); - assert_eq!(S3Operation::SelectObjectContent.to_event_name(), Some(EventName::ObjectAccessedGet)); - assert_eq!( - S3Operation::AbortMultipartUpload.to_event_name(), - Some(EventName::ObjectRemovedAbortMultipartUpload) - ); - } - - #[test] - fn test_event_name_to_s3_operation() { - assert_eq!(EventName::ObjectCreatedPut.to_s3_operation(), Some(S3Operation::PutObject)); - assert_eq!(EventName::ObjectAclPut.to_s3_operation(), Some(S3Operation::PutObjectAcl)); - assert_eq!(EventName::ObjectTaggingPut.to_s3_operation(), Some(S3Operation::PutObjectTagging)); - assert_eq!(EventName::ObjectTaggingDelete.to_s3_operation(), Some(S3Operation::DeleteObjectTagging)); - assert_eq!(EventName::ObjectAccessedGet.to_s3_operation(), Some(S3Operation::GetObject)); - assert_eq!(EventName::BucketCreated.to_s3_operation(), Some(S3Operation::CreateBucket)); - assert_eq!(EventName::Everything.to_s3_operation(), None); - assert_eq!(EventName::ObjectRestorePost.to_s3_operation(), Some(S3Operation::RestoreObject)); - assert_eq!(EventName::ObjectCreatedPost.to_s3_operation(), Some(S3Operation::PutObject)); - assert_eq!( - EventName::ObjectRemovedAbortMultipartUpload.to_s3_operation(), - Some(S3Operation::AbortMultipartUpload) - ); - } - #[test] fn test_event_name_aliases_parse_to_aws_compatible_variants() { assert_eq!(EventName::parse("s3:ObjectCreated:PutTagging").unwrap(), EventName::ObjectTaggingPut); @@ -673,4 +466,18 @@ mod tests { ] ); } + + #[test] + fn test_event_schema_version_mapping() { + assert_eq!(event_schema_version(EventName::ObjectCreatedPut), "2.1"); + assert_eq!(event_schema_version(EventName::ObjectReplicationFailed), "2.2"); + assert_eq!(event_schema_version(EventName::LifecycleTransition), "2.3"); + } + + #[test] + fn test_try_from_event_str_matches_parse() { + let parsed = EventName::try_from_event_str("s3:ObjectCreated:Put").unwrap(); + assert_eq!(parsed, EventName::ObjectCreatedPut); + assert!(EventName::try_from_event_str("s3:Invalid").is_err()); + } } diff --git a/crates/appauth/src/lib.rs b/crates/s3-types/src/lib.rs similarity index 86% rename from crates/appauth/src/lib.rs rename to crates/s3-types/src/lib.rs index a17997f48..36026a837 100644 --- a/crates/appauth/src/lib.rs +++ b/crates/s3-types/src/lib.rs @@ -12,4 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod token; +mod event_name; + +pub use event_name::{EventName, ParseEventNameError, event_schema_version}; diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index c8a87e79e..b699e4d7e 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -51,6 +51,7 @@ http = { workspace = true } rand = { workspace = true } s3s = { workspace = true } metrics = { workspace = true } +rustfs-data-usage = { workspace = true } [dev-dependencies] tracing-subscriber = { workspace = true } diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index d9addf4d9..bd01ca06d 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -12,14 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use path_clean::PathClean; use s3s::dto::BucketLifecycleConfiguration; use serde::{Deserialize, Serialize}; use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, future::Future, - hash::{DefaultHasher, Hash, Hasher}, - path::Path, sync::{Arc, LazyLock, Once}, time::SystemTime, }; @@ -27,6 +24,10 @@ use std::{ use http::HeaderMap; use metrics::{counter, describe_counter, describe_histogram, histogram}; use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; +pub use rustfs_data_usage::{ + BucketTargetUsageInfo, BucketUsageInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, hash_path, +}; +use rustfs_data_usage::{DataUsageCache as SharedDataUsageCache, DataUsageCacheInfo as SharedDataUsageCacheInfo}; use rustfs_ecstore::{ StorageAPI, bucket::{lifecycle::lifecycle::TRANSITION_COMPLETE, replication::ReplicationConfig}, @@ -133,90 +134,6 @@ impl AllTierStats { } } -/// Bucket target usage info provides replication statistics -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct BucketTargetUsageInfo { - pub replication_pending_size: u64, - pub replication_failed_size: u64, - pub replicated_size: u64, - pub replica_size: u64, - pub replication_pending_count: u64, - pub replication_failed_count: u64, - pub replicated_count: u64, -} - -/// Bucket usage info provides bucket-level statistics -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct BucketUsageInfo { - pub size: u64, - // Following five fields suffixed with V1 are here for backward compatibility - // Total Size for objects that have not yet been replicated - pub replication_pending_size_v1: u64, - // Total size for objects that have witness one or more failures and will be retried - pub replication_failed_size_v1: u64, - // Total size for objects that have been replicated to destination - pub replicated_size_v1: u64, - // Total number of objects pending replication - pub replication_pending_count_v1: u64, - // Total number of objects that failed replication - pub replication_failed_count_v1: u64, - - pub objects_count: u64, - pub object_size_histogram: HashMap, - pub object_versions_histogram: HashMap, - pub versions_count: u64, - pub delete_markers_count: u64, - pub replica_size: u64, - pub replica_count: u64, - pub replication_info: HashMap, -} - -/// DataUsageInfo represents data usage stats of the underlying storage -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct DataUsageInfo { - /// Total capacity - pub total_capacity: u64, - /// Total used capacity - pub total_used_capacity: u64, - /// Total free capacity - pub total_free_capacity: u64, - - /// LastUpdate is the timestamp of when the data usage info was last updated - pub last_update: Option, - - /// Objects total count across all buckets - pub objects_total_count: u64, - /// Versions total count across all buckets - pub versions_total_count: u64, - /// Delete markers total count across all buckets - pub delete_markers_total_count: u64, - /// Objects total size across all buckets - pub objects_total_size: u64, - /// Replication info across all buckets - pub replication_info: HashMap, - - /// Total number of buckets in this cluster - pub buckets_count: u64, - /// Buckets usage info provides following information across all buckets - pub buckets_usage: HashMap, - /// Deprecated kept here for backward compatibility reasons - pub bucket_sizes: HashMap, - /// Per-disk snapshot information when available - #[serde(default)] - pub disk_usage_status: Vec, -} - -/// Metadata describing the status of a disk-level data usage snapshot. -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct DiskUsageStatus { - pub disk_id: String, - pub pool_index: Option, - pub set_index: Option, - pub disk_index: Option, - pub last_update: Option, - pub snapshot_exists: bool, -} - /// Size summary for a single object or group of objects #[derive(Debug, Default, Clone)] pub struct SizeSummary { @@ -293,292 +210,6 @@ pub struct ReplTargetSizeSummary { // ===== Cache-related data structures ===== -/// Data usage hash for path-based caching -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct DataUsageHash(pub String); - -impl DataUsageHash { - pub fn string(&self) -> String { - self.0.clone() - } - - pub fn key(&self) -> String { - self.0.clone() - } - - pub fn mod_(&self, cycle: u32, cycles: u32) -> bool { - if cycles <= 1 { - return cycles == 1; - } - - let hash = self.calculate_hash(); - hash as u32 % cycles == cycle % cycles - } - - pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool { - if cycles <= 1 { - return cycles == 1; - } - - let hash = self.calculate_hash(); - - (hash >> 32) as u32 % cycles == cycle % cycles - } - - fn calculate_hash(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - self.0.hash(&mut hasher); - hasher.finish() - } -} - -/// Data usage hash map type -pub type DataUsageHashMap = HashSet; - -/// Size histogram for object size distribution -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SizeHistogram(Vec); - -impl Default for SizeHistogram { - fn default() -> Self { - Self(vec![0; 11]) // DATA_USAGE_BUCKET_LEN = 11 - } -} - -impl SizeHistogram { - pub fn add(&mut self, size: u64) { - let intervals = [ - (0, 1024), // LESS_THAN_1024_B - (1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB - (64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB - (256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB - (512 * 1024, 1024 * 1024 - 1), // BETWEEN_512_KB_AND_1_MB - (1024, 1024 * 1024 - 1), // BETWEEN_1024B_AND_1_MB - (1024 * 1024, 10 * 1024 * 1024 - 1), // BETWEEN_1_MB_AND_10_MB - (10 * 1024 * 1024, 64 * 1024 * 1024 - 1), // BETWEEN_10_MB_AND_64_MB - (64 * 1024 * 1024, 128 * 1024 * 1024 - 1), // BETWEEN_64_MB_AND_128_MB - (128 * 1024 * 1024, 512 * 1024 * 1024 - 1), // BETWEEN_128_MB_AND_512_MB - (512 * 1024 * 1024, u64::MAX), // GREATER_THAN_512_MB - ]; - - for (idx, (start, end)) in intervals.iter().enumerate() { - if size >= *start && size <= *end { - self.0[idx] += 1; - break; - } - } - } - - pub fn to_map(&self) -> HashMap { - let names = [ - "LESS_THAN_1024_B", - "BETWEEN_1024_B_AND_64_KB", - "BETWEEN_64_KB_AND_256_KB", - "BETWEEN_256_KB_AND_512_KB", - "BETWEEN_512_KB_AND_1_MB", - "BETWEEN_1024B_AND_1_MB", - "BETWEEN_1_MB_AND_10_MB", - "BETWEEN_10_MB_AND_64_MB", - "BETWEEN_64_MB_AND_128_MB", - "BETWEEN_128_MB_AND_512_MB", - "GREATER_THAN_512_MB", - ]; - - let mut res = HashMap::new(); - let mut spl_count = 0; - for (count, name) in self.0.iter().zip(names.iter()) { - if name == &"BETWEEN_1024B_AND_1_MB" { - res.insert(name.to_string(), spl_count); - } else if name.starts_with("BETWEEN_") && name.contains("_KB_") && name.contains("_MB") { - spl_count += count; - res.insert(name.to_string(), *count); - } else { - res.insert(name.to_string(), *count); - } - } - res - } -} - -/// Versions histogram for version count distribution -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct VersionsHistogram(Vec); - -impl Default for VersionsHistogram { - fn default() -> Self { - Self(vec![0; 7]) // DATA_USAGE_VERSION_LEN = 7 - } -} - -impl VersionsHistogram { - pub fn add(&mut self, count: u64) { - let intervals = [ - (0, 0), // UNVERSIONED - (1, 1), // SINGLE_VERSION - (2, 9), // BETWEEN_2_AND_10 - (10, 99), // BETWEEN_10_AND_100 - (100, 999), // BETWEEN_100_AND_1000 - (1000, 9999), // BETWEEN_1000_AND_10000 - (10000, u64::MAX), // GREATER_THAN_10000 - ]; - - for (idx, (start, end)) in intervals.iter().enumerate() { - if count >= *start && count <= *end { - self.0[idx] += 1; - break; - } - } - } - - pub fn to_map(&self) -> HashMap { - let names = [ - "UNVERSIONED", - "SINGLE_VERSION", - "BETWEEN_2_AND_10", - "BETWEEN_10_AND_100", - "BETWEEN_100_AND_1000", - "BETWEEN_1000_AND_10000", - "GREATER_THAN_10000", - ]; - - let mut res = HashMap::new(); - for (count, name) in self.0.iter().zip(names.iter()) { - res.insert(name.to_string(), *count); - } - res - } -} - -/// Replication statistics for a single target -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ReplicationStats { - pub pending_size: u64, - pub replicated_size: u64, - pub failed_size: u64, - pub failed_count: u64, - pub pending_count: u64, - pub missed_threshold_size: u64, - pub after_threshold_size: u64, - pub missed_threshold_count: u64, - pub after_threshold_count: u64, - pub replicated_count: u64, -} - -impl ReplicationStats { - pub fn empty(&self) -> bool { - self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0 - } -} - -/// Replication statistics for all targets -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct ReplicationAllStats { - pub targets: HashMap, - pub replica_size: u64, - pub replica_count: u64, -} - -impl ReplicationAllStats { - pub fn empty(&self) -> bool { - if self.replica_size != 0 && self.replica_count != 0 { - return false; - } - for (_, v) in self.targets.iter() { - if !v.empty() { - return false; - } - } - true - } -} - -/// Data usage cache entry -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct DataUsageEntry { - pub children: DataUsageHashMap, - // These fields do not include any children. - pub size: usize, - pub objects: usize, - pub versions: usize, - pub delete_markers: usize, - pub obj_sizes: SizeHistogram, - pub obj_versions: VersionsHistogram, - pub replication_stats: Option, - pub compacted: bool, - /// Number of objects that failed to scan (e.g., IO errors) - #[serde(default)] - pub failed_objects: usize, -} - -impl DataUsageEntry { - pub fn add_child(&mut self, hash: &DataUsageHash) { - if self.children.contains(&hash.key()) { - return; - } - self.children.insert(hash.key()); - } - - pub fn add_sizes(&mut self, summary: &SizeSummary) { - self.size += summary.total_size; - self.versions += summary.versions; - self.delete_markers += summary.delete_markers; - self.obj_sizes.add(summary.total_size as u64); - self.obj_versions.add(summary.versions as u64); - - let replication_stats = self.replication_stats.get_or_insert_with(ReplicationAllStats::default); - replication_stats.replica_size += summary.replica_size as u64; - replication_stats.replica_count += summary.replica_count as u64; - - for (arn, st) in &summary.repl_target_stats { - let tgt_stat = replication_stats.targets.entry(arn.to_string()).or_default(); - tgt_stat.pending_size += st.pending_size as u64; - tgt_stat.failed_size += st.failed_size as u64; - tgt_stat.replicated_size += st.replicated_size as u64; - tgt_stat.replicated_count += st.replicated_count as u64; - tgt_stat.failed_count += st.failed_count as u64; - tgt_stat.pending_count += st.pending_count as u64; - } - } - - pub fn merge(&mut self, other: &DataUsageEntry) { - self.objects += other.objects; - self.versions += other.versions; - self.delete_markers += other.delete_markers; - self.size += other.size; - self.failed_objects += other.failed_objects; - - if let Some(o_rep) = &other.replication_stats { - if self.replication_stats.is_none() { - self.replication_stats = Some(ReplicationAllStats::default()); - } - let s_rep = self.replication_stats.as_mut().unwrap(); - s_rep.targets.clear(); - s_rep.replica_size += o_rep.replica_size; - s_rep.replica_count += o_rep.replica_count; - for (arn, stat) in o_rep.targets.iter() { - let st = s_rep.targets.entry(arn.clone()).or_default(); - *st = ReplicationStats { - pending_size: stat.pending_size + st.pending_size, - failed_size: stat.failed_size + st.failed_size, - replicated_size: stat.replicated_size + st.replicated_size, - pending_count: stat.pending_count + st.pending_count, - failed_count: stat.failed_count + st.failed_count, - replicated_count: stat.replicated_count + st.replicated_count, - ..Default::default() - }; - } - } - - for (i, v) in other.obj_sizes.0.iter().enumerate() { - self.obj_sizes.0[i] += v; - } - - for (i, v) in other.obj_versions.0.iter().enumerate() { - self.obj_versions.0[i] += v; - } - } -} - #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct DataUsageEntryInfo { pub name: String, @@ -607,6 +238,28 @@ pub struct DataUsageCache { } impl DataUsageCache { + fn as_shared(&self) -> SharedDataUsageCache { + SharedDataUsageCache { + info: SharedDataUsageCacheInfo { + name: self.info.name.clone(), + next_cycle: self.info.next_cycle, + last_update: self.info.last_update, + skip_healing: self.info.skip_healing, + failed_objects: self.info.failed_objects.clone(), + }, + cache: self.cache.clone(), + } + } + + fn apply_shared_state(&mut self, shared: SharedDataUsageCache) { + self.info.name = shared.info.name; + self.info.next_cycle = shared.info.next_cycle; + self.info.last_update = shared.info.last_update; + self.info.skip_healing = shared.info.skip_healing; + self.info.failed_objects = shared.info.failed_objects; + self.cache = shared.cache; + } + fn ensure_cache_save_metrics_registered() { CACHE_SAVE_METRICS_ONCE.call_once(|| { describe_counter!( @@ -633,24 +286,15 @@ impl DataUsageCache { } pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) { - let hash = hash_path(path); - self.cache.insert(hash.key(), e); - if !parent.is_empty() { - let phash = hash_path(parent); - let p = { - let p = self.cache.entry(phash.key()).or_default(); - p.add_child(&hash); - p.clone() - }; - self.cache.insert(phash.key(), p); - } + let mut shared = self.as_shared(); + shared.replace(path, parent, e); + self.apply_shared_state(shared); } pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option, e: &DataUsageEntry) { - self.cache.insert(hash.key(), e.clone()); - if let Some(parent) = parent { - self.cache.entry(parent.key()).or_default().add_child(hash); - } + let mut shared = self.as_shared(); + shared.replace_hashed(hash, parent, e); + self.apply_shared_state(shared); } pub fn find(&self, path: &str) -> Option<&DataUsageEntry> { @@ -658,285 +302,73 @@ impl DataUsageCache { } pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap { - self.cache.entry(h.string()).or_default().children.clone() + let mut shared = self.as_shared(); + let children = shared.find_children_copy(h); + self.apply_shared_state(shared); + children } pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry { - let mut root = root.clone(); - for id in root.children.clone().iter() { - if let Some(e) = self.cache.get(id) { - let mut e = e.clone(); - if !e.children.is_empty() { - e = self.flatten(&e); - } - root.merge(&e); - } - } - root.children.clear(); - root + self.as_shared().flatten(root) } pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option) { - if let Some(e) = src.cache.get(&hash.string()) { - self.cache.insert(hash.key(), e.clone()); - for ch in e.children.iter() { - if *ch == hash.key() { - return; - } - self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone())); - } - if let Some(parent) = parent { - let p = self.cache.entry(parent.key()).or_default(); - p.add_child(hash); - } - } + let mut shared = self.as_shared(); + shared.copy_with_children(&src.as_shared(), hash, parent); + self.apply_shared_state(shared); } pub fn delete_recursive(&mut self, hash: &DataUsageHash) { - let mut need_remove = Vec::new(); - if let Some(v) = self.cache.get(&hash.string()) { - for child in v.children.iter() { - need_remove.push(child.clone()); - } - } - self.cache.remove(&hash.string()); - need_remove.iter().for_each(|child| { - self.delete_recursive(&DataUsageHash(child.to_string())); - }); + let mut shared = self.as_shared(); + shared.delete_recursive(hash); + self.apply_shared_state(shared); } pub fn size_recursive(&self, path: &str) -> Option { - match self.find(path) { - Some(root) => { - if root.children.is_empty() { - return Some(root.clone()); - } - let mut flat = self.flatten(root); - if flat.replication_stats.as_ref().is_some_and(|rs| rs.empty()) { - flat.replication_stats = None; - } - Some(flat) - } - None => None, - } + self.as_shared().size_recursive(path) } pub fn search_parent(&self, hash: &DataUsageHash) -> Option { - let want = hash.key(); - if let Some(last_index) = want.rfind('/') - && let Some(v) = self.find(&want[0..last_index]) - && v.children.contains(&want) - { - let found = hash_path(&want[0..last_index]); - return Some(found); - } - - for (k, v) in self.cache.iter() { - if v.children.contains(&want) { - let found = DataUsageHash(k.clone()); - return Some(found); - } - } - None + self.as_shared().search_parent(hash) } pub fn is_compacted(&self, hash: &DataUsageHash) -> bool { - match self.cache.get(&hash.key()) { - Some(due) => due.compacted, - None => false, - } + self.as_shared().is_compacted(hash) } pub fn force_compact(&mut self, limit: usize) { - if self.cache.len() < limit { - return; - } - let top = hash_path(&self.info.name).key(); - let top_e = match self.find(&top) { - Some(e) => e.clone(), - None => return, - }; - // Note: DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS constant would need to be passed as parameter - // or defined in common crate if needed - if top_e.children.len() > 250_000 { - // DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS - self.reduce_children_of(&hash_path(&self.info.name), limit, true); - } - if self.cache.len() <= limit { - return; - } - - let mut found = HashSet::new(); - found.insert(top); - mark(self, &top_e, &mut found); - self.cache.retain(|k, _| { - if !found.contains(k) { - return false; - } - true - }); + let mut shared = self.as_shared(); + shared.force_compact(limit); + self.apply_shared_state(shared); } pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) { - let e = match self.cache.get(&path.key()) { - Some(e) => e, - None => return, - }; - - if e.compacted { - return; - } - - if e.children.len() > limit && compact_self { - let mut flat = self.size_recursive(&path.key()).unwrap_or_default(); - flat.compacted = true; - self.delete_recursive(path); - self.replace_hashed(path, &None, &flat); - return; - } - let total = self.total_children_rec(&path.key()); - if total < limit { - return; - } - - let mut leaves = Vec::new(); - let mut remove = total - limit; - add(self, path, &mut leaves); - leaves.sort_by_key(|a| a.objects); - - while remove > 0 && !leaves.is_empty() { - let e = leaves.first().unwrap(); - let candidate = e.path.clone(); - if candidate == *path && !compact_self { - break; - } - let removing = self.total_children_rec(&candidate.key()); - let mut flat = match self.size_recursive(&candidate.key()) { - Some(flat) => flat, - None => { - leaves.remove(0); - continue; - } - }; - - flat.compacted = true; - self.delete_recursive(&candidate); - self.replace_hashed(&candidate, &None, &flat); - - remove -= removing; - leaves.remove(0); - } + let mut shared = self.as_shared(); + shared.reduce_children_of(path, limit, compact_self); + self.apply_shared_state(shared); } pub fn total_children_rec(&self, path: &str) -> usize { - let Some(root) = self.find(path) else { - return 0; - }; - if root.children.is_empty() { - return 0; - } - - let mut n = root.children.len(); - for ch in root.children.iter() { - n += self.total_children_rec(ch); - } - n + self.as_shared().total_children_rec(path) } pub fn merge(&mut self, o: &DataUsageCache) { - let mut existing_root = self.root(); - let other_root = o.root(); - if existing_root.is_none() && other_root.is_none() { - return; - } - if other_root.is_none() { - return; - } - if existing_root.is_none() { - *self = o.clone(); - return; - } - if o.info.last_update.gt(&self.info.last_update) { - self.info.last_update = o.info.last_update; - } - - existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap()); - self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap()); - let e_hash = self.root_hash(); - for key in other_root.as_ref().unwrap().children.iter() { - let entry = &o.cache[key]; - let flat = o.flatten(entry); - let mut existing = self.cache[key].clone(); - existing.merge(&flat); - self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing); - } + let mut shared = self.as_shared(); + shared.merge(&o.as_shared()); + self.apply_shared_state(shared); } pub fn root_hash(&self) -> DataUsageHash { - hash_path(&self.info.name) + self.as_shared().root_hash() } pub fn root(&self) -> Option { - self.find(&self.info.name).cloned() + self.as_shared().root() } /// Convert cache to DataUsageInfo for a specific path pub fn dui(&self, path: &str, buckets: &[String]) -> DataUsageInfo { - let e = match self.find(path) { - Some(e) => e, - None => return DataUsageInfo::default(), - }; - let flat = self.flatten(e); - - let mut buckets_usage = HashMap::new(); - for bucket_name in buckets.iter() { - let e = match self.find(bucket_name) { - Some(e) => e, - None => continue, - }; - let flat = self.flatten(e); - let mut bui = BucketUsageInfo { - size: flat.size as u64, - versions_count: flat.versions as u64, - objects_count: flat.objects as u64, - delete_markers_count: flat.delete_markers as u64, - object_size_histogram: flat.obj_sizes.to_map(), - object_versions_histogram: flat.obj_versions.to_map(), - ..Default::default() - }; - - if let Some(rs) = &flat.replication_stats { - bui.replica_size = rs.replica_size; - bui.replica_count = rs.replica_count; - - for (arn, stat) in rs.targets.iter() { - bui.replication_info.insert( - arn.clone(), - BucketTargetUsageInfo { - replication_pending_size: stat.pending_size, - replicated_size: stat.replicated_size, - replication_failed_size: stat.failed_size, - replication_pending_count: stat.pending_count, - replication_failed_count: stat.failed_count, - replicated_count: stat.replicated_count, - ..Default::default() - }, - ); - } - } - buckets_usage.insert(bucket_name.clone(), bui); - } - - DataUsageInfo { - last_update: self.info.last_update, - objects_total_count: flat.objects as u64, - versions_total_count: flat.versions as u64, - delete_markers_total_count: flat.delete_markers as u64, - objects_total_size: flat.size as u64, - buckets_count: e.children.len() as u64, - buckets_usage, - ..Default::default() - } + self.as_shared().dui(path, buckets) } pub fn marshal_msg(&self) -> Result, Box> { @@ -1252,366 +684,6 @@ pub trait DataUsageCacheStorage { async fn save(&self, name: &str) -> Result<(), Box>; } -// Helper structs and functions for cache operations -#[derive(Default, Clone)] -struct Inner { - objects: usize, - path: DataUsageHash, -} - -fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec) { - let e = match data_usage_cache.cache.get(&path.key()) { - Some(e) => e, - None => return, - }; - if !e.children.is_empty() { - return; - } - - let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default(); - leaves.push(Inner { - objects: sz.objects, - path: path.clone(), - }); - for ch in e.children.iter() { - add(data_usage_cache, &DataUsageHash(ch.clone()), leaves); - } -} - -fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet) { - for k in entry.children.iter() { - found.insert(k.to_string()); - if let Some(ch) = duc.cache.get(k) { - mark(duc, ch, found); - } - } -} - -/// Hash a path for data usage caching -pub fn hash_path(data: &str) -> DataUsageHash { - DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string()) -} - -impl DataUsageInfo { - /// Create a new DataUsageInfo - pub fn new() -> Self { - Self::default() - } - - /// Add object metadata to data usage statistics - pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) { - // This method is kept for backward compatibility - // For accurate version counting, use add_object_from_file_meta instead - let bucket_name = match self.extract_bucket_from_path(object_path) { - Ok(name) => name, - Err(_) => return, - }; - - // Update bucket statistics - if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) { - bucket_usage.size += meta_object.size as u64; - bucket_usage.objects_count += 1; - bucket_usage.versions_count += 1; // Simplified: assume 1 version per object - - // Update size histogram - let total_size = meta_object.size as u64; - let size_ranges = [ - ("0-1KB", 0, 1024), - ("1KB-1MB", 1024, 1024 * 1024), - ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), - ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), - ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), - ("1GB+", 1024 * 1024 * 1024, u64::MAX), - ]; - - for (range_name, min_size, max_size) in size_ranges { - if total_size >= min_size && total_size < max_size { - *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1; - break; - } - } - - // Update version histogram (simplified - count as single version) - *bucket_usage - .object_versions_histogram - .entry("SINGLE_VERSION".to_string()) - .or_insert(0) += 1; - } else { - // Create new bucket usage - let mut bucket_usage = BucketUsageInfo { - size: meta_object.size as u64, - objects_count: 1, - versions_count: 1, - ..Default::default() - }; - bucket_usage.object_size_histogram.insert("0-1KB".to_string(), 1); - bucket_usage.object_versions_histogram.insert("SINGLE_VERSION".to_string(), 1); - self.buckets_usage.insert(bucket_name, bucket_usage); - } - - // Update global statistics - self.objects_total_size += meta_object.size as u64; - self.objects_total_count += 1; - self.versions_total_count += 1; - } - - /// Add object from FileMeta for accurate version counting - pub fn add_object_from_file_meta(&mut self, object_path: &str, file_meta: &rustfs_filemeta::FileMeta) { - let bucket_name = match self.extract_bucket_from_path(object_path) { - Ok(name) => name, - Err(_) => return, - }; - - // Calculate accurate statistics from all versions - let mut total_size = 0u64; - let mut versions_count = 0u64; - let mut delete_markers_count = 0u64; - let mut latest_object_size = 0u64; - - // Process all versions to get accurate counts - for version in &file_meta.versions { - match rustfs_filemeta::FileMetaVersion::try_from(version.clone()) { - Ok(ver) => { - if let Some(obj) = ver.object { - total_size += obj.size as u64; - versions_count += 1; - latest_object_size = obj.size as u64; // Keep track of latest object size - } else if ver.delete_marker.is_some() { - delete_markers_count += 1; - } - } - Err(_) => { - // Skip invalid versions - continue; - } - } - } - - // Update bucket statistics - if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) { - bucket_usage.size += total_size; - bucket_usage.objects_count += 1; - bucket_usage.versions_count += versions_count; - bucket_usage.delete_markers_count += delete_markers_count; - - // Update size histogram based on latest object size - let size_ranges = [ - ("0-1KB", 0, 1024), - ("1KB-1MB", 1024, 1024 * 1024), - ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), - ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), - ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), - ("1GB+", 1024 * 1024 * 1024, u64::MAX), - ]; - - for (range_name, min_size, max_size) in size_ranges { - if latest_object_size >= min_size && latest_object_size < max_size { - *bucket_usage.object_size_histogram.entry(range_name.to_string()).or_insert(0) += 1; - break; - } - } - - // Update version histogram based on actual version count - let version_ranges = [ - ("1", 1, 1), - ("2-5", 2, 5), - ("6-10", 6, 10), - ("11-50", 11, 50), - ("51-100", 51, 100), - ("100+", 101, usize::MAX), - ]; - - for (range_name, min_versions, max_versions) in version_ranges { - if versions_count as usize >= min_versions && versions_count as usize <= max_versions { - *bucket_usage - .object_versions_histogram - .entry(range_name.to_string()) - .or_insert(0) += 1; - break; - } - } - } else { - // Create new bucket usage - let mut bucket_usage = BucketUsageInfo { - size: total_size, - objects_count: 1, - versions_count, - delete_markers_count, - ..Default::default() - }; - - // Set size histogram - let size_ranges = [ - ("0-1KB", 0, 1024), - ("1KB-1MB", 1024, 1024 * 1024), - ("1MB-10MB", 1024 * 1024, 10 * 1024 * 1024), - ("10MB-100MB", 10 * 1024 * 1024, 100 * 1024 * 1024), - ("100MB-1GB", 100 * 1024 * 1024, 1024 * 1024 * 1024), - ("1GB+", 1024 * 1024 * 1024, u64::MAX), - ]; - - for (range_name, min_size, max_size) in size_ranges { - if latest_object_size >= min_size && latest_object_size < max_size { - bucket_usage.object_size_histogram.insert(range_name.to_string(), 1); - break; - } - } - - // Set version histogram - let version_ranges = [ - ("1", 1, 1), - ("2-5", 2, 5), - ("6-10", 6, 10), - ("11-50", 11, 50), - ("51-100", 51, 100), - ("100+", 101, usize::MAX), - ]; - - for (range_name, min_versions, max_versions) in version_ranges { - if versions_count as usize >= min_versions && versions_count as usize <= max_versions { - bucket_usage.object_versions_histogram.insert(range_name.to_string(), 1); - break; - } - } - - self.buckets_usage.insert(bucket_name, bucket_usage); - // Update buckets count when adding new bucket - self.buckets_count = self.buckets_usage.len() as u64; - } - - // Update global statistics - self.objects_total_size += total_size; - self.objects_total_count += 1; - self.versions_total_count += versions_count; - self.delete_markers_total_count += delete_markers_count; - } - - /// Extract bucket name from object path - pub fn extract_bucket_from_path(&self, object_path: &str) -> Result> { - let parts: Vec<&str> = object_path.split('/').collect(); - if parts.is_empty() { - return Err("Invalid object path: empty".into()); - } - Ok(parts[0].to_string()) - } - - /// Update capacity information - pub fn update_capacity(&mut self, total: u64, used: u64, free: u64) { - self.total_capacity = total; - self.total_used_capacity = used; - self.total_free_capacity = free; - self.last_update = Some(SystemTime::now()); - } - - /// Add bucket usage info - pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) { - self.buckets_usage.insert(bucket, usage); - self.buckets_count = self.buckets_usage.len() as u64; - self.last_update = Some(SystemTime::now()); - } - - /// Get bucket usage info - pub fn get_bucket_usage(&self, bucket: &str) -> Option<&BucketUsageInfo> { - self.buckets_usage.get(bucket) - } - - /// Calculate total statistics from all buckets - pub fn calculate_totals(&mut self) { - self.objects_total_count = 0; - self.versions_total_count = 0; - self.delete_markers_total_count = 0; - self.objects_total_size = 0; - - for usage in self.buckets_usage.values() { - self.objects_total_count += usage.objects_count; - self.versions_total_count += usage.versions_count; - self.delete_markers_total_count += usage.delete_markers_count; - self.objects_total_size += usage.size; - } - } - - /// Merge another DataUsageInfo into this one - pub fn merge(&mut self, other: &DataUsageInfo) { - // Merge bucket usage - for (bucket, usage) in &other.buckets_usage { - if let Some(existing) = self.buckets_usage.get_mut(bucket) { - existing.merge(usage); - } else { - self.buckets_usage.insert(bucket.clone(), usage.clone()); - } - } - - self.disk_usage_status.extend(other.disk_usage_status.iter().cloned()); - - // Recalculate totals - self.calculate_totals(); - - // Ensure buckets_count stays consistent with buckets_usage - self.buckets_count = self.buckets_usage.len() as u64; - - // Update last update time - if let Some(other_update) = other.last_update - && (self.last_update.is_none() || other_update > self.last_update.unwrap()) - { - self.last_update = Some(other_update); - } - } -} - -impl BucketUsageInfo { - /// Create a new BucketUsageInfo - pub fn new() -> Self { - Self::default() - } - - /// Add size summary to this bucket usage - pub fn add_size_summary(&mut self, summary: &SizeSummary) { - self.size += summary.total_size as u64; - self.versions_count += summary.versions as u64; - self.delete_markers_count += summary.delete_markers as u64; - self.replica_size += summary.replica_size as u64; - self.replica_count += summary.replica_count as u64; - } - - /// Merge another BucketUsageInfo into this one - pub fn merge(&mut self, other: &BucketUsageInfo) { - self.size += other.size; - self.objects_count += other.objects_count; - self.versions_count += other.versions_count; - self.delete_markers_count += other.delete_markers_count; - self.replica_size += other.replica_size; - self.replica_count += other.replica_count; - - // Merge histograms - for (key, value) in &other.object_size_histogram { - *self.object_size_histogram.entry(key.clone()).or_insert(0) += value; - } - - for (key, value) in &other.object_versions_histogram { - *self.object_versions_histogram.entry(key.clone()).or_insert(0) += value; - } - - // Merge replication info - for (target, info) in &other.replication_info { - let entry = self.replication_info.entry(target.clone()).or_default(); - entry.replicated_size += info.replicated_size; - entry.replica_size += info.replica_size; - entry.replication_pending_size += info.replication_pending_size; - entry.replication_failed_size += info.replication_failed_size; - entry.replication_pending_count += info.replication_pending_count; - entry.replication_failed_count += info.replication_failed_count; - entry.replicated_count += info.replicated_count; - } - - // Merge backward compatibility fields - self.replication_pending_size_v1 += other.replication_pending_size_v1; - self.replication_failed_size_v1 += other.replication_failed_size_v1; - self.replicated_size_v1 += other.replicated_size_v1; - self.replication_pending_count_v1 += other.replication_pending_count_v1; - self.replication_failed_count_v1 += other.replication_failed_count_v1; - } -} - impl SizeSummary { /// Create a new SizeSummary pub fn new() -> Self { diff --git a/crates/scanner/src/last_minute.rs b/crates/scanner/src/last_minute.rs deleted file mode 100644 index b4a776f90..000000000 --- a/crates/scanner/src/last_minute.rs +++ /dev/null @@ -1,886 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -#[allow(dead_code)] -#[derive(Debug, Default)] -pub struct TimedAction { - count: u64, - acc_time: u64, - min_time: Option, - max_time: Option, - bytes: u64, -} - -#[allow(dead_code)] -impl TimedAction { - // Avg returns the average time spent on the action. - pub fn avg(&self) -> Option { - if self.count == 0 { - return None; - } - Some(Duration::from_nanos(self.acc_time / self.count)) - } - - // AvgBytes returns the average bytes processed. - pub fn avg_bytes(&self) -> u64 { - if self.count == 0 { - return 0; - } - self.bytes / self.count - } - - // Merge other into t. - pub fn merge(&mut self, other: TimedAction) { - self.count += other.count; - self.acc_time += other.acc_time; - self.bytes += other.bytes; - - if self.count == 0 { - self.min_time = other.min_time; - } - if let Some(other_min) = other.min_time { - self.min_time = self.min_time.map_or(Some(other_min), |min| Some(min.min(other_min))); - } - - self.max_time = self - .max_time - .map_or(other.max_time, |max| Some(max.max(other.max_time.unwrap_or(0)))); - } -} - -#[allow(dead_code)] -#[derive(Debug)] -enum SizeCategory { - SizeLessThan1KiB = 0, - SizeLessThan1MiB, - SizeLessThan10MiB, - SizeLessThan100MiB, - SizeLessThan1GiB, - SizeGreaterThan1GiB, - // Add new entries here - SizeLastElemMarker, -} - -impl std::fmt::Display for SizeCategory { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match *self { - SizeCategory::SizeLessThan1KiB => "SizeLessThan1KiB", - SizeCategory::SizeLessThan1MiB => "SizeLessThan1MiB", - SizeCategory::SizeLessThan10MiB => "SizeLessThan10MiB", - SizeCategory::SizeLessThan100MiB => "SizeLessThan100MiB", - SizeCategory::SizeLessThan1GiB => "SizeLessThan1GiB", - SizeCategory::SizeGreaterThan1GiB => "SizeGreaterThan1GiB", - SizeCategory::SizeLastElemMarker => "SizeLastElemMarker", - }; - write!(f, "{s}") - } -} - -#[derive(Clone, Debug, Default, Copy)] -pub struct AccElem { - pub total: u64, - pub size: u64, - pub n: u64, -} - -impl AccElem { - pub fn add(&mut self, dur: &Duration) { - let dur = dur.as_secs(); - self.total = self.total.wrapping_add(dur); - self.n = self.n.wrapping_add(1); - } - - pub fn merge(&mut self, b: &AccElem) { - self.n = self.n.wrapping_add(b.n); - self.total = self.total.wrapping_add(b.total); - self.size = self.size.wrapping_add(b.size); - } - - pub fn avg(&self) -> Duration { - if self.n >= 1 && self.total > 0 { - return Duration::from_secs(self.total / self.n); - } - Duration::from_secs(0) - } -} - -#[derive(Clone, Debug)] -pub struct LastMinuteLatency { - pub totals: Vec, - pub last_sec: u64, -} - -impl Default for LastMinuteLatency { - fn default() -> Self { - Self { - totals: vec![AccElem::default(); 60], - last_sec: Default::default(), - } - } -} - -impl LastMinuteLatency { - pub fn merge(&mut self, o: &LastMinuteLatency) -> LastMinuteLatency { - let mut merged = LastMinuteLatency::default(); - let mut x = o.clone(); - if self.last_sec > o.last_sec { - x.forward_to(self.last_sec); - merged.last_sec = self.last_sec; - } else { - self.forward_to(o.last_sec); - merged.last_sec = o.last_sec; - } - - for i in 0..merged.totals.len() { - merged.totals[i] = AccElem { - total: self.totals[i].total + o.totals[i].total, - n: self.totals[i].n + o.totals[i].n, - size: self.totals[i].size + o.totals[i].size, - } - } - merged - } - - pub fn add(&mut self, t: &Duration) { - let sec = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - self.forward_to(sec); - let win_idx = sec % 60; - self.totals[win_idx as usize].add(t); - self.last_sec = sec; - } - - pub fn add_all(&mut self, sec: u64, a: &AccElem) { - self.forward_to(sec); - let win_idx = sec % 60; - self.totals[win_idx as usize].merge(a); - self.last_sec = sec; - } - - pub fn get_total(&mut self) -> AccElem { - let mut res = AccElem::default(); - let sec = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - self.forward_to(sec); - for elem in self.totals.iter() { - res.merge(elem); - } - res - } - - pub fn forward_to(&mut self, t: u64) { - if self.last_sec >= t { - return; - } - if t - self.last_sec >= 60 { - self.totals = vec![AccElem::default(); 60]; - self.last_sec = t; - return; - } - while self.last_sec != t { - let idx = (self.last_sec + 1) % 60; - self.totals[idx as usize] = AccElem::default(); - self.last_sec += 1; - } - } -} -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - #[test] - fn test_acc_elem_default() { - let elem = AccElem::default(); - assert_eq!(elem.total, 0); - assert_eq!(elem.size, 0); - assert_eq!(elem.n, 0); - } - - #[test] - fn test_acc_elem_add_single_duration() { - let mut elem = AccElem::default(); - let duration = Duration::from_secs(5); - - elem.add(&duration); - - assert_eq!(elem.total, 5); - assert_eq!(elem.n, 1); - assert_eq!(elem.size, 0); // size is not modified by add - } - - #[test] - fn test_acc_elem_add_multiple_durations() { - let mut elem = AccElem::default(); - - elem.add(&Duration::from_secs(3)); - elem.add(&Duration::from_secs(7)); - elem.add(&Duration::from_secs(2)); - - assert_eq!(elem.total, 12); - assert_eq!(elem.n, 3); - assert_eq!(elem.size, 0); - } - - #[test] - fn test_acc_elem_add_zero_duration() { - let mut elem = AccElem::default(); - let duration = Duration::from_secs(0); - - elem.add(&duration); - - assert_eq!(elem.total, 0); - assert_eq!(elem.n, 1); - } - - #[test] - fn test_acc_elem_add_subsecond_duration() { - let mut elem = AccElem::default(); - // Duration less than 1 second should be truncated to 0 - let duration = Duration::from_millis(500); - - elem.add(&duration); - - assert_eq!(elem.total, 0); // as_secs() truncates subsecond values - assert_eq!(elem.n, 1); - } - - #[test] - fn test_acc_elem_merge_empty_elements() { - let mut elem1 = AccElem::default(); - let elem2 = AccElem::default(); - - elem1.merge(&elem2); - - assert_eq!(elem1.total, 0); - assert_eq!(elem1.size, 0); - assert_eq!(elem1.n, 0); - } - - #[test] - fn test_acc_elem_merge_with_data() { - let mut elem1 = AccElem { - total: 10, - size: 100, - n: 2, - }; - let elem2 = AccElem { - total: 15, - size: 200, - n: 3, - }; - - elem1.merge(&elem2); - - assert_eq!(elem1.total, 25); - assert_eq!(elem1.size, 300); - assert_eq!(elem1.n, 5); - } - - #[test] - fn test_acc_elem_merge_one_empty() { - let mut elem1 = AccElem { - total: 10, - size: 100, - n: 2, - }; - let elem2 = AccElem::default(); - - elem1.merge(&elem2); - - assert_eq!(elem1.total, 10); - assert_eq!(elem1.size, 100); - assert_eq!(elem1.n, 2); - } - - #[test] - fn test_acc_elem_avg_with_data() { - let elem = AccElem { - total: 15, - size: 0, - n: 3, - }; - - let avg = elem.avg(); - assert_eq!(avg, Duration::from_secs(5)); // 15 / 3 = 5 - } - - #[test] - fn test_acc_elem_avg_zero_count() { - let elem = AccElem { - total: 10, - size: 0, - n: 0, - }; - - let avg = elem.avg(); - assert_eq!(avg, Duration::from_secs(0)); - } - - #[test] - fn test_acc_elem_avg_zero_total() { - let elem = AccElem { total: 0, size: 0, n: 5 }; - - let avg = elem.avg(); - assert_eq!(avg, Duration::from_secs(0)); - } - - #[test] - fn test_acc_elem_avg_rounding() { - let elem = AccElem { - total: 10, - size: 0, - n: 3, - }; - - let avg = elem.avg(); - assert_eq!(avg, Duration::from_secs(3)); // 10 / 3 = 3 (integer division) - } - - #[test] - fn test_last_minute_latency_default() { - let latency = LastMinuteLatency::default(); - - assert_eq!(latency.totals.len(), 60); - assert_eq!(latency.last_sec, 0); - - // All elements should be default (empty) - for elem in &latency.totals { - assert_eq!(elem.total, 0); - assert_eq!(elem.size, 0); - assert_eq!(elem.n, 0); - } - } - - #[test] - fn test_last_minute_latency_forward_to_same_time() { - let mut latency = LastMinuteLatency { - last_sec: 100, - ..Default::default() - }; - - // Add some data to verify it's not cleared - latency.totals[0].total = 10; - latency.totals[0].n = 1; - - latency.forward_to(100); // Same time - - assert_eq!(latency.last_sec, 100); - assert_eq!(latency.totals[0].total, 10); // Data should remain - assert_eq!(latency.totals[0].n, 1); - } - - #[test] - fn test_last_minute_latency_forward_to_past_time() { - let mut latency = LastMinuteLatency { - last_sec: 100, - ..Default::default() - }; - - // Add some data to verify it's not cleared - latency.totals[0].total = 10; - latency.totals[0].n = 1; - - latency.forward_to(50); // Past time - - assert_eq!(latency.last_sec, 100); // Should not change - assert_eq!(latency.totals[0].total, 10); // Data should remain - assert_eq!(latency.totals[0].n, 1); - } - - #[test] - fn test_last_minute_latency_forward_to_large_gap() { - let mut latency = LastMinuteLatency { - last_sec: 100, - ..Default::default() - }; - - // Add some data to verify it's cleared - latency.totals[0].total = 10; - latency.totals[0].n = 1; - - latency.forward_to(200); // Gap >= 60 seconds - - assert_eq!(latency.last_sec, 200); // last_sec should be updated to target time - - // All data should be cleared - for elem in &latency.totals { - assert_eq!(elem.total, 0); - assert_eq!(elem.size, 0); - assert_eq!(elem.n, 0); - } - } - - #[test] - fn test_last_minute_latency_forward_to_small_gap() { - let mut latency = LastMinuteLatency { - last_sec: 100, - ..Default::default() - }; - - // Add data at specific indices - latency.totals[41].total = 10; // (100 + 1) % 60 = 41 - latency.totals[42].total = 20; // (100 + 2) % 60 = 42 - - latency.forward_to(102); // Forward by 2 seconds - - assert_eq!(latency.last_sec, 102); - - // The slots that were advanced should be cleared - assert_eq!(latency.totals[41].total, 0); // Cleared during forward - assert_eq!(latency.totals[42].total, 0); // Cleared during forward - } - - #[test] - fn test_last_minute_latency_add_all() { - let mut latency = LastMinuteLatency::default(); - let acc_elem = AccElem { - total: 15, - size: 100, - n: 3, - }; - - latency.add_all(1000, &acc_elem); - - assert_eq!(latency.last_sec, 1000); - let idx = 1000 % 60; // Should be 40 - assert_eq!(latency.totals[idx as usize].total, 15); - assert_eq!(latency.totals[idx as usize].size, 100); - assert_eq!(latency.totals[idx as usize].n, 3); - } - - #[test] - fn test_last_minute_latency_add_all_multiple() { - let mut latency = LastMinuteLatency::default(); - - let acc_elem1 = AccElem { - total: 10, - size: 50, - n: 2, - }; - let acc_elem2 = AccElem { - total: 20, - size: 100, - n: 4, - }; - - latency.add_all(1000, &acc_elem1); - latency.add_all(1000, &acc_elem2); // Same second - - let idx = 1000 % 60; - assert_eq!(latency.totals[idx as usize].total, 30); // 10 + 20 - assert_eq!(latency.totals[idx as usize].size, 150); // 50 + 100 - assert_eq!(latency.totals[idx as usize].n, 6); // 2 + 4 - } - - #[test] - fn test_last_minute_latency_merge_same_time() { - let mut latency1 = LastMinuteLatency::default(); - let mut latency2 = LastMinuteLatency::default(); - - latency1.last_sec = 1000; - latency2.last_sec = 1000; - - // Add data to both - latency1.totals[0].total = 10; - latency1.totals[0].n = 2; - latency2.totals[0].total = 20; - latency2.totals[0].n = 3; - - let merged = latency1.merge(&latency2); - - assert_eq!(merged.last_sec, 1000); - assert_eq!(merged.totals[0].total, 30); // 10 + 20 - assert_eq!(merged.totals[0].n, 5); // 2 + 3 - } - - #[test] - fn test_last_minute_latency_merge_different_times() { - let mut latency1 = LastMinuteLatency::default(); - let mut latency2 = LastMinuteLatency::default(); - - latency1.last_sec = 1000; - latency2.last_sec = 1010; // 10 seconds later - - // Add data to both - latency1.totals[0].total = 10; - latency2.totals[0].total = 20; - - let merged = latency1.merge(&latency2); - - assert_eq!(merged.last_sec, 1010); // Should use the later time - assert_eq!(merged.totals[0].total, 30); - } - - #[test] - fn test_last_minute_latency_merge_empty() { - let mut latency1 = LastMinuteLatency::default(); - let latency2 = LastMinuteLatency::default(); - - let merged = latency1.merge(&latency2); - - assert_eq!(merged.last_sec, 0); - for elem in &merged.totals { - assert_eq!(elem.total, 0); - assert_eq!(elem.size, 0); - assert_eq!(elem.n, 0); - } - } - - #[test] - fn test_last_minute_latency_window_wraparound() { - let mut latency = LastMinuteLatency::default(); - - // Test that indices wrap around correctly - for sec in 0..120 { - // Test for 2 minutes - let acc_elem = AccElem { - total: sec, - size: 0, - n: 1, - }; - latency.add_all(sec, &acc_elem); - - let expected_idx = sec % 60; - assert_eq!(latency.totals[expected_idx as usize].total, sec); - } - } - - #[test] - fn test_last_minute_latency_time_progression() { - let mut latency = LastMinuteLatency::default(); - - // Add data at time 1000 - latency.add_all( - 1000, - &AccElem { - total: 10, - size: 0, - n: 1, - }, - ); - - // Forward to time 1030 (30 seconds later) - latency.forward_to(1030); - - // Original data should still be there - let idx_1000 = 1000 % 60; - assert_eq!(latency.totals[idx_1000 as usize].total, 10); - - // Forward to time 1070 (70 seconds from original, > 60 seconds) - latency.forward_to(1070); - - // All data should be cleared due to large gap - for elem in &latency.totals { - assert_eq!(elem.total, 0); - assert_eq!(elem.n, 0); - } - } - - #[test] - fn test_last_minute_latency_realistic_scenario() { - let mut latency = LastMinuteLatency::default(); - let base_time = 1000u64; - - // Add data for exactly 60 seconds to fill the window - for i in 0..60 { - let current_time = base_time + i; - let duration_secs = i % 10 + 1; // Varying durations 1-10 seconds - let acc_elem = AccElem { - total: duration_secs, - size: 1024 * (i % 5 + 1), // Varying sizes - n: 1, - }; - - latency.add_all(current_time, &acc_elem); - } - - // Count non-empty slots after filling the window - let mut non_empty_count = 0; - let mut total_n = 0; - let mut total_sum = 0; - - for elem in &latency.totals { - if elem.n > 0 { - non_empty_count += 1; - total_n += elem.n; - total_sum += elem.total; - } - } - - // We should have exactly 60 non-empty slots (one for each second in the window) - assert_eq!(non_empty_count, 60); - assert_eq!(total_n, 60); // 60 data points total - assert!(total_sum > 0); - - // Test manual total calculation (get_total uses system time which interferes with test) - let mut manual_total = AccElem::default(); - for elem in &latency.totals { - manual_total.merge(elem); - } - assert_eq!(manual_total.n, 60); - assert_eq!(manual_total.total, total_sum); - } - - #[test] - fn test_acc_elem_clone_and_debug() { - let elem = AccElem { - total: 100, - size: 200, - n: 5, - }; - - let cloned = elem; - assert_eq!(elem.total, cloned.total); - assert_eq!(elem.size, cloned.size); - assert_eq!(elem.n, cloned.n); - - // Test Debug trait - let debug_str = format!("{elem:?}"); - assert!(debug_str.contains("100")); - assert!(debug_str.contains("200")); - assert!(debug_str.contains("5")); - } - - #[test] - fn test_last_minute_latency_clone() { - let mut latency = LastMinuteLatency { - last_sec: 1000, - ..Default::default() - }; - latency.totals[0].total = 100; - latency.totals[0].n = 5; - - let cloned = latency.clone(); - assert_eq!(latency.last_sec, cloned.last_sec); - assert_eq!(latency.totals[0].total, cloned.totals[0].total); - assert_eq!(latency.totals[0].n, cloned.totals[0].n); - } - - #[test] - fn test_edge_case_max_values() { - let mut elem = AccElem { - total: u64::MAX - 50, - size: u64::MAX - 50, - n: u64::MAX - 50, - }; - - let other = AccElem { - total: 100, - size: 100, - n: 100, - }; - - // This should not panic due to overflow, values will wrap around - elem.merge(&other); - - // Values should wrap around due to overflow (wrapping_add behavior) - assert_eq!(elem.total, 49); // (u64::MAX - 50) + 100 wraps to 49 - assert_eq!(elem.size, 49); - assert_eq!(elem.n, 49); - } - - #[test] - fn test_forward_to_boundary_conditions() { - let mut latency = LastMinuteLatency { - last_sec: 59, - ..Default::default() - }; - - // Add data at the last slot - latency.totals[59].total = 100; - latency.totals[59].n = 1; - - // Forward exactly 60 seconds (boundary case) - latency.forward_to(119); - - // All data should be cleared - for elem in &latency.totals { - assert_eq!(elem.total, 0); - assert_eq!(elem.n, 0); - } - } - - #[test] - fn test_get_total_with_data() { - let mut latency = LastMinuteLatency::default(); - - // Set a recent timestamp to avoid forward_to clearing data - let current_time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - latency.last_sec = current_time; - - // Add data to multiple slots - latency.totals[0] = AccElem { - total: 10, - size: 100, - n: 1, - }; - latency.totals[1] = AccElem { - total: 20, - size: 200, - n: 2, - }; - latency.totals[59] = AccElem { - total: 30, - size: 300, - n: 3, - }; - - let total = latency.get_total(); - - assert_eq!(total.total, 60); - assert_eq!(total.size, 600); - assert_eq!(total.n, 6); - } - - #[test] - fn test_window_index_calculation() { - // Test that window index calculation works correctly - let _latency = LastMinuteLatency::default(); - - let acc_elem = AccElem { total: 1, size: 1, n: 1 }; - - // Test various timestamps - let test_cases = [(0, 0), (1, 1), (59, 59), (60, 0), (61, 1), (119, 59), (120, 0)]; - - for (timestamp, expected_idx) in test_cases { - let mut test_latency = LastMinuteLatency::default(); - test_latency.add_all(timestamp, &acc_elem); - - assert_eq!( - test_latency.totals[expected_idx].n, 1, - "Failed for timestamp {timestamp} (expected index {expected_idx})" - ); - } - } - - #[test] - fn test_concurrent_safety_simulation() { - // Simulate concurrent access patterns - let mut latency = LastMinuteLatency::default(); - - // Use current time to ensure data doesn't get cleared by get_total - let current_time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - - // Simulate rapid additions within a 60-second window - for i in 0..1000 { - let acc_elem = AccElem { - total: (i % 10) + 1, // Ensure non-zero values - size: (i % 100) + 1, - n: 1, - }; - // Keep all timestamps within the current minute window - latency.add_all(current_time - (i % 60), &acc_elem); - } - - let total = latency.get_total(); - assert!(total.n > 0, "Total count should be greater than 0"); - assert!(total.total > 0, "Total time should be greater than 0"); - } - - #[test] - fn test_acc_elem_debug_format() { - let elem = AccElem { - total: 123, - size: 456, - n: 789, - }; - - let debug_str = format!("{elem:?}"); - assert!(debug_str.contains("123")); - assert!(debug_str.contains("456")); - assert!(debug_str.contains("789")); - } - - #[test] - fn test_large_values() { - let mut elem = AccElem::default(); - - // Test with large duration values - let large_duration = Duration::from_secs(u64::MAX / 2); - elem.add(&large_duration); - - assert_eq!(elem.total, u64::MAX / 2); - assert_eq!(elem.n, 1); - - // Test average calculation with large values - let avg = elem.avg(); - assert_eq!(avg, Duration::from_secs(u64::MAX / 2)); - } - - #[test] - fn test_zero_duration_handling() { - let mut elem = AccElem::default(); - - let zero_duration = Duration::from_secs(0); - elem.add(&zero_duration); - - assert_eq!(elem.total, 0); - assert_eq!(elem.n, 1); - assert_eq!(elem.avg(), Duration::from_secs(0)); - } -} - -const SIZE_LAST_ELEM_MARKER: usize = 10; // Assumed marker size is 10, modify according to actual situation - -#[allow(dead_code)] -#[derive(Debug, Default)] -pub struct LastMinuteHistogram { - histogram: Vec, - size: u32, -} - -impl LastMinuteHistogram { - pub fn merge(&mut self, other: &LastMinuteHistogram) { - for i in 0..self.histogram.len() { - self.histogram[i].merge(&other.histogram[i]); - } - } - - pub fn add(&mut self, size: i64, t: Duration) { - let index = size_to_tag(size); - self.histogram[index].add(&t); - } - - pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] { - let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER]; - for (i, elem) in self.histogram.iter_mut().enumerate() { - res[i] = elem.get_total(); - } - res - } -} - -fn size_to_tag(size: i64) -> usize { - match size { - _ if size < 1024 => 0, // sizeLessThan1KiB - _ if size < 1024 * 1024 => 1, // sizeLessThan1MiB - _ if size < 10 * 1024 * 1024 => 2, // sizeLessThan10MiB - _ if size < 100 * 1024 * 1024 => 3, // sizeLessThan100MiB - _ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB - _ => 5, // sizeGreaterThan1GiB - } -} diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index c9e8a182d..57bf92180 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -22,7 +22,6 @@ pub mod data_usage_define; pub mod error; -pub mod last_minute; pub mod scanner; pub mod scanner_folder; pub mod scanner_io; @@ -30,6 +29,7 @@ pub mod sleeper; pub use data_usage_define::*; pub use error::ScannerError; +pub use rustfs_common::last_minute; pub use scanner::init_data_scanner; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 997fbc9eb..7cfa6469a 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -115,6 +115,32 @@ fn warn_inline_heal_compat_requested() { }); } +fn non_negative_i64_to_u64(value: i64) -> u64 { + value.max(0) as u64 +} + +fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) { + into.size += summary.total_size; + into.versions += summary.versions; + into.delete_markers += summary.delete_markers; + into.obj_sizes.add(summary.total_size as u64); + into.obj_versions.add(summary.versions as u64); + + let replication_stats = into.replication_stats.get_or_insert_with(Default::default); + replication_stats.replica_size += non_negative_i64_to_u64(summary.replica_size); + replication_stats.replica_count += summary.replica_count as u64; + + for (arn, st) in &summary.repl_target_stats { + let tgt_stat = replication_stats.targets.entry(arn.clone()).or_default(); + tgt_stat.pending_size += non_negative_i64_to_u64(st.pending_size); + tgt_stat.failed_size += non_negative_i64_to_u64(st.failed_size); + tgt_stat.replicated_size += non_negative_i64_to_u64(st.replicated_size); + tgt_stat.replicated_count += st.replicated_count as u64; + tgt_stat.failed_count += st.failed_count as u64; + tgt_stat.pending_count += st.pending_count as u64; + } +} + /// Cached folder information for scanning #[derive(Clone, Debug)] pub struct CachedFolder { @@ -963,7 +989,7 @@ impl FolderScanner { abandoned_children.remove(&path_join_buf(&[&item.bucket, &item.object_path()])); - into.add_sizes(&sz); + apply_scanner_size_summary(into, &sz); into.objects += 1; object_count += 1; diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index daa7de6d7..787ddb408 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -9,13 +9,14 @@ homepage.workspace = true description = "Notification target abstraction and implementations for RustFS" keywords = ["file-system", "notification", "target", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "filesystem"] -documentation = "https://docs.rs/rustfs-target/latest/rustfs_target/" +documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/" [dependencies] rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] } rustfs-ecstore = { workspace = true } rustfs-utils = { workspace = true, features = ["notify", "tls"] } -rustfs-s3-common = { workspace = true } +rustfs-s3-types = { workspace = true } +rustfs-s3-ops = { workspace = true } async-trait = { workspace = true } async-nats = { workspace = true } deadpool-postgres = { workspace = true } diff --git a/crates/targets/src/lib.rs b/crates/targets/src/lib.rs index a173816fc..0fb7db9cc 100644 --- a/crates/targets/src/lib.rs +++ b/crates/targets/src/lib.rs @@ -61,7 +61,7 @@ pub use runtime::{ sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability}, start_replay_worker, }; -pub use rustfs_s3_common::EventName; +pub use rustfs_s3_types::EventName; use serde::{Deserialize, Serialize}; pub use sys::user_agent::*; pub use target::{Target, TargetDeliverySnapshot}; diff --git a/crates/targets/src/target/amqp.rs b/crates/targets/src/target/amqp.rs index 1173bdb2c..0c9c01627 100644 --- a/crates/targets/src/target/amqp.rs +++ b/crates/targets/src/target/amqp.rs @@ -501,7 +501,7 @@ where #[cfg(test)] mod tests { use super::*; - use rustfs_s3_common::EventName; + use rustfs_s3_types::EventName; use serde_json::json; use std::path::PathBuf; use std::sync::Arc; diff --git a/crates/targets/src/target/mod.rs b/crates/targets/src/target/mod.rs index 351d06a41..f4a51ac46 100644 --- a/crates/targets/src/target/mod.rs +++ b/crates/targets/src/target/mod.rs @@ -16,7 +16,7 @@ use crate::arn::TargetID; use crate::store::{Key, QueueStore, Store}; use crate::{StoreError, TargetError, TargetLog}; use async_trait::async_trait; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::fmt::Formatter; diff --git a/crates/targets/src/target/mysql.rs b/crates/targets/src/target/mysql.rs index 1450ade22..ed2116988 100644 --- a/crates/targets/src/target/mysql.rs +++ b/crates/targets/src/target/mysql.rs @@ -1015,7 +1015,7 @@ mod tests { let entity = EntityTarget { object_name: "bucket%2Fobj.txt".to_string(), bucket_name: "testbucket".to_string(), - event_name: rustfs_s3_common::EventName::ObjectCreatedPut, + event_name: rustfs_s3_types::EventName::ObjectCreatedPut, data: serde_json::json!({"eventTime": "2026-05-03T10:00:00Z"}), }; @@ -1061,7 +1061,7 @@ mod tests { let body = br#"{"Records":[]}"#.to_vec(); let meta = QueuedPayloadMeta::new( - rustfs_s3_common::EventName::ObjectCreatedPut, + rustfs_s3_types::EventName::ObjectCreatedPut, "testbucket".to_string(), "obj.txt".to_string(), "application/json", @@ -1120,7 +1120,7 @@ mod tests { br#"{"EventName":"s3:ObjectCreated:Put","Key":"bucket/obj.txt","Records":[{"eventTime":"2026-05-03T10:00:00Z"}]}"# .to_vec(); let meta = QueuedPayloadMeta::new( - rustfs_s3_common::EventName::ObjectCreatedPut, + rustfs_s3_types::EventName::ObjectCreatedPut, "testbucket".to_string(), "obj.txt".to_string(), "application/json", diff --git a/crates/targets/src/target/postgres.rs b/crates/targets/src/target/postgres.rs index 51a4cca64..344a34ba8 100644 --- a/crates/targets/src/target/postgres.rs +++ b/crates/targets/src/target/postgres.rs @@ -1042,7 +1042,7 @@ mod tests { "Records": [] }); let meta = QueuedPayloadMeta::new( - rustfs_s3_common::EventName::ObjectCreatedPut, + rustfs_s3_types::EventName::ObjectCreatedPut, "bucket-a".to_string(), "fallback%2Fvalue.txt".to_string(), "application/json", @@ -1059,7 +1059,7 @@ mod tests { "Records": [] }); let meta = QueuedPayloadMeta::new( - rustfs_s3_common::EventName::ObjectCreatedPut, + rustfs_s3_types::EventName::ObjectCreatedPut, "bucket-a".to_string(), "hello+world%2Ftest.txt".to_string(), "application/json", diff --git a/crates/targets/src/target/redis.rs b/crates/targets/src/target/redis.rs index 133b84001..e749934ca 100644 --- a/crates/targets/src/target/redis.rs +++ b/crates/targets/src/target/redis.rs @@ -969,7 +969,7 @@ mod tests { let payload = build_queued_payload(&EntityTarget { object_name: "greeting+file+%282%29.csv".to_string(), bucket_name: "bucket".to_string(), - event_name: rustfs_s3_common::EventName::ObjectCreatedPut, + event_name: rustfs_s3_types::EventName::ObjectCreatedPut, data: "payload-data".to_string(), }) .expect("payload should build"); @@ -1071,7 +1071,7 @@ mod tests { let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); let meta = QueuedPayloadMeta::new( - rustfs_s3_common::EventName::ObjectCreatedPut, + rustfs_s3_types::EventName::ObjectCreatedPut, "bucket".to_string(), "object".to_string(), "application/json", @@ -1110,7 +1110,7 @@ mod tests { let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); let meta = QueuedPayloadMeta::new( - rustfs_s3_common::EventName::ObjectCreatedPut, + rustfs_s3_types::EventName::ObjectCreatedPut, "bucket".to_string(), "object".to_string(), "application/json", @@ -1149,7 +1149,7 @@ mod tests { let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); let meta = QueuedPayloadMeta::new( - rustfs_s3_common::EventName::ObjectCreatedPut, + rustfs_s3_types::EventName::ObjectCreatedPut, "bucket".to_string(), "object".to_string(), "application/json", diff --git a/crates/targets/tests/amqp_integration.rs b/crates/targets/tests/amqp_integration.rs index 1f4de9146..97a9924b5 100644 --- a/crates/targets/tests/amqp_integration.rs +++ b/crates/targets/tests/amqp_integration.rs @@ -29,7 +29,7 @@ use lapin::{ options::{BasicAckOptions, BasicGetOptions, QueueBindOptions, QueueDeclareOptions, QueueDeleteOptions}, types::FieldTable, }; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::Target; use rustfs_targets::check_amqp_broker_available; use rustfs_targets::target::EntityTarget; diff --git a/crates/targets/tests/mysql_integration.rs b/crates/targets/tests/mysql_integration.rs index f2f4d77d5..ef0ccc2c8 100644 --- a/crates/targets/tests/mysql_integration.rs +++ b/crates/targets/tests/mysql_integration.rs @@ -70,7 +70,7 @@ fn make_args(dsn: &str, table: &str, queue_dir: &str) -> MySqlArgs { } } -fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_common::EventName) -> EntityTarget { +fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_types::EventName) -> EntityTarget { EntityTarget { object_name: object.to_string(), bucket_name: bucket.to_string(), @@ -117,7 +117,7 @@ async fn direct_write_and_read() { target.init().await.expect("init"); - let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut); target.save(Arc::new(entity)).await.expect("save"); let pool = build_test_pool(&dsn); @@ -141,10 +141,10 @@ async fn delete_appends_row_does_not_remove_old() { target.init().await.expect("init"); - let put = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + let put = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut); target.save(Arc::new(put)).await.expect("save put"); - let delete = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectRemovedDelete); + let delete = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectRemovedDelete); target.save(Arc::new(delete)).await.expect("save delete"); let pool = build_test_pool(&dsn); @@ -166,7 +166,7 @@ async fn queue_store_saves_entry_and_replays() { let target: MySqlTarget = MySqlTarget::new("queue".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target"); - let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut); target.save(Arc::new(entity)).await.expect("save to queue"); let store = target.store().expect("store should exist"); @@ -206,7 +206,7 @@ async fn duplicate_replay_produces_duplicate_rows() { let target: MySqlTarget = MySqlTarget::new("dupe".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target"); - let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut); target.save(Arc::new(entity)).await.expect("save to queue"); target.init().await.expect("init"); diff --git a/crates/targets/tests/postgres_integration.rs b/crates/targets/tests/postgres_integration.rs index 89d6072fe..5f4c2a463 100644 --- a/crates/targets/tests/postgres_integration.rs +++ b/crates/targets/tests/postgres_integration.rs @@ -38,7 +38,7 @@ //! podman rm -f rustfs-pg-test //! ``` -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_targets::Target; use rustfs_targets::check_postgres_server_available; use rustfs_targets::target::EntityTarget; diff --git a/crates/workers/README.md b/crates/workers/README.md deleted file mode 100644 index c78c27d98..000000000 --- a/crates/workers/README.md +++ /dev/null @@ -1,37 +0,0 @@ -[![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com) - -# RustFS Workers - Background Job System - -

- Distributed background job processing system for RustFS object storage -

- -

- CI - ๐Ÿ“– Documentation - ยท ๐Ÿ› Bug Reports - ยท ๐Ÿ’ฌ Discussions -

- ---- - -## ๐Ÿ“– Overview - -**RustFS Workers** provides distributed background job processing capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs). - -## โœจ Features - -- Distributed job execution across cluster nodes -- Priority-based job scheduling and queue management -- Built-in workers for replication, cleanup, healing, and indexing -- Automatic retry logic with exponential backoff -- Horizontal scaling with load balancing -- Real-time job monitoring and administrative interface - -## ๐Ÿ“š Documentation - -For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs). - -## ๐Ÿ“„ License - -This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details. diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 7d7f5413d..00bb57e12 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -61,7 +61,6 @@ workspace = true [dependencies] # RustFS Internal Crates rustfs-heal = { workspace = true } -rustfs-appauth = { workspace = true } rustfs-audit = { workspace = true } rustfs-common = { workspace = true } rustfs-config = { workspace = true, features = ["constants", "notify"] } @@ -80,7 +79,9 @@ rustfs-policy = { workspace = true } rustfs-protocols = { workspace = true } rustfs-protos = { workspace = true } rustfs-rio.workspace = true -rustfs-s3-common = { workspace = true } +rustfs-s3-types = { workspace = true } +rustfs-s3-ops = { workspace = true } +rustfs-data-usage = { workspace = true } rustfs-s3select-api = { workspace = true } rustfs-s3select-query = { workspace = true } rustfs-targets = { workspace = true } diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 736e0a457..a45ee805e 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -27,8 +27,8 @@ use http::{HeaderMap, StatusCode}; use hyper::Method; use matchit::Params; use percent_encoding::percent_decode_str; -use rustfs_common::data_usage::TierStats; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; +use rustfs_data_usage::TierStats; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState; use rustfs_ecstore::{ bucket::lifecycle::tier_last_day_stats::DailyAllTierStats, diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index d3e6a7201..960ef47f1 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -65,7 +65,7 @@ use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_madmin::utils::parse_duration; use rustfs_notify::{Event as NotificationEvent, notification_system}; use rustfs_policy::policy::action::{Action, S3Action}; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use rustfs_signer::pre_sign_v4; use rustfs_utils::http::{ SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, @@ -3803,7 +3803,7 @@ mod tests { fn event_matches_listen_notification_respects_bucket_event_and_object_filters() { let filter = ListenNotificationFilter { bucket: Some("demo-bucket".to_string()), - event_mask: EventName::ObjectCreatedPut.mask() | EventName::ObjectCreatedPost.mask(), + event_mask: rustfs_s3_ops::put_object_created_event_mask(), prefix: Some("logs/".to_string()), suffix: Some(".json".to_string()), }; diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index bff361f57..66b559916 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -17,7 +17,7 @@ use crate::app::context::{AppContext, get_global_app_context}; use crate::capacity::resolve_admin_used_capacity; use crate::error::ApiError; -use rustfs_common::data_usage::DataUsageInfo; +use rustfs_data_usage::DataUsageInfo; use rustfs_ecstore::admin_server_info::get_server_info; use rustfs_ecstore::data_usage::load_data_usage_from_backend; use rustfs_ecstore::endpoints::EndpointServerPools; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 29f836149..ae750ae0f 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -64,7 +64,7 @@ use rustfs_policy::policy::{ action::{Action, S3Action}, {BucketPolicy, BucketPolicyArgs, Effect, Validator}, }; -use rustfs_s3_common::S3Operation; +use rustfs_s3_ops::S3Operation; use rustfs_targets::{ EventName, arn::{ARN, TargetIDError}, diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index f99543c68..82f8e03bf 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -52,7 +52,7 @@ use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_rio::{CompressReader, EncryptReader, HashReader}; #[cfg(test)] use rustfs_rio::{DecryptReader, HardLimitReader, boxed_reader, wrap_reader}; -use rustfs_s3_common::S3Operation; +use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::{ diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index c1efcb8c3..5a157fe90 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -86,7 +86,7 @@ use rustfs_io_metrics; use rustfs_notify::EventArgsBuilder; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_rio::{CompressReader, DynReader, EncryptReader, HashReader, wrap_reader}; -use rustfs_s3_common::S3Operation; +use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object}; use rustfs_s3select_api::{ object_store::bytes_stream, query::{Context, Query}, @@ -1625,9 +1625,9 @@ impl DefaultObjectUsecase { fn put_object_execution_context(req: &S3Request) -> (EventName, QuotaOperation, &'static str) { if req.extensions.get::().is_some() { - (EventName::ObjectCreatedPost, QuotaOperation::PostObject, "POST") + (put_event_name_for_post_object(true), QuotaOperation::PostObject, "POST") } else { - (EventName::ObjectCreatedPut, QuotaOperation::PutObject, "PUT") + (put_event_name_for_post_object(false), QuotaOperation::PutObject, "PUT") } } @@ -3141,11 +3141,7 @@ impl DefaultObjectUsecase { let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify); for res in delete_results { if let Some(dobj) = res.delete_object { - let event_name = if dobj.delete_marker { - EventName::ObjectRemovedDeleteMarkerCreated - } else { - EventName::ObjectRemovedDelete - }; + let event_name = delete_event_name_for_marker(dobj.delete_marker); let event_args = EventArgsBuilder::new( event_name, bucket.clone(), @@ -3310,7 +3306,7 @@ impl DefaultObjectUsecase { } // Prefix/force-delete returns empty ObjectInfo; still emit bucket notification so webhooks match S3 DELETE. helper = helper - .event_name(EventName::ObjectRemovedDelete) + .event_name(delete_event_name_for_marker(false)) .object(ObjectInfo { name: key.clone(), bucket: bucket.clone(), @@ -3378,11 +3374,7 @@ impl DefaultObjectUsecase { ..Default::default() }; - let event_name = if delete_marker { - EventName::ObjectRemovedDeleteMarkerCreated - } else { - EventName::ObjectRemovedDelete - }; + let event_name = delete_event_name_for_marker(delete_marker); helper = helper.event_name(event_name); helper = helper @@ -4341,7 +4333,7 @@ impl DefaultObjectUsecase { }; let event_args = rustfs_notify::EventArgs { - event_name: EventName::ObjectCreatedPut, + event_name: put_event_name_for_post_object(false), bucket_name: bucket.clone(), object: obj_info.clone(), req_params: req_params.clone(), diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index 2ae596e17..e64868dcb 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_appauth::token::{Token, parse_license_with_public_key}; +use rustfs_crypto::{Token, parse_license_with_public_key}; use std::fmt; use std::io::{Error, ErrorKind, Result}; use std::sync::Arc; @@ -328,7 +328,7 @@ mod tests { RsaPrivateKey, RsaPublicKey, pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding}, }; - use rustfs_appauth::token::sign_license_token; + use rustfs_crypto::sign_license_token; use serial_test::serial; #[test] diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index 6443ec934..4b39a0914 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -16,7 +16,7 @@ use super::{module_switch::resolve_notify_module_state, refresh_persisted_module use crate::app::context::resolve_server_config; use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook}; use rustfs_notify::EventArgs as NotifyEventArgs; -use rustfs_s3_common::EventName; +use rustfs_s3_types::EventName; use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::spawn; @@ -38,14 +38,26 @@ pub fn is_notify_module_enabled() -> bool { NOTIFY_MODULE_ENABLED.load(Ordering::Relaxed) } -fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs { +fn convert_ecstore_event_args(args: EcstoreEventArgs) -> Option { let version_id = args.object.version_id.map(|v| v.to_string()).unwrap_or_default(); let (host, port) = parse_host_and_port(args.host); let req_params = args.req_params.into_iter().collect(); let resp_elements = args.resp_elements.into_iter().collect(); + let event_name = match EventName::try_from_event_str(args.event_name.as_str()) { + Ok(event_name) => event_name, + Err(err) => { + warn!( + event_name = args.event_name, + bucket = args.bucket_name, + error = %err, + "dropping ecstore event with invalid event name" + ); + return None; + } + }; - NotifyEventArgs { - event_name: EventName::from(args.event_name.as_str()), + Some(NotifyEventArgs { + event_name, bucket_name: args.bucket_name, object: args.object, req_params, @@ -54,7 +66,7 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs { host, port, user_agent: args.user_agent, - } + }) } fn parse_host_and_port(host: String) -> (String, u16) { @@ -77,7 +89,9 @@ fn parse_host_and_port(host: String) -> (String, u16) { fn install_ecstore_event_dispatch_hook() { let installed = register_event_dispatch_hook(|args| { - let notify_args = convert_ecstore_event_args(args); + let Some(notify_args) = convert_ecstore_event_args(args) else { + return; + }; spawn(async move { rustfs_notify::notifier_global::notify(notify_args).await; }); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 190a7daa1..c781c05e8 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -41,7 +41,8 @@ use rustfs_ecstore::{ new_object_layer_fn, store_api::{BucketOperations, BucketOptions, ObjectLockRetentionOptions, ObjectOperations, ObjectOptions}, }; -use rustfs_s3_common::{S3Operation, record_s3_op}; +use rustfs_io_metrics::record_s3_op; +use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::http::headers::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, @@ -71,7 +72,7 @@ impl Default for FS { impl FS { pub fn new() -> Self { - rustfs_s3_common::init_s3_metrics(); + rustfs_io_metrics::init_s3_metrics(); Self {} } diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index 5f85a3169..2a1c5e510 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -23,9 +23,10 @@ use rustfs_audit::{ global::AuditLogger, }; use rustfs_ecstore::store_api::ObjectInfo; +use rustfs_io_metrics::record_s3_op; use rustfs_notify::{EventArgsBuilder, notifier_global}; -use rustfs_s3_common::record_s3_op; -use rustfs_s3_common::{EventName, S3Operation}; +use rustfs_s3_ops::{S3Operation, operation_matches_event_name}; +use rustfs_s3_types::EventName; use rustfs_utils::{ extract_params_header, extract_req_params, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, http::headers::AMZ_REQUEST_ID, @@ -34,7 +35,7 @@ use s3s::{S3Request, S3Response, S3Result}; use serde_json::Value; use std::future::Future; use tokio::runtime::{Builder, Handle}; -use tracing::{Instrument, info_span}; +use tracing::{Instrument, info_span, warn}; /// Schedules an asynchronous task on the current runtime; /// if there is no runtime, creates a minimal runtime execution on a new thread. @@ -89,6 +90,22 @@ pub struct EnabledOperationHelper { impl OperationHelper { /// Create a new OperationHelper for S3 requests. pub fn new(req: &S3Request, event: EventName, op: S3Operation) -> Self { + let op_event_matches = operation_matches_event_name(op, event); + debug_assert!(op_event_matches, "operation/event mismatch: op={} event={}", op.as_str(), event.as_str()); + if !op_event_matches { + counter!( + "rustfs_log_chain_op_event_mismatch_total", + "op" => op.as_str(), + "event" => event.as_str().to_string() + ) + .increment(1); + warn!( + op = op.as_str(), + event = event.as_str(), + "operation/event mismatch detected; check S3 semantic mapping" + ); + } + let audit_enabled = is_audit_module_enabled(); let notify_enabled = should_build_notification_event(is_notify_module_enabled()); diff --git a/rustfs/src/storage/rpc/disk.rs b/rustfs/src/storage/rpc/disk.rs index ec6cf4ef4..0b4c4cd6d 100644 --- a/rustfs/src/storage/rpc/disk.rs +++ b/rustfs/src/storage/rpc/disk.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use rustfs_common::internode_metrics::{ +use rustfs_io_metrics::internode_metrics::{ INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, global_internode_metrics, }; use serde::de::DeserializeOwned; diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index e28c256fc..90824cde5 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -18,15 +18,15 @@ use futures_util::TryStreamExt; use http::{HeaderMap, Method, Request, Response, StatusCode, Uri}; use http_body_util::{BodyExt, Limited}; use hyper::body::Incoming; -use rustfs_common::internode_metrics::{ - INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, - global_internode_metrics, -}; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_ecstore::disk::{DiskAPI, WalkDirOptions}; use rustfs_ecstore::rpc::verify_rpc_signature; use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; use rustfs_ecstore::store::find_local_disk_by_ref; +use rustfs_io_metrics::internode_metrics::{ + INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, + global_internode_metrics, +}; use rustfs_utils::net::bytes_stream; use s3s::Body; use s3s::dto::StreamingBlob;