gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 03:46:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(s3): consolidate semantic boundaries and remove s3-common (#3012)

Browse Source 
* refactor(common): introduce rustfs-data-usage core crate

* refactor(concurrency): migrate workers crate into concurrency

* refactor(crypto): migrate appauth token APIs into crypto

* fix docs urls

* remove unused crate

* refactor(data-usage): switch consumers to rustfs-data-usage

* chore(fmt): apply cargo fmt and lockfile sync

* refactor(common): remove data_usage compatibility re-export

* refactor(capacity): move capacity_scope to object-capacity

* refactor(io-metrics): relocate internode metrics from common

* refactor(common): decouple scanner report from madmin

* chore(fmt): normalize import ordering after pre-commit

* refactor(s3): split s3 types and ops crates

* refactor(s3): centralize event version and safe parsing

* refactor(s3): add op-event compatibility guardrails

* refactor(s3): add runtime op-event mismatch observability

* refactor(s3): extract delete event mapping helper

* refactor(s3): extract put event mapping helper

* refactor(s3): consolidate remaining event semantic helpers

* refactor(s3): add op-event coverage checks and observability alerts

* refactor(s3-ops): consolidate op-event semantic mapping

* refactor(scanner): remove last_minute wrapper module

* refactor(scanner): consolidate duplicated data usage models
This commit is contained in:
houseme
2026-05-19 20:50:25 +08:00
committed by GitHub
parent d6fc70eb12
commit 73bde843d6
100 changed files with 982 additions and 2520 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/targets/Cargo.toml
+3 -2
View File
@@ -9,13 +9,14 @@ homepage.workspace = true
description = "Notification target abstraction and implementations for RustFS"
keywords = ["file-system", "notification", "target", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "filesystem"]
documentation = "https://docs.rs/rustfs-target/latest/rustfs_target/"
documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
[dependencies]
rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] }
rustfs-ecstore = { workspace = true }
rustfs-utils = { workspace = true, features = ["notify", "tls"] }
rustfs-s3-common = { workspace = true }
rustfs-s3-types = { workspace = true }
rustfs-s3-ops = { workspace = true }
async-trait = { workspace = true }
async-nats = { workspace = true }
deadpool-postgres = { workspace = true }
crates/targets/src/lib.rs
+1 -1
View File
@@ -61,7 +61,7 @@ pub use runtime::{
sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability},
start_replay_worker,
};
pub use rustfs_s3_common::EventName;
pub use rustfs_s3_types::EventName;
use serde::{Deserialize, Serialize};
pub use sys::user_agent::*;
pub use target::{Target, TargetDeliverySnapshot};
crates/targets/src/target/amqp.rs
+1 -1
View File
@@ -501,7 +501,7 @@ where
#[cfg(test)]
mod tests {
use super::*;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use serde_json::json;
use std::path::PathBuf;
use std::sync::Arc;
crates/targets/src/target/mod.rs
+1 -1
View File
@@ -16,7 +16,7 @@ use crate::arn::TargetID;
use crate::store::{Key, QueueStore, Store};
use crate::{StoreError, TargetError, TargetLog};
use async_trait::async_trait;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::fmt::Formatter;
crates/targets/src/target/mysql.rs
+3 -3
View File
@@ -1015,7 +1015,7 @@ mod tests {
let entity = EntityTarget {
object_name: "bucket%2Fobj.txt".to_string(),
bucket_name: "testbucket".to_string(),
event_name: rustfs_s3_common::EventName::ObjectCreatedPut,
event_name: rustfs_s3_types::EventName::ObjectCreatedPut,
data: serde_json::json!({"eventTime": "2026-05-03T10:00:00Z"}),
};
@@ -1061,7 +1061,7 @@ mod tests {
let body = br#"{"Records":[]}"#.to_vec();
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"testbucket".to_string(),
"obj.txt".to_string(),
"application/json",
@@ -1120,7 +1120,7 @@ mod tests {
br#"{"EventName":"s3:ObjectCreated:Put","Key":"bucket/obj.txt","Records":[{"eventTime":"2026-05-03T10:00:00Z"}]}"#
.to_vec();
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"testbucket".to_string(),
"obj.txt".to_string(),
"application/json",
crates/targets/src/target/postgres.rs
+2 -2
View File
@@ -1042,7 +1042,7 @@ mod tests {
"Records": []
});
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket-a".to_string(),
"fallback%2Fvalue.txt".to_string(),
"application/json",
@@ -1059,7 +1059,7 @@ mod tests {
"Records": []
});
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket-a".to_string(),
"hello+world%2Ftest.txt".to_string(),
"application/json",
crates/targets/src/target/redis.rs
+4 -4
View File
@@ -969,7 +969,7 @@ mod tests {
let payload = build_queued_payload(&EntityTarget {
object_name: "greeting+file+%282%29.csv".to_string(),
bucket_name: "bucket".to_string(),
event_name: rustfs_s3_common::EventName::ObjectCreatedPut,
event_name: rustfs_s3_types::EventName::ObjectCreatedPut,
data: "payload-data".to_string(),
})
.expect("payload should build");
@@ -1071,7 +1071,7 @@ mod tests {
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket".to_string(),
"object".to_string(),
"application/json",
@@ -1110,7 +1110,7 @@ mod tests {
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket".to_string(),
"object".to_string(),
"application/json",
@@ -1149,7 +1149,7 @@ mod tests {
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket".to_string(),
"object".to_string(),
"application/json",
crates/targets/tests/amqp_integration.rs
+1 -1
View File
@@ -29,7 +29,7 @@ use lapin::{
options::{BasicAckOptions, BasicGetOptions, QueueBindOptions, QueueDeclareOptions, QueueDeleteOptions},
types::FieldTable,
};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::Target;
use rustfs_targets::check_amqp_broker_available;
use rustfs_targets::target::EntityTarget;
crates/targets/tests/mysql_integration.rs
+6 -6
View File
@@ -70,7 +70,7 @@ fn make_args(dsn: &str, table: &str, queue_dir: &str) -> MySqlArgs {
}
}
fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_common::EventName) -> EntityTarget<serde_json::Value> {
fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_types::EventName) -> EntityTarget<serde_json::Value> {
EntityTarget {
object_name: object.to_string(),
bucket_name: bucket.to_string(),
@@ -117,7 +117,7 @@ async fn direct_write_and_read() {
target.init().await.expect("init");
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
target.save(Arc::new(entity)).await.expect("save");
let pool = build_test_pool(&dsn);
@@ -141,10 +141,10 @@ async fn delete_appends_row_does_not_remove_old() {
target.init().await.expect("init");
let put = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
let put = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
target.save(Arc::new(put)).await.expect("save put");
let delete = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectRemovedDelete);
let delete = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectRemovedDelete);
target.save(Arc::new(delete)).await.expect("save delete");
let pool = build_test_pool(&dsn);
@@ -166,7 +166,7 @@ async fn queue_store_saves_entry_and_replays() {
let target: MySqlTarget<serde_json::Value> =
MySqlTarget::new("queue".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target");
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
target.save(Arc::new(entity)).await.expect("save to queue");
let store = target.store().expect("store should exist");
@@ -206,7 +206,7 @@ async fn duplicate_replay_produces_duplicate_rows() {
let target: MySqlTarget<serde_json::Value> =
MySqlTarget::new("dupe".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target");
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_types::EventName::ObjectCreatedPut);
target.save(Arc::new(entity)).await.expect("save to queue");
target.init().await.expect("init");
crates/targets/tests/postgres_integration.rs
+1 -1
View File
@@ -38,7 +38,7 @@
//! podman rm -f rustfs-pg-test
//! ```
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::Target;
use rustfs_targets::check_postgres_server_available;
use rustfs_targets::target::EntityTarget;