refactor(s3): consolidate semantic boundaries and remove s3-common (#3012)

* refactor(common): introduce rustfs-data-usage core crate

* refactor(concurrency): migrate workers crate into concurrency

* refactor(crypto): migrate appauth token APIs into crypto

* fix docs urls

* remove unused crate

* refactor(data-usage): switch consumers to rustfs-data-usage

* chore(fmt): apply cargo fmt and lockfile sync

* refactor(common): remove data_usage compatibility re-export

* refactor(capacity): move capacity_scope to object-capacity

* refactor(io-metrics): relocate internode metrics from common

* refactor(common): decouple scanner report from madmin

* chore(fmt): normalize import ordering after pre-commit

* refactor(s3): split s3 types and ops crates

* refactor(s3): centralize event version and safe parsing

* refactor(s3): add op-event compatibility guardrails

* refactor(s3): add runtime op-event mismatch observability

* refactor(s3): extract delete event mapping helper

* refactor(s3): extract put event mapping helper

* refactor(s3): consolidate remaining event semantic helpers

* refactor(s3): add op-event coverage checks and observability alerts

* refactor(s3-ops): consolidate op-event semantic mapping

* refactor(scanner): remove last_minute wrapper module

* refactor(scanner): consolidate duplicated data usage models
This commit is contained in:
houseme
2026-05-19 20:50:25 +08:00
committed by GitHub
parent d6fc70eb12
commit 73bde843d6
100 changed files with 982 additions and 2520 deletions
+1 -1
View File
@@ -61,7 +61,7 @@ pub use runtime::{
sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability},
start_replay_worker,
};
pub use rustfs_s3_common::EventName;
pub use rustfs_s3_types::EventName;
use serde::{Deserialize, Serialize};
pub use sys::user_agent::*;
pub use target::{Target, TargetDeliverySnapshot};
+1 -1
View File
@@ -501,7 +501,7 @@ where
#[cfg(test)]
mod tests {
use super::*;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use serde_json::json;
use std::path::PathBuf;
use std::sync::Arc;
+1 -1
View File
@@ -16,7 +16,7 @@ use crate::arn::TargetID;
use crate::store::{Key, QueueStore, Store};
use crate::{StoreError, TargetError, TargetLog};
use async_trait::async_trait;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::fmt::Formatter;
+3 -3
View File
@@ -1015,7 +1015,7 @@ mod tests {
let entity = EntityTarget {
object_name: "bucket%2Fobj.txt".to_string(),
bucket_name: "testbucket".to_string(),
event_name: rustfs_s3_common::EventName::ObjectCreatedPut,
event_name: rustfs_s3_types::EventName::ObjectCreatedPut,
data: serde_json::json!({"eventTime": "2026-05-03T10:00:00Z"}),
};
@@ -1061,7 +1061,7 @@ mod tests {
let body = br#"{"Records":[]}"#.to_vec();
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"testbucket".to_string(),
"obj.txt".to_string(),
"application/json",
@@ -1120,7 +1120,7 @@ mod tests {
br#"{"EventName":"s3:ObjectCreated:Put","Key":"bucket/obj.txt","Records":[{"eventTime":"2026-05-03T10:00:00Z"}]}"#
.to_vec();
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"testbucket".to_string(),
"obj.txt".to_string(),
"application/json",
+2 -2
View File
@@ -1042,7 +1042,7 @@ mod tests {
"Records": []
});
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket-a".to_string(),
"fallback%2Fvalue.txt".to_string(),
"application/json",
@@ -1059,7 +1059,7 @@ mod tests {
"Records": []
});
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket-a".to_string(),
"hello+world%2Ftest.txt".to_string(),
"application/json",
+4 -4
View File
@@ -969,7 +969,7 @@ mod tests {
let payload = build_queued_payload(&EntityTarget {
object_name: "greeting+file+%282%29.csv".to_string(),
bucket_name: "bucket".to_string(),
event_name: rustfs_s3_common::EventName::ObjectCreatedPut,
event_name: rustfs_s3_types::EventName::ObjectCreatedPut,
data: "payload-data".to_string(),
})
.expect("payload should build");
@@ -1071,7 +1071,7 @@ mod tests {
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket".to_string(),
"object".to_string(),
"application/json",
@@ -1110,7 +1110,7 @@ mod tests {
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket".to_string(),
"object".to_string(),
"application/json",
@@ -1149,7 +1149,7 @@ mod tests {
let target = RedisTarget::<String>::new("redis:test".to_string(), args).expect("target should build");
let meta = QueuedPayloadMeta::new(
rustfs_s3_common::EventName::ObjectCreatedPut,
rustfs_s3_types::EventName::ObjectCreatedPut,
"bucket".to_string(),
"object".to_string(),
"application/json",