gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 07:36:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(s3): consolidate semantic boundaries and remove s3-common (#3012)

Browse Source 
* refactor(common): introduce rustfs-data-usage core crate

* refactor(concurrency): migrate workers crate into concurrency

* refactor(crypto): migrate appauth token APIs into crypto

* fix docs urls

* remove unused crate

* refactor(data-usage): switch consumers to rustfs-data-usage

* chore(fmt): apply cargo fmt and lockfile sync

* refactor(common): remove data_usage compatibility re-export

* refactor(capacity): move capacity_scope to object-capacity

* refactor(io-metrics): relocate internode metrics from common

* refactor(common): decouple scanner report from madmin

* chore(fmt): normalize import ordering after pre-commit

* refactor(s3): split s3 types and ops crates

* refactor(s3): centralize event version and safe parsing

* refactor(s3): add op-event compatibility guardrails

* refactor(s3): add runtime op-event mismatch observability

* refactor(s3): extract delete event mapping helper

* refactor(s3): extract put event mapping helper

* refactor(s3): consolidate remaining event semantic helpers

* refactor(s3): add op-event coverage checks and observability alerts

* refactor(s3-ops): consolidate op-event semantic mapping

* refactor(scanner): remove last_minute wrapper module

* refactor(scanner): consolidate duplicated data usage models
This commit is contained in:
houseme
2026-05-19 20:50:25 +08:00
committed by GitHub
parent d6fc70eb12
commit 73bde843d6
100 changed files with 982 additions and 2520 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/notify/src/bucket_config_manager.rs
+2 -2
View File
@@ -17,7 +17,7 @@ use crate::{
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
rules::ParseConfigError,
};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use std::sync::Arc;
use tracing::{debug, info, warn};
@@ -91,7 +91,7 @@ mod tests {
BucketNotificationConfig, integration::NotificationMetrics,
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use std::sync::Arc;
crates/notify/src/event.rs
+6 -27
View File
@@ -14,7 +14,8 @@
use chrono::{DateTime, SecondsFormat, Utc};
use hashbrown::HashMap;
use rustfs_s3_common::EventName;
use rustfs_s3_ops::is_object_removed_event;
use rustfs_s3_types::{EventName, event_schema_version};
use serde::{Deserialize, Serialize};
use url::form_urlencoded;
@@ -135,25 +136,6 @@ pub struct Event {
}
impl Event {
fn event_version_for(event_name: EventName) -> &'static str {
match event_name {
EventName::ObjectReplicationFailed
| EventName::ObjectReplicationComplete
| EventName::ObjectReplicationMissedThreshold
| EventName::ObjectReplicationReplicatedAfterThreshold
| EventName::ObjectReplicationNotTracked => "2.2",
EventName::ObjectRestoreCompleted
| EventName::ObjectAclPut
| EventName::ObjectTaggingPut
| EventName::ObjectTaggingDelete
| EventName::LifecycleExpirationDelete
| EventName::LifecycleExpirationDeleteMarkerCreated
| EventName::LifecycleTransition
| EventName::IntelligentTiering => "2.3",
_ => "2.1",
}
}
/// Creates a test event for a given bucket and object
pub fn new_test_event(bucket: &str, key: &str, event_name: EventName) -> Self {
let mut user_metadata = HashMap::new();
@@ -176,7 +158,7 @@ impl Event {
user_metadata.insert("x-request-time".to_string(), Utc::now().to_rfc3339());
Event {
event_version: Self::event_version_for(event_name).to_string(),
event_version: event_schema_version(event_name).to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: "us-east-1".to_string(),
event_time: Utc::now(),
@@ -256,10 +238,7 @@ impl Event {
},
};
let is_removed_event = matches!(
args.event_name,
EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated
);
let is_removed_event = is_object_removed_event(args.event_name);
if !is_removed_event {
s3_metadata.object.size = Some(args.object.size);
@@ -293,7 +272,7 @@ impl Event {
};
Self {
event_version: Self::event_version_for(args.event_name).to_string(),
event_version: event_schema_version(args.event_name).to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
event_time: event_time.and_utc(),
@@ -542,7 +521,7 @@ mod event_args_tests {
use super::EventArgs;
use hashbrown::HashMap;
use rustfs_ecstore::store_api::ObjectInfo;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
fn args_with_headers(pairs: &[(&str, &str)]) -> EventArgs {
let mut req_params = HashMap::new();
crates/notify/src/global.rs
+1 -1
View File
@@ -17,7 +17,7 @@ use crate::{
NotificationSystem, NotificationTargetMetricSnapshot,
};
use rustfs_ecstore::config::Config;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use std::sync::{Arc, OnceLock};
use tracing::error;
crates/notify/src/integration.rs
+2 -2
View File
@@ -23,7 +23,7 @@ use hashbrown::HashMap;
use metrics::{counter, gauge};
use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
use rustfs_ecstore::config::{Config, KVS};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget};
use std::sync::Arc;
@@ -402,7 +402,7 @@ pub async fn load_config_from_file(path: &str, system: &NotificationSystem) -> R
#[cfg(test)]
mod tests {
use super::*;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
#[test]
fn live_event_history_snapshots_from_sequence() {
crates/notify/src/notification_system_subscriber.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::BucketNotificationConfig;
use crate::rules::{BucketRulesSnapshot, DynRulesContainer, SubscriberIndex};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
/// NotificationSystemSubscriberView - Provides an interface to manage and query
/// the subscription status of buckets in the notification system.
crates/notify/src/notifier.rs
+1 -1
View File
@@ -308,7 +308,7 @@ mod tests {
use super::*;
use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap};
use async_trait::async_trait;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::StoreError;
use rustfs_targets::{
TargetError,
crates/notify/src/pipeline.rs
+1 -1
View File
@@ -105,7 +105,7 @@ pub type NotifyEventBridge = NotifyPipeline;
mod tests {
use super::{LiveEventHistory, NotifyPipeline};
use crate::{Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use std::sync::Arc;
use tokio::sync::{RwLock, broadcast};
crates/notify/src/rule_engine.rs
+2 -2
View File
@@ -14,7 +14,7 @@
use crate::rules::{RulesMap, TargetIdSet};
use percent_encoding::percent_decode_str;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
use std::sync::Arc;
@@ -99,7 +99,7 @@ impl Default for NotifyRuleEngine {
mod tests {
use super::NotifyRuleEngine;
use crate::rules::RulesMap;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
#[tokio::test]
crates/notify/src/rules/config.rs
+1 -1
View File
@@ -16,7 +16,7 @@ use super::rules_map::RulesMap;
use super::xml_config::ParseConfigError as BucketNotificationConfigError;
use crate::rules::NotificationConfiguration;
use crate::rules::subscriber_snapshot::{BucketRulesSnapshot, DynRulesContainer, RuleEvents, RulesContainer};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use serde::{Deserialize, Serialize};
use std::io::Read;
crates/notify/src/rules/config_test.rs
+1 -1
View File
@@ -18,7 +18,7 @@
//! to event matching, including filter rules with prefix and suffix.
use super::*;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::{ARN, TargetID};
use std::io::Cursor;
crates/notify/src/rules/pattern_rules_test.rs
+1 -1
View File
@@ -19,7 +19,7 @@
//! configuration to event matching.
use super::*;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
#[cfg(test)]
crates/notify/src/rules/rules_map.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::rules::{PatternRules, TargetIdSet};
use hashbrown::HashMap;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use serde::{Deserialize, Serialize};
crates/notify/src/rules/subscriber_index.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::rules::{BucketRulesSnapshot, BucketSnapshotRef, DynRulesContainer};
use arc_swap::ArcSwap;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use starshard::{DEFAULT_SHARDS, ShardedHashMap};
use std::fmt;
use std::sync::Arc;
crates/notify/src/rules/subscriber_snapshot.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use std::sync::Arc;
/// Let the rules structure provide "what events it is subscribed to".
crates/notify/src/rules/xml_config.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::rules::pattern;
use hashbrown::HashSet;
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::{ARN, ArnError, TargetIDError};
use serde::{Deserialize, Serialize};
use std::io::Read;