gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 15:46:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(s3): consolidate semantic boundaries and remove s3-common (#3012)

Browse Source 
* refactor(common): introduce rustfs-data-usage core crate

* refactor(concurrency): migrate workers crate into concurrency

* refactor(crypto): migrate appauth token APIs into crypto

* fix docs urls

* remove unused crate

* refactor(data-usage): switch consumers to rustfs-data-usage

* chore(fmt): apply cargo fmt and lockfile sync

* refactor(common): remove data_usage compatibility re-export

* refactor(capacity): move capacity_scope to object-capacity

* refactor(io-metrics): relocate internode metrics from common

* refactor(common): decouple scanner report from madmin

* chore(fmt): normalize import ordering after pre-commit

* refactor(s3): split s3 types and ops crates

* refactor(s3): centralize event version and safe parsing

* refactor(s3): add op-event compatibility guardrails

* refactor(s3): add runtime op-event mismatch observability

* refactor(s3): extract delete event mapping helper

* refactor(s3): extract put event mapping helper

* refactor(s3): consolidate remaining event semantic helpers

* refactor(s3): add op-event coverage checks and observability alerts

* refactor(s3-ops): consolidate op-event semantic mapping

* refactor(scanner): remove last_minute wrapper module

* refactor(scanner): consolidate duplicated data usage models
This commit is contained in:
houseme
2026-05-19 20:50:25 +08:00
committed by GitHub
parent d6fc70eb12
commit 73bde843d6
100 changed files with 982 additions and 2520 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+2 -2
View File
@@ -44,7 +44,6 @@ use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
use http::HeaderMap;
use lazy_static::lazy_static;
use rustfs_common::data_usage::TierStats;
use rustfs_common::heal_channel::rep_has_active_rules;
use rustfs_common::metrics::{IlmAction, Metrics};
use rustfs_config::{
@@ -52,11 +51,12 @@ use rustfs_config::{
DEFAULT_TRANSITION_WORKERS_CAP, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, ENV_TRANSITION_QUEUE_CAPACITY,
ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
};
use rustfs_data_usage::TierStats;
use rustfs_filemeta::{
FileInfo, FileInfoOpts, NULL_VERSION_ID, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicationState, RestoreStatusOps,
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs
+1 -1
View File
@@ -18,7 +18,7 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use rustfs_common::data_usage::TierStats;
use rustfs_data_usage::TierStats;
use sha2::Sha256;
use std::collections::HashMap;
use std::ops::Sub;
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+1 -1
View File
@@ -56,7 +56,7 @@ use rustfs_filemeta::{
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
crates/ecstore/src/data_usage.rs
+2 -2
View File
@@ -27,7 +27,7 @@ pub use local_snapshot::{
data_usage_dir, data_usage_state_dir, ensure_data_usage_layout, read_snapshot as read_local_snapshot, snapshot_file_name,
snapshot_object_path, snapshot_path, write_snapshot as write_local_snapshot,
};
use rustfs_common::data_usage::{
use rustfs_data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
};
use rustfs_io_metrics::record_system_path_failure;
@@ -686,7 +686,7 @@ pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate:
#[cfg(test)]
mod tests {
use super::*;
use rustfs_common::data_usage::BucketUsageInfo;
use rustfs_data_usage::BucketUsageInfo;
fn aggregate_for_test(
inputs: Vec<(DiskUsageStatus, Result<Option<LocalUsageSnapshot>, Error>)>,
crates/ecstore/src/event/name.rs
+2 -2
View File
@@ -13,6 +13,6 @@
// limitations under the License.
//! Compatibility re-export for the legacy `rustfs_ecstore::event::name::EventName` path.
//! The canonical event definition now lives in `rustfs_s3_common::EventName`.
//! The canonical event definition now lives in `rustfs_s3_types::EventName`.
pub use rustfs_s3_common::EventName;
pub use rustfs_s3_types::EventName;
crates/ecstore/src/metrics_realtime.rs
+52 -6
View File
@@ -14,11 +14,12 @@
use crate::{admin_server_info::get_local_server_property, new_object_layer_fn, store_api::StorageAPI};
use chrono::Utc;
use rustfs_common::{
GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, internode_metrics::global_internode_metrics,
metrics::global_metrics,
use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::DriveState, metrics::global_metrics};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_madmin::metrics::{
DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics,
ScannerMetrics as MadminScannerMetrics, TimedAction as MadminTimedAction,
};
use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics};
use rustfs_utils::os::get_drive_stats;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
@@ -56,6 +57,51 @@ impl MetricType {
}
}
fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
MadminScannerMetrics {
collected_at: metrics.collected_at,
current_cycle: metrics.current_cycle,
current_started: metrics.current_started,
cycles_completed_at: metrics.cycles_completed_at,
ongoing_buckets: metrics.ongoing_buckets,
life_time_ops: metrics.life_time_ops,
life_time_ilm: metrics.life_time_ilm,
last_minute: MadminLastMinute {
actions: metrics
.last_minute
.actions
.into_iter()
.map(|(key, value)| {
(
key,
MadminTimedAction {
count: value.count,
acc_time: value.acc_time,
bytes: value.bytes,
},
)
})
.collect(),
ilm: metrics
.last_minute
.ilm
.into_iter()
.map(|(key, value)| {
(
key,
MadminTimedAction {
count: value.count,
acc_time: value.acc_time,
bytes: value.bytes,
},
)
})
.collect(),
},
active_paths: metrics.active_paths,
}
}
impl MetricType {
fn contains(&self, x: &MetricType) -> bool {
(self.0 & x.0) == x.0
@@ -114,7 +160,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
if let Some(init_time) = rustfs_common::get_global_init_time().await {
metrics.current_started = init_time;
}
real_time_metrics.aggregated.scanner = Some(metrics);
real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
}
// if types.contains(&MetricType::OS) {}
@@ -252,7 +298,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
#[cfg(test)]
mod test {
use super::*;
use rustfs_common::internode_metrics::global_internode_metrics;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use std::time::Duration;
#[test]
crates/ecstore/src/pools.rs
+1 -1
View File
@@ -51,9 +51,9 @@ use rmp_serde::Deserializer;
use rmp_serde::Serializer;
use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_concurrency::workers::Workers;
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
use rustfs_workers::workers::Workers;
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
crates/ecstore/src/rpc/remote_disk.rs
+2 -2
View File
@@ -36,10 +36,10 @@ use bytes::Bytes;
use futures::lock::Mutex;
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
use metrics::counter;
use rustfs_common::internode_metrics::{
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, global_internode_metrics,
};
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
crates/ecstore/src/set_disk.rs
+4 -3
View File
@@ -68,7 +68,6 @@ use http::HeaderMap;
use md5::{Digest as Md5Digest, Md5};
use rand::{Rng, seq::SliceRandom};
use regex::Regex;
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope};
use rustfs_common::heal_channel::{DriveState, HealChannelPriority, HealItemType, HealOpts, HealScanMode, send_heal_disk};
use rustfs_config::MI_B;
use rustfs_filemeta::{
@@ -80,8 +79,11 @@ use rustfs_lock::fast_lock::types::LockResult;
use rustfs_lock::local_lock::LocalLock;
use rustfs_lock::{FastLockGuard, LockManager, NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper, ObjectKey};
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
use rustfs_s3_common::EventName;
use rustfs_s3_types::EventName;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{
@@ -97,7 +99,6 @@ use rustfs_utils::{
crypto::hex,
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
};
use rustfs_workers::workers::Workers;
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE};
use sha2::{Digest, Sha256};
use std::hash::Hash;