diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index db91990d4..ce6b1dad3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -93,7 +93,7 @@ refactors. | Domain | Current workspace crates | Responsibility | |--------|--------------------------|----------------| | Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-contracts`, `utils` | Shared configuration, data-usage models, heal/scanner domain contracts, utilities, and checksums. | -| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, and I/O pipelines. | +| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. | | Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. | | Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. | | Operations and integration | `audit`, `notify`, `obs`, `targets`, `zip` | Auditing, observability, event delivery, notification targets, and archive support. | @@ -138,15 +138,19 @@ default build (lifecycle: `BackpressureSettings` copy that lingered in io-metrics was removed (rustfs/backlog#1833). -4. **ecstore does not know about HTTP or S3 protocol details.** It operates on - storage-level abstractions (objects, buckets, disks, pools). - - ⚠️ VIOLATED: 58 files under `crates/ecstore/src` reference `s3s` - (`rg -l 's3s' crates/ecstore/src | wc -l`), `crates/ecstore/src/client/` - is a ~9.4K-line embedded S3 HTTP client, and `crates/ecstore/Cargo.toml` - depends on `s3s`, `http`, `hyper`/`hyper-util`/`hyper-rustls`, and - `reqwest`. Target state: the engine's need to act as an S3 client - (tiering, replication targets) is served by an extracted client crate, - and ecstore holds no wire or DTO types. +4. **ecstore does not *serve* HTTP or the S3 wire protocol.** It operates on + storage-level abstractions (objects, buckets, disks, pools) and holds no + wire or DTO types of the serving surface. *Consuming* remote S3-compatible + endpoints (ILM tier warm backends, transition targets) is a legitimate + engine capability, but it lives in the dedicated `rustfs-s3-client` crate + (`crates/s3-client`, extracted from the formerly embedded + `crates/ecstore/src/client/` by rustfs/backlog#1842), not inside ecstore. + - ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore + (bucket metadata/replication/lifecycle DTOs and error mapping). The + count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh` + (`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to + storage-level types as the first ratchet step). Target state: the + baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`. 5. **The `rustfs` binary crate is the only place that wires everything together.** Individual crates should be testable in isolation. diff --git a/Cargo.lock b/Cargo.lock index b2db988dd..63e94b893 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9389,6 +9389,7 @@ dependencies = [ "rustfs-protocols", "rustfs-protos", "rustfs-rio", + "rustfs-s3-client", "rustfs-s3-ops", "rustfs-s3-types", "rustfs-s3select-api", diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index f8d9ac9d4..558a3a984 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -255,26 +255,8 @@ pub mod capacity { pub use crate::store::utils::is_reserved_or_invalid_bucket; } -pub mod client { - pub mod admin_handler_utils { - pub use crate::client::admin_handler_utils::AdminError; - } - - pub mod api_put_object { - pub use crate::client::api_put_object::{AdvancedPutOptions, PutObjectOptions}; - } - - pub mod object_api_utils { - pub use crate::client::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag}; - } - - pub mod transition_api { - pub use crate::client::transition_api::{ - BucketLookupType, CreateBucketConfiguration, LocationConstraint, ObjectInfo, ObjectMultipartInfo, Options, - PutObjectPartOptions, ReadCloser, ReaderImpl, RequestMetadata, RestoreInfo, SendRequest, TransitionClient, - TransitionCore, UploadInfo, to_object_info, - }; - } +pub mod object_api_utils { + pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag}; } pub mod cluster { diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index c3c1f67f0..d0e5cec70 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -939,7 +939,7 @@ impl ExpiryState { let version_count = u64::try_from(v.versions.len()).unwrap_or(u64::MAX); let trace = LifecycleExpiryTrace::for_batch(&v.bucket, &v.event, &v.src, version_count); trace.emit(EVENT_LIFECYCLE_DELETE_DISPATCHED, "delete_dispatched", None); - crate::client::object_handlers_common::delete_object_versions( + crate::bucket::lifecycle::object_handlers_common::delete_object_versions( &api, &v.bucket, &v.versions, @@ -5431,8 +5431,6 @@ mod tests { use crate::bucket::lifecycle::tier_sweeper::Jentry; use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG}; use crate::bucket::metadata_sys; - #[cfg(feature = "test-util")] - use crate::client::transition_api::ReaderImpl; use crate::disk::endpoint::Endpoint; use crate::disk::{RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE}; use crate::error::{Error, is_err_invalid_upload_id}; @@ -5464,6 +5462,8 @@ mod tests { use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX; use rustfs_data_usage::TierStats; use rustfs_filemeta::{FileInfo, FileMeta}; + #[cfg(feature = "test-util")] + use rustfs_s3_client::transition_api::ReaderImpl; use rustfs_scanner_contracts::metrics::{IlmAction, global_metrics}; use s3s::dto::{ BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, @@ -6352,7 +6352,7 @@ mod tests { assert_eq!(err.kind(), std::io::ErrorKind::Other); let admin_err = err .get_ref() - .and_then(|source| source.downcast_ref::()) + .and_then(|source| source.downcast_ref::()) .expect("identity mismatch should retain the typed tier error"); assert_eq!(admin_err.code, crate::services::tier::tier::ERR_TIER_INVALID_CONFIG.code); assert_eq!(new_backend.get_count().await, 0); @@ -11563,7 +11563,7 @@ mod tests { lease .put( "remote/object", - crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")), + rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")), 9, ) .await diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 20956ca70..5006dce1b 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -21,6 +21,7 @@ pub mod evaluator; pub mod manual_transition_job; mod metadata_boundary; pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs}; +mod object_handlers_common; mod object_lock_boundary; pub use self::core as lifecycle; mod replication_sink; diff --git a/crates/ecstore/src/client/object_handlers_common.rs b/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs similarity index 95% rename from crates/ecstore/src/client/object_handlers_common.rs rename to crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs index fa4739252..95f332e64 100644 --- a/crates/ecstore/src/client/object_handlers_common.rs +++ b/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs @@ -21,7 +21,7 @@ const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped"; const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed"; use crate::bucket::lifecycle::lifecycle; -use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationObjectBridge}; +use crate::bucket::lifecycle::replication_sink::{self, ReplicationObjectBridge}; use crate::object_api::ObjectOptions; use crate::storage_api_contracts::object::{ObjectOperations as _, ObjectToDelete}; use crate::store::ECStore; @@ -84,7 +84,7 @@ pub async fn delete_object_versions( if deleted_obj.replication_state.is_none() { continue; } - ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_obj.clone()).await; + replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone()).await; } for (i, err) in errors.iter().enumerate() { diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index b2786fb18..0d622576d 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -22,11 +22,11 @@ use super::runtime_boundary as runtime_sources; use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp; use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts}; use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry; -use crate::client::signer_error::error_chain_contains_signer_header_marker; use crate::object_api::ObjectInfo; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease}; use crate::storage_api_contracts::lifecycle::TransitionedObject; use crate::store::ECStore; +use rustfs_s3_client::signer_error::error_chain_contains_signer_header_marker; use rustfs_utils::get_env_usize; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -670,7 +670,7 @@ pub(crate) fn transitioned_delete_journal_entry_for_source( #[cfg(test)] mod test { - use crate::client::signer_error::invalid_utf8_header_error; + use rustfs_s3_client::signer_error::invalid_utf8_header_error; use super::{ CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, diff --git a/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs b/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs index 9126f3563..1fa113f9c 100644 --- a/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs @@ -18,7 +18,6 @@ use tokio_util::sync::CancellationToken; use super::replication_error_boundary::Error; use super::replication_filemeta_boundary::{replication_state_from_filemeta, version_purge_status_from_filemeta}; pub(crate) type ReplicationObjectStore = crate::store::ECStore; -pub(crate) use crate::client::api_get_options::{AdvancedGetOptions, StatObjectOptions}; pub(crate) use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; pub(crate) use crate::storage_api_contracts::list::{ ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, @@ -29,6 +28,7 @@ pub(crate) use crate::storage_api_contracts::object::{ }; pub(crate) use crate::storage_api_contracts::range::HTTPRangeSpec; pub(crate) use rustfs_replication::{DeletedObject as ReplicationDeletedObject, ObjectToDelete as ReplicationObjectToDelete}; +pub(crate) use rustfs_s3_client::api_get_options::{AdvancedGetOptions, StatObjectOptions}; type ListObjectsV2Info = StorageListObjectsV2Info; type ListObjectVersionsInfo = StorageListObjectVersionsInfo; diff --git a/crates/ecstore/src/client/mod.rs b/crates/ecstore/src/client/mod.rs deleted file mode 100644 index de0c2f7a5..000000000 --- a/crates/ecstore/src/client/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The S3-consuming client moved to the `rustfs-s3-client` crate -// (rustfs/backlog#1842). This shim keeps `crate::client::*` paths working for -// in-crate consumers during the migration window; it is deleted once every -// consumer imports `rustfs_s3_client` directly. Only the two server-side -// modules below (misfiled here historically) remain as real ecstore code. - -pub use rustfs_s3_client::{ - admin_handler_utils, api_get_options, api_list, api_put_object, api_remove, api_s3_datatypes, credentials, provider_versions, - signer_error, transition_api, -}; - -pub mod object_api_utils; -pub mod object_handlers_common; diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index da4abbd39..bd2f1fa58 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -56,7 +56,6 @@ mod storage_api_contracts; mod store; // pub mod checksum; -mod client; mod event; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 0099047c1..769ade684 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -14,6 +14,8 @@ // #730: object API readers keep staged compatibility paths during facade migration. +pub mod object_api_utils; + use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::replication::{ DeleteReplicationConfigSnapshot, ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, diff --git a/crates/ecstore/src/client/object_api_utils.rs b/crates/ecstore/src/object_api/object_api_utils.rs similarity index 100% rename from crates/ecstore/src/client/object_api_utils.rs rename to crates/ecstore/src/object_api/object_api_utils.rs diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index d9623684d..3dc112057 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -68,7 +68,6 @@ use tokio::io::AsyncReadExt; use tokio::sync::{Mutex, Notify, RwLock}; use uuid::Uuid; -use crate::client::transition_api::{ReadCloser, ReaderImpl}; use crate::disk::endpoint::Endpoint; use crate::disk::format::FormatV3; use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk}; @@ -78,6 +77,7 @@ use crate::services::tier::warm_backend::{ TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, }; use rustfs_filemeta::FileMeta; +use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl}; use rustfs_utils::path::path_join_buf; /// One-shot barrier before rejected transition cleanup resolves its ECStore. diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index ab5e02823..bac56bd0b 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -47,7 +47,6 @@ use tokio::{ }; use tracing::{debug, error, info, warn}; -use crate::client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities}; use crate::error::{Error, Result, StorageError}; use crate::services::tier::{ tier_admin::TierCreds, @@ -80,6 +79,7 @@ use crate::{ }; use rustfs_filemeta::FileInfo; use rustfs_rio::HashReader; +use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; use s3s::S3ErrorCode; @@ -1529,14 +1529,14 @@ impl WarmBackend for SharedWarmBackendProxy { self.0.validate_remote_version_id(remote_version_id) } - async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result { + async fn put(&self, object: &str, r: rustfs_s3_client::transition_api::ReaderImpl, length: i64) -> io::Result { self.0.put(object, r, length).await } async fn put_with_meta( &self, object: &str, - r: crate::client::transition_api::ReaderImpl, + r: rustfs_s3_client::transition_api::ReaderImpl, length: i64, meta: HashMap, ) -> io::Result { @@ -1548,7 +1548,7 @@ impl WarmBackend for SharedWarmBackendProxy { object: &str, rv: &str, opts: crate::services::tier::warm_backend::WarmBackendGetOpts, - ) -> io::Result { + ) -> io::Result { self.0.get(object, rv, opts).await } @@ -5778,8 +5778,8 @@ mod tests { // lets us drive `remove`/`verify` through every branch. // --------------------------------------------------------------------- - use crate::client::transition_api::{ReadCloser, ReaderImpl}; use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts}; + use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl}; fn empty_mgr() -> TierConfigMgr { TierConfigMgr { diff --git a/crates/ecstore/src/services/tier/tier_handlers.rs b/crates/ecstore/src/services/tier/tier_handlers.rs index 07768c7ae..b68d19886 100644 --- a/crates/ecstore/src/services/tier/tier_handlers.rs +++ b/crates/ecstore/src/services/tier/tier_handlers.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::client::admin_handler_utils::AdminError; use http::status::StatusCode; use lazy_static::lazy_static; +use rustfs_s3_client::admin_handler_utils::AdminError; lazy_static! { pub static ref ERR_TIER_ALREADY_EXISTS: AdminError = AdminError { diff --git a/crates/ecstore/src/services/tier/tier_mutation_peer.rs b/crates/ecstore/src/services/tier/tier_mutation_peer.rs index e0207f070..a7f261e2f 100644 --- a/crates/ecstore/src/services/tier/tier_mutation_peer.rs +++ b/crates/ecstore/src/services/tier/tier_mutation_peer.rs @@ -22,9 +22,9 @@ use super::tier_mutation_intent::{ MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent, load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent, }; -use crate::client::admin_handler_utils::AdminError; use crate::error::{Error, StorageError}; use crate::store::ECStore; +use rustfs_s3_client::admin_handler_utils::AdminError; pub const MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE; diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 464dd41a9..8e0a59546 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -18,11 +18,6 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::{AdvancedPutOptions, PutObjectOptions}, - transition_api::{ReadCloser, ReaderImpl}, -}; use crate::error::is_err_bucket_not_found; use crate::services::tier::{ tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, @@ -41,6 +36,11 @@ use crate::services::tier::{ }; use bytes::Bytes; use http::StatusCode; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::{AdvancedPutOptions, PutObjectOptions}, + transition_api::{ReadCloser, ReaderImpl}, +}; use rustfs_utils::http::headers::{ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, }; diff --git a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs index ddf283742..410913c14 100644 --- a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, -}; use crate::services::tier::{ tier_config::TierAliyun, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; diff --git a/crates/ecstore/src/services/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs index 5dd0b8c67..61b3d2e9e 100644 --- a/crates/ecstore/src/services/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, -}; use crate::services::tier::{ tier_config::TierAzure, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index 35aecfb6d..adf6bf11b 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -30,15 +30,15 @@ use google_cloud_storage::client::Storage; use google_cloud_storage::client::StorageControl; use std::convert::TryFrom; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - transition_api::{Options, ReadCloser, ReaderImpl}, -}; use crate::services::tier::{ tier_config::TierGCS, warm_backend::{WarmBackend, WarmBackendGetOpts}, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + transition_api::{Options, ReadCloser, ReaderImpl}, +}; use tracing::warn; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; diff --git a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index e33736750..4e0900fd9 100644 --- a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, -}; use crate::services::tier::{ tier_config::TierHuaweicloud, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; diff --git a/crates/ecstore/src/services/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs index a97a01baf..8205a3e56 100644 --- a/crates/ecstore/src/services/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, -}; use crate::services::tier::{ tier_config::TierMinIO, warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; diff --git a/crates/ecstore/src/services/tier/warm_backend_r2.rs b/crates/ecstore/src/services/tier/warm_backend_r2.rs index cee1e1108..685c3338e 100644 --- a/crates/ecstore/src/services/tier/warm_backend_r2.rs +++ b/crates/ecstore/src/services/tier/warm_backend_r2.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, -}; use crate::services::tier::{ tier_config::TierR2, warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; diff --git a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs index 503f1238e..dc1f4aec5 100644 --- a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, -}; use crate::services::tier::{ tier_config::TierRustFS, warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; const MAX_PARTS_COUNT: i64 = 10000; diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 814947987..55a29418c 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -22,7 +22,15 @@ use std::collections::HashMap; use std::sync::Arc; use url::Url; -use crate::client::{ +use crate::services::tier::{ + tier_config::TierS3, + warm_backend::{ + TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts, + build_transition_put_options, + }, +}; +use http::HeaderMap; +use rustfs_s3_client::{ api_get_options::GetObjectOptions, api_list::ListObjectsOptions, api_put_object::PutObjectOptions, @@ -33,14 +41,6 @@ use crate::client::{ transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}, transition_api::{ReadCloser, ReaderImpl}, }; -use crate::services::tier::{ - tier_config::TierS3, - warm_backend::{ - TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts, - build_transition_put_options, - }, -}; -use http::HeaderMap; use rustfs_utils::egress::validate_outbound_url; use rustfs_utils::path::SLASH_SEPARATOR; use s3s::dto::BucketVersioningStatus; @@ -379,7 +379,7 @@ impl TransitionCandidateVersions { #[cfg(test)] mod tests { use super::*; - use crate::client::api_s3_datatypes::{ListVersionsResult, Version}; + use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version}; #[tokio::test] async fn new_rejects_loopback_endpoint_before_network_setup() { diff --git a/crates/ecstore/src/services/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs index 9b5fa33a9..3d7e856e4 100644 --- a/crates/ecstore/src/services/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -21,17 +21,17 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::client::{ - admin_handler_utils::AdminError, - api_put_object::PutObjectOptions, - credentials::{Credentials, SignatureType, Static, Value}, - transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, -}; use crate::services::tier::{ tier_config::TierTencent, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; diff --git a/crates/ecstore/src/services/tier/warm_backend_wasabi.rs b/crates/ecstore/src/services/tier/warm_backend_wasabi.rs index cb367bcdc..2311e6576 100644 --- a/crates/ecstore/src/services/tier/warm_backend_wasabi.rs +++ b/crates/ecstore/src/services/tier/warm_backend_wasabi.rs @@ -21,14 +21,12 @@ use std::{ use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID}; use uuid::Uuid; -use crate::{ - client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl}, - services::tier::{ - tier_config::{TierS3, TierWasabi}, - warm_backend::{WarmBackend, WarmBackendGetOpts}, - warm_backend_s3::WarmBackendS3, - }, +use crate::services::tier::{ + tier_config::{TierS3, TierWasabi}, + warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend_s3::WarmBackendS3, }; +use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl}; const WASABI_VERSIONING_DRIFT_ERROR: &str = "Wasabi tier bucket versioning changed after configuration"; @@ -184,7 +182,7 @@ impl WarmBackend for WarmBackendWasabi { #[cfg(test)] mod tests { use super::*; - use crate::client::{ + use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, TransitionCore}, }; @@ -221,7 +219,7 @@ mod tests { async fn backend_for_endpoint(endpoint: &str, max_retries: i64) -> WarmBackendWasabi { let client = Arc::new( - crate::client::transition_api::TransitionClient::new( + rustfs_s3_client::transition_api::TransitionClient::new( endpoint, Options { creds: Credentials::new(Static(Value { @@ -653,7 +651,7 @@ mod tests { assert_ne!(read, 0, "connection closed before request headers were received"); request.extend_from_slice(&buffer[..read]); } - let body = vec![b'x'; crate::client::transition_api::MAX_S3_ERROR_RESPONSE_SIZE + 1]; + let body = vec![b'x'; rustfs_s3_client::transition_api::MAX_S3_ERROR_RESPONSE_SIZE + 1]; let head = format!( "HTTP/1.1 500 Internal Server Error\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len() diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 71fbf5aee..0358ae613 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -55,7 +55,6 @@ use crate::bucket::replication::{ }; use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; -use crate::client::{object_api_utils::get_raw_etag, transition_api::ObjectReader, transition_api::ReaderImpl}; use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::data_usage::record_compression_total_memory; use crate::diagnostics::get::{ @@ -88,6 +87,7 @@ use crate::error::{GenericError, ObjectApiError, is_err_object_not_found}; use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_reader_from_bytes, create_bitrot_writer}; use crate::object_api::ObjectOptions; use crate::object_api::get_object_body_cache_hook; +use crate::object_api::object_api_utils::get_raw_etag; use crate::runtime::instance::{InstanceContext, bootstrap_ctx}; use crate::runtime::sources as runtime_sources; use crate::services::batch_processor::AsyncBatchProcessor; @@ -151,6 +151,7 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use rustfs_object_capacity::capacity_scope::{ CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope, }; +use rustfs_s3_client::transition_api::{ObjectReader, ReaderImpl}; use rustfs_s3_types::EventName; #[cfg(test)] use rustfs_utils::http::SSEC_ALGORITHM_HEADER; diff --git a/docs/architecture/ecstore-module-split-plan.md b/docs/architecture/ecstore-module-split-plan.md index 6d8d7f85a..ef5f12362 100644 --- a/docs/architecture/ecstore-module-split-plan.md +++ b/docs/architecture/ecstore-module-split-plan.md @@ -13,6 +13,7 @@ and rollback steps. | Bucket replication | `crates/ecstore/src/bucket/replication/` | 15,619 lines | Contracts extracted; runtime move pending | | Set disks | `crates/ecstore/src/set_disk/` | state carrier plus operation modules | Keep in ECStore | | Public ECStore facade | `crates/ecstore/src/api/mod.rs` | broad compatibility surface | Shrink only through guarded PRs | +| Embedded S3 client | `crates/s3-client/` (`rustfs-s3-client`) | ~8.4K lines | Extracted (rustfs/backlog#1842) | Measured 2026-08-12: the whole crate is 265 files / ~288K lines (roughly half is inline `#[cfg(test)]` code). The largest single files are `disk/local.rs` @@ -37,6 +38,12 @@ list, multipart, lock, heal, and replication code live in separate modules. The remaining large surface is the shared `SetDisks` state and cross-cutting contracts, not only file layout. +## Completed: S3 Client Extraction (rustfs/backlog#1842) + +`crates/ecstore/src/client/` was a ~8.4K-line hand-written S3 HTTP client the engine uses to *consume* remote S3-compatible endpoints (ILM tier warm backends, transition targets). It was a legitimate engine capability misfiled inside the engine: it pulled `s3s`/`hyper` wire types into ecstore against ARCHITECTURE.md invariant 4, which distinguishes serving the S3 wire protocol (forbidden in ecstore) from consuming it (allowed, but in a dedicated crate). + +The extraction landed as: pure move of the 21 client modules to `crates/s3-client` (`rustfs-s3-client`) with a temporary re-export shim, then direct `rustfs_s3_client::` imports and shim deletion. The two server-side modules historically misfiled under `client/` stayed in ecstore and moved to their real homes: `object_api_utils.rs` under `object_api/`, `object_handlers_common.rs` under `bucket/lifecycle/` (behind the `replication_sink` boundary). The remaining serving-side `s3s` references in ecstore are ratcheted shrink-only by the `S3S_ECSTORE_FILES_BASELINE` counter in `scripts/check_s3s_footprint.sh`; per-module conversions to storage-level types (first: `bucket/object_lock/`) lower the baseline in the same change. + ## Non-Negotiable Rules - Do not split crates in the same PR that moves runtime state or changes diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 670c4b395..3d7dddf62 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -234,6 +234,7 @@ rustfs-policy = { workspace = true } rustfs-protocols = { workspace = true } rustfs-protos = { workspace = true } rustfs-rio = { workspace = true } +rustfs-s3-client = { workspace = true } rustfs-s3-types = { workspace = true } rustfs-s3-ops = { workspace = true } rustfs-security-governance = { workspace = true } diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 3f623eaa6..acab79166 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -416,7 +416,8 @@ pub(crate) mod ecstore_capacity { } pub(crate) mod ecstore_client { - pub(crate) use rustfs_ecstore::api::client::{admin_handler_utils, object_api_utils}; + pub(crate) use rustfs_ecstore::api::object_api_utils; + pub(crate) use rustfs_s3_client::admin_handler_utils; } pub(crate) mod ecstore_compression { diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 1f6490665..d36cf982f 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -595,8 +595,8 @@ require_source_line \ "ECStore bucket metadata public facade explicit module" require_source_line \ "crates/ecstore/src/api/mod.rs" \ - " pub mod admin_handler_utils {" \ - "ECStore client admin handler public facade explicit module" + "pub mod object_api_utils {" \ + "ECStore object-api utils public facade explicit module" require_source_line \ "crates/ecstore/src/api/mod.rs" \ " pub mod com {" \ @@ -609,7 +609,7 @@ require_source_line \ "crates/ecstore/src/api/mod.rs" \ " pub mod tier_config {" \ "ECStore tier config public facade explicit module" -for ecstore_explicit_facade in bucket client; do +for ecstore_explicit_facade in bucket; do if grep -qF "pub use crate::${ecstore_explicit_facade}::{" "${ROOT_DIR}/crates/ecstore/src/api/mod.rs"; then report_failure "ECStore ${ecstore_explicit_facade} public facade must expose explicit submodules instead of whole owner module passthroughs" fi @@ -626,7 +626,6 @@ fi for ecstore_private_module in \ bucket \ cache_value \ - client \ config \ data_usage \ diagnostics \ diff --git a/scripts/ecstore-module-lint-register.txt b/scripts/ecstore-module-lint-register.txt index 69dd097dc..1033461f3 100644 --- a/scripts/ecstore-module-lint-register.txt +++ b/scripts/ecstore-module-lint-register.txt @@ -67,9 +67,9 @@ crates/s3-client/src/constants.rs|unused_variables crates/s3-client/src/credentials.rs|clippy::all crates/s3-client/src/credentials.rs|unused_must_use crates/s3-client/src/credentials.rs|unused_variables -crates/ecstore/src/client/object_api_utils.rs|clippy::all -crates/ecstore/src/client/object_api_utils.rs|unused_must_use -crates/ecstore/src/client/object_api_utils.rs|unused_variables +crates/ecstore/src/object_api/object_api_utils.rs|clippy::all +crates/ecstore/src/object_api/object_api_utils.rs|unused_must_use +crates/ecstore/src/object_api/object_api_utils.rs|unused_variables crates/s3-client/src/transition_api.rs|clippy::all crates/s3-client/src/transition_api.rs|unused_must_use crates/s3-client/src/transition_api.rs|unused_variables