mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-30 00:47:13 +00:00
refactor(ecstore): drop the client shim, import rustfs-s3-client directly (#6668)
* refactor(ecstore): drop the client shim, import rustfs-s3-client directly Completes the migration window opened by the rustfs-s3-client extraction (rustfs/backlog#1842 PR3): every consumer now imports the client crate directly and the crate::client shim is deleted. - All in-crate crate::client:: paths (tier warm backends, tier core, lifecycle tier_sweeper, replication storage boundary, set_disk) now import rustfs_s3_client::* directly; crates/ecstore/src/client/mod.rs and the lib.rs mod client declaration are gone. - The two server-side modules historically misfiled under client/ move to their real homes: object_api_utils.rs to crates/ecstore/src/object_api/ (it builds engine-side object readers/writers), and object_handlers_common.rs to crates/ecstore/src/bucket/lifecycle/ (it is the lifecycle noncurrent-version cleanup helper). The latter now routes its replication calls through the lifecycle replication_sink boundary (schedule_delete wrapper and the sink's ReplicationObjectBridge re-export), as the lifecycle guard requires. - The ecstore public facade drops api::client: object_api_utils is exposed as api::object_api_utils, and the rustfs crate takes admin_handler_utils (AdminError) from rustfs-s3-client directly (new dependency). - Guard updates: the migration guard no longer pins mod client in ecstore's lib.rs or the admin_handler_utils facade module (it pins the new api::object_api_utils facade instead), and the module-lint register follows object_api_utils.rs to its new path. Verification: cargo check -p rustfs-ecstore --all-targets and -p rustfs; cargo fmt --all; tier/transition/lifecycle-focused nextest (626 passed) and the decommission/rebalance/heal families in a filtered run (603 passed; the full-suite parallel run only fails on this machine's known decommission/rebalance baseline flakes, which pass in filtered reruns and fail identically on pristine origin/main); layer/migration/s3s/logging/error-format/doc-path guard scripts all pass. * docs(architecture): record the S3 client extraction and reword invariant 4 (#6669) Closes the documentation step of rustfs/backlog#1842. ARCHITECTURE.md invariant 4 now states the serving-vs-consuming distinction the adversarial ruling asked for: ecstore must not serve HTTP/S3 wire types, while consuming remote S3 endpoints is a legitimate engine capability that lives in the extracted rustfs-s3-client crate. The violation note is updated from the pre-extraction snapshot (58 files, embedded client) to the current ratcheted state (shrink-only S3S_ECSTORE_FILES_BASELINE in scripts/check_s3s_footprint.sh, object_lock converted first), and the crate map gains s3-client. ecstore-module-split-plan.md gets the client-directory entry the plan was missing: a Current Shape row and a completed-extraction section describing the pure-move + shim + direct-import sequence and the re-homing of the two misfiled server-side modules.
This commit is contained in:
@@ -68,7 +68,6 @@ use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Mutex, Notify, RwLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::client::transition_api::{ReadCloser, ReaderImpl};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::format::FormatV3;
|
||||
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
|
||||
@@ -78,6 +77,7 @@ use crate::services::tier::warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
|
||||
|
||||
@@ -47,7 +47,6 @@ use tokio::{
|
||||
};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use crate::client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
|
||||
use crate::error::{Error, Result, StorageError};
|
||||
use crate::services::tier::{
|
||||
tier_admin::TierCreds,
|
||||
@@ -80,6 +79,7 @@ use crate::{
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
@@ -1529,14 +1529,14 @@ impl WarmBackend for SharedWarmBackendProxy {
|
||||
self.0.validate_remote_version_id(remote_version_id)
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
|
||||
async fn put(&self, object: &str, r: rustfs_s3_client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
|
||||
self.0.put(object, r, length).await
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: crate::client::transition_api::ReaderImpl,
|
||||
r: rustfs_s3_client::transition_api::ReaderImpl,
|
||||
length: i64,
|
||||
meta: HashMap<String, String>,
|
||||
) -> io::Result<String> {
|
||||
@@ -1548,7 +1548,7 @@ impl WarmBackend for SharedWarmBackendProxy {
|
||||
object: &str,
|
||||
rv: &str,
|
||||
opts: crate::services::tier::warm_backend::WarmBackendGetOpts,
|
||||
) -> io::Result<crate::client::transition_api::ReadCloser> {
|
||||
) -> io::Result<rustfs_s3_client::transition_api::ReadCloser> {
|
||||
self.0.get(object, rv, opts).await
|
||||
}
|
||||
|
||||
@@ -5778,8 +5778,8 @@ mod tests {
|
||||
// lets us drive `remove`/`verify` through every branch.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
use crate::client::transition_api::{ReadCloser, ReaderImpl};
|
||||
use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
|
||||
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
|
||||
|
||||
fn empty_mgr() -> TierConfigMgr {
|
||||
TierConfigMgr {
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::client::admin_handler_utils::AdminError;
|
||||
use http::status::StatusCode;
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_s3_client::admin_handler_utils::AdminError;
|
||||
|
||||
lazy_static! {
|
||||
pub static ref ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
|
||||
|
||||
@@ -22,9 +22,9 @@ use super::tier_mutation_intent::{
|
||||
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
|
||||
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
|
||||
};
|
||||
use crate::client::admin_handler_utils::AdminError;
|
||||
use crate::error::{Error, StorageError};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_s3_client::admin_handler_utils::AdminError;
|
||||
|
||||
pub const MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE;
|
||||
|
||||
|
||||
@@ -18,11 +18,6 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::{AdvancedPutOptions, PutObjectOptions},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
use crate::services::tier::{
|
||||
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
|
||||
@@ -41,6 +36,11 @@ use crate::services::tier::{
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::{AdvancedPutOptions, PutObjectOptions},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierAliyun,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierAzure,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
|
||||
@@ -30,15 +30,15 @@ use google_cloud_storage::client::Storage;
|
||||
use google_cloud_storage::client::StorageControl;
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
transition_api::{Options, ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierGCS,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
transition_api::{Options, ReadCloser, ReaderImpl},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierHuaweicloud,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierMinIO,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierR2,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierRustFS,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
|
||||
@@ -22,7 +22,15 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use url::Url;
|
||||
|
||||
use crate::client::{
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options,
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_s3_client::{
|
||||
api_get_options::GetObjectOptions,
|
||||
api_list::ListObjectsOptions,
|
||||
api_put_object::PutObjectOptions,
|
||||
@@ -33,14 +41,6 @@ use crate::client::{
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options,
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::dto::BucketVersioningStatus;
|
||||
@@ -379,7 +379,7 @@ impl TransitionCandidateVersions {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::client::api_s3_datatypes::{ListVersionsResult, Version};
|
||||
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
|
||||
@@ -21,17 +21,17 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use crate::services::tier::{
|
||||
tier_config::TierTencent,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
|
||||
@@ -21,14 +21,12 @@ use std::{
|
||||
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl},
|
||||
services::tier::{
|
||||
tier_config::{TierS3, TierWasabi},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
},
|
||||
use crate::services::tier::{
|
||||
tier_config::{TierS3, TierWasabi},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
|
||||
|
||||
const WASABI_VERSIONING_DRIFT_ERROR: &str = "Wasabi tier bucket versioning changed after configuration";
|
||||
|
||||
@@ -184,7 +182,7 @@ impl WarmBackend for WarmBackendWasabi {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::client::{
|
||||
use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, TransitionCore},
|
||||
};
|
||||
@@ -221,7 +219,7 @@ mod tests {
|
||||
|
||||
async fn backend_for_endpoint(endpoint: &str, max_retries: i64) -> WarmBackendWasabi {
|
||||
let client = Arc::new(
|
||||
crate::client::transition_api::TransitionClient::new(
|
||||
rustfs_s3_client::transition_api::TransitionClient::new(
|
||||
endpoint,
|
||||
Options {
|
||||
creds: Credentials::new(Static(Value {
|
||||
@@ -653,7 +651,7 @@ mod tests {
|
||||
assert_ne!(read, 0, "connection closed before request headers were received");
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
}
|
||||
let body = vec![b'x'; crate::client::transition_api::MAX_S3_ERROR_RESPONSE_SIZE + 1];
|
||||
let body = vec![b'x'; rustfs_s3_client::transition_api::MAX_S3_ERROR_RESPONSE_SIZE + 1];
|
||||
let head = format!(
|
||||
"HTTP/1.1 500 Internal Server Error\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
|
||||
Reference in New Issue
Block a user