gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 08:27:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ecstore): drop the client shim, import rustfs-s3-client directly (#6668)

Browse Source 
* refactor(ecstore): drop the client shim, import rustfs-s3-client directly

Completes the migration window opened by the rustfs-s3-client extraction (rustfs/backlog#1842 PR3): every consumer now imports the client crate directly and the crate::client shim is deleted.

- All in-crate crate::client:: paths (tier warm backends, tier core, lifecycle tier_sweeper, replication storage boundary, set_disk) now import rustfs_s3_client::* directly; crates/ecstore/src/client/mod.rs and the lib.rs mod client declaration are gone.
- The two server-side modules historically misfiled under client/ move to their real homes: object_api_utils.rs to crates/ecstore/src/object_api/ (it builds engine-side object readers/writers), and object_handlers_common.rs to crates/ecstore/src/bucket/lifecycle/ (it is the lifecycle noncurrent-version cleanup helper). The latter now routes its replication calls through the lifecycle replication_sink boundary (schedule_delete wrapper and the sink's ReplicationObjectBridge re-export), as the lifecycle guard requires.
- The ecstore public facade drops api::client: object_api_utils is exposed as api::object_api_utils, and the rustfs crate takes admin_handler_utils (AdminError) from rustfs-s3-client directly (new dependency).
- Guard updates: the migration guard no longer pins mod client in ecstore's lib.rs or the admin_handler_utils facade module (it pins the new api::object_api_utils facade instead), and the module-lint register follows object_api_utils.rs to its new path.

Verification: cargo check -p rustfs-ecstore --all-targets and -p rustfs; cargo fmt --all; tier/transition/lifecycle-focused nextest (626 passed) and the decommission/rebalance/heal families in a filtered run (603 passed; the full-suite parallel run only fails on this machine's known decommission/rebalance baseline flakes, which pass in filtered reruns and fail identically on pristine origin/main); layer/migration/s3s/logging/error-format/doc-path guard scripts all pass.

* docs(architecture): record the S3 client extraction and reword invariant 4 (#6669)

Closes the documentation step of rustfs/backlog#1842. ARCHITECTURE.md invariant 4 now states the serving-vs-consuming distinction the adversarial ruling asked for: ecstore must not serve HTTP/S3 wire types, while consuming remote S3 endpoints is a legitimate engine capability that lives in the extracted rustfs-s3-client crate. The violation note is updated from the pre-extraction snapshot (58 files, embedded client) to the current ratcheted state (shrink-only S3S_ECSTORE_FILES_BASELINE in scripts/check_s3s_footprint.sh, object_lock converted first), and the crate map gains s3-client. ecstore-module-split-plan.md gets the client-directory entry the plan was missing: a Current Shape row and a completed-extraction section describing the pure-move + shim + direct-import sequence and the re-homing of the two misfiled server-side modules.
This commit is contained in:
Zhengchao An
2026-08-26 22:02:36 +08:00
committed by GitHub
parent 0e56ef4f1c
commit 2ebf8bc138
33 changed files with 126 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/api/mod.rs
+2 -20
View File
@@ -255,26 +255,8 @@ pub mod capacity {
pub use crate::store::utils::is_reserved_or_invalid_bucket;
}
pub mod client {
pub mod admin_handler_utils {
pub use crate::client::admin_handler_utils::AdminError;
}
pub mod api_put_object {
pub use crate::client::api_put_object::{AdvancedPutOptions, PutObjectOptions};
}
pub mod object_api_utils {
pub use crate::client::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag};
}
pub mod transition_api {
pub use crate::client::transition_api::{
BucketLookupType, CreateBucketConfiguration, LocationConstraint, ObjectInfo, ObjectMultipartInfo, Options,
PutObjectPartOptions, ReadCloser, ReaderImpl, RequestMetadata, RestoreInfo, SendRequest, TransitionClient,
TransitionCore, UploadInfo, to_object_info,
};
}
pub mod object_api_utils {
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag};
}
pub mod cluster {
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+5 -5
View File
@@ -939,7 +939,7 @@ impl ExpiryState {
let version_count = u64::try_from(v.versions.len()).unwrap_or(u64::MAX);
let trace = LifecycleExpiryTrace::for_batch(&v.bucket, &v.event, &v.src, version_count);
trace.emit(EVENT_LIFECYCLE_DELETE_DISPATCHED, "delete_dispatched", None);
crate::client::object_handlers_common::delete_object_versions(
crate::bucket::lifecycle::object_handlers_common::delete_object_versions(
&api,
&v.bucket,
&v.versions,
@@ -5431,8 +5431,6 @@ mod tests {
use crate::bucket::lifecycle::tier_sweeper::Jentry;
use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG};
use crate::bucket::metadata_sys;
#[cfg(feature = "test-util")]
use crate::client::transition_api::ReaderImpl;
use crate::disk::endpoint::Endpoint;
use crate::disk::{RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
use crate::error::{Error, is_err_invalid_upload_id};
@@ -5464,6 +5462,8 @@ mod tests {
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_data_usage::TierStats;
use rustfs_filemeta::{FileInfo, FileMeta};
#[cfg(feature = "test-util")]
use rustfs_s3_client::transition_api::ReaderImpl;
use rustfs_scanner_contracts::metrics::{IlmAction, global_metrics};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
@@ -6352,7 +6352,7 @@ mod tests {
assert_eq!(err.kind(), std::io::ErrorKind::Other);
let admin_err = err
.get_ref()
.and_then(|source| source.downcast_ref::<crate::client::admin_handler_utils::AdminError>())
.and_then(|source| source.downcast_ref::<rustfs_s3_client::admin_handler_utils::AdminError>())
.expect("identity mismatch should retain the typed tier error");
assert_eq!(admin_err.code, crate::services::tier::tier::ERR_TIER_INVALID_CONFIG.code);
assert_eq!(new_backend.get_count().await, 0);
@@ -11563,7 +11563,7 @@ mod tests {
lease
.put(
"remote/object",
crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")),
rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")),
9,
)
.await
crates/ecstore/src/bucket/lifecycle/mod.rs
+1
View File
@@ -21,6 +21,7 @@ pub mod evaluator;
pub mod manual_transition_job;
mod metadata_boundary;
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
mod object_handlers_common;
mod object_lock_boundary;
pub use self::core as lifecycle;
mod replication_sink;
crates/ecstore/src/client/object_handlers_common.rs → crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs
+2 -2
View File
@@ -21,7 +21,7 @@ const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped";
const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed";
use crate::bucket::lifecycle::lifecycle;
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationObjectBridge};
use crate::bucket::lifecycle::replication_sink::{self, ReplicationObjectBridge};
use crate::object_api::ObjectOptions;
use crate::storage_api_contracts::object::{ObjectOperations as _, ObjectToDelete};
use crate::store::ECStore;
@@ -84,7 +84,7 @@ pub async fn delete_object_versions(
if deleted_obj.replication_state.is_none() {
continue;
}
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
}
for (i, err) in errors.iter().enumerate() {
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs
+2 -2
View File
@@ -22,11 +22,11 @@ use super::runtime_boundary as runtime_sources;
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts};
use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry;
use crate::client::signer_error::error_chain_contains_signer_header_marker;
use crate::object_api::ObjectInfo;
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease};
use crate::storage_api_contracts::lifecycle::TransitionedObject;
use crate::store::ECStore;
use rustfs_s3_client::signer_error::error_chain_contains_signer_header_marker;
use rustfs_utils::get_env_usize;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
@@ -670,7 +670,7 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
#[cfg(test)]
mod test {
use crate::client::signer_error::invalid_utf8_header_error;
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
use super::{
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
crates/ecstore/src/bucket/replication/replication_storage_boundary.rs
+1 -1
View File
@@ -18,7 +18,6 @@ use tokio_util::sync::CancellationToken;
use super::replication_error_boundary::Error;
use super::replication_filemeta_boundary::{replication_state_from_filemeta, version_purge_status_from_filemeta};
pub(crate) type ReplicationObjectStore = crate::store::ECStore;
pub(crate) use crate::client::api_get_options::{AdvancedGetOptions, StatObjectOptions};
pub(crate) use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
pub(crate) use crate::storage_api_contracts::list::{
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
@@ -29,6 +28,7 @@ pub(crate) use crate::storage_api_contracts::object::{
};
pub(crate) use crate::storage_api_contracts::range::HTTPRangeSpec;
pub(crate) use rustfs_replication::{DeletedObject as ReplicationDeletedObject, ObjectToDelete as ReplicationObjectToDelete};
pub(crate) use rustfs_s3_client::api_get_options::{AdvancedGetOptions, StatObjectOptions};
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
crates/ecstore/src/client/mod.rs
-27
View File
@@ -1,27 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The S3-consuming client moved to the `rustfs-s3-client` crate
// (rustfs/backlog#1842). This shim keeps `crate::client::*` paths working for
// in-crate consumers during the migration window; it is deleted once every
// consumer imports `rustfs_s3_client` directly. Only the two server-side
// modules below (misfiled here historically) remain as real ecstore code.
pub use rustfs_s3_client::{
admin_handler_utils, api_get_options, api_list, api_put_object, api_remove, api_s3_datatypes, credentials, provider_versions,
signer_error, transition_api,
};
pub mod object_api_utils;
pub mod object_handlers_common;
crates/ecstore/src/lib.rs
-1
View File
@@ -56,7 +56,6 @@ mod storage_api_contracts;
mod store;
// pub mod checksum;
mod client;
mod event;
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
crates/ecstore/src/object_api/mod.rs
+2
View File
@@ -14,6 +14,8 @@
// #730: object API readers keep staged compatibility paths during facade migration.
pub mod object_api_utils;
use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::replication::{
DeleteReplicationConfigSnapshot, ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
crates/ecstore/src/client/object_api_utils.rs → crates/ecstore/src/object_api/object_api_utils.rs
View File
crates/ecstore/src/services/tier/test_util.rs
+1 -1
View File
@@ -68,7 +68,6 @@ use tokio::io::AsyncReadExt;
use tokio::sync::{Mutex, Notify, RwLock};
use uuid::Uuid;
use crate::client::transition_api::{ReadCloser, ReaderImpl};
use crate::disk::endpoint::Endpoint;
use crate::disk::format::FormatV3;
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
@@ -78,6 +77,7 @@ use crate::services::tier::warm_backend::{
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
use rustfs_filemeta::FileMeta;
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
use rustfs_utils::path::path_join_buf;
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
crates/ecstore/src/services/tier/tier.rs
+5 -5
View File
@@ -47,7 +47,6 @@ use tokio::{
};
use tracing::{debug, error, info, warn};
use crate::client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
use crate::error::{Error, Result, StorageError};
use crate::services::tier::{
tier_admin::TierCreds,
@@ -80,6 +79,7 @@ use crate::{
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
@@ -1529,14 +1529,14 @@ impl WarmBackend for SharedWarmBackendProxy {
self.0.validate_remote_version_id(remote_version_id)
}
async fn put(&self, object: &str, r: crate::client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
async fn put(&self, object: &str, r: rustfs_s3_client::transition_api::ReaderImpl, length: i64) -> io::Result<String> {
self.0.put(object, r, length).await
}
async fn put_with_meta(
&self,
object: &str,
r: crate::client::transition_api::ReaderImpl,
r: rustfs_s3_client::transition_api::ReaderImpl,
length: i64,
meta: HashMap<String, String>,
) -> io::Result<String> {
@@ -1548,7 +1548,7 @@ impl WarmBackend for SharedWarmBackendProxy {
object: &str,
rv: &str,
opts: crate::services::tier::warm_backend::WarmBackendGetOpts,
) -> io::Result<crate::client::transition_api::ReadCloser> {
) -> io::Result<rustfs_s3_client::transition_api::ReadCloser> {
self.0.get(object, rv, opts).await
}
@@ -5778,8 +5778,8 @@ mod tests {
// lets us drive `remove`/`verify` through every branch.
// ---------------------------------------------------------------------
use crate::client::transition_api::{ReadCloser, ReaderImpl};
use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
fn empty_mgr() -> TierConfigMgr {
TierConfigMgr {
crates/ecstore/src/services/tier/tier_handlers.rs
+1 -1
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::client::admin_handler_utils::AdminError;
use http::status::StatusCode;
use lazy_static::lazy_static;
use rustfs_s3_client::admin_handler_utils::AdminError;
lazy_static! {
pub static ref ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
crates/ecstore/src/services/tier/tier_mutation_peer.rs
+1 -1
View File
@@ -22,9 +22,9 @@ use super::tier_mutation_intent::{
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
};
use crate::client::admin_handler_utils::AdminError;
use crate::error::{Error, StorageError};
use crate::store::ECStore;
use rustfs_s3_client::admin_handler_utils::AdminError;
pub const MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE;
crates/ecstore/src/services/tier/warm_backend.rs
+5 -5
View File
@@ -18,11 +18,6 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::{AdvancedPutOptions, PutObjectOptions},
transition_api::{ReadCloser, ReaderImpl},
};
use crate::error::is_err_bucket_not_found;
use crate::services::tier::{
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
@@ -41,6 +36,11 @@ use crate::services::tier::{
};
use bytes::Bytes;
use http::StatusCode;
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::{AdvancedPutOptions, PutObjectOptions},
transition_api::{ReadCloser, ReaderImpl},
};
use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
};
crates/ecstore/src/services/tier/warm_backend_aliyun.rs
+6 -6
View File
@@ -21,17 +21,17 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::services::tier::{
tier_config::TierAliyun,
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
crates/ecstore/src/services/tier/warm_backend_azure.rs
+6 -6
View File
@@ -21,17 +21,17 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::services::tier::{
tier_config::TierAzure,
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
crates/ecstore/src/services/tier/warm_backend_gcs.rs
+5 -5
View File
@@ -30,15 +30,15 @@ use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use std::convert::TryFrom;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
transition_api::{Options, ReadCloser, ReaderImpl},
};
use crate::services::tier::{
tier_config::TierGCS,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
transition_api::{Options, ReadCloser, ReaderImpl},
};
use tracing::warn;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs
+6 -6
View File
@@ -21,17 +21,17 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::services::tier::{
tier_config::TierHuaweicloud,
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
crates/ecstore/src/services/tier/warm_backend_minio.rs
+6 -6
View File
@@ -21,17 +21,17 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::services::tier::{
tier_config::TierMinIO,
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
crates/ecstore/src/services/tier/warm_backend_r2.rs
+6 -6
View File
@@ -21,17 +21,17 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::services::tier::{
tier_config::TierR2,
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
crates/ecstore/src/services/tier/warm_backend_rustfs.rs
+6 -6
View File
@@ -21,17 +21,17 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::services::tier::{
tier_config::TierRustFS,
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
const MAX_PARTS_COUNT: i64 = 10000;
crates/ecstore/src/services/tier/warm_backend_s3.rs
+10 -10
View File
@@ -22,7 +22,15 @@ use std::collections::HashMap;
use std::sync::Arc;
use url::Url;
use crate::client::{
use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options,
},
};
use http::HeaderMap;
use rustfs_s3_client::{
api_get_options::GetObjectOptions,
api_list::ListObjectsOptions,
api_put_object::PutObjectOptions,
@@ -33,14 +41,6 @@ use crate::client::{
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl},
};
use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options,
},
};
use http::HeaderMap;
use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::path::SLASH_SEPARATOR;
use s3s::dto::BucketVersioningStatus;
@@ -379,7 +379,7 @@ impl TransitionCandidateVersions {
#[cfg(test)]
mod tests {
use super::*;
use crate::client::api_s3_datatypes::{ListVersionsResult, Version};
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
#[tokio::test]
async fn new_rejects_loopback_endpoint_before_network_setup() {
crates/ecstore/src/services/tier/warm_backend_tencent.rs
+6 -6
View File
@@ -21,17 +21,17 @@
use std::collections::HashMap;
use std::sync::Arc;
use crate::client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use crate::services::tier::{
tier_config::TierTencent,
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_put_object::PutObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
};
use tracing::warn;
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
crates/ecstore/src/services/tier/warm_backend_wasabi.rs
+8 -10
View File
@@ -21,14 +21,12 @@ use std::{
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
use uuid::Uuid;
use crate::{
client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl},
services::tier::{
tier_config::{TierS3, TierWasabi},
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
},
use crate::services::tier::{
tier_config::{TierS3, TierWasabi},
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
const WASABI_VERSIONING_DRIFT_ERROR: &str = "Wasabi tier bucket versioning changed after configuration";
@@ -184,7 +182,7 @@ impl WarmBackend for WarmBackendWasabi {
#[cfg(test)]
mod tests {
use super::*;
use crate::client::{
use rustfs_s3_client::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, TransitionCore},
};
@@ -221,7 +219,7 @@ mod tests {
async fn backend_for_endpoint(endpoint: &str, max_retries: i64) -> WarmBackendWasabi {
let client = Arc::new(
crate::client::transition_api::TransitionClient::new(
rustfs_s3_client::transition_api::TransitionClient::new(
endpoint,
Options {
creds: Credentials::new(Static(Value {
@@ -653,7 +651,7 @@ mod tests {
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
}
let body = vec![b'x'; crate::client::transition_api::MAX_S3_ERROR_RESPONSE_SIZE + 1];
let body = vec![b'x'; rustfs_s3_client::transition_api::MAX_S3_ERROR_RESPONSE_SIZE + 1];
let head = format!(
"HTTP/1.1 500 Internal Server Error\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
crates/ecstore/src/set_disk/mod.rs
+2 -1
View File
@@ -55,7 +55,6 @@ use crate::bucket::replication::{
};
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::client::{object_api_utils::get_raw_etag, transition_api::ObjectReader, transition_api::ReaderImpl};
use crate::cluster::rpc::heal_bucket_local_on_disks;
use crate::data_usage::record_compression_total_memory;
use crate::diagnostics::get::{
@@ -88,6 +87,7 @@ use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_reader_from_bytes, create_bitrot_writer};
use crate::object_api::ObjectOptions;
use crate::object_api::get_object_body_cache_hook;
use crate::object_api::object_api_utils::get_raw_etag;
use crate::runtime::instance::{InstanceContext, bootstrap_ctx};
use crate::runtime::sources as runtime_sources;
use crate::services::batch_processor::AsyncBatchProcessor;
@@ -151,6 +151,7 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_s3_client::transition_api::{ObjectReader, ReaderImpl};
use rustfs_s3_types::EventName;
#[cfg(test)]
use rustfs_utils::http::SSEC_ALGORITHM_HEADER;