refactor(ecstore): extract the embedded S3 client into rustfs-s3-client (#6627)

The storage engine embedded a ~8.4K-line hand-written S3 HTTP client under crates/ecstore/src/client (rustfs/backlog#1842). That client is a legitimate engine capability — it consumes remote S3-compatible endpoints for ILM tier warm backends and transition targets — but it was misfiled inside the engine, dragging s3s/hyper wire types into ecstore and blocking ARCHITECTURE.md invariant 4.

This PR is the pure-move step: 21 modules move verbatim to the new crates/s3-client crate (rustfs-s3-client), and crates/ecstore/src/client/mod.rs becomes a re-export shim so every in-crate crate::client:: path keeps working. The two server-side modules that were historically misfiled under client/ — object_api_utils.rs and object_handlers_common.rs — stay in ecstore.

Three reverse dependencies from the client into engine internals are severed so the move can be pure:

- transition_api::ReaderImpl::ObjectBody held ecstore's GetObjectReader; the client only ever reads the body, so the variant now holds an ObjectReader newtype over Box<dyn AsyncRead + Send + Sync + Unpin> with the same read_all() surface. The single production construction site (set_disk transition upload) and the two engine-side consumers were adjusted.
- api_list/api_remove used ecstore's storage_api_contracts / object_api types; api_list now imports BucketInfo from rustfs-storage-api directly, and api_remove uses the client's own transition_api::ObjectInfo (only .name/.version_id were read; the error-path bucket name is now threaded as a parameter instead of read from the deleted objects).
- the api_put_object_streaming regression tests built a GetObjectReader by hand; they now wrap the duplex stream in ObjectReader::new.

Guard updates: the s3s footprint ratchet gains an ecstore-scoped counter (42 files, shrink-only, per rustfs/backlog#1842), the ecstore module-lint-blanket register follows the moved files into crates/s3-client so the blanket ratchet keeps covering them, the logging guardrail path pin follows transition_api.rs, and the ::other(format!) baseline is regenerated (moved call sites left ecstore).

Verification: cargo check -p rustfs-s3-client -p rustfs-ecstore; cargo nextest run -p rustfs-s3-client (43 passed) and -p rustfs-ecstore (4515/4523; the 8 failures reproduce identically on pristine origin/main on the same machine); cargo clippy --all-targets; scripts/check_layer_dependencies.sh, check_architecture_migration_rules.sh, check_s3s_footprint.sh, check_logging_guardrails.sh, check_error_other_format_ratchet.sh, check_doc_paths.sh, check_ci_paths_sync.sh all pass.
This commit is contained in:
Zhengchao An
2026-08-26 12:38:52 +08:00
committed by GitHub
parent 65a7cc9cd4
commit 8f0d4a20d1
35 changed files with 352 additions and 182 deletions
+172
View File
@@ -0,0 +1,172 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use quick_xml;
use serde::de::Deserialize;
use std::fmt::{Display, Formatter};
use std::io::{Error, ErrorKind};
use time::OffsetDateTime;
#[derive(Debug, Default, Clone, Eq, PartialEq)]
pub enum SignatureType {
#[default]
SignatureDefault,
SignatureV4,
SignatureV2,
SignatureV4Streaming,
SignatureAnonymous,
}
#[derive(Debug, Clone, Default)]
pub struct Credentials<P: Provider + Default> {
creds: Value,
force_refresh: bool,
provider: P,
}
impl<P: Provider + Default> Credentials<P> {
pub fn new(provider: P) -> Self {
Self {
provider,
force_refresh: true,
..Default::default()
}
}
pub fn get(&mut self) -> Result<Value, std::io::Error> {
self.get_with_context(None)
}
pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
if self.is_expired() {
let creds = self.provider.retrieve_with_cred_context(cc.unwrap_or(CredContext {
endpoint: "".to_string(),
}));
self.creds = creds;
self.force_refresh = false;
}
Ok(self.creds.clone())
}
#[allow(
dead_code,
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
)]
fn expire(&mut self) {
self.force_refresh = true;
}
pub fn is_expired(&self) -> bool {
self.force_refresh || self.provider.is_expired()
}
}
#[derive(Debug, Clone)]
pub struct Value {
pub access_key_id: String,
pub secret_access_key: String,
pub session_token: String,
pub expiration: OffsetDateTime,
pub signer_type: SignatureType,
}
impl Default for Value {
fn default() -> Self {
Self {
access_key_id: "".to_string(),
secret_access_key: "".to_string(),
session_token: "".to_string(),
expiration: OffsetDateTime::now_utc(),
signer_type: SignatureType::SignatureDefault,
}
}
}
pub struct CredContext {
//pub client: SendRequest,
pub endpoint: String,
}
pub trait Provider {
fn retrieve(&self) -> Value;
fn retrieve_with_cred_context(&self, _: CredContext) -> Value;
fn is_expired(&self) -> bool;
}
#[derive(Debug, Clone, Default)]
pub struct Static(pub Value);
impl Provider for Static {
fn retrieve(&self) -> Value {
if self.0.access_key_id == "" || self.0.secret_access_key == "" {
return Value {
signer_type: SignatureType::SignatureAnonymous,
..Default::default()
};
}
self.0.clone()
}
fn retrieve_with_cred_context(&self, _: CredContext) -> Value {
self.retrieve()
}
fn is_expired(&self) -> bool {
false
}
}
#[derive(Debug, Clone, Default)]
pub struct STSError {
#[allow(
dead_code,
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
)]
pub r#type: String,
pub code: String,
pub message: String,
}
#[derive(Debug, Clone, thiserror::Error)]
pub struct ErrorResponse {
pub sts_error: STSError,
#[allow(
dead_code,
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
)]
pub request_id: String,
}
impl Display for ErrorResponse {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.error())
}
}
impl ErrorResponse {
fn error(&self) -> String {
if self.sts_error.message == "" {
return format!("Error response code {}.", self.sts_error.code);
}
return self.sts_error.message.clone();
}
}