diff --git a/Cargo.lock b/Cargo.lock index 85c794e23..c2f7fdd3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9552,6 +9552,7 @@ dependencies = [ "rustfs-replication", "rustfs-rio", "rustfs-rio-v2", + "rustfs-s3-client", "rustfs-s3-types", "rustfs-signer", "rustfs-storage-api", @@ -10316,6 +10317,51 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustfs-s3-client" +version = "1.0.0-rc.3" +dependencies = [ + "base64-simd", + "bytes", + "enumset", + "futures", + "futures-util", + "hex-simd", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "lazy_static", + "md-5 0.11.0", + "quick-xml 0.42.0", + "rand 0.10.2", + "rustfs-checksums", + "rustfs-config", + "rustfs-rio", + "rustfs-signer", + "rustfs-storage-api", + "rustfs-tls-runtime", + "rustfs-utils", + "rustls", + "rustls-pki-types", + "s3s", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", + "thiserror 2.0.20", + "time", + "tokio", + "tokio-util", + "tower", + "tracing", + "url", + "urlencoding", + "uuid", +] + [[package]] name = "rustfs-s3-ops" version = "1.0.0-rc.3" diff --git a/Cargo.toml b/Cargo.toml index d66551c31..a7d289245 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build) "crates/replication", # Replication contracts and wire formats "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling + "crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets) "crates/s3-types", # S3 event type definitions "crates/s3-ops", # S3 operation definitions and mapping "crates/s3select-api", # S3 Select API interface @@ -121,6 +122,7 @@ rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" } rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" } rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" } rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" } +rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.3" } rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" } rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" } rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 0bdc39a37..fb223489c 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -147,6 +147,7 @@ rustfs-policy.workspace = true rustfs-protos.workspace = true rustfs-replication.workspace = true rustfs-lifecycle.workspace = true +rustfs-s3-client = { workspace = true } rustfs-s3-types = { workspace = true } rustfs-data-usage.workspace = true rustfs-object-capacity.workspace = true diff --git a/crates/ecstore/src/client/mod.rs b/crates/ecstore/src/client/mod.rs index 1b9977a97..de0c2f7a5 100644 --- a/crates/ecstore/src/client/mod.rs +++ b/crates/ecstore/src/client/mod.rs @@ -12,28 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades. +// The S3-consuming client moved to the `rustfs-s3-client` crate +// (rustfs/backlog#1842). This shim keeps `crate::client::*` paths working for +// in-crate consumers during the migration window; it is deleted once every +// consumer imports `rustfs_s3_client` directly. Only the two server-side +// modules below (misfiled here historically) remain as real ecstore code. + +pub use rustfs_s3_client::{ + admin_handler_utils, api_get_options, api_list, api_put_object, api_remove, api_s3_datatypes, credentials, provider_versions, + signer_error, transition_api, +}; -pub mod admin_handler_utils; -pub mod api_error_response; -pub mod api_get_object; -pub mod api_get_options; -pub mod api_list; -pub mod api_put_object; -pub mod api_put_object_common; -pub mod api_put_object_multipart; -pub mod api_put_object_streaming; -pub mod api_remove; -pub mod api_s3_datatypes; -pub mod api_stat; -pub mod bucket_cache; -pub mod checksum; -pub mod constants; -pub mod credentials; pub mod object_api_utils; pub mod object_handlers_common; -pub(crate) mod provider_versions; -pub(crate) mod runtime_sources; -pub mod signer_error; -pub mod transition_api; -pub mod utils; diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index 751f2e92b..d9623684d 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -596,9 +596,9 @@ impl MockWarmBackend { if let Some(limit) = limit { let limit = u64::try_from(limit).map_err(|_| std::io::Error::other("mock PUT read limit exceeds u64::MAX"))?; - reader.stream.take(limit).read_to_end(&mut buf).await?; + (&mut reader).take(limit).read_to_end(&mut buf).await?; } else { - reader.stream.read_to_end(&mut buf).await?; + reader.read_to_end(&mut buf).await?; } Ok(buf) } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index e504fb2c4..e26fcdd40 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -55,7 +55,7 @@ use crate::bucket::replication::{ }; use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; -use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl}; +use crate::client::{object_api_utils::get_raw_etag, transition_api::ObjectReader, transition_api::ReaderImpl}; use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::data_usage::record_compression_total_memory; use crate::diagnostics::get::{ @@ -1559,8 +1559,8 @@ impl SetDisks { } } -/// Get lock acquire timeout from environment variable RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT (in seconds) -/// Defaults to 5 seconds if not set or invalid +/// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds) +/// Defaults to 30 seconds if not set or invalid /// Lock acquisition timeout. Cached: this is consulted on every object /// lock acquisition and `std::env::var` takes a process-global lock. In test /// builds the env var is read directly so `temp_env` overrides take effect. diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 2de84eaae..87f46e868 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -7332,12 +7332,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let expected_size = u64::try_from(fi.size).map_err(|_| StorageError::FileCorrupt)?; let (pr, pw) = tokio::io::duplex(fi.erasure.block_size); let consumed = Arc::new(AtomicU64::new(0)); - let reader = ReaderImpl::ObjectBody(GetObjectReader { - stream: Box::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))), - object_info: oi, - buffered_body: None, - body_source: GetObjectBodySource::Unprobed, - }); + let reader = ReaderImpl::ObjectBody(ObjectReader::new(TransitionUploadReader::new(pr, Arc::clone(&consumed)))); let cloned_bucket = bucket.to_string(); let cloned_object = object.to_string(); diff --git a/crates/s3-client/Cargo.toml b/crates/s3-client/Cargo.toml new file mode 100644 index 000000000..af63ae896 --- /dev/null +++ b/crates/s3-client/Cargo.toml @@ -0,0 +1,73 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "rustfs-s3-client" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true +homepage.workspace = true +description = "S3 client used by RustFS when the storage engine consumes remote S3-compatible endpoints (tier warm backends, transition targets)." +keywords = ["s3", "client", "tiering", "rustfs", "Minio"] +categories = ["web-programming", "development-tools", "network-programming"] +documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/" + +[dependencies] +rustfs-checksums.workspace = true +rustfs-config.workspace = true +rustfs-rio.workspace = true +rustfs-signer.workspace = true +rustfs-storage-api.workspace = true +rustfs-tls-runtime.workspace = true +rustfs-utils = { workspace = true, features = ["full"] } +base64-simd.workspace = true +bytes = { workspace = true, features = ["serde"] } +hex-simd = { workspace = true } +enumset = { workspace = true } +futures.workspace = true +futures-util.workspace = true +http.workspace = true +http-body = { workspace = true } +http-body-util.workspace = true +hyper = { workspace = true, features = ["http2", "http1", "server"] } +hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] } +hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] } +lazy_static.workspace = true +md-5.workspace = true +quick-xml = { workspace = true, features = ["serialize", "async-tokio"] } +rand = { workspace = true, features = ["serde"] } +rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } +rustls-pki-types.workspace = true +s3s = { workspace = true, features = ["minio"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["raw_value"] } +sha1 = { workspace = true } +sha2 = { workspace = true } +thiserror.workspace = true +time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] } +tokio = { workspace = true, features = ["io-util", "sync", "fs", "rt-multi-thread"] } +tokio-util = { workspace = true, features = ["io", "compat"] } +tower = { workspace = true, features = ["timeout"] } +tracing.workspace = true +url.workspace = true +urlencoding = { workspace = true } +uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] } + +[lints] +workspace = true + +[lib] +doctest = false diff --git a/crates/ecstore/src/client/admin_handler_utils.rs b/crates/s3-client/src/admin_handler_utils.rs similarity index 100% rename from crates/ecstore/src/client/admin_handler_utils.rs rename to crates/s3-client/src/admin_handler_utils.rs diff --git a/crates/ecstore/src/client/api_error_response.rs b/crates/s3-client/src/api_error_response.rs similarity index 100% rename from crates/ecstore/src/client/api_error_response.rs rename to crates/s3-client/src/api_error_response.rs diff --git a/crates/ecstore/src/client/api_get_object.rs b/crates/s3-client/src/api_get_object.rs similarity index 99% rename from crates/ecstore/src/client/api_get_object.rs rename to crates/s3-client/src/api_get_object.rs index 1eb6122ca..9f6f49d59 100644 --- a/crates/ecstore/src/client/api_get_object.rs +++ b/crates/s3-client/src/api_get_object.rs @@ -27,7 +27,7 @@ use std::task::{Context, Poll}; use tokio::io::BufReader; use tokio_util::io::StreamReader; -use crate::client::{ +use crate::{ api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider}, diff --git a/crates/ecstore/src/client/api_get_options.rs b/crates/s3-client/src/api_get_options.rs similarity index 99% rename from crates/ecstore/src/client/api_get_options.rs rename to crates/s3-client/src/api_get_options.rs index 3438453b3..f85fd3d30 100644 --- a/crates/ecstore/src/client/api_get_options.rs +++ b/crates/s3-client/src/api_get_options.rs @@ -25,7 +25,7 @@ use std::collections::HashMap; use time::OffsetDateTime; use tracing::warn; -use crate::client::api_error_response::err_invalid_argument; +use crate::api_error_response::err_invalid_argument; #[derive(Default)] pub struct AdvancedGetOptions { diff --git a/crates/ecstore/src/client/api_list.rs b/crates/s3-client/src/api_list.rs similarity index 99% rename from crates/ecstore/src/client/api_list.rs rename to crates/s3-client/src/api_list.rs index a7f894a00..3bd1f1edd 100644 --- a/crates/ecstore/src/client/api_list.rs +++ b/crates/s3-client/src/api_list.rs @@ -18,7 +18,7 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use crate::client::{ +use crate::{ api_error_response::http_resp_to_error_response, api_s3_datatypes::{ ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart, @@ -26,12 +26,12 @@ use crate::client::{ credentials, transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body}, }; -use crate::storage_api_contracts::bucket::BucketInfo; use http::{HeaderMap, StatusCode}; use http_body_util::BodyExt; use hyper::body::Body; use hyper::body::Bytes; use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE; +use rustfs_storage_api::BucketInfo; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use std::collections::HashMap; use std::io::ErrorKind; diff --git a/crates/ecstore/src/client/api_put_object.rs b/crates/s3-client/src/api_put_object.rs similarity index 99% rename from crates/ecstore/src/client/api_put_object.rs rename to crates/s3-client/src/api_put_object.rs index bf9efe4a2..82797a22f 100644 --- a/crates/ecstore/src/client/api_put_object.rs +++ b/crates/s3-client/src/api_put_object.rs @@ -30,9 +30,9 @@ use s3s::header::{ X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION, }; //use crate::disk::{BufferReader, Reader}; -use crate::client::checksum::ChecksumMode; -use crate::client::utils::base64_encode; -use crate::client::{ +use crate::checksum::ChecksumMode; +use crate::utils::base64_encode; +use crate::{ api_error_response::{err_entity_too_large, err_invalid_argument}, api_put_object_common::optimal_part_info, api_put_object_multipart::UploadPartParams, diff --git a/crates/ecstore/src/client/api_put_object_common.rs b/crates/s3-client/src/api_put_object_common.rs similarity index 99% rename from crates/ecstore/src/client/api_put_object_common.rs rename to crates/s3-client/src/api_put_object_common.rs index b0c405b43..3c019450e 100644 --- a/crates/ecstore/src/client/api_put_object_common.rs +++ b/crates/s3-client/src/api_put_object_common.rs @@ -18,7 +18,7 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use crate::client::{ +use crate::{ api_error_response::{err_entity_too_large, err_invalid_argument}, api_put_object::PutObjectOptions, constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE}, diff --git a/crates/ecstore/src/client/api_put_object_multipart.rs b/crates/s3-client/src/api_put_object_multipart.rs similarity index 98% rename from crates/ecstore/src/client/api_put_object_multipart.rs rename to crates/s3-client/src/api_put_object_multipart.rs index dbb1b3c40..17efdbf3a 100644 --- a/crates/ecstore/src/client/api_put_object_multipart.rs +++ b/crates/s3-client/src/api_put_object_multipart.rs @@ -26,9 +26,9 @@ use time::OffsetDateTime; use tracing::warn; use uuid::Uuid; -use crate::client::checksum::ChecksumMode; -use crate::client::utils::base64_encode; -use crate::client::{ +use crate::checksum::ChecksumMode; +use crate::utils::base64_encode; +use crate::{ api_error_response::{ err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response, }, @@ -448,7 +448,7 @@ pub struct UploadPartParams { #[cfg(test)] mod tests { - use crate::client::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult}; + use crate::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult}; #[test] fn complete_multipart_upload_serializes_s3_part_elements() { diff --git a/crates/ecstore/src/client/api_put_object_streaming.rs b/crates/s3-client/src/api_put_object_streaming.rs similarity index 97% rename from crates/ecstore/src/client/api_put_object_streaming.rs rename to crates/s3-client/src/api_put_object_streaming.rs index b0a665707..9ae54b893 100644 --- a/crates/ecstore/src/client/api_put_object_streaming.rs +++ b/crates/s3-client/src/api_put_object_streaming.rs @@ -31,8 +31,8 @@ use tokio_util::sync::CancellationToken; use tracing::warn; use uuid::Uuid; -use crate::client::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum}; -use crate::client::{ +use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum}; +use crate::{ api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response}, api_put_object::PutObjectOptions, api_put_object_common::{is_object, optimal_part_info}, @@ -42,7 +42,7 @@ use crate::client::{ transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo}, }; -use crate::client::utils::base64_encode; +use crate::utils::base64_encode; use rustfs_utils::path::trim_etag; use s3s::header::X_AMZ_EXPIRATION; @@ -620,7 +620,7 @@ fn collect_complete_parts(parts_info: &HashMap, total_parts_cou #[cfg(test)] mod tests { use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part}; - use crate::object_api::GetObjectReader; + use crate::transition_api::ObjectReader; use bytes::Bytes; use rustfs_utils::hash::HashAlgorithm; use std::collections::HashMap; @@ -654,12 +654,7 @@ mod tests { let data: Vec = (0..total).map(|i| i as u8).collect(); w.write_all(&data).await.unwrap(); }); - let reader = ReaderImpl::ObjectBody(GetObjectReader { - stream: Box::new(r), - object_info: Default::default(), - buffered_body: None, - body_source: Default::default(), - }); + let reader = ReaderImpl::ObjectBody(ObjectReader::new(r)); let sizes = collect_part_sizes(reader, total, 100, 50).await; assert_eq!(sizes, vec![100, 100, 50]); @@ -684,12 +679,7 @@ mod tests { use tokio::io::AsyncWriteExt; w.write_all(&[1u8; 30]).await.unwrap(); }); - let mut reader = ReaderImpl::ObjectBody(GetObjectReader { - stream: Box::new(r), - object_info: Default::default(), - buffered_body: None, - body_source: Default::default(), - }); + let mut reader = ReaderImpl::ObjectBody(ObjectReader::new(r)); let buf = read_multipart_part(&mut reader, 100).await.unwrap(); assert_eq!(buf.len(), 30); } diff --git a/crates/ecstore/src/client/api_remove.rs b/crates/s3-client/src/api_remove.rs similarity index 97% rename from crates/ecstore/src/client/api_remove.rs rename to crates/s3-client/src/api_remove.rs index 573a2eed1..3002e71ca 100644 --- a/crates/ecstore/src/client/api_remove.rs +++ b/crates/s3-client/src/api_remove.rs @@ -36,16 +36,13 @@ use time::OffsetDateTime; use tokio::sync::mpsc::{self, Receiver, Sender}; use tracing::Instrument; -use crate::client::utils::base64_encode; -use crate::client::{ +use crate::transition_api::ObjectInfo; +use crate::utils::base64_encode; +use crate::{ api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response}, api_s3_datatypes::{DeleteMultiObjects, DeleteObject}, transition_api::{ReaderImpl, RequestMetadata, TransitionClient}, }; -use crate::{ - disk::DiskAPI, - object_api::{GetObjectReader, ObjectInfo}, -}; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; pub struct RemoveBucketOptions { @@ -366,7 +363,13 @@ impl TransitionClient { body_vec.extend_from_slice(data); } } - process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), &batch, result_tx.clone()).await; + process_remove_multi_objects_response( + ReaderImpl::Body(Bytes::from(body_vec)), + bucket_name, + &batch, + result_tx.clone(), + ) + .await; } Ok(()) } @@ -553,6 +556,7 @@ pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec pub async fn process_remove_multi_objects_response( body: ReaderImpl, + bucket_name: &str, objects: &[ObjectInfo], result_tx: Sender, ) { @@ -575,7 +579,7 @@ pub async fn process_remove_multi_objects_response( err: Some(std::io::Error::other(ErrorResponse { code: S3ErrorCode::Custom("ReadDeleteResponseFailed".into()), message: format!("read multi remove response failed: {err}"), - bucket_name: object.bucket.clone(), + bucket_name: bucket_name.to_string(), key: object.name.clone(), resource: "".to_string(), request_id: "".to_string(), @@ -647,7 +651,7 @@ pub async fn process_remove_multi_objects_response( "unmarshal multi remove response failed: {err}; response_body={}", body.chars().take(DELETE_RESPONSE_PREVIEW_LEN).collect::() ), - bucket_name: object.bucket.clone(), + bucket_name: bucket_name.to_string(), key: object.name.clone(), resource: "".to_string(), request_id: "".to_string(), @@ -707,13 +711,7 @@ pub async fn process_remove_multi_objects_response( } for (object_name, object_version_id) in pending { - let bucket_name = objects - .iter() - .find(|object| { - object.name == object_name && object.version_id.as_ref().map(|v| v.to_string()) == Some(object_version_id.clone()) - }) - .map(|o| o.bucket.clone()) - .unwrap_or_default(); + let bucket_name = bucket_name.to_string(); let object_name = object_name; let object_version_id = object_version_id; let error_message = format!( @@ -750,7 +748,7 @@ fn has_invalid_xml_char(str: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::client::{ + use crate::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options}, }; @@ -808,7 +806,6 @@ mod tests { #[tokio::test] async fn multi_object_delete_request_uses_lowercase_hex_sha256_header() { let objects = vec![ObjectInfo { - bucket: "bucket".to_string(), name: "object.txt".to_string(), ..Default::default() }]; diff --git a/crates/ecstore/src/client/api_s3_datatypes.rs b/crates/s3-client/src/api_s3_datatypes.rs similarity index 98% rename from crates/ecstore/src/client/api_s3_datatypes.rs rename to crates/s3-client/src/api_s3_datatypes.rs index 6703fc4b3..2f6475815 100644 --- a/crates/ecstore/src/client/api_s3_datatypes.rs +++ b/crates/s3-client/src/api_s3_datatypes.rs @@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use time::OffsetDateTime; -use crate::client::checksum::ChecksumMode; -use crate::client::transition_api::ObjectMultipartInfo; -use crate::client::utils::base64_decode; +use crate::checksum::ChecksumMode; +use crate::transition_api::ObjectMultipartInfo; +use crate::utils::base64_decode; use super::transition_api; diff --git a/crates/ecstore/src/client/api_stat.rs b/crates/s3-client/src/api_stat.rs similarity index 99% rename from crates/ecstore/src/client/api_stat.rs rename to crates/s3-client/src/api_stat.rs index e68b21d5e..c4bc8c9d8 100644 --- a/crates/ecstore/src/client/api_stat.rs +++ b/crates/s3-client/src/api_stat.rs @@ -28,7 +28,7 @@ use tokio::io::BufReader; use tracing::warn; use uuid::Uuid; -use crate::client::{ +use crate::{ api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response}, api_get_options::GetObjectOptions, transition_api::{ diff --git a/crates/ecstore/src/client/bucket_cache.rs b/crates/s3-client/src/bucket_cache.rs similarity index 99% rename from crates/ecstore/src/client/bucket_cache.rs rename to crates/s3-client/src/bucket_cache.rs index 739c0f72d..c891d65a0 100644 --- a/crates/ecstore/src/client/bucket_cache.rs +++ b/crates/s3-client/src/bucket_cache.rs @@ -20,7 +20,7 @@ use super::constants::UNSIGNED_PAYLOAD; use super::credentials::SignatureType; -use crate::client::{ +use crate::{ api_error_response::http_resp_to_error_response, signer_error, transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient}, diff --git a/crates/ecstore/src/client/checksum.rs b/crates/s3-client/src/checksum.rs similarity index 98% rename from crates/ecstore/src/client/checksum.rs rename to crates/s3-client/src/checksum.rs index 7bb96a6a3..6b4de15d7 100644 --- a/crates/ecstore/src/client/checksum.rs +++ b/crates/s3-client/src/checksum.rs @@ -23,10 +23,9 @@ use lazy_static::lazy_static; use rustfs_checksums::ChecksumAlgorithm; use std::collections::HashMap; -use crate::client::utils::base64_decode; -use crate::client::utils::base64_encode; -use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart}; -use crate::{disk::DiskAPI, object_api::GetObjectReader}; +use crate::utils::base64_decode; +use crate::utils::base64_encode; +use crate::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart}; // s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives // in rustfs-utils' headers module. use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME; diff --git a/crates/ecstore/src/client/constants.rs b/crates/s3-client/src/constants.rs similarity index 100% rename from crates/ecstore/src/client/constants.rs rename to crates/s3-client/src/constants.rs diff --git a/crates/ecstore/src/client/credentials.rs b/crates/s3-client/src/credentials.rs similarity index 100% rename from crates/ecstore/src/client/credentials.rs rename to crates/s3-client/src/credentials.rs diff --git a/crates/s3-client/src/lib.rs b/crates/s3-client/src/lib.rs new file mode 100644 index 000000000..6b8247a45 --- /dev/null +++ b/crates/s3-client/src/lib.rs @@ -0,0 +1,41 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S3 client used by the storage engine when it *consumes* remote S3-compatible +//! endpoints (ILM tier warm backends, transition targets). Extracted from +//! `crates/ecstore/src/client` (rustfs/backlog#1842) so the engine no longer +//! embeds an S3 HTTP client; ecstore re-exports these modules during the +//! migration window. + +pub mod admin_handler_utils; +pub mod api_error_response; +pub mod api_get_object; +pub mod api_get_options; +pub mod api_list; +pub mod api_put_object; +pub mod api_put_object_common; +pub mod api_put_object_multipart; +pub mod api_put_object_streaming; +pub mod api_remove; +pub mod api_s3_datatypes; +pub mod api_stat; +pub mod bucket_cache; +pub mod checksum; +pub mod constants; +pub mod credentials; +pub mod provider_versions; +pub mod runtime_sources; +pub mod signer_error; +pub mod transition_api; +pub mod utils; diff --git a/crates/ecstore/src/client/provider_versions.rs b/crates/s3-client/src/provider_versions.rs similarity index 94% rename from crates/ecstore/src/client/provider_versions.rs rename to crates/s3-client/src/provider_versions.rs index ceb30bb9c..8b36de5fc 100644 --- a/crates/ecstore/src/client/provider_versions.rs +++ b/crates/s3-client/src/provider_versions.rs @@ -24,7 +24,7 @@ const MAX_REMOTE_VERSION_ID_LEN: usize = 1024; #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")] -pub(crate) enum BucketVersioningState { +pub enum BucketVersioningState { Unknown, Disabled, Suspended, @@ -32,7 +32,7 @@ pub(crate) enum BucketVersioningState { } #[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) enum RemoteVersion { +pub enum RemoteVersion { Unknown, Disabled, SuspendedNull, @@ -40,7 +40,7 @@ pub(crate) enum RemoteVersion { } impl RemoteVersion { - pub(crate) fn exact_id(&self) -> Option<&str> { + pub fn exact_id(&self) -> Option<&str> { match self { Self::SuspendedNull => Some("null"), Self::Exact(version_id) => Some(version_id), @@ -49,7 +49,7 @@ impl RemoteVersion { } #[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")] - pub(crate) fn exact_request_id(&self) -> Result, Error> { + pub fn exact_request_id(&self) -> Result, Error> { match self { Self::Unknown => Err(Error::new( ErrorKind::InvalidData, @@ -63,23 +63,23 @@ impl RemoteVersion { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum ConditionalCreateCapability { +pub enum ConditionalCreateCapability { Unsupported, IfNoneMatchStar, GenerationMatchZero, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct ProviderVersionCapabilities { +pub struct ProviderVersionCapabilities { raw_version_header: Option<&'static str>, - pub(crate) bucket_versioning_state: bool, - pub(crate) list_object_versions: bool, - pub(crate) conditional_create: ConditionalCreateCapability, - pub(crate) exact_get_delete: bool, + pub bucket_versioning_state: bool, + pub list_object_versions: bool, + pub conditional_create: ConditionalCreateCapability, + pub exact_get_delete: bool, } impl ProviderVersionCapabilities { - pub(crate) fn for_tier_type(tier_type: &str) -> Self { + pub fn for_tier_type(tier_type: &str) -> Self { if tier_type.eq_ignore_ascii_case("s3") || tier_type.eq_ignore_ascii_case("rustfs") || tier_type.eq_ignore_ascii_case("minio") @@ -144,7 +144,7 @@ impl ProviderVersionCapabilities { } } - pub(crate) fn raw_version_id(self, headers: &HeaderMap) -> Result, Error> { + pub fn raw_version_id(self, headers: &HeaderMap) -> Result, Error> { let Some(header_name) = self.raw_version_header else { return Ok(None); }; @@ -158,7 +158,7 @@ impl ProviderVersionCapabilities { Ok(Some(value)) } - pub(crate) fn remote_version(self, headers: &HeaderMap, versioning: BucketVersioningState) -> Result { + pub fn remote_version(self, headers: &HeaderMap, versioning: BucketVersioningState) -> Result { let Some(value) = self.raw_version_id(headers)? else { return Ok(match versioning { BucketVersioningState::Disabled => RemoteVersion::Disabled, @@ -174,7 +174,7 @@ impl ProviderVersionCapabilities { } } -pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> { +pub fn validate_remote_version_id(version_id: &str) -> Result<(), Error> { if version_id.is_empty() { return Err(Error::new( ErrorKind::InvalidData, diff --git a/crates/ecstore/src/client/runtime_sources.rs b/crates/s3-client/src/runtime_sources.rs similarity index 84% rename from crates/ecstore/src/client/runtime_sources.rs rename to crates/s3-client/src/runtime_sources.rs index 1acb5db2b..df368157c 100644 --- a/crates/ecstore/src/client/runtime_sources.rs +++ b/crates/s3-client/src/runtime_sources.rs @@ -16,10 +16,10 @@ use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_t const ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER: &str = "ecstore_transition_client"; -pub(crate) async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState { +pub async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState { load_global_outbound_tls_state().await } -pub(crate) fn record_transition_client_tls_generation(generation: u64) { +pub fn record_transition_client_tls_generation(generation: u64) { record_tls_generation(ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER, generation); } diff --git a/crates/ecstore/src/client/signer_error.rs b/crates/s3-client/src/signer_error.rs similarity index 88% rename from crates/ecstore/src/client/signer_error.rs rename to crates/s3-client/src/signer_error.rs index 384e0432d..dddccfea8 100644 --- a/crates/ecstore/src/client/signer_error.rs +++ b/crates/s3-client/src/signer_error.rs @@ -16,7 +16,7 @@ use std::error::Error as StdError; use std::fmt::{Display, Formatter}; use std::io::{Error, ErrorKind}; -pub(crate) const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error"; +pub const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error"; #[derive(Debug)] struct SignerHeaderError { @@ -45,18 +45,18 @@ impl Display for SignerHeaderError { impl StdError for SignerHeaderError {} -pub(crate) fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error { +pub fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error { Error::new(ErrorKind::InvalidInput, SignerHeaderError::new(scope, header_name)) } -pub(crate) fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error { +pub fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error { match error { rustfs_signer::SignV4Error::InvalidHeaderValue { name } => invalid_utf8_header_error(scope, &name), other => Error::other(format!("{scope}: {other}")), } } -pub(crate) fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool { +pub fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool { let mut current = Some(err); while let Some(source) = current { if source.downcast_ref::().is_some() { diff --git a/crates/ecstore/src/client/transition_api.rs b/crates/s3-client/src/transition_api.rs similarity index 96% rename from crates/ecstore/src/client/transition_api.rs rename to crates/s3-client/src/transition_api.rs index be8e0a510..87614ee84 100644 --- a/crates/ecstore/src/client/transition_api.rs +++ b/crates/s3-client/src/transition_api.rs @@ -18,8 +18,9 @@ #![allow(unused_must_use)] #![allow(clippy::all)] -use crate::client::bucket_cache::BucketLocationCache; -use crate::client::{ +use crate::bucket_cache::BucketLocationCache; +use crate::checksum::ChecksumMode; +use crate::{ api_error_response::ErrorResponse, api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response}, api_get_options::GetObjectOptions, @@ -34,7 +35,6 @@ use crate::client::{ provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion}, signer_error, }; -use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader}; use futures::{Future, StreamExt}; use http::{HeaderMap, HeaderName}; use http::{ @@ -79,16 +79,17 @@ use std::{ use time::Duration; use time::OffsetDateTime; use tokio::io::BufReader; +use tokio::io::{AsyncRead, AsyncReadExt}; use tracing::{debug, error, warn}; use url::{Url, form_urlencoded}; use uuid::Uuid; const C_USER_AGENT: &str = "RustFS (linux; x86)"; -pub(crate) const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024; +pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024; const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT]; -pub(crate) async fn collect_response_body(body: B, limit: usize) -> Result, std::io::Error> +pub async fn collect_response_body(body: B, limit: usize) -> Result, std::io::Error> where B: Body, B::Error: Into>, @@ -135,10 +136,42 @@ fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> s signer_error::signer_error_to_io_error(scope, error) } -//pub type ReaderImpl = Box; +/// Streaming object body handed to the client by the storage engine. The +/// client only ever reads it to completion, so the engine-side reader type +/// (e.g. ecstore's `GetObjectReader`) stays behind this boxed `AsyncRead`. +pub struct ObjectReader(Box); + +impl ObjectReader { + pub fn new(reader: impl AsyncRead + Send + Sync + Unpin + 'static) -> Self { + Self(Box::new(reader)) + } + + pub async fn read_all(&mut self) -> Result, std::io::Error> { + let mut data = Vec::new(); + self.0.read_to_end(&mut data).await?; + Ok(data) + } +} + +impl std::fmt::Debug for ObjectReader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ObjectReader") + } +} + +impl AsyncRead for ObjectReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.0).poll_read(cx, buf) + } +} + pub enum ReaderImpl { Body(Bytes), - ObjectBody(GetObjectReader), + ObjectBody(ObjectReader), } pub type ReadCloser = BufReader>>; @@ -223,8 +256,8 @@ where async fn build_tls_config() -> Result { with_rustls_init_guard(|| Ok(()))?; - let outbound_tls = crate::client::runtime_sources::transition_client_outbound_tls_state().await; - crate::client::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0); + let outbound_tls = crate::runtime_sources::transition_client_outbound_tls_state().await; + crate::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0); let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) @@ -336,15 +369,15 @@ impl TransitionClient { self.endpoint_url.clone() } - pub(crate) fn provider_version_capabilities(&self) -> ProviderVersionCapabilities { + pub fn provider_version_capabilities(&self) -> ProviderVersionCapabilities { ProviderVersionCapabilities::for_tier_type(&self.tier_type) } - pub(crate) fn raw_version_id<'a>(&self, headers: &'a HeaderMap) -> Result, std::io::Error> { + pub fn raw_version_id<'a>(&self, headers: &'a HeaderMap) -> Result, std::io::Error> { self.provider_version_capabilities().raw_version_id(headers) } - pub(crate) fn remote_version( + pub fn remote_version( &self, headers: &HeaderMap, versioning: BucketVersioningState, @@ -352,7 +385,7 @@ impl TransitionClient { self.provider_version_capabilities().remote_version(headers, versioning) } - pub(crate) fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result { + pub fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result { Ok(self .remote_version(headers, BucketVersioningState::Unknown)? .exact_id() @@ -934,8 +967,8 @@ impl TransitionCore { // part_id, start_offset, length, metadata) Err(std::io::Error::new( std::io::ErrorKind::Unsupported, - crate::client::credentials::ErrorResponse { - sts_error: crate::client::credentials::STSError { + crate::credentials::ErrorResponse { + sts_error: crate::credentials::STSError { r#type: "".to_string(), code: "NotImplemented".to_string(), message: format!( @@ -1147,7 +1180,7 @@ impl Default for ObjectInfo { impl ObjectInfo { #[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")] - pub(crate) fn remote_version( + pub fn remote_version( &self, capabilities: ProviderVersionCapabilities, versioning: BucketVersioningState, @@ -1229,7 +1262,7 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re to_object_info_for_provider(bucket_name, object_name, h, ProviderVersionCapabilities::for_tier_type("s3")) } -pub(crate) fn to_object_info_for_provider( +pub fn to_object_info_for_provider( bucket_name: &str, object_name: &str, h: &HeaderMap, @@ -1475,7 +1508,7 @@ mod tests { MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body, signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard, }; - use crate::client::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion}; + use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion}; use http::{HeaderMap, HeaderValue}; use http_body_util::Full; use hyper::body::Bytes; diff --git a/crates/ecstore/src/client/utils.rs b/crates/s3-client/src/utils.rs similarity index 100% rename from crates/ecstore/src/client/utils.rs rename to crates/s3-client/src/utils.rs diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index d510d49a6..bb6309d38 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -5494,7 +5494,7 @@ ECSTORE_LINT_EXPECTED="${TMP_DIR}/ecstore_lint_expected.txt" ( cd "$ROOT_DIR" - rg -n '^#!\[allow\(dead_code\)\]' crates/ecstore/src/ 2>/dev/null || true + rg -n '^#!\[allow\(dead_code\)\]' crates/ecstore/src/ crates/s3-client/src/ 2>/dev/null || true ) >"$ECSTORE_DEAD_CODE_HITS" if [[ -s "$ECSTORE_DEAD_CODE_HITS" ]]; then @@ -5503,7 +5503,7 @@ fi ( cd "$ROOT_DIR" - rg -n '^#!\[allow\((unused_variables|unused_must_use|clippy::all)\)\]' crates/ecstore/src/ 2>/dev/null | + rg -n '^#!\[allow\((unused_variables|unused_must_use|clippy::all)\)\]' crates/ecstore/src/ crates/s3-client/src/ 2>/dev/null | sed -E 's#^([^:]+):[0-9]+:\#!\[allow\(([^)]+)\)\]#\1|\2#' | sort -u ) >"$ECSTORE_LINT_ACTUAL" diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index 335075f39..c506b1990 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -60,7 +60,7 @@ checked_files=( "crates/targets/src/target/webhook.rs" "crates/ecstore/src/store/peer.rs" "crates/ecstore/src/store/init.rs" - "crates/ecstore/src/client/transition_api.rs" + "crates/s3-client/src/transition_api.rs" "crates/ecstore/src/services/tier/tier.rs" "crates/heal/src/heal/manager.rs" "crates/heal/src/heal/storage.rs" diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index 1ebd59f5b..07e826176 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -27,6 +27,12 @@ cd "$(dirname "$0")/.." # to verify S3 behavior and does not widen the production s3s surface. S3S_IMPORT_FILES_BASELINE=211 S3_ERROR_LINES_BASELINE=1620 +# ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not +# know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming* +# client was extracted to crates/s3-client, where s3s usage is legitimate; +# this counter ratchets the remaining serving-side s3s references out of +# crates/ecstore. Baseline verified on 2026-08-26. +S3S_ECSTORE_FILES_BASELINE=42 S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::' E2E_TEST_GLOB='--glob=!crates/e2e_test/**' @@ -47,11 +53,13 @@ run_rg_to() { run_rg_to "$TMP_DIR/import_files" -l "$S3S_PATH_PATTERN" --type rust $E2E_TEST_GLOB run_rg_to "$TMP_DIR/error_lines" -c 's3_error!' --type rust $E2E_TEST_GLOB +run_rg_to "$TMP_DIR/ecstore_files" -l "$S3S_PATH_PATTERN" --type rust crates/ecstore/src s3s_import_files="$(grep -c . "$TMP_DIR/import_files" || true)" s3_error_lines="$(awk -F: '{sum += $NF} END {print sum + 0}' "$TMP_DIR/error_lines")" +s3s_ecstore_files="$(grep -c . "$TMP_DIR/ecstore_files" || true)" -for value in "$s3s_import_files" "$s3_error_lines"; do +for value in "$s3s_import_files" "$s3_error_lines" "$s3s_ecstore_files"; do if ! [[ "$value" =~ ^[0-9]+$ ]]; then echo "error: could not compute s3s footprint counts (got: '$value')" >&2 exit 1 @@ -82,6 +90,8 @@ check_ratchet "files importing s3s" "$s3s_import_files" "$S3S_IMPORT_FILES_BASEL "rg -l '$S3S_PATH_PATTERN' --type rust $E2E_TEST_GLOB" check_ratchet "s3_error! invocation lines" "$s3_error_lines" "$S3_ERROR_LINES_BASELINE" \ "rg -c 's3_error!' --type rust $E2E_TEST_GLOB" +check_ratchet "ecstore files referencing s3s" "$s3s_ecstore_files" "$S3S_ECSTORE_FILES_BASELINE" \ + "rg -l '$S3S_PATH_PATTERN' --type rust crates/ecstore/src" if ((status != 0)); then exit 1 diff --git a/scripts/ecstore-module-lint-register.txt b/scripts/ecstore-module-lint-register.txt index a6d2ac8d4..69dd097dc 100644 --- a/scripts/ecstore-module-lint-register.txt +++ b/scripts/ecstore-module-lint-register.txt @@ -23,56 +23,56 @@ crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables -crates/ecstore/src/client/api_error_response.rs|clippy::all -crates/ecstore/src/client/api_error_response.rs|unused_must_use -crates/ecstore/src/client/api_error_response.rs|unused_variables -crates/ecstore/src/client/api_get_object.rs|clippy::all -crates/ecstore/src/client/api_get_object.rs|unused_must_use -crates/ecstore/src/client/api_get_object.rs|unused_variables -crates/ecstore/src/client/api_get_options.rs|clippy::all -crates/ecstore/src/client/api_get_options.rs|unused_must_use -crates/ecstore/src/client/api_get_options.rs|unused_variables -crates/ecstore/src/client/api_list.rs|clippy::all -crates/ecstore/src/client/api_list.rs|unused_must_use -crates/ecstore/src/client/api_list.rs|unused_variables -crates/ecstore/src/client/api_put_object.rs|clippy::all -crates/ecstore/src/client/api_put_object.rs|unused_must_use -crates/ecstore/src/client/api_put_object.rs|unused_variables -crates/ecstore/src/client/api_put_object_common.rs|clippy::all -crates/ecstore/src/client/api_put_object_common.rs|unused_must_use -crates/ecstore/src/client/api_put_object_common.rs|unused_variables -crates/ecstore/src/client/api_put_object_multipart.rs|clippy::all -crates/ecstore/src/client/api_put_object_multipart.rs|unused_must_use -crates/ecstore/src/client/api_put_object_multipart.rs|unused_variables -crates/ecstore/src/client/api_put_object_streaming.rs|clippy::all -crates/ecstore/src/client/api_put_object_streaming.rs|unused_must_use -crates/ecstore/src/client/api_put_object_streaming.rs|unused_variables -crates/ecstore/src/client/api_remove.rs|clippy::all -crates/ecstore/src/client/api_remove.rs|unused_must_use -crates/ecstore/src/client/api_remove.rs|unused_variables -crates/ecstore/src/client/api_s3_datatypes.rs|clippy::all -crates/ecstore/src/client/api_s3_datatypes.rs|unused_must_use -crates/ecstore/src/client/api_s3_datatypes.rs|unused_variables -crates/ecstore/src/client/api_stat.rs|clippy::all -crates/ecstore/src/client/api_stat.rs|unused_must_use -crates/ecstore/src/client/api_stat.rs|unused_variables -crates/ecstore/src/client/bucket_cache.rs|clippy::all -crates/ecstore/src/client/bucket_cache.rs|unused_must_use -crates/ecstore/src/client/bucket_cache.rs|unused_variables -crates/ecstore/src/client/checksum.rs|clippy::all -crates/ecstore/src/client/checksum.rs|unused_must_use -crates/ecstore/src/client/checksum.rs|unused_variables -crates/ecstore/src/client/constants.rs|unused_must_use -crates/ecstore/src/client/constants.rs|unused_variables -crates/ecstore/src/client/credentials.rs|clippy::all -crates/ecstore/src/client/credentials.rs|unused_must_use -crates/ecstore/src/client/credentials.rs|unused_variables +crates/s3-client/src/api_error_response.rs|clippy::all +crates/s3-client/src/api_error_response.rs|unused_must_use +crates/s3-client/src/api_error_response.rs|unused_variables +crates/s3-client/src/api_get_object.rs|clippy::all +crates/s3-client/src/api_get_object.rs|unused_must_use +crates/s3-client/src/api_get_object.rs|unused_variables +crates/s3-client/src/api_get_options.rs|clippy::all +crates/s3-client/src/api_get_options.rs|unused_must_use +crates/s3-client/src/api_get_options.rs|unused_variables +crates/s3-client/src/api_list.rs|clippy::all +crates/s3-client/src/api_list.rs|unused_must_use +crates/s3-client/src/api_list.rs|unused_variables +crates/s3-client/src/api_put_object.rs|clippy::all +crates/s3-client/src/api_put_object.rs|unused_must_use +crates/s3-client/src/api_put_object.rs|unused_variables +crates/s3-client/src/api_put_object_common.rs|clippy::all +crates/s3-client/src/api_put_object_common.rs|unused_must_use +crates/s3-client/src/api_put_object_common.rs|unused_variables +crates/s3-client/src/api_put_object_multipart.rs|clippy::all +crates/s3-client/src/api_put_object_multipart.rs|unused_must_use +crates/s3-client/src/api_put_object_multipart.rs|unused_variables +crates/s3-client/src/api_put_object_streaming.rs|clippy::all +crates/s3-client/src/api_put_object_streaming.rs|unused_must_use +crates/s3-client/src/api_put_object_streaming.rs|unused_variables +crates/s3-client/src/api_remove.rs|clippy::all +crates/s3-client/src/api_remove.rs|unused_must_use +crates/s3-client/src/api_remove.rs|unused_variables +crates/s3-client/src/api_s3_datatypes.rs|clippy::all +crates/s3-client/src/api_s3_datatypes.rs|unused_must_use +crates/s3-client/src/api_s3_datatypes.rs|unused_variables +crates/s3-client/src/api_stat.rs|clippy::all +crates/s3-client/src/api_stat.rs|unused_must_use +crates/s3-client/src/api_stat.rs|unused_variables +crates/s3-client/src/bucket_cache.rs|clippy::all +crates/s3-client/src/bucket_cache.rs|unused_must_use +crates/s3-client/src/bucket_cache.rs|unused_variables +crates/s3-client/src/checksum.rs|clippy::all +crates/s3-client/src/checksum.rs|unused_must_use +crates/s3-client/src/checksum.rs|unused_variables +crates/s3-client/src/constants.rs|unused_must_use +crates/s3-client/src/constants.rs|unused_variables +crates/s3-client/src/credentials.rs|clippy::all +crates/s3-client/src/credentials.rs|unused_must_use +crates/s3-client/src/credentials.rs|unused_variables crates/ecstore/src/client/object_api_utils.rs|clippy::all crates/ecstore/src/client/object_api_utils.rs|unused_must_use crates/ecstore/src/client/object_api_utils.rs|unused_variables -crates/ecstore/src/client/transition_api.rs|clippy::all -crates/ecstore/src/client/transition_api.rs|unused_must_use -crates/ecstore/src/client/transition_api.rs|unused_variables +crates/s3-client/src/transition_api.rs|clippy::all +crates/s3-client/src/transition_api.rs|unused_must_use +crates/s3-client/src/transition_api.rs|unused_variables crates/ecstore/src/services/event_notification.rs|unused_variables crates/ecstore/src/services/tier/tier.rs|clippy::all crates/ecstore/src/services/tier/tier.rs|unused_must_use diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 9e4a057a0..9eec2d2ec 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -19,11 +19,6 @@ 1|crates/ecstore/src/bucket/replication/replication_object_config.rs 2|crates/ecstore/src/bucket/replication/replication_pool.rs 2|crates/ecstore/src/bucket/replication/replication_target_boundary.rs -1|crates/ecstore/src/client/api_put_object_multipart.rs -2|crates/ecstore/src/client/api_put_object_streaming.rs -1|crates/ecstore/src/client/api_s3_datatypes.rs -1|crates/ecstore/src/client/signer_error.rs -5|crates/ecstore/src/client/transition_api.rs 3|crates/ecstore/src/cluster/rpc/http_auth.rs 2|crates/ecstore/src/cluster/rpc/internode_data_transport.rs 11|crates/ecstore/src/cluster/rpc/peer_rest_client.rs