mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
refactor(ecstore): extract the embedded S3 client into rustfs-s3-client (#6627)
The storage engine embedded a ~8.4K-line hand-written S3 HTTP client under crates/ecstore/src/client (rustfs/backlog#1842). That client is a legitimate engine capability — it consumes remote S3-compatible endpoints for ILM tier warm backends and transition targets — but it was misfiled inside the engine, dragging s3s/hyper wire types into ecstore and blocking ARCHITECTURE.md invariant 4. This PR is the pure-move step: 21 modules move verbatim to the new crates/s3-client crate (rustfs-s3-client), and crates/ecstore/src/client/mod.rs becomes a re-export shim so every in-crate crate::client:: path keeps working. The two server-side modules that were historically misfiled under client/ — object_api_utils.rs and object_handlers_common.rs — stay in ecstore. Three reverse dependencies from the client into engine internals are severed so the move can be pure: - transition_api::ReaderImpl::ObjectBody held ecstore's GetObjectReader; the client only ever reads the body, so the variant now holds an ObjectReader newtype over Box<dyn AsyncRead + Send + Sync + Unpin> with the same read_all() surface. The single production construction site (set_disk transition upload) and the two engine-side consumers were adjusted. - api_list/api_remove used ecstore's storage_api_contracts / object_api types; api_list now imports BucketInfo from rustfs-storage-api directly, and api_remove uses the client's own transition_api::ObjectInfo (only .name/.version_id were read; the error-path bucket name is now threaded as a parameter instead of read from the deleted objects). - the api_put_object_streaming regression tests built a GetObjectReader by hand; they now wrap the duplex stream in ObjectReader::new. Guard updates: the s3s footprint ratchet gains an ecstore-scoped counter (42 files, shrink-only, per rustfs/backlog#1842), the ecstore module-lint-blanket register follows the moved files into crates/s3-client so the blanket ratchet keeps covering them, the logging guardrail path pin follows transition_api.rs, and the ::other(format!) baseline is regenerated (moved call sites left ecstore). Verification: cargo check -p rustfs-s3-client -p rustfs-ecstore; cargo nextest run -p rustfs-s3-client (43 passed) and -p rustfs-ecstore (4515/4523; the 8 failures reproduce identically on pristine origin/main on the same machine); cargo clippy --all-targets; scripts/check_layer_dependencies.sh, check_architecture_migration_rules.sh, check_s3s_footprint.sh, check_logging_guardrails.sh, check_error_other_format_ratchet.sh, check_doc_paths.sh, check_ci_paths_sync.sh all pass.
This commit is contained in:
Generated
+46
@@ -9552,6 +9552,7 @@ dependencies = [
|
||||
"rustfs-replication",
|
||||
"rustfs-rio",
|
||||
"rustfs-rio-v2",
|
||||
"rustfs-s3-client",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-signer",
|
||||
"rustfs-storage-api",
|
||||
@@ -10316,6 +10317,51 @@ dependencies = [
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-client"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"enumset",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"hex-simd",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"lazy_static",
|
||||
"md-5 0.11.0",
|
||||
"quick-xml 0.42.0",
|
||||
"rand 0.10.2",
|
||||
"rustfs-checksums",
|
||||
"rustfs-config",
|
||||
"rustfs-rio",
|
||||
"rustfs-signer",
|
||||
"rustfs-storage-api",
|
||||
"rustfs-tls-runtime",
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tracing",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.3"
|
||||
|
||||
@@ -45,6 +45,7 @@ members = [
|
||||
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
"crates/s3-ops", # S3 operation definitions and mapping
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
@@ -121,6 +122,7 @@ rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
||||
|
||||
@@ -147,6 +147,7 @@ rustfs-policy.workspace = true
|
||||
rustfs-protos.workspace = true
|
||||
rustfs-replication.workspace = true
|
||||
rustfs-lifecycle.workspace = true
|
||||
rustfs-s3-client = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
|
||||
@@ -12,28 +12,16 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
||||
// The S3-consuming client moved to the `rustfs-s3-client` crate
|
||||
// (rustfs/backlog#1842). This shim keeps `crate::client::*` paths working for
|
||||
// in-crate consumers during the migration window; it is deleted once every
|
||||
// consumer imports `rustfs_s3_client` directly. Only the two server-side
|
||||
// modules below (misfiled here historically) remain as real ecstore code.
|
||||
|
||||
pub use rustfs_s3_client::{
|
||||
admin_handler_utils, api_get_options, api_list, api_put_object, api_remove, api_s3_datatypes, credentials, provider_versions,
|
||||
signer_error, transition_api,
|
||||
};
|
||||
|
||||
pub mod admin_handler_utils;
|
||||
pub mod api_error_response;
|
||||
pub mod api_get_object;
|
||||
pub mod api_get_options;
|
||||
pub mod api_list;
|
||||
pub mod api_put_object;
|
||||
pub mod api_put_object_common;
|
||||
pub mod api_put_object_multipart;
|
||||
pub mod api_put_object_streaming;
|
||||
pub mod api_remove;
|
||||
pub mod api_s3_datatypes;
|
||||
pub mod api_stat;
|
||||
pub mod bucket_cache;
|
||||
pub mod checksum;
|
||||
pub mod constants;
|
||||
pub mod credentials;
|
||||
pub mod object_api_utils;
|
||||
pub mod object_handlers_common;
|
||||
pub(crate) mod provider_versions;
|
||||
pub(crate) mod runtime_sources;
|
||||
pub mod signer_error;
|
||||
pub mod transition_api;
|
||||
pub mod utils;
|
||||
|
||||
@@ -596,9 +596,9 @@ impl MockWarmBackend {
|
||||
if let Some(limit) = limit {
|
||||
let limit =
|
||||
u64::try_from(limit).map_err(|_| std::io::Error::other("mock PUT read limit exceeds u64::MAX"))?;
|
||||
reader.stream.take(limit).read_to_end(&mut buf).await?;
|
||||
(&mut reader).take(limit).read_to_end(&mut buf).await?;
|
||||
} else {
|
||||
reader.stream.read_to_end(&mut buf).await?;
|
||||
reader.read_to_end(&mut buf).await?;
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ use crate::bucket::replication::{
|
||||
};
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
|
||||
use crate::client::{object_api_utils::get_raw_etag, transition_api::ObjectReader, transition_api::ReaderImpl};
|
||||
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
||||
use crate::data_usage::record_compression_total_memory;
|
||||
use crate::diagnostics::get::{
|
||||
@@ -1559,8 +1559,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get lock acquire timeout from environment variable RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT (in seconds)
|
||||
/// Defaults to 5 seconds if not set or invalid
|
||||
/// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
|
||||
/// Defaults to 30 seconds if not set or invalid
|
||||
/// Lock acquisition timeout. Cached: this is consulted on every object
|
||||
/// lock acquisition and `std::env::var` takes a process-global lock. In test
|
||||
/// builds the env var is read directly so `temp_env` overrides take effect.
|
||||
|
||||
@@ -7332,12 +7332,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let expected_size = u64::try_from(fi.size).map_err(|_| StorageError::FileCorrupt)?;
|
||||
let (pr, pw) = tokio::io::duplex(fi.erasure.block_size);
|
||||
let consumed = Arc::new(AtomicU64::new(0));
|
||||
let reader = ReaderImpl::ObjectBody(GetObjectReader {
|
||||
stream: Box::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))),
|
||||
object_info: oi,
|
||||
buffered_body: None,
|
||||
body_source: GetObjectBodySource::Unprobed,
|
||||
});
|
||||
let reader = ReaderImpl::ObjectBody(ObjectReader::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))));
|
||||
|
||||
let cloned_bucket = bucket.to_string();
|
||||
let cloned_object = object.to_string();
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-s3-client"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "S3 client used by RustFS when the storage engine consumes remote S3-compatible endpoints (tier warm backends, transition targets)."
|
||||
keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
|
||||
|
||||
[dependencies]
|
||||
rustfs-checksums.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
rustfs-rio.workspace = true
|
||||
rustfs-signer.workspace = true
|
||||
rustfs-storage-api.workspace = true
|
||||
rustfs-tls-runtime.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
base64-simd.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
enumset = { workspace = true }
|
||||
futures.workspace = true
|
||||
futures-util.workspace = true
|
||||
http.workspace = true
|
||||
http-body = { workspace = true }
|
||||
http-body-util.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
lazy_static.workspace = true
|
||||
md-5.workspace = true
|
||||
quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
sha1 = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
thiserror.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
tokio = { workspace = true, features = ["io-util", "sync", "fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tower = { workspace = true, features = ["timeout"] }
|
||||
tracing.workspace = true
|
||||
url.workspace = true
|
||||
urlencoding = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
+1
-1
@@ -27,7 +27,7 @@ use std::task::{Context, Poll};
|
||||
use tokio::io::BufReader;
|
||||
use tokio_util::io::StreamReader;
|
||||
|
||||
use crate::client::{
|
||||
use crate::{
|
||||
api_error_response::err_invalid_argument,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
|
||||
+1
-1
@@ -25,7 +25,7 @@ use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::client::api_error_response::err_invalid_argument;
|
||||
use crate::api_error_response::err_invalid_argument;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct AdvancedGetOptions {
|
||||
@@ -18,7 +18,7 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
use crate::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
api_s3_datatypes::{
|
||||
ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
|
||||
@@ -26,12 +26,12 @@ use crate::client::{
|
||||
credentials,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
|
||||
};
|
||||
use crate::storage_api_contracts::bucket::BucketInfo;
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_storage_api::BucketInfo;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use std::collections::HashMap;
|
||||
use std::io::ErrorKind;
|
||||
+3
-3
@@ -30,9 +30,9 @@ use s3s::header::{
|
||||
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
|
||||
};
|
||||
//use crate::disk::{BufferReader, Reader};
|
||||
use crate::client::checksum::ChecksumMode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::utils::base64_encode;
|
||||
use crate::{
|
||||
api_error_response::{err_entity_too_large, err_invalid_argument},
|
||||
api_put_object_common::optimal_part_info,
|
||||
api_put_object_multipart::UploadPartParams,
|
||||
+1
-1
@@ -18,7 +18,7 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
use crate::{
|
||||
api_error_response::{err_entity_too_large, err_invalid_argument},
|
||||
api_put_object::PutObjectOptions,
|
||||
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
|
||||
+4
-4
@@ -26,9 +26,9 @@ use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::client::checksum::ChecksumMode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::utils::base64_encode;
|
||||
use crate::{
|
||||
api_error_response::{
|
||||
err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
|
||||
},
|
||||
@@ -448,7 +448,7 @@ pub struct UploadPartParams {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::client::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult};
|
||||
use crate::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult};
|
||||
|
||||
#[test]
|
||||
fn complete_multipart_upload_serializes_s3_part_elements() {
|
||||
+6
-16
@@ -31,8 +31,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::client::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
|
||||
use crate::client::{
|
||||
use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
|
||||
use crate::{
|
||||
api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
|
||||
api_put_object::PutObjectOptions,
|
||||
api_put_object_common::{is_object, optimal_part_info},
|
||||
@@ -42,7 +42,7 @@ use crate::client::{
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
|
||||
};
|
||||
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::utils::base64_encode;
|
||||
use rustfs_utils::path::trim_etag;
|
||||
use s3s::header::X_AMZ_EXPIRATION;
|
||||
|
||||
@@ -620,7 +620,7 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
|
||||
use crate::object_api::GetObjectReader;
|
||||
use crate::transition_api::ObjectReader;
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::hash::HashAlgorithm;
|
||||
use std::collections::HashMap;
|
||||
@@ -654,12 +654,7 @@ mod tests {
|
||||
let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
|
||||
w.write_all(&data).await.unwrap();
|
||||
});
|
||||
let reader = ReaderImpl::ObjectBody(GetObjectReader {
|
||||
stream: Box::new(r),
|
||||
object_info: Default::default(),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
let reader = ReaderImpl::ObjectBody(ObjectReader::new(r));
|
||||
|
||||
let sizes = collect_part_sizes(reader, total, 100, 50).await;
|
||||
assert_eq!(sizes, vec![100, 100, 50]);
|
||||
@@ -684,12 +679,7 @@ mod tests {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
w.write_all(&[1u8; 30]).await.unwrap();
|
||||
});
|
||||
let mut reader = ReaderImpl::ObjectBody(GetObjectReader {
|
||||
stream: Box::new(r),
|
||||
object_info: Default::default(),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
let mut reader = ReaderImpl::ObjectBody(ObjectReader::new(r));
|
||||
let buf = read_multipart_part(&mut reader, 100).await.unwrap();
|
||||
assert_eq!(buf.len(), 30);
|
||||
}
|
||||
@@ -36,16 +36,13 @@ use time::OffsetDateTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
use tracing::Instrument;
|
||||
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{
|
||||
use crate::transition_api::ObjectInfo;
|
||||
use crate::utils::base64_encode;
|
||||
use crate::{
|
||||
api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
|
||||
api_s3_datatypes::{DeleteMultiObjects, DeleteObject},
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use crate::{
|
||||
disk::DiskAPI,
|
||||
object_api::{GetObjectReader, ObjectInfo},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
pub struct RemoveBucketOptions {
|
||||
@@ -366,7 +363,13 @@ impl TransitionClient {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), &batch, result_tx.clone()).await;
|
||||
process_remove_multi_objects_response(
|
||||
ReaderImpl::Body(Bytes::from(body_vec)),
|
||||
bucket_name,
|
||||
&batch,
|
||||
result_tx.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -553,6 +556,7 @@ pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8>
|
||||
|
||||
pub async fn process_remove_multi_objects_response(
|
||||
body: ReaderImpl,
|
||||
bucket_name: &str,
|
||||
objects: &[ObjectInfo],
|
||||
result_tx: Sender<RemoveObjectResult>,
|
||||
) {
|
||||
@@ -575,7 +579,7 @@ pub async fn process_remove_multi_objects_response(
|
||||
err: Some(std::io::Error::other(ErrorResponse {
|
||||
code: S3ErrorCode::Custom("ReadDeleteResponseFailed".into()),
|
||||
message: format!("read multi remove response failed: {err}"),
|
||||
bucket_name: object.bucket.clone(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object.name.clone(),
|
||||
resource: "".to_string(),
|
||||
request_id: "".to_string(),
|
||||
@@ -647,7 +651,7 @@ pub async fn process_remove_multi_objects_response(
|
||||
"unmarshal multi remove response failed: {err}; response_body={}",
|
||||
body.chars().take(DELETE_RESPONSE_PREVIEW_LEN).collect::<String>()
|
||||
),
|
||||
bucket_name: object.bucket.clone(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object.name.clone(),
|
||||
resource: "".to_string(),
|
||||
request_id: "".to_string(),
|
||||
@@ -707,13 +711,7 @@ pub async fn process_remove_multi_objects_response(
|
||||
}
|
||||
|
||||
for (object_name, object_version_id) in pending {
|
||||
let bucket_name = objects
|
||||
.iter()
|
||||
.find(|object| {
|
||||
object.name == object_name && object.version_id.as_ref().map(|v| v.to_string()) == Some(object_version_id.clone())
|
||||
})
|
||||
.map(|o| o.bucket.clone())
|
||||
.unwrap_or_default();
|
||||
let bucket_name = bucket_name.to_string();
|
||||
let object_name = object_name;
|
||||
let object_version_id = object_version_id;
|
||||
let error_message = format!(
|
||||
@@ -750,7 +748,7 @@ fn has_invalid_xml_char(str: &str) -> bool {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::client::{
|
||||
use crate::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options},
|
||||
};
|
||||
@@ -808,7 +806,6 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn multi_object_delete_request_uses_lowercase_hex_sha256_header() {
|
||||
let objects = vec![ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object.txt".to_string(),
|
||||
..Default::default()
|
||||
}];
|
||||
+3
-3
@@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::client::checksum::ChecksumMode;
|
||||
use crate::client::transition_api::ObjectMultipartInfo;
|
||||
use crate::client::utils::base64_decode;
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::transition_api::ObjectMultipartInfo;
|
||||
use crate::utils::base64_decode;
|
||||
|
||||
use super::transition_api;
|
||||
|
||||
@@ -28,7 +28,7 @@ use tokio::io::BufReader;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::client::{
|
||||
use crate::{
|
||||
api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response},
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{
|
||||
@@ -20,7 +20,7 @@
|
||||
|
||||
use super::constants::UNSIGNED_PAYLOAD;
|
||||
use super::credentials::SignatureType;
|
||||
use crate::client::{
|
||||
use crate::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
signer_error,
|
||||
transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
|
||||
@@ -23,10 +23,9 @@ use lazy_static::lazy_static;
|
||||
use rustfs_checksums::ChecksumAlgorithm;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::utils::base64_decode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||
use crate::{disk::DiskAPI, object_api::GetObjectReader};
|
||||
use crate::utils::base64_decode;
|
||||
use crate::utils::base64_encode;
|
||||
use crate::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||
// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
|
||||
// in rustfs-utils' headers module.
|
||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! S3 client used by the storage engine when it *consumes* remote S3-compatible
|
||||
//! endpoints (ILM tier warm backends, transition targets). Extracted from
|
||||
//! `crates/ecstore/src/client` (rustfs/backlog#1842) so the engine no longer
|
||||
//! embeds an S3 HTTP client; ecstore re-exports these modules during the
|
||||
//! migration window.
|
||||
|
||||
pub mod admin_handler_utils;
|
||||
pub mod api_error_response;
|
||||
pub mod api_get_object;
|
||||
pub mod api_get_options;
|
||||
pub mod api_list;
|
||||
pub mod api_put_object;
|
||||
pub mod api_put_object_common;
|
||||
pub mod api_put_object_multipart;
|
||||
pub mod api_put_object_streaming;
|
||||
pub mod api_remove;
|
||||
pub mod api_s3_datatypes;
|
||||
pub mod api_stat;
|
||||
pub mod bucket_cache;
|
||||
pub mod checksum;
|
||||
pub mod constants;
|
||||
pub mod credentials;
|
||||
pub mod provider_versions;
|
||||
pub mod runtime_sources;
|
||||
pub mod signer_error;
|
||||
pub mod transition_api;
|
||||
pub mod utils;
|
||||
+14
-14
@@ -24,7 +24,7 @@ const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
||||
pub(crate) enum BucketVersioningState {
|
||||
pub enum BucketVersioningState {
|
||||
Unknown,
|
||||
Disabled,
|
||||
Suspended,
|
||||
@@ -32,7 +32,7 @@ pub(crate) enum BucketVersioningState {
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum RemoteVersion {
|
||||
pub enum RemoteVersion {
|
||||
Unknown,
|
||||
Disabled,
|
||||
SuspendedNull,
|
||||
@@ -40,7 +40,7 @@ pub(crate) enum RemoteVersion {
|
||||
}
|
||||
|
||||
impl RemoteVersion {
|
||||
pub(crate) fn exact_id(&self) -> Option<&str> {
|
||||
pub fn exact_id(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::SuspendedNull => Some("null"),
|
||||
Self::Exact(version_id) => Some(version_id),
|
||||
@@ -49,7 +49,7 @@ impl RemoteVersion {
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||
pub fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||
match self {
|
||||
Self::Unknown => Err(Error::new(
|
||||
ErrorKind::InvalidData,
|
||||
@@ -63,23 +63,23 @@ impl RemoteVersion {
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum ConditionalCreateCapability {
|
||||
pub enum ConditionalCreateCapability {
|
||||
Unsupported,
|
||||
IfNoneMatchStar,
|
||||
GenerationMatchZero,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct ProviderVersionCapabilities {
|
||||
pub struct ProviderVersionCapabilities {
|
||||
raw_version_header: Option<&'static str>,
|
||||
pub(crate) bucket_versioning_state: bool,
|
||||
pub(crate) list_object_versions: bool,
|
||||
pub(crate) conditional_create: ConditionalCreateCapability,
|
||||
pub(crate) exact_get_delete: bool,
|
||||
pub bucket_versioning_state: bool,
|
||||
pub list_object_versions: bool,
|
||||
pub conditional_create: ConditionalCreateCapability,
|
||||
pub exact_get_delete: bool,
|
||||
}
|
||||
|
||||
impl ProviderVersionCapabilities {
|
||||
pub(crate) fn for_tier_type(tier_type: &str) -> Self {
|
||||
pub fn for_tier_type(tier_type: &str) -> Self {
|
||||
if tier_type.eq_ignore_ascii_case("s3")
|
||||
|| tier_type.eq_ignore_ascii_case("rustfs")
|
||||
|| tier_type.eq_ignore_ascii_case("minio")
|
||||
@@ -144,7 +144,7 @@ impl ProviderVersionCapabilities {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn raw_version_id(self, headers: &HeaderMap) -> Result<Option<&str>, Error> {
|
||||
pub fn raw_version_id(self, headers: &HeaderMap) -> Result<Option<&str>, Error> {
|
||||
let Some(header_name) = self.raw_version_header else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -158,7 +158,7 @@ impl ProviderVersionCapabilities {
|
||||
Ok(Some(value))
|
||||
}
|
||||
|
||||
pub(crate) fn remote_version(self, headers: &HeaderMap, versioning: BucketVersioningState) -> Result<RemoteVersion, Error> {
|
||||
pub fn remote_version(self, headers: &HeaderMap, versioning: BucketVersioningState) -> Result<RemoteVersion, Error> {
|
||||
let Some(value) = self.raw_version_id(headers)? else {
|
||||
return Ok(match versioning {
|
||||
BucketVersioningState::Disabled => RemoteVersion::Disabled,
|
||||
@@ -174,7 +174,7 @@ impl ProviderVersionCapabilities {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
pub fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
if version_id.is_empty() {
|
||||
return Err(Error::new(
|
||||
ErrorKind::InvalidData,
|
||||
+2
-2
@@ -16,10 +16,10 @@ use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_t
|
||||
|
||||
const ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER: &str = "ecstore_transition_client";
|
||||
|
||||
pub(crate) async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
|
||||
pub async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
|
||||
load_global_outbound_tls_state().await
|
||||
}
|
||||
|
||||
pub(crate) fn record_transition_client_tls_generation(generation: u64) {
|
||||
pub fn record_transition_client_tls_generation(generation: u64) {
|
||||
record_tls_generation(ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER, generation);
|
||||
}
|
||||
@@ -16,7 +16,7 @@ use std::error::Error as StdError;
|
||||
use std::fmt::{Display, Formatter};
|
||||
use std::io::{Error, ErrorKind};
|
||||
|
||||
pub(crate) const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error";
|
||||
pub const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error";
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SignerHeaderError {
|
||||
@@ -45,18 +45,18 @@ impl Display for SignerHeaderError {
|
||||
|
||||
impl StdError for SignerHeaderError {}
|
||||
|
||||
pub(crate) fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error {
|
||||
pub fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error {
|
||||
Error::new(ErrorKind::InvalidInput, SignerHeaderError::new(scope, header_name))
|
||||
}
|
||||
|
||||
pub(crate) fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error {
|
||||
pub fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error {
|
||||
match error {
|
||||
rustfs_signer::SignV4Error::InvalidHeaderValue { name } => invalid_utf8_header_error(scope, &name),
|
||||
other => Error::other(format!("{scope}: {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool {
|
||||
pub fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool {
|
||||
let mut current = Some(err);
|
||||
while let Some(source) = current {
|
||||
if source.downcast_ref::<SignerHeaderError>().is_some() {
|
||||
+51
-18
@@ -18,8 +18,9 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::bucket_cache::BucketLocationCache;
|
||||
use crate::client::{
|
||||
use crate::bucket_cache::BucketLocationCache;
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::{
|
||||
api_error_response::ErrorResponse,
|
||||
api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response},
|
||||
api_get_options::GetObjectOptions,
|
||||
@@ -34,7 +35,6 @@ use crate::client::{
|
||||
provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion},
|
||||
signer_error,
|
||||
};
|
||||
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
|
||||
use futures::{Future, StreamExt};
|
||||
use http::{HeaderMap, HeaderName};
|
||||
use http::{
|
||||
@@ -79,16 +79,17 @@ use std::{
|
||||
use time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::{debug, error, warn};
|
||||
use url::{Url, form_urlencoded};
|
||||
use uuid::Uuid;
|
||||
|
||||
const C_USER_AGENT: &str = "RustFS (linux; x86)";
|
||||
pub(crate) const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
|
||||
pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
|
||||
|
||||
const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
|
||||
|
||||
pub(crate) async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
|
||||
pub async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
|
||||
where
|
||||
B: Body<Data = Bytes>,
|
||||
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
@@ -135,10 +136,42 @@ fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> s
|
||||
signer_error::signer_error_to_io_error(scope, error)
|
||||
}
|
||||
|
||||
//pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>;
|
||||
/// Streaming object body handed to the client by the storage engine. The
|
||||
/// client only ever reads it to completion, so the engine-side reader type
|
||||
/// (e.g. ecstore's `GetObjectReader`) stays behind this boxed `AsyncRead`.
|
||||
pub struct ObjectReader(Box<dyn AsyncRead + Send + Sync + Unpin>);
|
||||
|
||||
impl ObjectReader {
|
||||
pub fn new(reader: impl AsyncRead + Send + Sync + Unpin + 'static) -> Self {
|
||||
Self(Box::new(reader))
|
||||
}
|
||||
|
||||
pub async fn read_all(&mut self) -> Result<Vec<u8>, std::io::Error> {
|
||||
let mut data = Vec::new();
|
||||
self.0.read_to_end(&mut data).await?;
|
||||
Ok(data)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ObjectReader {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("ObjectReader")
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for ObjectReader {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::pin::Pin::new(&mut self.0).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum ReaderImpl {
|
||||
Body(Bytes),
|
||||
ObjectBody(GetObjectReader),
|
||||
ObjectBody(ObjectReader),
|
||||
}
|
||||
|
||||
pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;
|
||||
@@ -223,8 +256,8 @@ where
|
||||
async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
|
||||
with_rustls_init_guard(|| Ok(()))?;
|
||||
|
||||
let outbound_tls = crate::client::runtime_sources::transition_client_outbound_tls_state().await;
|
||||
crate::client::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0);
|
||||
let outbound_tls = crate::runtime_sources::transition_client_outbound_tls_state().await;
|
||||
crate::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0);
|
||||
let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
|
||||
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
|
||||
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
|
||||
@@ -336,15 +369,15 @@ impl TransitionClient {
|
||||
self.endpoint_url.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn provider_version_capabilities(&self) -> ProviderVersionCapabilities {
|
||||
pub fn provider_version_capabilities(&self) -> ProviderVersionCapabilities {
|
||||
ProviderVersionCapabilities::for_tier_type(&self.tier_type)
|
||||
}
|
||||
|
||||
pub(crate) fn raw_version_id<'a>(&self, headers: &'a HeaderMap) -> Result<Option<&'a str>, std::io::Error> {
|
||||
pub fn raw_version_id<'a>(&self, headers: &'a HeaderMap) -> Result<Option<&'a str>, std::io::Error> {
|
||||
self.provider_version_capabilities().raw_version_id(headers)
|
||||
}
|
||||
|
||||
pub(crate) fn remote_version(
|
||||
pub fn remote_version(
|
||||
&self,
|
||||
headers: &HeaderMap,
|
||||
versioning: BucketVersioningState,
|
||||
@@ -352,7 +385,7 @@ impl TransitionClient {
|
||||
self.provider_version_capabilities().remote_version(headers, versioning)
|
||||
}
|
||||
|
||||
pub(crate) fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result<String, std::io::Error> {
|
||||
pub fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result<String, std::io::Error> {
|
||||
Ok(self
|
||||
.remote_version(headers, BucketVersioningState::Unknown)?
|
||||
.exact_id()
|
||||
@@ -934,8 +967,8 @@ impl TransitionCore {
|
||||
// part_id, start_offset, length, metadata)
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
crate::client::credentials::ErrorResponse {
|
||||
sts_error: crate::client::credentials::STSError {
|
||||
crate::credentials::ErrorResponse {
|
||||
sts_error: crate::credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!(
|
||||
@@ -1147,7 +1180,7 @@ impl Default for ObjectInfo {
|
||||
|
||||
impl ObjectInfo {
|
||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||
pub(crate) fn remote_version(
|
||||
pub fn remote_version(
|
||||
&self,
|
||||
capabilities: ProviderVersionCapabilities,
|
||||
versioning: BucketVersioningState,
|
||||
@@ -1229,7 +1262,7 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re
|
||||
to_object_info_for_provider(bucket_name, object_name, h, ProviderVersionCapabilities::for_tier_type("s3"))
|
||||
}
|
||||
|
||||
pub(crate) fn to_object_info_for_provider(
|
||||
pub fn to_object_info_for_provider(
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
h: &HeaderMap,
|
||||
@@ -1475,7 +1508,7 @@ mod tests {
|
||||
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
|
||||
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
|
||||
};
|
||||
use crate::client::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
|
||||
use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::Full;
|
||||
use hyper::body::Bytes;
|
||||
@@ -5494,7 +5494,7 @@ ECSTORE_LINT_EXPECTED="${TMP_DIR}/ecstore_lint_expected.txt"
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n '^#!\[allow\(dead_code\)\]' crates/ecstore/src/ 2>/dev/null || true
|
||||
rg -n '^#!\[allow\(dead_code\)\]' crates/ecstore/src/ crates/s3-client/src/ 2>/dev/null || true
|
||||
) >"$ECSTORE_DEAD_CODE_HITS"
|
||||
|
||||
if [[ -s "$ECSTORE_DEAD_CODE_HITS" ]]; then
|
||||
@@ -5503,7 +5503,7 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n '^#!\[allow\((unused_variables|unused_must_use|clippy::all)\)\]' crates/ecstore/src/ 2>/dev/null |
|
||||
rg -n '^#!\[allow\((unused_variables|unused_must_use|clippy::all)\)\]' crates/ecstore/src/ crates/s3-client/src/ 2>/dev/null |
|
||||
sed -E 's#^([^:]+):[0-9]+:\#!\[allow\(([^)]+)\)\]#\1|\2#' | sort -u
|
||||
) >"$ECSTORE_LINT_ACTUAL"
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ checked_files=(
|
||||
"crates/targets/src/target/webhook.rs"
|
||||
"crates/ecstore/src/store/peer.rs"
|
||||
"crates/ecstore/src/store/init.rs"
|
||||
"crates/ecstore/src/client/transition_api.rs"
|
||||
"crates/s3-client/src/transition_api.rs"
|
||||
"crates/ecstore/src/services/tier/tier.rs"
|
||||
"crates/heal/src/heal/manager.rs"
|
||||
"crates/heal/src/heal/storage.rs"
|
||||
|
||||
@@ -27,6 +27,12 @@ cd "$(dirname "$0")/.."
|
||||
# to verify S3 behavior and does not widen the production s3s surface.
|
||||
S3S_IMPORT_FILES_BASELINE=211
|
||||
S3_ERROR_LINES_BASELINE=1620
|
||||
# ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not
|
||||
# know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming*
|
||||
# client was extracted to crates/s3-client, where s3s usage is legitimate;
|
||||
# this counter ratchets the remaining serving-side s3s references out of
|
||||
# crates/ecstore. Baseline verified on 2026-08-26.
|
||||
S3S_ECSTORE_FILES_BASELINE=42
|
||||
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
|
||||
E2E_TEST_GLOB='--glob=!crates/e2e_test/**'
|
||||
|
||||
@@ -47,11 +53,13 @@ run_rg_to() {
|
||||
|
||||
run_rg_to "$TMP_DIR/import_files" -l "$S3S_PATH_PATTERN" --type rust $E2E_TEST_GLOB
|
||||
run_rg_to "$TMP_DIR/error_lines" -c 's3_error!' --type rust $E2E_TEST_GLOB
|
||||
run_rg_to "$TMP_DIR/ecstore_files" -l "$S3S_PATH_PATTERN" --type rust crates/ecstore/src
|
||||
|
||||
s3s_import_files="$(grep -c . "$TMP_DIR/import_files" || true)"
|
||||
s3_error_lines="$(awk -F: '{sum += $NF} END {print sum + 0}' "$TMP_DIR/error_lines")"
|
||||
s3s_ecstore_files="$(grep -c . "$TMP_DIR/ecstore_files" || true)"
|
||||
|
||||
for value in "$s3s_import_files" "$s3_error_lines"; do
|
||||
for value in "$s3s_import_files" "$s3_error_lines" "$s3s_ecstore_files"; do
|
||||
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
|
||||
echo "error: could not compute s3s footprint counts (got: '$value')" >&2
|
||||
exit 1
|
||||
@@ -82,6 +90,8 @@ check_ratchet "files importing s3s" "$s3s_import_files" "$S3S_IMPORT_FILES_BASEL
|
||||
"rg -l '$S3S_PATH_PATTERN' --type rust $E2E_TEST_GLOB"
|
||||
check_ratchet "s3_error! invocation lines" "$s3_error_lines" "$S3_ERROR_LINES_BASELINE" \
|
||||
"rg -c 's3_error!' --type rust $E2E_TEST_GLOB"
|
||||
check_ratchet "ecstore files referencing s3s" "$s3s_ecstore_files" "$S3S_ECSTORE_FILES_BASELINE" \
|
||||
"rg -l '$S3S_PATH_PATTERN' --type rust crates/ecstore/src"
|
||||
|
||||
if ((status != 0)); then
|
||||
exit 1
|
||||
|
||||
@@ -23,56 +23,56 @@ crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables
|
||||
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all
|
||||
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use
|
||||
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables
|
||||
crates/ecstore/src/client/api_error_response.rs|clippy::all
|
||||
crates/ecstore/src/client/api_error_response.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_error_response.rs|unused_variables
|
||||
crates/ecstore/src/client/api_get_object.rs|clippy::all
|
||||
crates/ecstore/src/client/api_get_object.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_get_object.rs|unused_variables
|
||||
crates/ecstore/src/client/api_get_options.rs|clippy::all
|
||||
crates/ecstore/src/client/api_get_options.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_get_options.rs|unused_variables
|
||||
crates/ecstore/src/client/api_list.rs|clippy::all
|
||||
crates/ecstore/src/client/api_list.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_list.rs|unused_variables
|
||||
crates/ecstore/src/client/api_put_object.rs|clippy::all
|
||||
crates/ecstore/src/client/api_put_object.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_put_object.rs|unused_variables
|
||||
crates/ecstore/src/client/api_put_object_common.rs|clippy::all
|
||||
crates/ecstore/src/client/api_put_object_common.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_put_object_common.rs|unused_variables
|
||||
crates/ecstore/src/client/api_put_object_multipart.rs|clippy::all
|
||||
crates/ecstore/src/client/api_put_object_multipart.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_put_object_multipart.rs|unused_variables
|
||||
crates/ecstore/src/client/api_put_object_streaming.rs|clippy::all
|
||||
crates/ecstore/src/client/api_put_object_streaming.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_put_object_streaming.rs|unused_variables
|
||||
crates/ecstore/src/client/api_remove.rs|clippy::all
|
||||
crates/ecstore/src/client/api_remove.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_remove.rs|unused_variables
|
||||
crates/ecstore/src/client/api_s3_datatypes.rs|clippy::all
|
||||
crates/ecstore/src/client/api_s3_datatypes.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_s3_datatypes.rs|unused_variables
|
||||
crates/ecstore/src/client/api_stat.rs|clippy::all
|
||||
crates/ecstore/src/client/api_stat.rs|unused_must_use
|
||||
crates/ecstore/src/client/api_stat.rs|unused_variables
|
||||
crates/ecstore/src/client/bucket_cache.rs|clippy::all
|
||||
crates/ecstore/src/client/bucket_cache.rs|unused_must_use
|
||||
crates/ecstore/src/client/bucket_cache.rs|unused_variables
|
||||
crates/ecstore/src/client/checksum.rs|clippy::all
|
||||
crates/ecstore/src/client/checksum.rs|unused_must_use
|
||||
crates/ecstore/src/client/checksum.rs|unused_variables
|
||||
crates/ecstore/src/client/constants.rs|unused_must_use
|
||||
crates/ecstore/src/client/constants.rs|unused_variables
|
||||
crates/ecstore/src/client/credentials.rs|clippy::all
|
||||
crates/ecstore/src/client/credentials.rs|unused_must_use
|
||||
crates/ecstore/src/client/credentials.rs|unused_variables
|
||||
crates/s3-client/src/api_error_response.rs|clippy::all
|
||||
crates/s3-client/src/api_error_response.rs|unused_must_use
|
||||
crates/s3-client/src/api_error_response.rs|unused_variables
|
||||
crates/s3-client/src/api_get_object.rs|clippy::all
|
||||
crates/s3-client/src/api_get_object.rs|unused_must_use
|
||||
crates/s3-client/src/api_get_object.rs|unused_variables
|
||||
crates/s3-client/src/api_get_options.rs|clippy::all
|
||||
crates/s3-client/src/api_get_options.rs|unused_must_use
|
||||
crates/s3-client/src/api_get_options.rs|unused_variables
|
||||
crates/s3-client/src/api_list.rs|clippy::all
|
||||
crates/s3-client/src/api_list.rs|unused_must_use
|
||||
crates/s3-client/src/api_list.rs|unused_variables
|
||||
crates/s3-client/src/api_put_object.rs|clippy::all
|
||||
crates/s3-client/src/api_put_object.rs|unused_must_use
|
||||
crates/s3-client/src/api_put_object.rs|unused_variables
|
||||
crates/s3-client/src/api_put_object_common.rs|clippy::all
|
||||
crates/s3-client/src/api_put_object_common.rs|unused_must_use
|
||||
crates/s3-client/src/api_put_object_common.rs|unused_variables
|
||||
crates/s3-client/src/api_put_object_multipart.rs|clippy::all
|
||||
crates/s3-client/src/api_put_object_multipart.rs|unused_must_use
|
||||
crates/s3-client/src/api_put_object_multipart.rs|unused_variables
|
||||
crates/s3-client/src/api_put_object_streaming.rs|clippy::all
|
||||
crates/s3-client/src/api_put_object_streaming.rs|unused_must_use
|
||||
crates/s3-client/src/api_put_object_streaming.rs|unused_variables
|
||||
crates/s3-client/src/api_remove.rs|clippy::all
|
||||
crates/s3-client/src/api_remove.rs|unused_must_use
|
||||
crates/s3-client/src/api_remove.rs|unused_variables
|
||||
crates/s3-client/src/api_s3_datatypes.rs|clippy::all
|
||||
crates/s3-client/src/api_s3_datatypes.rs|unused_must_use
|
||||
crates/s3-client/src/api_s3_datatypes.rs|unused_variables
|
||||
crates/s3-client/src/api_stat.rs|clippy::all
|
||||
crates/s3-client/src/api_stat.rs|unused_must_use
|
||||
crates/s3-client/src/api_stat.rs|unused_variables
|
||||
crates/s3-client/src/bucket_cache.rs|clippy::all
|
||||
crates/s3-client/src/bucket_cache.rs|unused_must_use
|
||||
crates/s3-client/src/bucket_cache.rs|unused_variables
|
||||
crates/s3-client/src/checksum.rs|clippy::all
|
||||
crates/s3-client/src/checksum.rs|unused_must_use
|
||||
crates/s3-client/src/checksum.rs|unused_variables
|
||||
crates/s3-client/src/constants.rs|unused_must_use
|
||||
crates/s3-client/src/constants.rs|unused_variables
|
||||
crates/s3-client/src/credentials.rs|clippy::all
|
||||
crates/s3-client/src/credentials.rs|unused_must_use
|
||||
crates/s3-client/src/credentials.rs|unused_variables
|
||||
crates/ecstore/src/client/object_api_utils.rs|clippy::all
|
||||
crates/ecstore/src/client/object_api_utils.rs|unused_must_use
|
||||
crates/ecstore/src/client/object_api_utils.rs|unused_variables
|
||||
crates/ecstore/src/client/transition_api.rs|clippy::all
|
||||
crates/ecstore/src/client/transition_api.rs|unused_must_use
|
||||
crates/ecstore/src/client/transition_api.rs|unused_variables
|
||||
crates/s3-client/src/transition_api.rs|clippy::all
|
||||
crates/s3-client/src/transition_api.rs|unused_must_use
|
||||
crates/s3-client/src/transition_api.rs|unused_variables
|
||||
crates/ecstore/src/services/event_notification.rs|unused_variables
|
||||
crates/ecstore/src/services/tier/tier.rs|clippy::all
|
||||
crates/ecstore/src/services/tier/tier.rs|unused_must_use
|
||||
|
||||
@@ -19,11 +19,6 @@
|
||||
1|crates/ecstore/src/bucket/replication/replication_object_config.rs
|
||||
2|crates/ecstore/src/bucket/replication/replication_pool.rs
|
||||
2|crates/ecstore/src/bucket/replication/replication_target_boundary.rs
|
||||
1|crates/ecstore/src/client/api_put_object_multipart.rs
|
||||
2|crates/ecstore/src/client/api_put_object_streaming.rs
|
||||
1|crates/ecstore/src/client/api_s3_datatypes.rs
|
||||
1|crates/ecstore/src/client/signer_error.rs
|
||||
5|crates/ecstore/src/client/transition_api.rs
|
||||
3|crates/ecstore/src/cluster/rpc/http_auth.rs
|
||||
2|crates/ecstore/src/cluster/rpc/internode_data_transport.rs
|
||||
11|crates/ecstore/src/cluster/rpc/peer_rest_client.rs
|
||||
|
||||
Reference in New Issue
Block a user