mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 07:57:01 +00:00
refactor(ecstore): extract the embedded S3 client into rustfs-s3-client (#6627)
The storage engine embedded a ~8.4K-line hand-written S3 HTTP client under crates/ecstore/src/client (rustfs/backlog#1842). That client is a legitimate engine capability — it consumes remote S3-compatible endpoints for ILM tier warm backends and transition targets — but it was misfiled inside the engine, dragging s3s/hyper wire types into ecstore and blocking ARCHITECTURE.md invariant 4. This PR is the pure-move step: 21 modules move verbatim to the new crates/s3-client crate (rustfs-s3-client), and crates/ecstore/src/client/mod.rs becomes a re-export shim so every in-crate crate::client:: path keeps working. The two server-side modules that were historically misfiled under client/ — object_api_utils.rs and object_handlers_common.rs — stay in ecstore. Three reverse dependencies from the client into engine internals are severed so the move can be pure: - transition_api::ReaderImpl::ObjectBody held ecstore's GetObjectReader; the client only ever reads the body, so the variant now holds an ObjectReader newtype over Box<dyn AsyncRead + Send + Sync + Unpin> with the same read_all() surface. The single production construction site (set_disk transition upload) and the two engine-side consumers were adjusted. - api_list/api_remove used ecstore's storage_api_contracts / object_api types; api_list now imports BucketInfo from rustfs-storage-api directly, and api_remove uses the client's own transition_api::ObjectInfo (only .name/.version_id were read; the error-path bucket name is now threaded as a parameter instead of read from the deleted objects). - the api_put_object_streaming regression tests built a GetObjectReader by hand; they now wrap the duplex stream in ObjectReader::new. Guard updates: the s3s footprint ratchet gains an ecstore-scoped counter (42 files, shrink-only, per rustfs/backlog#1842), the ecstore module-lint-blanket register follows the moved files into crates/s3-client so the blanket ratchet keeps covering them, the logging guardrail path pin follows transition_api.rs, and the ::other(format!) baseline is regenerated (moved call sites left ecstore). Verification: cargo check -p rustfs-s3-client -p rustfs-ecstore; cargo nextest run -p rustfs-s3-client (43 passed) and -p rustfs-ecstore (4515/4523; the 8 failures reproduce identically on pristine origin/main on the same machine); cargo clippy --all-targets; scripts/check_layer_dependencies.sh, check_architecture_migration_rules.sh, check_s3s_footprint.sh, check_logging_guardrails.sh, check_error_other_format_ratchet.sh, check_doc_paths.sh, check_ci_paths_sync.sh all pass.
This commit is contained in:
Generated
+46
@@ -9552,6 +9552,7 @@ dependencies = [
|
|||||||
"rustfs-replication",
|
"rustfs-replication",
|
||||||
"rustfs-rio",
|
"rustfs-rio",
|
||||||
"rustfs-rio-v2",
|
"rustfs-rio-v2",
|
||||||
|
"rustfs-s3-client",
|
||||||
"rustfs-s3-types",
|
"rustfs-s3-types",
|
||||||
"rustfs-signer",
|
"rustfs-signer",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
@@ -10316,6 +10317,51 @@ dependencies = [
|
|||||||
"walkdir",
|
"walkdir",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustfs-s3-client"
|
||||||
|
version = "1.0.0-rc.3"
|
||||||
|
dependencies = [
|
||||||
|
"base64-simd",
|
||||||
|
"bytes",
|
||||||
|
"enumset",
|
||||||
|
"futures",
|
||||||
|
"futures-util",
|
||||||
|
"hex-simd",
|
||||||
|
"http 1.5.0",
|
||||||
|
"http-body 1.1.0",
|
||||||
|
"http-body-util",
|
||||||
|
"hyper",
|
||||||
|
"hyper-rustls",
|
||||||
|
"hyper-util",
|
||||||
|
"lazy_static",
|
||||||
|
"md-5 0.11.0",
|
||||||
|
"quick-xml 0.42.0",
|
||||||
|
"rand 0.10.2",
|
||||||
|
"rustfs-checksums",
|
||||||
|
"rustfs-config",
|
||||||
|
"rustfs-rio",
|
||||||
|
"rustfs-signer",
|
||||||
|
"rustfs-storage-api",
|
||||||
|
"rustfs-tls-runtime",
|
||||||
|
"rustfs-utils",
|
||||||
|
"rustls",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"s3s",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"sha1 0.11.0",
|
||||||
|
"sha2 0.11.0",
|
||||||
|
"thiserror 2.0.20",
|
||||||
|
"time",
|
||||||
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
|
"tower",
|
||||||
|
"tracing",
|
||||||
|
"url",
|
||||||
|
"urlencoding",
|
||||||
|
"uuid",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustfs-s3-ops"
|
name = "rustfs-s3-ops"
|
||||||
version = "1.0.0-rc.3"
|
version = "1.0.0-rc.3"
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ members = [
|
|||||||
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
||||||
"crates/replication", # Replication contracts and wire formats
|
"crates/replication", # Replication contracts and wire formats
|
||||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||||
|
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
|
||||||
"crates/s3-types", # S3 event type definitions
|
"crates/s3-types", # S3 event type definitions
|
||||||
"crates/s3-ops", # S3 operation definitions and mapping
|
"crates/s3-ops", # S3 operation definitions and mapping
|
||||||
"crates/s3select-api", # S3 Select API interface
|
"crates/s3select-api", # S3 Select API interface
|
||||||
@@ -121,6 +122,7 @@ rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" }
|
|||||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
||||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
||||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
||||||
|
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.3" }
|
||||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
||||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
||||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
||||||
|
|||||||
@@ -147,6 +147,7 @@ rustfs-policy.workspace = true
|
|||||||
rustfs-protos.workspace = true
|
rustfs-protos.workspace = true
|
||||||
rustfs-replication.workspace = true
|
rustfs-replication.workspace = true
|
||||||
rustfs-lifecycle.workspace = true
|
rustfs-lifecycle.workspace = true
|
||||||
|
rustfs-s3-client = { workspace = true }
|
||||||
rustfs-s3-types = { workspace = true }
|
rustfs-s3-types = { workspace = true }
|
||||||
rustfs-data-usage.workspace = true
|
rustfs-data-usage.workspace = true
|
||||||
rustfs-object-capacity.workspace = true
|
rustfs-object-capacity.workspace = true
|
||||||
|
|||||||
@@ -12,28 +12,16 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
|
// The S3-consuming client moved to the `rustfs-s3-client` crate
|
||||||
|
// (rustfs/backlog#1842). This shim keeps `crate::client::*` paths working for
|
||||||
|
// in-crate consumers during the migration window; it is deleted once every
|
||||||
|
// consumer imports `rustfs_s3_client` directly. Only the two server-side
|
||||||
|
// modules below (misfiled here historically) remain as real ecstore code.
|
||||||
|
|
||||||
|
pub use rustfs_s3_client::{
|
||||||
|
admin_handler_utils, api_get_options, api_list, api_put_object, api_remove, api_s3_datatypes, credentials, provider_versions,
|
||||||
|
signer_error, transition_api,
|
||||||
|
};
|
||||||
|
|
||||||
pub mod admin_handler_utils;
|
|
||||||
pub mod api_error_response;
|
|
||||||
pub mod api_get_object;
|
|
||||||
pub mod api_get_options;
|
|
||||||
pub mod api_list;
|
|
||||||
pub mod api_put_object;
|
|
||||||
pub mod api_put_object_common;
|
|
||||||
pub mod api_put_object_multipart;
|
|
||||||
pub mod api_put_object_streaming;
|
|
||||||
pub mod api_remove;
|
|
||||||
pub mod api_s3_datatypes;
|
|
||||||
pub mod api_stat;
|
|
||||||
pub mod bucket_cache;
|
|
||||||
pub mod checksum;
|
|
||||||
pub mod constants;
|
|
||||||
pub mod credentials;
|
|
||||||
pub mod object_api_utils;
|
pub mod object_api_utils;
|
||||||
pub mod object_handlers_common;
|
pub mod object_handlers_common;
|
||||||
pub(crate) mod provider_versions;
|
|
||||||
pub(crate) mod runtime_sources;
|
|
||||||
pub mod signer_error;
|
|
||||||
pub mod transition_api;
|
|
||||||
pub mod utils;
|
|
||||||
|
|||||||
@@ -596,9 +596,9 @@ impl MockWarmBackend {
|
|||||||
if let Some(limit) = limit {
|
if let Some(limit) = limit {
|
||||||
let limit =
|
let limit =
|
||||||
u64::try_from(limit).map_err(|_| std::io::Error::other("mock PUT read limit exceeds u64::MAX"))?;
|
u64::try_from(limit).map_err(|_| std::io::Error::other("mock PUT read limit exceeds u64::MAX"))?;
|
||||||
reader.stream.take(limit).read_to_end(&mut buf).await?;
|
(&mut reader).take(limit).read_to_end(&mut buf).await?;
|
||||||
} else {
|
} else {
|
||||||
reader.stream.read_to_end(&mut buf).await?;
|
reader.read_to_end(&mut buf).await?;
|
||||||
}
|
}
|
||||||
Ok(buf)
|
Ok(buf)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ use crate::bucket::replication::{
|
|||||||
};
|
};
|
||||||
use crate::bucket::versioning::VersioningApi;
|
use crate::bucket::versioning::VersioningApi;
|
||||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||||
use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
|
use crate::client::{object_api_utils::get_raw_etag, transition_api::ObjectReader, transition_api::ReaderImpl};
|
||||||
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
||||||
use crate::data_usage::record_compression_total_memory;
|
use crate::data_usage::record_compression_total_memory;
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
@@ -1559,8 +1559,8 @@ impl SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get lock acquire timeout from environment variable RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT (in seconds)
|
/// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
|
||||||
/// Defaults to 5 seconds if not set or invalid
|
/// Defaults to 30 seconds if not set or invalid
|
||||||
/// Lock acquisition timeout. Cached: this is consulted on every object
|
/// Lock acquisition timeout. Cached: this is consulted on every object
|
||||||
/// lock acquisition and `std::env::var` takes a process-global lock. In test
|
/// lock acquisition and `std::env::var` takes a process-global lock. In test
|
||||||
/// builds the env var is read directly so `temp_env` overrides take effect.
|
/// builds the env var is read directly so `temp_env` overrides take effect.
|
||||||
|
|||||||
@@ -7332,12 +7332,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
|||||||
let expected_size = u64::try_from(fi.size).map_err(|_| StorageError::FileCorrupt)?;
|
let expected_size = u64::try_from(fi.size).map_err(|_| StorageError::FileCorrupt)?;
|
||||||
let (pr, pw) = tokio::io::duplex(fi.erasure.block_size);
|
let (pr, pw) = tokio::io::duplex(fi.erasure.block_size);
|
||||||
let consumed = Arc::new(AtomicU64::new(0));
|
let consumed = Arc::new(AtomicU64::new(0));
|
||||||
let reader = ReaderImpl::ObjectBody(GetObjectReader {
|
let reader = ReaderImpl::ObjectBody(ObjectReader::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))));
|
||||||
stream: Box::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))),
|
|
||||||
object_info: oi,
|
|
||||||
buffered_body: None,
|
|
||||||
body_source: GetObjectBodySource::Unprobed,
|
|
||||||
});
|
|
||||||
|
|
||||||
let cloned_bucket = bucket.to_string();
|
let cloned_bucket = bucket.to_string();
|
||||||
let cloned_object = object.to_string();
|
let cloned_object = object.to_string();
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# Copyright 2024 RustFS Team
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
[package]
|
||||||
|
name = "rustfs-s3-client"
|
||||||
|
edition.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
repository.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
version.workspace = true
|
||||||
|
homepage.workspace = true
|
||||||
|
description = "S3 client used by RustFS when the storage engine consumes remote S3-compatible endpoints (tier warm backends, transition targets)."
|
||||||
|
keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
|
||||||
|
categories = ["web-programming", "development-tools", "network-programming"]
|
||||||
|
documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
rustfs-checksums.workspace = true
|
||||||
|
rustfs-config.workspace = true
|
||||||
|
rustfs-rio.workspace = true
|
||||||
|
rustfs-signer.workspace = true
|
||||||
|
rustfs-storage-api.workspace = true
|
||||||
|
rustfs-tls-runtime.workspace = true
|
||||||
|
rustfs-utils = { workspace = true, features = ["full"] }
|
||||||
|
base64-simd.workspace = true
|
||||||
|
bytes = { workspace = true, features = ["serde"] }
|
||||||
|
hex-simd = { workspace = true }
|
||||||
|
enumset = { workspace = true }
|
||||||
|
futures.workspace = true
|
||||||
|
futures-util.workspace = true
|
||||||
|
http.workspace = true
|
||||||
|
http-body = { workspace = true }
|
||||||
|
http-body-util.workspace = true
|
||||||
|
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||||
|
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
|
||||||
|
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||||
|
lazy_static.workspace = true
|
||||||
|
md-5.workspace = true
|
||||||
|
quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
|
||||||
|
rand = { workspace = true, features = ["serde"] }
|
||||||
|
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||||
|
rustls-pki-types.workspace = true
|
||||||
|
s3s = { workspace = true, features = ["minio"] }
|
||||||
|
serde = { workspace = true, features = ["derive"] }
|
||||||
|
serde_json = { workspace = true, features = ["raw_value"] }
|
||||||
|
sha1 = { workspace = true }
|
||||||
|
sha2 = { workspace = true }
|
||||||
|
thiserror.workspace = true
|
||||||
|
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||||
|
tokio = { workspace = true, features = ["io-util", "sync", "fs", "rt-multi-thread"] }
|
||||||
|
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||||
|
tower = { workspace = true, features = ["timeout"] }
|
||||||
|
tracing.workspace = true
|
||||||
|
url.workspace = true
|
||||||
|
urlencoding = { workspace = true }
|
||||||
|
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
doctest = false
|
||||||
+1
-1
@@ -27,7 +27,7 @@ use std::task::{Context, Poll};
|
|||||||
use tokio::io::BufReader;
|
use tokio::io::BufReader;
|
||||||
use tokio_util::io::StreamReader;
|
use tokio_util::io::StreamReader;
|
||||||
|
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::err_invalid_argument,
|
api_error_response::err_invalid_argument,
|
||||||
api_get_options::GetObjectOptions,
|
api_get_options::GetObjectOptions,
|
||||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
|
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
|
||||||
+1
-1
@@ -25,7 +25,7 @@ use std::collections::HashMap;
|
|||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::client::api_error_response::err_invalid_argument;
|
use crate::api_error_response::err_invalid_argument;
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub struct AdvancedGetOptions {
|
pub struct AdvancedGetOptions {
|
||||||
@@ -18,7 +18,7 @@
|
|||||||
#![allow(unused_must_use)]
|
#![allow(unused_must_use)]
|
||||||
#![allow(clippy::all)]
|
#![allow(clippy::all)]
|
||||||
|
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::http_resp_to_error_response,
|
api_error_response::http_resp_to_error_response,
|
||||||
api_s3_datatypes::{
|
api_s3_datatypes::{
|
||||||
ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
|
ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
|
||||||
@@ -26,12 +26,12 @@ use crate::client::{
|
|||||||
credentials,
|
credentials,
|
||||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
|
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
|
||||||
};
|
};
|
||||||
use crate::storage_api_contracts::bucket::BucketInfo;
|
|
||||||
use http::{HeaderMap, StatusCode};
|
use http::{HeaderMap, StatusCode};
|
||||||
use http_body_util::BodyExt;
|
use http_body_util::BodyExt;
|
||||||
use hyper::body::Body;
|
use hyper::body::Body;
|
||||||
use hyper::body::Bytes;
|
use hyper::body::Bytes;
|
||||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||||
|
use rustfs_storage_api::BucketInfo;
|
||||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
+3
-3
@@ -30,9 +30,9 @@ use s3s::header::{
|
|||||||
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
|
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
|
||||||
};
|
};
|
||||||
//use crate::disk::{BufferReader, Reader};
|
//use crate::disk::{BufferReader, Reader};
|
||||||
use crate::client::checksum::ChecksumMode;
|
use crate::checksum::ChecksumMode;
|
||||||
use crate::client::utils::base64_encode;
|
use crate::utils::base64_encode;
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::{err_entity_too_large, err_invalid_argument},
|
api_error_response::{err_entity_too_large, err_invalid_argument},
|
||||||
api_put_object_common::optimal_part_info,
|
api_put_object_common::optimal_part_info,
|
||||||
api_put_object_multipart::UploadPartParams,
|
api_put_object_multipart::UploadPartParams,
|
||||||
+1
-1
@@ -18,7 +18,7 @@
|
|||||||
#![allow(unused_must_use)]
|
#![allow(unused_must_use)]
|
||||||
#![allow(clippy::all)]
|
#![allow(clippy::all)]
|
||||||
|
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::{err_entity_too_large, err_invalid_argument},
|
api_error_response::{err_entity_too_large, err_invalid_argument},
|
||||||
api_put_object::PutObjectOptions,
|
api_put_object::PutObjectOptions,
|
||||||
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
|
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
|
||||||
+4
-4
@@ -26,9 +26,9 @@ use time::OffsetDateTime;
|
|||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::client::checksum::ChecksumMode;
|
use crate::checksum::ChecksumMode;
|
||||||
use crate::client::utils::base64_encode;
|
use crate::utils::base64_encode;
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::{
|
api_error_response::{
|
||||||
err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
|
err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
|
||||||
},
|
},
|
||||||
@@ -448,7 +448,7 @@ pub struct UploadPartParams {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::client::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult};
|
use crate::api_s3_datatypes::{CompleteMultipartUpload, CompletePart, InitiateMultipartUploadResult};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn complete_multipart_upload_serializes_s3_part_elements() {
|
fn complete_multipart_upload_serializes_s3_part_elements() {
|
||||||
+6
-16
@@ -31,8 +31,8 @@ use tokio_util::sync::CancellationToken;
|
|||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::client::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
|
use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
|
api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
|
||||||
api_put_object::PutObjectOptions,
|
api_put_object::PutObjectOptions,
|
||||||
api_put_object_common::{is_object, optimal_part_info},
|
api_put_object_common::{is_object, optimal_part_info},
|
||||||
@@ -42,7 +42,7 @@ use crate::client::{
|
|||||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
|
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::client::utils::base64_encode;
|
use crate::utils::base64_encode;
|
||||||
use rustfs_utils::path::trim_etag;
|
use rustfs_utils::path::trim_etag;
|
||||||
use s3s::header::X_AMZ_EXPIRATION;
|
use s3s::header::X_AMZ_EXPIRATION;
|
||||||
|
|
||||||
@@ -620,7 +620,7 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
|
use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
|
||||||
use crate::object_api::GetObjectReader;
|
use crate::transition_api::ObjectReader;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use rustfs_utils::hash::HashAlgorithm;
|
use rustfs_utils::hash::HashAlgorithm;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -654,12 +654,7 @@ mod tests {
|
|||||||
let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
|
let data: Vec<u8> = (0..total).map(|i| i as u8).collect();
|
||||||
w.write_all(&data).await.unwrap();
|
w.write_all(&data).await.unwrap();
|
||||||
});
|
});
|
||||||
let reader = ReaderImpl::ObjectBody(GetObjectReader {
|
let reader = ReaderImpl::ObjectBody(ObjectReader::new(r));
|
||||||
stream: Box::new(r),
|
|
||||||
object_info: Default::default(),
|
|
||||||
buffered_body: None,
|
|
||||||
body_source: Default::default(),
|
|
||||||
});
|
|
||||||
|
|
||||||
let sizes = collect_part_sizes(reader, total, 100, 50).await;
|
let sizes = collect_part_sizes(reader, total, 100, 50).await;
|
||||||
assert_eq!(sizes, vec![100, 100, 50]);
|
assert_eq!(sizes, vec![100, 100, 50]);
|
||||||
@@ -684,12 +679,7 @@ mod tests {
|
|||||||
use tokio::io::AsyncWriteExt;
|
use tokio::io::AsyncWriteExt;
|
||||||
w.write_all(&[1u8; 30]).await.unwrap();
|
w.write_all(&[1u8; 30]).await.unwrap();
|
||||||
});
|
});
|
||||||
let mut reader = ReaderImpl::ObjectBody(GetObjectReader {
|
let mut reader = ReaderImpl::ObjectBody(ObjectReader::new(r));
|
||||||
stream: Box::new(r),
|
|
||||||
object_info: Default::default(),
|
|
||||||
buffered_body: None,
|
|
||||||
body_source: Default::default(),
|
|
||||||
});
|
|
||||||
let buf = read_multipart_part(&mut reader, 100).await.unwrap();
|
let buf = read_multipart_part(&mut reader, 100).await.unwrap();
|
||||||
assert_eq!(buf.len(), 30);
|
assert_eq!(buf.len(), 30);
|
||||||
}
|
}
|
||||||
@@ -36,16 +36,13 @@ use time::OffsetDateTime;
|
|||||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||||
use tracing::Instrument;
|
use tracing::Instrument;
|
||||||
|
|
||||||
use crate::client::utils::base64_encode;
|
use crate::transition_api::ObjectInfo;
|
||||||
use crate::client::{
|
use crate::utils::base64_encode;
|
||||||
|
use crate::{
|
||||||
api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
|
api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
|
||||||
api_s3_datatypes::{DeleteMultiObjects, DeleteObject},
|
api_s3_datatypes::{DeleteMultiObjects, DeleteObject},
|
||||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||||
};
|
};
|
||||||
use crate::{
|
|
||||||
disk::DiskAPI,
|
|
||||||
object_api::{GetObjectReader, ObjectInfo},
|
|
||||||
};
|
|
||||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||||
|
|
||||||
pub struct RemoveBucketOptions {
|
pub struct RemoveBucketOptions {
|
||||||
@@ -366,7 +363,13 @@ impl TransitionClient {
|
|||||||
body_vec.extend_from_slice(data);
|
body_vec.extend_from_slice(data);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), &batch, result_tx.clone()).await;
|
process_remove_multi_objects_response(
|
||||||
|
ReaderImpl::Body(Bytes::from(body_vec)),
|
||||||
|
bucket_name,
|
||||||
|
&batch,
|
||||||
|
result_tx.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -553,6 +556,7 @@ pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8>
|
|||||||
|
|
||||||
pub async fn process_remove_multi_objects_response(
|
pub async fn process_remove_multi_objects_response(
|
||||||
body: ReaderImpl,
|
body: ReaderImpl,
|
||||||
|
bucket_name: &str,
|
||||||
objects: &[ObjectInfo],
|
objects: &[ObjectInfo],
|
||||||
result_tx: Sender<RemoveObjectResult>,
|
result_tx: Sender<RemoveObjectResult>,
|
||||||
) {
|
) {
|
||||||
@@ -575,7 +579,7 @@ pub async fn process_remove_multi_objects_response(
|
|||||||
err: Some(std::io::Error::other(ErrorResponse {
|
err: Some(std::io::Error::other(ErrorResponse {
|
||||||
code: S3ErrorCode::Custom("ReadDeleteResponseFailed".into()),
|
code: S3ErrorCode::Custom("ReadDeleteResponseFailed".into()),
|
||||||
message: format!("read multi remove response failed: {err}"),
|
message: format!("read multi remove response failed: {err}"),
|
||||||
bucket_name: object.bucket.clone(),
|
bucket_name: bucket_name.to_string(),
|
||||||
key: object.name.clone(),
|
key: object.name.clone(),
|
||||||
resource: "".to_string(),
|
resource: "".to_string(),
|
||||||
request_id: "".to_string(),
|
request_id: "".to_string(),
|
||||||
@@ -647,7 +651,7 @@ pub async fn process_remove_multi_objects_response(
|
|||||||
"unmarshal multi remove response failed: {err}; response_body={}",
|
"unmarshal multi remove response failed: {err}; response_body={}",
|
||||||
body.chars().take(DELETE_RESPONSE_PREVIEW_LEN).collect::<String>()
|
body.chars().take(DELETE_RESPONSE_PREVIEW_LEN).collect::<String>()
|
||||||
),
|
),
|
||||||
bucket_name: object.bucket.clone(),
|
bucket_name: bucket_name.to_string(),
|
||||||
key: object.name.clone(),
|
key: object.name.clone(),
|
||||||
resource: "".to_string(),
|
resource: "".to_string(),
|
||||||
request_id: "".to_string(),
|
request_id: "".to_string(),
|
||||||
@@ -707,13 +711,7 @@ pub async fn process_remove_multi_objects_response(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (object_name, object_version_id) in pending {
|
for (object_name, object_version_id) in pending {
|
||||||
let bucket_name = objects
|
let bucket_name = bucket_name.to_string();
|
||||||
.iter()
|
|
||||||
.find(|object| {
|
|
||||||
object.name == object_name && object.version_id.as_ref().map(|v| v.to_string()) == Some(object_version_id.clone())
|
|
||||||
})
|
|
||||||
.map(|o| o.bucket.clone())
|
|
||||||
.unwrap_or_default();
|
|
||||||
let object_name = object_name;
|
let object_name = object_name;
|
||||||
let object_version_id = object_version_id;
|
let object_version_id = object_version_id;
|
||||||
let error_message = format!(
|
let error_message = format!(
|
||||||
@@ -750,7 +748,7 @@ fn has_invalid_xml_char(str: &str) -> bool {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::client::{
|
use crate::{
|
||||||
credentials::{Credentials, SignatureType, Static, Value},
|
credentials::{Credentials, SignatureType, Static, Value},
|
||||||
transition_api::{BucketLookupType, Options},
|
transition_api::{BucketLookupType, Options},
|
||||||
};
|
};
|
||||||
@@ -808,7 +806,6 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn multi_object_delete_request_uses_lowercase_hex_sha256_header() {
|
async fn multi_object_delete_request_uses_lowercase_hex_sha256_header() {
|
||||||
let objects = vec![ObjectInfo {
|
let objects = vec![ObjectInfo {
|
||||||
bucket: "bucket".to_string(),
|
|
||||||
name: "object.txt".to_string(),
|
name: "object.txt".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}];
|
}];
|
||||||
+3
-3
@@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize};
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
use crate::client::checksum::ChecksumMode;
|
use crate::checksum::ChecksumMode;
|
||||||
use crate::client::transition_api::ObjectMultipartInfo;
|
use crate::transition_api::ObjectMultipartInfo;
|
||||||
use crate::client::utils::base64_decode;
|
use crate::utils::base64_decode;
|
||||||
|
|
||||||
use super::transition_api;
|
use super::transition_api;
|
||||||
|
|
||||||
@@ -28,7 +28,7 @@ use tokio::io::BufReader;
|
|||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response},
|
api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response},
|
||||||
api_get_options::GetObjectOptions,
|
api_get_options::GetObjectOptions,
|
||||||
transition_api::{
|
transition_api::{
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
|
|
||||||
use super::constants::UNSIGNED_PAYLOAD;
|
use super::constants::UNSIGNED_PAYLOAD;
|
||||||
use super::credentials::SignatureType;
|
use super::credentials::SignatureType;
|
||||||
use crate::client::{
|
use crate::{
|
||||||
api_error_response::http_resp_to_error_response,
|
api_error_response::http_resp_to_error_response,
|
||||||
signer_error,
|
signer_error,
|
||||||
transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
|
transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
|
||||||
@@ -23,10 +23,9 @@ use lazy_static::lazy_static;
|
|||||||
use rustfs_checksums::ChecksumAlgorithm;
|
use rustfs_checksums::ChecksumAlgorithm;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use crate::client::utils::base64_decode;
|
use crate::utils::base64_decode;
|
||||||
use crate::client::utils::base64_encode;
|
use crate::utils::base64_encode;
|
||||||
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
use crate::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||||
use crate::{disk::DiskAPI, object_api::GetObjectReader};
|
|
||||||
// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
|
// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
|
||||||
// in rustfs-utils' headers module.
|
// in rustfs-utils' headers module.
|
||||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
|
use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
//! S3 client used by the storage engine when it *consumes* remote S3-compatible
|
||||||
|
//! endpoints (ILM tier warm backends, transition targets). Extracted from
|
||||||
|
//! `crates/ecstore/src/client` (rustfs/backlog#1842) so the engine no longer
|
||||||
|
//! embeds an S3 HTTP client; ecstore re-exports these modules during the
|
||||||
|
//! migration window.
|
||||||
|
|
||||||
|
pub mod admin_handler_utils;
|
||||||
|
pub mod api_error_response;
|
||||||
|
pub mod api_get_object;
|
||||||
|
pub mod api_get_options;
|
||||||
|
pub mod api_list;
|
||||||
|
pub mod api_put_object;
|
||||||
|
pub mod api_put_object_common;
|
||||||
|
pub mod api_put_object_multipart;
|
||||||
|
pub mod api_put_object_streaming;
|
||||||
|
pub mod api_remove;
|
||||||
|
pub mod api_s3_datatypes;
|
||||||
|
pub mod api_stat;
|
||||||
|
pub mod bucket_cache;
|
||||||
|
pub mod checksum;
|
||||||
|
pub mod constants;
|
||||||
|
pub mod credentials;
|
||||||
|
pub mod provider_versions;
|
||||||
|
pub mod runtime_sources;
|
||||||
|
pub mod signer_error;
|
||||||
|
pub mod transition_api;
|
||||||
|
pub mod utils;
|
||||||
+14
-14
@@ -24,7 +24,7 @@ const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
|
|||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
|
||||||
pub(crate) enum BucketVersioningState {
|
pub enum BucketVersioningState {
|
||||||
Unknown,
|
Unknown,
|
||||||
Disabled,
|
Disabled,
|
||||||
Suspended,
|
Suspended,
|
||||||
@@ -32,7 +32,7 @@ pub(crate) enum BucketVersioningState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub(crate) enum RemoteVersion {
|
pub enum RemoteVersion {
|
||||||
Unknown,
|
Unknown,
|
||||||
Disabled,
|
Disabled,
|
||||||
SuspendedNull,
|
SuspendedNull,
|
||||||
@@ -40,7 +40,7 @@ pub(crate) enum RemoteVersion {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RemoteVersion {
|
impl RemoteVersion {
|
||||||
pub(crate) fn exact_id(&self) -> Option<&str> {
|
pub fn exact_id(&self) -> Option<&str> {
|
||||||
match self {
|
match self {
|
||||||
Self::SuspendedNull => Some("null"),
|
Self::SuspendedNull => Some("null"),
|
||||||
Self::Exact(version_id) => Some(version_id),
|
Self::Exact(version_id) => Some(version_id),
|
||||||
@@ -49,7 +49,7 @@ impl RemoteVersion {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
pub fn exact_request_id(&self) -> Result<Option<&str>, Error> {
|
||||||
match self {
|
match self {
|
||||||
Self::Unknown => Err(Error::new(
|
Self::Unknown => Err(Error::new(
|
||||||
ErrorKind::InvalidData,
|
ErrorKind::InvalidData,
|
||||||
@@ -63,23 +63,23 @@ impl RemoteVersion {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub(crate) enum ConditionalCreateCapability {
|
pub enum ConditionalCreateCapability {
|
||||||
Unsupported,
|
Unsupported,
|
||||||
IfNoneMatchStar,
|
IfNoneMatchStar,
|
||||||
GenerationMatchZero,
|
GenerationMatchZero,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub(crate) struct ProviderVersionCapabilities {
|
pub struct ProviderVersionCapabilities {
|
||||||
raw_version_header: Option<&'static str>,
|
raw_version_header: Option<&'static str>,
|
||||||
pub(crate) bucket_versioning_state: bool,
|
pub bucket_versioning_state: bool,
|
||||||
pub(crate) list_object_versions: bool,
|
pub list_object_versions: bool,
|
||||||
pub(crate) conditional_create: ConditionalCreateCapability,
|
pub conditional_create: ConditionalCreateCapability,
|
||||||
pub(crate) exact_get_delete: bool,
|
pub exact_get_delete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProviderVersionCapabilities {
|
impl ProviderVersionCapabilities {
|
||||||
pub(crate) fn for_tier_type(tier_type: &str) -> Self {
|
pub fn for_tier_type(tier_type: &str) -> Self {
|
||||||
if tier_type.eq_ignore_ascii_case("s3")
|
if tier_type.eq_ignore_ascii_case("s3")
|
||||||
|| tier_type.eq_ignore_ascii_case("rustfs")
|
|| tier_type.eq_ignore_ascii_case("rustfs")
|
||||||
|| tier_type.eq_ignore_ascii_case("minio")
|
|| tier_type.eq_ignore_ascii_case("minio")
|
||||||
@@ -144,7 +144,7 @@ impl ProviderVersionCapabilities {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn raw_version_id(self, headers: &HeaderMap) -> Result<Option<&str>, Error> {
|
pub fn raw_version_id(self, headers: &HeaderMap) -> Result<Option<&str>, Error> {
|
||||||
let Some(header_name) = self.raw_version_header else {
|
let Some(header_name) = self.raw_version_header else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
@@ -158,7 +158,7 @@ impl ProviderVersionCapabilities {
|
|||||||
Ok(Some(value))
|
Ok(Some(value))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn remote_version(self, headers: &HeaderMap, versioning: BucketVersioningState) -> Result<RemoteVersion, Error> {
|
pub fn remote_version(self, headers: &HeaderMap, versioning: BucketVersioningState) -> Result<RemoteVersion, Error> {
|
||||||
let Some(value) = self.raw_version_id(headers)? else {
|
let Some(value) = self.raw_version_id(headers)? else {
|
||||||
return Ok(match versioning {
|
return Ok(match versioning {
|
||||||
BucketVersioningState::Disabled => RemoteVersion::Disabled,
|
BucketVersioningState::Disabled => RemoteVersion::Disabled,
|
||||||
@@ -174,7 +174,7 @@ impl ProviderVersionCapabilities {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
pub fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||||
if version_id.is_empty() {
|
if version_id.is_empty() {
|
||||||
return Err(Error::new(
|
return Err(Error::new(
|
||||||
ErrorKind::InvalidData,
|
ErrorKind::InvalidData,
|
||||||
+2
-2
@@ -16,10 +16,10 @@ use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, load_global_outbound_t
|
|||||||
|
|
||||||
const ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER: &str = "ecstore_transition_client";
|
const ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER: &str = "ecstore_transition_client";
|
||||||
|
|
||||||
pub(crate) async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
|
pub async fn transition_client_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
|
||||||
load_global_outbound_tls_state().await
|
load_global_outbound_tls_state().await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn record_transition_client_tls_generation(generation: u64) {
|
pub fn record_transition_client_tls_generation(generation: u64) {
|
||||||
record_tls_generation(ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER, generation);
|
record_tls_generation(ECSTORE_TRANSITION_CLIENT_TLS_CONSUMER, generation);
|
||||||
}
|
}
|
||||||
@@ -16,7 +16,7 @@ use std::error::Error as StdError;
|
|||||||
use std::fmt::{Display, Formatter};
|
use std::fmt::{Display, Formatter};
|
||||||
use std::io::{Error, ErrorKind};
|
use std::io::{Error, ErrorKind};
|
||||||
|
|
||||||
pub(crate) const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error";
|
pub const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error";
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct SignerHeaderError {
|
struct SignerHeaderError {
|
||||||
@@ -45,18 +45,18 @@ impl Display for SignerHeaderError {
|
|||||||
|
|
||||||
impl StdError for SignerHeaderError {}
|
impl StdError for SignerHeaderError {}
|
||||||
|
|
||||||
pub(crate) fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error {
|
pub fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error {
|
||||||
Error::new(ErrorKind::InvalidInput, SignerHeaderError::new(scope, header_name))
|
Error::new(ErrorKind::InvalidInput, SignerHeaderError::new(scope, header_name))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error {
|
pub fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error {
|
||||||
match error {
|
match error {
|
||||||
rustfs_signer::SignV4Error::InvalidHeaderValue { name } => invalid_utf8_header_error(scope, &name),
|
rustfs_signer::SignV4Error::InvalidHeaderValue { name } => invalid_utf8_header_error(scope, &name),
|
||||||
other => Error::other(format!("{scope}: {other}")),
|
other => Error::other(format!("{scope}: {other}")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool {
|
pub fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool {
|
||||||
let mut current = Some(err);
|
let mut current = Some(err);
|
||||||
while let Some(source) = current {
|
while let Some(source) = current {
|
||||||
if source.downcast_ref::<SignerHeaderError>().is_some() {
|
if source.downcast_ref::<SignerHeaderError>().is_some() {
|
||||||
+51
-18
@@ -18,8 +18,9 @@
|
|||||||
#![allow(unused_must_use)]
|
#![allow(unused_must_use)]
|
||||||
#![allow(clippy::all)]
|
#![allow(clippy::all)]
|
||||||
|
|
||||||
use crate::client::bucket_cache::BucketLocationCache;
|
use crate::bucket_cache::BucketLocationCache;
|
||||||
use crate::client::{
|
use crate::checksum::ChecksumMode;
|
||||||
|
use crate::{
|
||||||
api_error_response::ErrorResponse,
|
api_error_response::ErrorResponse,
|
||||||
api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response},
|
api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response},
|
||||||
api_get_options::GetObjectOptions,
|
api_get_options::GetObjectOptions,
|
||||||
@@ -34,7 +35,6 @@ use crate::client::{
|
|||||||
provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion},
|
provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion},
|
||||||
signer_error,
|
signer_error,
|
||||||
};
|
};
|
||||||
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
|
|
||||||
use futures::{Future, StreamExt};
|
use futures::{Future, StreamExt};
|
||||||
use http::{HeaderMap, HeaderName};
|
use http::{HeaderMap, HeaderName};
|
||||||
use http::{
|
use http::{
|
||||||
@@ -79,16 +79,17 @@ use std::{
|
|||||||
use time::Duration;
|
use time::Duration;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use tokio::io::BufReader;
|
use tokio::io::BufReader;
|
||||||
|
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||||
use tracing::{debug, error, warn};
|
use tracing::{debug, error, warn};
|
||||||
use url::{Url, form_urlencoded};
|
use url::{Url, form_urlencoded};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const C_USER_AGENT: &str = "RustFS (linux; x86)";
|
const C_USER_AGENT: &str = "RustFS (linux; x86)";
|
||||||
pub(crate) const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
|
pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
|
||||||
|
|
||||||
const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
|
const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
|
||||||
|
|
||||||
pub(crate) async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
|
pub async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
|
||||||
where
|
where
|
||||||
B: Body<Data = Bytes>,
|
B: Body<Data = Bytes>,
|
||||||
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||||
@@ -135,10 +136,42 @@ fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> s
|
|||||||
signer_error::signer_error_to_io_error(scope, error)
|
signer_error::signer_error_to_io_error(scope, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
//pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>;
|
/// Streaming object body handed to the client by the storage engine. The
|
||||||
|
/// client only ever reads it to completion, so the engine-side reader type
|
||||||
|
/// (e.g. ecstore's `GetObjectReader`) stays behind this boxed `AsyncRead`.
|
||||||
|
pub struct ObjectReader(Box<dyn AsyncRead + Send + Sync + Unpin>);
|
||||||
|
|
||||||
|
impl ObjectReader {
|
||||||
|
pub fn new(reader: impl AsyncRead + Send + Sync + Unpin + 'static) -> Self {
|
||||||
|
Self(Box::new(reader))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn read_all(&mut self) -> Result<Vec<u8>, std::io::Error> {
|
||||||
|
let mut data = Vec::new();
|
||||||
|
self.0.read_to_end(&mut data).await?;
|
||||||
|
Ok(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for ObjectReader {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.write_str("ObjectReader")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncRead for ObjectReader {
|
||||||
|
fn poll_read(
|
||||||
|
mut self: std::pin::Pin<&mut Self>,
|
||||||
|
cx: &mut std::task::Context<'_>,
|
||||||
|
buf: &mut tokio::io::ReadBuf<'_>,
|
||||||
|
) -> std::task::Poll<std::io::Result<()>> {
|
||||||
|
std::pin::Pin::new(&mut self.0).poll_read(cx, buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub enum ReaderImpl {
|
pub enum ReaderImpl {
|
||||||
Body(Bytes),
|
Body(Bytes),
|
||||||
ObjectBody(GetObjectReader),
|
ObjectBody(ObjectReader),
|
||||||
}
|
}
|
||||||
|
|
||||||
pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;
|
pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;
|
||||||
@@ -223,8 +256,8 @@ where
|
|||||||
async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
|
async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
|
||||||
with_rustls_init_guard(|| Ok(()))?;
|
with_rustls_init_guard(|| Ok(()))?;
|
||||||
|
|
||||||
let outbound_tls = crate::client::runtime_sources::transition_client_outbound_tls_state().await;
|
let outbound_tls = crate::runtime_sources::transition_client_outbound_tls_state().await;
|
||||||
crate::client::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0);
|
crate::runtime_sources::record_transition_client_tls_generation(outbound_tls.generation.0);
|
||||||
let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
|
let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
|
||||||
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
|
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
|
||||||
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
|
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
|
||||||
@@ -336,15 +369,15 @@ impl TransitionClient {
|
|||||||
self.endpoint_url.clone()
|
self.endpoint_url.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn provider_version_capabilities(&self) -> ProviderVersionCapabilities {
|
pub fn provider_version_capabilities(&self) -> ProviderVersionCapabilities {
|
||||||
ProviderVersionCapabilities::for_tier_type(&self.tier_type)
|
ProviderVersionCapabilities::for_tier_type(&self.tier_type)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn raw_version_id<'a>(&self, headers: &'a HeaderMap) -> Result<Option<&'a str>, std::io::Error> {
|
pub fn raw_version_id<'a>(&self, headers: &'a HeaderMap) -> Result<Option<&'a str>, std::io::Error> {
|
||||||
self.provider_version_capabilities().raw_version_id(headers)
|
self.provider_version_capabilities().raw_version_id(headers)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn remote_version(
|
pub fn remote_version(
|
||||||
&self,
|
&self,
|
||||||
headers: &HeaderMap,
|
headers: &HeaderMap,
|
||||||
versioning: BucketVersioningState,
|
versioning: BucketVersioningState,
|
||||||
@@ -352,7 +385,7 @@ impl TransitionClient {
|
|||||||
self.provider_version_capabilities().remote_version(headers, versioning)
|
self.provider_version_capabilities().remote_version(headers, versioning)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result<String, std::io::Error> {
|
pub fn legacy_remote_version_id(&self, headers: &HeaderMap) -> Result<String, std::io::Error> {
|
||||||
Ok(self
|
Ok(self
|
||||||
.remote_version(headers, BucketVersioningState::Unknown)?
|
.remote_version(headers, BucketVersioningState::Unknown)?
|
||||||
.exact_id()
|
.exact_id()
|
||||||
@@ -934,8 +967,8 @@ impl TransitionCore {
|
|||||||
// part_id, start_offset, length, metadata)
|
// part_id, start_offset, length, metadata)
|
||||||
Err(std::io::Error::new(
|
Err(std::io::Error::new(
|
||||||
std::io::ErrorKind::Unsupported,
|
std::io::ErrorKind::Unsupported,
|
||||||
crate::client::credentials::ErrorResponse {
|
crate::credentials::ErrorResponse {
|
||||||
sts_error: crate::client::credentials::STSError {
|
sts_error: crate::credentials::STSError {
|
||||||
r#type: "".to_string(),
|
r#type: "".to_string(),
|
||||||
code: "NotImplemented".to_string(),
|
code: "NotImplemented".to_string(),
|
||||||
message: format!(
|
message: format!(
|
||||||
@@ -1147,7 +1180,7 @@ impl Default for ObjectInfo {
|
|||||||
|
|
||||||
impl ObjectInfo {
|
impl ObjectInfo {
|
||||||
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
|
||||||
pub(crate) fn remote_version(
|
pub fn remote_version(
|
||||||
&self,
|
&self,
|
||||||
capabilities: ProviderVersionCapabilities,
|
capabilities: ProviderVersionCapabilities,
|
||||||
versioning: BucketVersioningState,
|
versioning: BucketVersioningState,
|
||||||
@@ -1229,7 +1262,7 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re
|
|||||||
to_object_info_for_provider(bucket_name, object_name, h, ProviderVersionCapabilities::for_tier_type("s3"))
|
to_object_info_for_provider(bucket_name, object_name, h, ProviderVersionCapabilities::for_tier_type("s3"))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn to_object_info_for_provider(
|
pub fn to_object_info_for_provider(
|
||||||
bucket_name: &str,
|
bucket_name: &str,
|
||||||
object_name: &str,
|
object_name: &str,
|
||||||
h: &HeaderMap,
|
h: &HeaderMap,
|
||||||
@@ -1475,7 +1508,7 @@ mod tests {
|
|||||||
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
|
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
|
||||||
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
|
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
|
||||||
};
|
};
|
||||||
use crate::client::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
|
use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
|
||||||
use http::{HeaderMap, HeaderValue};
|
use http::{HeaderMap, HeaderValue};
|
||||||
use http_body_util::Full;
|
use http_body_util::Full;
|
||||||
use hyper::body::Bytes;
|
use hyper::body::Bytes;
|
||||||
@@ -5494,7 +5494,7 @@ ECSTORE_LINT_EXPECTED="${TMP_DIR}/ecstore_lint_expected.txt"
|
|||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
rg -n '^#!\[allow\(dead_code\)\]' crates/ecstore/src/ 2>/dev/null || true
|
rg -n '^#!\[allow\(dead_code\)\]' crates/ecstore/src/ crates/s3-client/src/ 2>/dev/null || true
|
||||||
) >"$ECSTORE_DEAD_CODE_HITS"
|
) >"$ECSTORE_DEAD_CODE_HITS"
|
||||||
|
|
||||||
if [[ -s "$ECSTORE_DEAD_CODE_HITS" ]]; then
|
if [[ -s "$ECSTORE_DEAD_CODE_HITS" ]]; then
|
||||||
@@ -5503,7 +5503,7 @@ fi
|
|||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
rg -n '^#!\[allow\((unused_variables|unused_must_use|clippy::all)\)\]' crates/ecstore/src/ 2>/dev/null |
|
rg -n '^#!\[allow\((unused_variables|unused_must_use|clippy::all)\)\]' crates/ecstore/src/ crates/s3-client/src/ 2>/dev/null |
|
||||||
sed -E 's#^([^:]+):[0-9]+:\#!\[allow\(([^)]+)\)\]#\1|\2#' | sort -u
|
sed -E 's#^([^:]+):[0-9]+:\#!\[allow\(([^)]+)\)\]#\1|\2#' | sort -u
|
||||||
) >"$ECSTORE_LINT_ACTUAL"
|
) >"$ECSTORE_LINT_ACTUAL"
|
||||||
|
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ checked_files=(
|
|||||||
"crates/targets/src/target/webhook.rs"
|
"crates/targets/src/target/webhook.rs"
|
||||||
"crates/ecstore/src/store/peer.rs"
|
"crates/ecstore/src/store/peer.rs"
|
||||||
"crates/ecstore/src/store/init.rs"
|
"crates/ecstore/src/store/init.rs"
|
||||||
"crates/ecstore/src/client/transition_api.rs"
|
"crates/s3-client/src/transition_api.rs"
|
||||||
"crates/ecstore/src/services/tier/tier.rs"
|
"crates/ecstore/src/services/tier/tier.rs"
|
||||||
"crates/heal/src/heal/manager.rs"
|
"crates/heal/src/heal/manager.rs"
|
||||||
"crates/heal/src/heal/storage.rs"
|
"crates/heal/src/heal/storage.rs"
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ cd "$(dirname "$0")/.."
|
|||||||
# to verify S3 behavior and does not widen the production s3s surface.
|
# to verify S3 behavior and does not widen the production s3s surface.
|
||||||
S3S_IMPORT_FILES_BASELINE=211
|
S3S_IMPORT_FILES_BASELINE=211
|
||||||
S3_ERROR_LINES_BASELINE=1620
|
S3_ERROR_LINES_BASELINE=1620
|
||||||
|
# ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not
|
||||||
|
# know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming*
|
||||||
|
# client was extracted to crates/s3-client, where s3s usage is legitimate;
|
||||||
|
# this counter ratchets the remaining serving-side s3s references out of
|
||||||
|
# crates/ecstore. Baseline verified on 2026-08-26.
|
||||||
|
S3S_ECSTORE_FILES_BASELINE=42
|
||||||
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
|
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
|
||||||
E2E_TEST_GLOB='--glob=!crates/e2e_test/**'
|
E2E_TEST_GLOB='--glob=!crates/e2e_test/**'
|
||||||
|
|
||||||
@@ -47,11 +53,13 @@ run_rg_to() {
|
|||||||
|
|
||||||
run_rg_to "$TMP_DIR/import_files" -l "$S3S_PATH_PATTERN" --type rust $E2E_TEST_GLOB
|
run_rg_to "$TMP_DIR/import_files" -l "$S3S_PATH_PATTERN" --type rust $E2E_TEST_GLOB
|
||||||
run_rg_to "$TMP_DIR/error_lines" -c 's3_error!' --type rust $E2E_TEST_GLOB
|
run_rg_to "$TMP_DIR/error_lines" -c 's3_error!' --type rust $E2E_TEST_GLOB
|
||||||
|
run_rg_to "$TMP_DIR/ecstore_files" -l "$S3S_PATH_PATTERN" --type rust crates/ecstore/src
|
||||||
|
|
||||||
s3s_import_files="$(grep -c . "$TMP_DIR/import_files" || true)"
|
s3s_import_files="$(grep -c . "$TMP_DIR/import_files" || true)"
|
||||||
s3_error_lines="$(awk -F: '{sum += $NF} END {print sum + 0}' "$TMP_DIR/error_lines")"
|
s3_error_lines="$(awk -F: '{sum += $NF} END {print sum + 0}' "$TMP_DIR/error_lines")"
|
||||||
|
s3s_ecstore_files="$(grep -c . "$TMP_DIR/ecstore_files" || true)"
|
||||||
|
|
||||||
for value in "$s3s_import_files" "$s3_error_lines"; do
|
for value in "$s3s_import_files" "$s3_error_lines" "$s3s_ecstore_files"; do
|
||||||
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
|
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
|
||||||
echo "error: could not compute s3s footprint counts (got: '$value')" >&2
|
echo "error: could not compute s3s footprint counts (got: '$value')" >&2
|
||||||
exit 1
|
exit 1
|
||||||
@@ -82,6 +90,8 @@ check_ratchet "files importing s3s" "$s3s_import_files" "$S3S_IMPORT_FILES_BASEL
|
|||||||
"rg -l '$S3S_PATH_PATTERN' --type rust $E2E_TEST_GLOB"
|
"rg -l '$S3S_PATH_PATTERN' --type rust $E2E_TEST_GLOB"
|
||||||
check_ratchet "s3_error! invocation lines" "$s3_error_lines" "$S3_ERROR_LINES_BASELINE" \
|
check_ratchet "s3_error! invocation lines" "$s3_error_lines" "$S3_ERROR_LINES_BASELINE" \
|
||||||
"rg -c 's3_error!' --type rust $E2E_TEST_GLOB"
|
"rg -c 's3_error!' --type rust $E2E_TEST_GLOB"
|
||||||
|
check_ratchet "ecstore files referencing s3s" "$s3s_ecstore_files" "$S3S_ECSTORE_FILES_BASELINE" \
|
||||||
|
"rg -l '$S3S_PATH_PATTERN' --type rust crates/ecstore/src"
|
||||||
|
|
||||||
if ((status != 0)); then
|
if ((status != 0)); then
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -23,56 +23,56 @@ crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs|unused_variables
|
|||||||
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all
|
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|clippy::all
|
||||||
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use
|
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_must_use
|
||||||
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables
|
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_error_response.rs|clippy::all
|
crates/s3-client/src/api_error_response.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_error_response.rs|unused_must_use
|
crates/s3-client/src/api_error_response.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_error_response.rs|unused_variables
|
crates/s3-client/src/api_error_response.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_get_object.rs|clippy::all
|
crates/s3-client/src/api_get_object.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_get_object.rs|unused_must_use
|
crates/s3-client/src/api_get_object.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_get_object.rs|unused_variables
|
crates/s3-client/src/api_get_object.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_get_options.rs|clippy::all
|
crates/s3-client/src/api_get_options.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_get_options.rs|unused_must_use
|
crates/s3-client/src/api_get_options.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_get_options.rs|unused_variables
|
crates/s3-client/src/api_get_options.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_list.rs|clippy::all
|
crates/s3-client/src/api_list.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_list.rs|unused_must_use
|
crates/s3-client/src/api_list.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_list.rs|unused_variables
|
crates/s3-client/src/api_list.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_put_object.rs|clippy::all
|
crates/s3-client/src/api_put_object.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_put_object.rs|unused_must_use
|
crates/s3-client/src/api_put_object.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_put_object.rs|unused_variables
|
crates/s3-client/src/api_put_object.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_put_object_common.rs|clippy::all
|
crates/s3-client/src/api_put_object_common.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_put_object_common.rs|unused_must_use
|
crates/s3-client/src/api_put_object_common.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_put_object_common.rs|unused_variables
|
crates/s3-client/src/api_put_object_common.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_put_object_multipart.rs|clippy::all
|
crates/s3-client/src/api_put_object_multipart.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_put_object_multipart.rs|unused_must_use
|
crates/s3-client/src/api_put_object_multipart.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_put_object_multipart.rs|unused_variables
|
crates/s3-client/src/api_put_object_multipart.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_put_object_streaming.rs|clippy::all
|
crates/s3-client/src/api_put_object_streaming.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_put_object_streaming.rs|unused_must_use
|
crates/s3-client/src/api_put_object_streaming.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_put_object_streaming.rs|unused_variables
|
crates/s3-client/src/api_put_object_streaming.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_remove.rs|clippy::all
|
crates/s3-client/src/api_remove.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_remove.rs|unused_must_use
|
crates/s3-client/src/api_remove.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_remove.rs|unused_variables
|
crates/s3-client/src/api_remove.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_s3_datatypes.rs|clippy::all
|
crates/s3-client/src/api_s3_datatypes.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_s3_datatypes.rs|unused_must_use
|
crates/s3-client/src/api_s3_datatypes.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_s3_datatypes.rs|unused_variables
|
crates/s3-client/src/api_s3_datatypes.rs|unused_variables
|
||||||
crates/ecstore/src/client/api_stat.rs|clippy::all
|
crates/s3-client/src/api_stat.rs|clippy::all
|
||||||
crates/ecstore/src/client/api_stat.rs|unused_must_use
|
crates/s3-client/src/api_stat.rs|unused_must_use
|
||||||
crates/ecstore/src/client/api_stat.rs|unused_variables
|
crates/s3-client/src/api_stat.rs|unused_variables
|
||||||
crates/ecstore/src/client/bucket_cache.rs|clippy::all
|
crates/s3-client/src/bucket_cache.rs|clippy::all
|
||||||
crates/ecstore/src/client/bucket_cache.rs|unused_must_use
|
crates/s3-client/src/bucket_cache.rs|unused_must_use
|
||||||
crates/ecstore/src/client/bucket_cache.rs|unused_variables
|
crates/s3-client/src/bucket_cache.rs|unused_variables
|
||||||
crates/ecstore/src/client/checksum.rs|clippy::all
|
crates/s3-client/src/checksum.rs|clippy::all
|
||||||
crates/ecstore/src/client/checksum.rs|unused_must_use
|
crates/s3-client/src/checksum.rs|unused_must_use
|
||||||
crates/ecstore/src/client/checksum.rs|unused_variables
|
crates/s3-client/src/checksum.rs|unused_variables
|
||||||
crates/ecstore/src/client/constants.rs|unused_must_use
|
crates/s3-client/src/constants.rs|unused_must_use
|
||||||
crates/ecstore/src/client/constants.rs|unused_variables
|
crates/s3-client/src/constants.rs|unused_variables
|
||||||
crates/ecstore/src/client/credentials.rs|clippy::all
|
crates/s3-client/src/credentials.rs|clippy::all
|
||||||
crates/ecstore/src/client/credentials.rs|unused_must_use
|
crates/s3-client/src/credentials.rs|unused_must_use
|
||||||
crates/ecstore/src/client/credentials.rs|unused_variables
|
crates/s3-client/src/credentials.rs|unused_variables
|
||||||
crates/ecstore/src/client/object_api_utils.rs|clippy::all
|
crates/ecstore/src/client/object_api_utils.rs|clippy::all
|
||||||
crates/ecstore/src/client/object_api_utils.rs|unused_must_use
|
crates/ecstore/src/client/object_api_utils.rs|unused_must_use
|
||||||
crates/ecstore/src/client/object_api_utils.rs|unused_variables
|
crates/ecstore/src/client/object_api_utils.rs|unused_variables
|
||||||
crates/ecstore/src/client/transition_api.rs|clippy::all
|
crates/s3-client/src/transition_api.rs|clippy::all
|
||||||
crates/ecstore/src/client/transition_api.rs|unused_must_use
|
crates/s3-client/src/transition_api.rs|unused_must_use
|
||||||
crates/ecstore/src/client/transition_api.rs|unused_variables
|
crates/s3-client/src/transition_api.rs|unused_variables
|
||||||
crates/ecstore/src/services/event_notification.rs|unused_variables
|
crates/ecstore/src/services/event_notification.rs|unused_variables
|
||||||
crates/ecstore/src/services/tier/tier.rs|clippy::all
|
crates/ecstore/src/services/tier/tier.rs|clippy::all
|
||||||
crates/ecstore/src/services/tier/tier.rs|unused_must_use
|
crates/ecstore/src/services/tier/tier.rs|unused_must_use
|
||||||
|
|||||||
@@ -19,11 +19,6 @@
|
|||||||
1|crates/ecstore/src/bucket/replication/replication_object_config.rs
|
1|crates/ecstore/src/bucket/replication/replication_object_config.rs
|
||||||
2|crates/ecstore/src/bucket/replication/replication_pool.rs
|
2|crates/ecstore/src/bucket/replication/replication_pool.rs
|
||||||
2|crates/ecstore/src/bucket/replication/replication_target_boundary.rs
|
2|crates/ecstore/src/bucket/replication/replication_target_boundary.rs
|
||||||
1|crates/ecstore/src/client/api_put_object_multipart.rs
|
|
||||||
2|crates/ecstore/src/client/api_put_object_streaming.rs
|
|
||||||
1|crates/ecstore/src/client/api_s3_datatypes.rs
|
|
||||||
1|crates/ecstore/src/client/signer_error.rs
|
|
||||||
5|crates/ecstore/src/client/transition_api.rs
|
|
||||||
3|crates/ecstore/src/cluster/rpc/http_auth.rs
|
3|crates/ecstore/src/cluster/rpc/http_auth.rs
|
||||||
2|crates/ecstore/src/cluster/rpc/internode_data_transport.rs
|
2|crates/ecstore/src/cluster/rpc/internode_data_transport.rs
|
||||||
11|crates/ecstore/src/cluster/rpc/peer_rest_client.rs
|
11|crates/ecstore/src/cluster/rpc/peer_rest_client.rs
|
||||||
|
|||||||
Reference in New Issue
Block a user