Files
rustfs/crates/s3-client/src/api_error_response.rs
T
Zhengchao An 8f0d4a20d1 refactor(ecstore): extract the embedded S3 client into rustfs-s3-client (#6627)
The storage engine embedded a ~8.4K-line hand-written S3 HTTP client under crates/ecstore/src/client (rustfs/backlog#1842). That client is a legitimate engine capability — it consumes remote S3-compatible endpoints for ILM tier warm backends and transition targets — but it was misfiled inside the engine, dragging s3s/hyper wire types into ecstore and blocking ARCHITECTURE.md invariant 4.

This PR is the pure-move step: 21 modules move verbatim to the new crates/s3-client crate (rustfs-s3-client), and crates/ecstore/src/client/mod.rs becomes a re-export shim so every in-crate crate::client:: path keeps working. The two server-side modules that were historically misfiled under client/ — object_api_utils.rs and object_handlers_common.rs — stay in ecstore.

Three reverse dependencies from the client into engine internals are severed so the move can be pure:

- transition_api::ReaderImpl::ObjectBody held ecstore's GetObjectReader; the client only ever reads the body, so the variant now holds an ObjectReader newtype over Box<dyn AsyncRead + Send + Sync + Unpin> with the same read_all() surface. The single production construction site (set_disk transition upload) and the two engine-side consumers were adjusted.
- api_list/api_remove used ecstore's storage_api_contracts / object_api types; api_list now imports BucketInfo from rustfs-storage-api directly, and api_remove uses the client's own transition_api::ObjectInfo (only .name/.version_id were read; the error-path bucket name is now threaded as a parameter instead of read from the deleted objects).
- the api_put_object_streaming regression tests built a GetObjectReader by hand; they now wrap the duplex stream in ObjectReader::new.

Guard updates: the s3s footprint ratchet gains an ecstore-scoped counter (42 files, shrink-only, per rustfs/backlog#1842), the ecstore module-lint-blanket register follows the moved files into crates/s3-client so the blanket ratchet keeps covering them, the logging guardrail path pin follows transition_api.rs, and the ::other(format!) baseline is regenerated (moved call sites left ecstore).

Verification: cargo check -p rustfs-s3-client -p rustfs-ecstore; cargo nextest run -p rustfs-s3-client (43 passed) and -p rustfs-ecstore (4515/4523; the 8 failures reproduce identically on pristine origin/main on the same machine); cargo clippy --all-targets; scripts/check_layer_dependencies.sh, check_architecture_migration_rules.sh, check_s3s_footprint.sh, check_logging_guardrails.sh, check_error_other_format_ratchet.sh, check_doc_paths.sh, check_ci_paths_sync.sh all pass.
2026-08-26 12:38:52 +08:00

329 lines
11 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use http::{HeaderMap, StatusCode};
use serde::{Deserialize, Serialize};
use serde::{de::Deserializer, ser::Serializer};
use std::fmt::Display;
use s3s::S3ErrorCode;
const _REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)]
#[serde(default, rename_all = "PascalCase")]
pub struct ErrorResponse {
#[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
pub code: S3ErrorCode,
pub message: String,
pub bucket_name: String,
pub key: String,
pub resource: String,
// External S3-style error response contract: keep `RequestId`.
#[serde(rename = "RequestId")]
pub request_id: String,
pub host_id: String,
pub region: String,
pub server: String,
#[serde(skip)]
pub status_code: StatusCode,
}
fn serialize_code<S>(_data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
s.serialize_str("")
}
fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
where
D: Deserializer<'de>,
{
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
}
impl Default for ErrorResponse {
fn default() -> Self {
ErrorResponse {
code: S3ErrorCode::Custom("".into()),
message: Default::default(),
bucket_name: Default::default(),
key: Default::default(),
resource: Default::default(),
request_id: Default::default(),
host_id: Default::default(),
region: Default::default(),
server: Default::default(),
status_code: Default::default(),
}
}
}
impl Display for ErrorResponse {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.message)
}
}
pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
if let Some(err) = err.get_ref() {
if err.is::<ErrorResponse>() {
err.downcast_ref::<ErrorResponse>().expect("err!").clone()
} else {
ErrorResponse::default()
}
} else {
ErrorResponse::default()
}
}
pub fn http_resp_to_error_response(
resp_status: StatusCode,
h: &HeaderMap,
b: Vec<u8>,
bucket_name: &str,
object_name: &str,
) -> ErrorResponse {
let err_body = String::from_utf8_lossy(&b).to_string();
if h.is_empty() || !(resp_status.is_client_error() || resp_status.is_server_error()) {
return ErrorResponse {
status_code: resp_status,
code: S3ErrorCode::ResponseInterrupted,
message: "Invalid HTTP response.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
};
}
let err_resp_ = quick_xml::de::from_str::<ErrorResponse>(&err_body);
let mut err_resp = ErrorResponse::default();
if err_resp_.is_err() {
match resp_status {
StatusCode::NOT_FOUND => {
if object_name == "" {
err_resp = ErrorResponse {
status_code: resp_status,
code: S3ErrorCode::NoSuchBucket,
message: "The specified bucket does not exist.".to_string(),
bucket_name: bucket_name.to_string(),
..Default::default()
};
} else {
err_resp = ErrorResponse {
status_code: resp_status,
code: S3ErrorCode::NoSuchKey,
message: "The specified key does not exist.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
};
}
}
StatusCode::FORBIDDEN => {
err_resp = ErrorResponse {
status_code: resp_status,
code: S3ErrorCode::AccessDenied,
message: "Access Denied.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
};
}
StatusCode::CONFLICT => {
err_resp = ErrorResponse {
status_code: resp_status,
code: S3ErrorCode::BucketNotEmpty,
message: "Bucket not empty.".to_string(),
bucket_name: bucket_name.to_string(),
..Default::default()
};
}
StatusCode::PRECONDITION_FAILED => {
err_resp = ErrorResponse {
status_code: resp_status,
code: S3ErrorCode::PreconditionFailed,
message: "Pre condition failed.".to_string(),
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
};
}
_ => {
let mut msg = resp_status.to_string();
if err_body.len() > 0 {
msg = err_body;
}
err_resp = ErrorResponse {
status_code: resp_status,
code: S3ErrorCode::Custom(resp_status.to_string().into()),
message: msg,
bucket_name: bucket_name.to_string(),
..Default::default()
};
}
}
} else if let Ok(parsed_resp) = err_resp_ {
err_resp = parsed_resp;
}
err_resp.status_code = resp_status;
if let Some(server_name) = h.get("Server") {
if let Ok(server_str) = server_name.to_str() {
err_resp.server = server_str.to_string();
}
}
if let Some(code) = h.get("x-minio-error-code") {
if let Ok(code_str) = code.to_str() {
err_resp.code = S3ErrorCode::Custom(code_str.into());
}
}
if let Some(desc) = h.get("x-minio-error-desc") {
if let Ok(desc_str) = desc.to_str() {
err_resp.message = desc_str.trim_matches('"').to_string();
}
}
if err_resp.request_id == "" {
if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
if let Ok(request_id_str) = x_amz_request_id.to_str() {
err_resp.request_id = request_id_str.to_string();
}
}
}
if err_resp.host_id == "" {
if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
if let Ok(host_id_str) = x_amz_id_2.to_str() {
err_resp.host_id = host_id_str.to_string();
}
}
}
if err_resp.region == "" {
if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
if let Ok(region_str) = x_amz_bucket_region.to_str() {
err_resp.region = region_str.to_string();
}
}
}
if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
err_resp.message = format!("Region does not match, expecting region {}.", err_resp.region);
}
err_resp
}
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!(
"Your proposed upload size {} exceeds the maximum allowed object size {} for single PUT operation.",
total_size, max_object_size
);
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooLarge,
message: msg,
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
}
}
pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!(
"Your proposed upload size {} is below the minimum allowed object size 0B for single PUT operation.",
total_size
);
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::EntityTooSmall,
message: msg,
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
}
}
pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!(
"Data read {} is not equal to the size {} of the input Reader.",
total_read, total_size
);
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::Custom("UnexpectedEOF".into()),
message: msg,
bucket_name: bucket_name.to_string(),
key: object_name.to_string(),
..Default::default()
}
}
pub fn err_invalid_argument(message: &str) -> ErrorResponse {
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::InvalidArgument,
message: message.to_string(),
request_id: "rustfs".to_string(),
..Default::default()
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::Value;
#[test]
fn error_response_serializes_request_id_as_pascal_case_contract() {
let response = ErrorResponse {
code: S3ErrorCode::InvalidArgument,
message: "bad request".to_string(),
bucket_name: "bucket".to_string(),
key: "key".to_string(),
resource: "/bucket/key".to_string(),
request_id: "req-xml-123".to_string(),
host_id: "host-1".to_string(),
region: "us-east-1".to_string(),
server: "rustfs".to_string(),
status_code: StatusCode::BAD_REQUEST,
};
let value = serde_json::to_value(response).expect("error response should serialize");
assert_eq!(value["RequestId"], Value::String("req-xml-123".to_string()));
assert!(value.get("request_id").is_none(), "external error contract must not expose request_id");
}
#[test]
fn parses_s3_error_code_from_client_error_response() {
let mut headers = HeaderMap::new();
headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
let response = http_resp_to_error_response(
StatusCode::NOT_FOUND,
&headers,
b"<Error><Code>NoSuchVersion</Code><Message>remote detail</Message></Error>".to_vec(),
"bucket",
"object",
);
assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
}
}