feat(gateway): serve GetObject through the owned read path

Register GetObject and share its authorization and native response facts
with the legacy edge. Preserve conditional headers, range completion,
stream errors, body ownership, SSE reads and request accounting. Pin the
gateway 304 response-override fix without unrelated dependency changes.
Keep the incoming native Put path and update its profiling guard source.

Validation on b7e7a6156:
- Normal and mutation-disabled controls: 9/9 each.
- Affected suite: 754/755; the profiling guard still read the old Put file.
  After correcting that source path, its targeted rerun passed 1/1.
- All-target Clippy passed with gateway-stack and default features.
- Formatting, script-tests, test wiring and prepush-quick passed.
- Authorization matrix: 40 refusals and 8 allowances across both edges.

Mutations: swapped version action, redirected resource, ignored denial,
and bypassed Served authorization failed the permission or HTTP checks.
Earlier effective mutations covered body copying, premature owner drop,
range status, missing 304 headers, a 304 body, wrapped stream error causes
and duplicate accounting. No assertions were removed or weakened.

Linux RDMA paths were not compiled or exercised. GetObjectAttributes,
PutObject gateway registration and the default-stack switch are not part
of this increment.
This commit is contained in:
overtrue
2026-11-15 07:40:48 +08:00
parent b7e7a6156c
commit 0567078f5b
29 changed files with 1954 additions and 547 deletions
Generated
+8 -8
View File
@@ -10268,7 +10268,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway"
version = "0.63.0"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"arc-swap",
"bytes",
@@ -10315,7 +10315,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-core"
version = "0.60.0"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"bytes",
"futures-core",
@@ -10336,7 +10336,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-http"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10354,7 +10354,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-macros"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"proc-macro2",
"quote",
@@ -10364,7 +10364,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-sig"
version = "0.15.0"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10379,7 +10379,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-stream"
version = "0.2.0"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"bitflags 2.13.2",
"bytes",
@@ -10390,7 +10390,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-types"
version = "0.40.0+aws.2026-10-02"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"bytes",
"crc-fast",
@@ -10413,7 +10413,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-xml"
version = "0.4.0"
source = "git+https://github.com/rustfs/gateway.git?rev=1afff6a91882ec47999f529245a717d10bc4c0fb#1afff6a91882ec47999f529245a717d10bc4c0fb"
source = "git+https://github.com/rustfs/gateway.git?rev=d75b6cebb77d43f0e314d250a1f6bd7c5437bf73#d75b6cebb77d43f0e314d250a1f6bd7c5437bf73"
dependencies = [
"quick-xml 0.41.0",
]
+4 -4
View File
@@ -151,14 +151,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "1afff6a91882ec47999f529245a717d10bc4c0fb", version = "0.40.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "1afff6a91882ec47999f529245a717d10bc4c0fb", version = "0.2.0", default-features = false }
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", version = "0.40.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "1afff6a91882ec47999f529245a717d10bc4c0fb", version = "0.63.0", default-features = false }
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", version = "0.63.0", default-features = false }
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "1afff6a91882ec47999f529245a717d10bc4c0fb", default-features = false }
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d75b6cebb77d43f0e314d250a1f6bd7c5437bf73", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
+7 -5
View File
@@ -18,9 +18,7 @@ use crate::admin::console::{is_console_path, make_console_server};
use crate::admin::handlers::oidc::is_oidc_path;
use crate::admin::handlers::s3_extension;
use crate::admin::operation::{AdminRequest, AdminResponse, Operation as OwnedOperation};
use crate::admin::runtime_sources::{
ServerContextSlot, current_object_store_handle, current_region, current_server_config, default_object_usecase,
};
use crate::admin::runtime_sources::{ServerContextSlot, current_object_store_handle, current_region, current_server_config};
use crate::admin::storage_api::access::ReqInfo;
use crate::admin::utils::{parse_query_pairs, query_value_exact, query_values_exact};
use crate::auth::{check_key_valid, constant_time_eq, get_session_token, reject_unsigned_amz_headers_on_sigv4_request};
@@ -896,8 +894,12 @@ async fn handle_object_lambda_request(req: &mut S3Request<Body>, bucket: &str, o
authorize_object_lambda_request(req, bucket, object).await?;
validate_object_lambda_query(&req.uri)?;
let get_req = authorized_object_lambda_get_request(req, bucket, object).await?;
let usecase = default_object_usecase();
let get_resp = usecase.execute_get_object(get_req).await?;
let slot = get_req
.extensions
.get::<Arc<ServerContextSlot>>()
.cloned()
.ok_or_else(server_context_not_ready_error)?;
let get_resp = s3s::S3::get_object(&crate::storage::ecfs::FS::with_server_ctx(slot), get_req).await?;
invoke_object_lambda_target(req, bucket, object, get_resp).await
}
-5
View File
@@ -20,7 +20,6 @@ pub(crate) use crate::app::admin_usecase::{
AdminPoolStatus, DefaultAdminUsecase, QueryPoolStatusRequest, QueryServerInfoRequest,
};
use crate::app::object_data_cache::ObjectDataCacheAdapter;
use crate::app::object_usecase::DefaultObjectUsecase;
use crate::runtime_sources as root_runtime_sources;
pub(crate) use crate::runtime_sources::{
AppContext, ServerContextSlot, current_action_credentials, current_boot_time, current_bucket_metadata_handle,
@@ -45,10 +44,6 @@ pub(crate) fn default_admin_usecase() -> DefaultAdminUsecase {
DefaultAdminUsecase::from_global()
}
pub(crate) fn default_object_usecase() -> DefaultObjectUsecase {
DefaultObjectUsecase::from_global()
}
pub(crate) fn current_app_context() -> Option<Arc<AppContext>> {
crate::runtime_sources::current_app_context()
}
+1 -4
View File
@@ -15,7 +15,6 @@
//! MinIO-compatible metadata listing extension routes.
use super::bucket_usecase::DefaultBucketUsecase;
use super::object_usecase::DefaultObjectUsecase;
use crate::app::runtime_sources::ServerContextSlot;
use crate::auth::{check_key_valid, get_session_token};
use crate::storage::access::access_owned::listing_multipart::authorize_bucket_listing;
@@ -376,9 +375,7 @@ async fn website_get_object(
req_info.is_owner = false;
request.extensions.insert(WebsiteRead);
S3Access::get_object(&crate::storage::ecfs::FS::with_server_ctx(Arc::clone(server_ctx)), &mut request).await?;
DefaultObjectUsecase::with_context(server_ctx.installed_app_context())
.execute_get_object(request)
.await
s3s::S3::get_object(&crate::storage::ecfs::FS::with_server_ctx(Arc::clone(server_ctx)), request).await
}
async fn website_head_redirect(
@@ -48,7 +48,7 @@ async fn read_once(usecase: &DefaultObjectUsecase, object: &str, etag: &str, kin
HeaderValue::from_str(&format!("\"{tag}\"")).expect("benchmark condition"),
);
}
match usecase.execute_get_object(req).await {
match usecase.execute_get_object_from_s3s(req).await {
Err(error) => {
assert_eq!(kind, "hit");
assert_eq!(error.code(), &S3ErrorCode::NotModified);
-15
View File
@@ -1417,21 +1417,6 @@ impl StagingSinkHandle {
}
}
/// Request-extension slot carrying the sink handle from request setup to the
/// body build; taken exactly once.
#[derive(Clone)]
pub(crate) struct RdmaSinkSlot(Arc<std::sync::Mutex<Option<StagingSinkHandle>>>);
impl RdmaSinkSlot {
pub(crate) fn new(handle: StagingSinkHandle) -> Self {
Self(Arc::new(std::sync::Mutex::new(Some(handle))))
}
pub(crate) fn take(&self) -> Option<StagingSinkHandle> {
self.0.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).take()
}
}
/// Completes a GET whose body went through the sink: drains the reader (which
/// only yields the producer's completion), then flushes the sink and reports
/// the bytes moved. A producer error aborts the sink first.
File diff suppressed because it is too large Load Diff
+112
View File
@@ -0,0 +1,112 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Native GetObject fields and response facts shared by its two protocol edges.
//! The use case owns reading and completion; these carriers preserve raw names,
//! stored ETags, and the existing in-memory body owner until an edge encodes them.
use super::head::ResponseEtag;
use bytes::Bytes;
use futures::Stream;
use rustfs_gateway_stream::{BoxPayloadStream, PayloadCaps, PayloadRead, PayloadStream, StreamError, TrailingHeaders};
use rustfs_gateway_types::{ByteRange, dto};
use std::pin::Pin;
use std::task::{Context, Poll};
/// The legacy edge must not validate names before the use case's existing checks.
/// No Debug implementation: the customer key is bearer material.
pub(crate) struct GetObjectFields {
pub(crate) bucket: String,
pub(crate) key: String,
pub(crate) version_id: Option<String>,
pub(crate) part_number: Option<i32>,
pub(crate) range: Option<ByteRange>,
pub(crate) sse_customer_key: Option<String>,
pub(crate) sse_customer_key_md5: Option<String>,
}
/// Body and ETag retain their native representation until the final edge.
#[derive(Debug, Default)]
pub(crate) struct GetObjectOutputFacts {
pub(crate) dto: dto::GetObjectOutput,
pub(crate) etag: Option<ResponseEtag>,
pub(crate) body: Option<GetObjectBody>,
}
/// Preserve the once-body's owner instead of hiding it inside a stream adapter.
pub(crate) enum GetObjectBody {
Bytes(Bytes),
Stream(BoxPayloadStream),
}
impl std::fmt::Debug for GetObjectBody {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let length = match self {
Self::Bytes(bytes) => Some(bytes.len() as u64),
Self::Stream(stream) => stream.len_hint(),
};
formatter
.debug_struct("GetObjectBody")
.field("length", &length)
.finish_non_exhaustive()
}
}
impl GetObjectBody {
pub(super) fn stream<S, E>(stream: S) -> Self
where
S: Stream<Item = Result<Bytes, E>> + Send + 'static,
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
Self::Stream(Box::pin(GetPayload {
stream: Box::pin(stream),
}))
}
}
struct GetPayload<S> {
stream: Pin<Box<S>>,
}
impl<S, E> PayloadStream for GetPayload<S>
where
S: Stream<Item = Result<Bytes, E>> + Send,
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
fn poll_read(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Result<PayloadRead, StreamError>> {
poll_stream(self.get_mut().stream.as_mut(), context)
}
fn caps(&self) -> PayloadCaps {
PayloadCaps::PUSH
}
fn len_hint(&self) -> Option<u64> {
None
}
}
pub(super) fn poll_stream<S, E>(stream: Pin<&mut S>, context: &mut Context<'_>) -> Poll<Result<PayloadRead, StreamError>>
where
S: Stream<Item = Result<Bytes, E>>,
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
match std::task::ready!(stream.poll_next(context)) {
Some(Ok(bytes)) => Poll::Ready(Ok(PayloadRead::Chunk(bytes))),
Some(Err(error)) => Poll::Ready(Err(StreamError::upstream(error.into()))),
None => Poll::Ready(Ok(PayloadRead::Eof {
trailers: TrailingHeaders::empty(),
})),
}
}
+706
View File
@@ -0,0 +1,706 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! GetObject typed, wire, completion, and body-handoff regression contracts.
//! Expected responses were measured on a279b4b before the owned migration.
use super::*;
use crate::app::storage_api::s3::dto::Range;
use crate::app::storage_api::test::ReqInfo;
use http::Method;
fn get_request(input: GetObjectInput) -> S3Request<GetObjectInput> {
let info = ReqInfo {
bucket: Some(input.bucket.clone()),
object: Some(input.key.clone()),
version_id: input.version_id.clone(),
..Default::default()
};
let mut request = build_request(input, Method::GET);
request.extensions.insert(info);
request
}
async fn wire_get(
server: &crate::embedded::RustFSServer,
path: &str,
headers: &[(&str, &str)],
admitted: bool,
) -> http::Response<Bytes> {
use http_body_util::BodyExt as _;
use hyper_util::rt::TokioIo;
let before = operation_counts();
let socket = tokio::net::TcpStream::connect(server.address()).await.unwrap();
let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(socket)).await.unwrap();
let transport = tokio::spawn(connection);
let mut builder = http::Request::builder()
.method(Method::GET)
.uri(path)
.header(http::header::HOST, "localhost");
for (name, value) in headers {
builder = builder.header(*name, *value);
}
let response = sender
.send_request(builder.body(http_body_util::Empty::<Bytes>::new()).unwrap())
.await
.unwrap();
let (parts, body) = response.into_parts();
let body = body.collect().await.unwrap().to_bytes();
drop(sender);
transport.abort();
let _ = transport.await;
let delta: std::collections::BTreeMap<_, _> = operation_counts()
.into_iter()
.filter_map(|(op, count)| {
let delta = count - before.get(op).copied().unwrap_or(0);
(delta > 0).then_some((op, delta))
})
.collect();
let expected = if admitted {
std::collections::BTreeMap::from([(rustfs_s3_ops::S3Operation::GetObject.as_str(), 1)])
} else {
std::collections::BTreeMap::new()
};
if !admitted {
assert_eq!(parts.status, StatusCode::FORBIDDEN, "authorization refusal for {path}");
}
assert_eq!(delta, expected, "the admitted operation is counted once, after authorization");
http::Response::from_parts(parts, body)
}
fn operation_counts() -> std::collections::BTreeMap<&'static str, u64> {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
.map(|row| (row.op, row.total))
.collect()
}
#[test]
#[serial_test::serial]
fn get_boundary_legacy() {
get_boundary(None);
}
#[cfg(feature = "gateway-stack")]
#[test]
#[serial_test::serial]
fn get_boundary_gateway() {
get_boundary(Some("gateway"));
}
fn get_boundary(stack: Option<&str>) {
temp_env::with_vars(
[
("RUSTFS_S3_STACK", stack),
("RUSTFS_AUDIT_ENABLE", Some("true")),
("RUSTFS_SSE_C_REQUIRE_TLS", Some("false")),
],
|| {
crate::app::gating_test_env::run_large_stack_test("get-boundary-observations", || async {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::embedded::{RustFSServerBuilder, find_available_port};
let server = RustFSServerBuilder::new()
.address(format!("127.0.0.1:{}", find_available_port().unwrap()))
.access_key("getfixture")
.secret_key("get-fixture-test-secret")
.build()
.await
.unwrap();
crate::server::refresh_audit_module_enabled();
let context = crate::app::runtime_sources::current_app_context().unwrap();
let store = context.object_store();
let usecase = DefaultObjectUsecase::with_context(Some(context));
let bucket = "get-boundary-facts";
let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(bucket);
store
.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.unwrap();
store.update_bucket_metadata_config(bucket, crate::app::storage_api::object_usecase::bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")}]})).unwrap()).await.unwrap();
for (index, etag) in ["abc", "W/\"\"", "\"a\\b\"", "\"a\"b\"", "é", "a\rb", "a\nb", "a\0b"]
.iter()
.enumerate()
{
let key = format!("object-{index}");
let info = store
.put_object(
bucket,
&key,
&mut PutObjReader::from_vec(b"payload".to_vec()),
&ObjectOptions {
versioned: true,
preserve_etag: Some((*etag).to_owned()),
user_defined: HashMap::from([
("cache-control".to_owned(), "max-age=7".to_owned()),
("content-location".to_owned(), "/snapshot".to_owned()),
("expires".to_owned(), "Wed, 21 Oct 2037 07:28:00 GMT".to_owned()),
("vary".to_owned(), "Origin".to_owned()),
]),
..Default::default()
},
)
.await
.unwrap();
let request = get_request(GetObjectInput {
bucket: bucket.to_owned(),
key: key.clone(),
..Default::default()
});
let response = crate::app::test_seams::get_object(&usecase, request).await.unwrap();
let tag = response.output.e_tag.unwrap();
assert_eq!(tag.value(), ["abc", "", "a\\b", "a\"b", "é", "a\rb", "a\nb", "a\0b"][index]);
assert_eq!(tag.as_weak().is_some(), index == 1);
assert_eq!(response.status, None);
for (case, suffix, headers) in [
("ordinary", String::new(), vec![]),
("latest304", String::new(), vec![("if-none-match", "*")]),
(
"version304",
format!("?versionId={}", info.version_id.unwrap()),
vec![("if-none-match", "*")],
),
("precondition", String::new(), vec![("if-match", "\"different\"")]),
("range", String::new(), vec![("range", "bytes=1-3")]),
("unsatisfied", String::new(), vec![("range", "bytes=99-100")]),
] {
let range = match case {
"range" => Some(Range::Int { first: 1, last: Some(3) }),
"unsatisfied" => Some(Range::Int {
first: 99,
last: Some(100),
}),
_ => None,
};
let input = GetObjectInput {
bucket: bucket.to_owned(),
key: key.clone(),
range,
version_id: (case == "version304").then(|| info.version_id.unwrap().to_string()),
..Default::default()
};
let mut request = get_request(input);
for (name, value) in &headers {
request.headers.insert(
http::header::HeaderName::from_bytes(name.as_bytes()).unwrap(),
HeaderValue::from_str(value).unwrap(),
);
}
match crate::app::test_seams::get_object(&usecase, request).await {
Ok(response) => {
assert!(matches!(case, "ordinary" | "range"));
assert_eq!(response.status, None);
assert_eq!(response.output.content_range.as_deref(), (case == "range").then_some("bytes 1-3/7"));
}
Err(error) => {
let expected = match case {
"latest304" | "version304" if index >= 5 => 500,
"latest304" | "version304" => 304,
"precondition" => 412,
"unsatisfied" => 416,
_ => panic!("unexpected typed error: {case}: {error}"),
};
assert_eq!(error.status_code().unwrap().as_u16(), expected);
if expected == 304 {
assert_eq!(error.code(), &S3ErrorCode::NotModified);
assert_conditional_headers(error.headers().unwrap(), index, &info);
} else {
assert!(error.headers().is_none());
assert_eq!(
error.code(),
&match expected {
412 => S3ErrorCode::PreconditionFailed,
416 => S3ErrorCode::InvalidRange,
_ => S3ErrorCode::InternalError,
}
);
}
}
}
let response = wire_get(&server, &format!("/{bucket}/{key}{suffix}"), &headers, true).await;
let expected = match case {
"ordinary" | "range" | "latest304" | "version304" if index >= 5 => 500,
"ordinary" => 200,
"range" => 206,
"latest304" | "version304" => 304,
"precondition" => 412,
"unsatisfied" => 416,
_ => unreachable!(),
};
assert_eq!(response.status().as_u16(), expected, "{index}: {case}");
match expected {
200 | 206 => {
assert_eq!(response.body().as_ref(), if expected == 200 { &b"payload"[..] } else { &b"ayl"[..] });
assert_eq!(response.headers()[http::header::ETAG].as_bytes(), normal_etag(index));
assert_eq!(
response.headers().get(http::header::CONTENT_RANGE).map(HeaderValue::as_bytes),
(expected == 206).then_some(&b"bytes 1-3/7"[..])
);
}
304 => {
assert!(response.body().is_empty());
assert_conditional_headers(response.headers(), index, &info);
}
412 => assert_error_body(&response, "PreconditionFailed"),
416 => {
assert_error_body(&response, "InvalidRange");
assert!(!response.headers().contains_key(http::header::CONTENT_RANGE));
}
_ => assert_error_body(&response, "InternalError"),
};
}
probe.wait_for((index + 1) * 13, 0).await;
}
let audits = probe
.audits
.lock()
.unwrap()
.iter()
.map(|entry| (entry.api.name.clone(), entry.api.status.clone(), entry.api.status_code))
.collect::<Vec<_>>();
assert_eq!(audits.len(), 8 * 13);
for (index, row) in audits.as_chunks::<13>().0.iter().enumerate() {
let conditional = if index < 5 { 304 } else { 500 };
let statuses = [
200,
200,
200,
conditional,
conditional,
conditional,
conditional,
412,
412,
200,
200,
416,
416,
];
for ((name, state, code), expected) in row.iter().zip(statuses) {
assert_eq!(name.as_deref(), Some("s3:GetObject"));
assert_eq!(*code, Some(expected));
assert_eq!(state.as_deref(), Some(if expected == 200 { "success" } else { "failure" }));
}
}
for (key, part, range) in [
("", Some(0), None),
("object-0", Some(0), None),
("object-0", Some(1), Some(Range::Int { first: 0, last: Some(1) })),
] {
let input = GetObjectInput {
bucket: "missing-boundary-bucket".to_owned(),
key: key.to_owned(),
part_number: part,
range,
..Default::default()
};
let Err(error) = crate::app::test_seams::get_object(&usecase, get_request(input)).await else {
panic!("invalid input must fail")
};
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST));
assert_eq!(
error.message(),
Some(if key.is_empty() {
"Invalid object key"
} else if part == Some(0) {
"GET: partNumber must be between 1 and 10000"
} else {
"range and part_number invalid"
})
);
}
let response = wire_get(&server, &format!("/{bucket}/missing"), &[], true).await;
assert_eq!(response.status(), StatusCode::NOT_FOUND);
assert_error_body(&response, "NoSuchKey");
signed_override_contract(&server, bucket).await;
for (case, headers) in [
("ordinary", vec![]),
("condition", vec![("if-none-match", "*")]),
("failed-match", vec![("if-match", "\"different\"")]),
("range", vec![("range", "bytes=1-3")]),
] {
let response = wire_get(&server, &format!("/{bucket}/missing"), &headers, true).await;
assert_eq!(response.status(), StatusCode::NOT_FOUND, "{case}");
assert_error_body(&response, "NoSuchKey");
}
let marker = store
.delete_object(
bucket,
"object-0",
ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.unwrap();
for path in [
format!("/{bucket}/object-0"),
format!("/{bucket}/object-0?versionId={}", marker.version_id.unwrap()),
] {
let response = wire_get(&server, &path, &[], true).await;
let explicit = path.contains('?');
assert_eq!(response.status().as_u16(), if explicit { 405 } else { 404 });
assert_eq!(response.headers()["x-amz-delete-marker"], "true");
assert_eq!(response.headers()["x-amz-version-id"], marker.version_id.unwrap().to_string());
assert_eq!(response.headers().contains_key(http::header::LAST_MODIFIED), explicit);
assert_error_body(&response, if explicit { "MethodNotAllowed" } else { "NoSuchKey" });
}
ssec_wire_contract(&usecase, &server, bucket).await;
for reserved in ["sthree-get-boundary", "get-boundary-s3alias"] {
store.make_bucket(reserved, &MakeBucketOptions::default()).await.unwrap();
store.update_bucket_metadata_config(reserved, crate::app::storage_api::object_usecase::bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{reserved}/*")}]})).unwrap()).await.unwrap();
store
.put_object(
reserved,
"object",
&mut PutObjReader::from_vec(b"payload".to_vec()),
&ObjectOptions::default(),
)
.await
.unwrap();
let typed = crate::app::test_seams::get_object(
&usecase,
get_request(GetObjectInput {
bucket: reserved.to_owned(),
key: "object".to_owned(),
..Default::default()
}),
)
.await
.unwrap();
let wire = wire_get(&server, &format!("/{reserved}/object"), &[], true).await;
assert_eq!(typed.status, None);
assert_eq!(wire.status(), StatusCode::OK);
assert_eq!(wire.body().as_ref(), b"payload");
}
store.update_bucket_metadata_config(bucket, crate::app::storage_api::object_usecase::bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")}]})).unwrap()).await.unwrap();
for key in ["object-1", "missing"] {
for (case, headers) in [
("ordinary", vec![]),
("condition", vec![("if-none-match", "*")]),
("failed-match", vec![("if-match", "\"different\"")]),
("range", vec![("range", "bytes=1-3")]),
] {
let response = wire_get(&server, &format!("/{bucket}/{key}"), &headers, false).await;
assert_eq!(response.status(), StatusCode::FORBIDDEN, "{key}: {case}");
assert_error_body(&response, "AccessDenied");
}
}
server.shutdown().await;
});
},
);
}
async fn signed_override_contract(server: &crate::embedded::RustFSServer, bucket: &str) {
use aws_sdk_s3::config::{Credentials, Region};
let client = aws_sdk_s3::Client::from_conf(
aws_sdk_s3::Config::builder()
.credentials_provider(Credentials::new(server.access_key(), server.secret_key(), None, None, "test"))
.region(Region::new("us-east-1"))
.endpoint_url(server.endpoint())
.force_path_style(true)
.behavior_version_latest()
.build(),
);
for (case, conditional, ranged) in [
("ordinary", false, false),
("range", false, true),
("conditional", true, false),
] {
let mut request = client
.get_object()
.bucket(bucket)
.key("object-0")
.response_cache_control("max-age=11")
.response_content_type("text/plain; charset=utf-8")
.response_content_disposition("attachment; filename=\"a b.txt\"")
.response_content_language("en-US")
.response_content_encoding("identity")
.response_expires(aws_sdk_s3::primitives::DateTime::from_secs(1700000000));
if conditional {
request = request.if_none_match("*");
}
if ranged {
request = request.range("bytes=1-3");
}
match request.send().await {
Ok(output) => {
assert!(!conditional);
assert_eq!(output.content_type(), Some("text/plain; charset=utf-8"));
assert_eq!(output.cache_control(), Some("max-age=11"));
assert_eq!(output.content_disposition(), Some("attachment; filename=\"a b.txt\""));
assert_eq!(output.content_language(), Some("en-US"));
assert_eq!(output.content_encoding(), Some("identity"));
assert_eq!(output.expires_string(), Some("Tue, 14 Nov 2023 22:13:20 GMT"));
assert_eq!(output.content_range(), ranged.then_some("bytes 1-3/7"));
let body = output.body.collect().await.unwrap().into_bytes();
assert_eq!(body.as_ref(), if ranged { &b"ayl"[..] } else { &b"payload"[..] }, "{case}");
}
Err(error) => {
let response = error.raw_response().unwrap();
assert!(conditional, "{case}: {error}");
assert_eq!(response.status().as_u16(), 304);
assert_eq!(response.headers().get("cache-control"), Some("max-age=7"));
assert_eq!(response.headers().get("expires"), Some("Wed, 21 Oct 2037 07:28:00 GMT"));
}
}
}
}
async fn ssec_wire_contract(usecase: &DefaultObjectUsecase, server: &crate::embedded::RustFSServer, bucket: &str) {
use crate::app::storage_api::s3::dto::PutObjectInput;
use md5::{Digest as _, Md5};
let key = [0x2a_u8; 32];
let encoded = base64_simd::STANDARD.encode_to_string(key);
let digest = base64_simd::STANDARD.encode_to_string(Md5::digest(key));
let wrong = [0x2b_u8; 32];
let wrong_encoded = base64_simd::STANDARD.encode_to_string(wrong);
let wrong_digest = base64_simd::STANDARD.encode_to_string(Md5::digest(wrong));
let input = PutObjectInput::builder()
.bucket(bucket.to_owned())
.key("encrypted".to_owned())
.body(Some(StreamingBlob::from_bytes(Bytes::from_static(b"protected payload"))))
.content_length(Some(17))
.sse_customer_algorithm(Some("AES256".to_owned()))
.sse_customer_key(Some(encoded.clone()))
.sse_customer_key_md5(Some(digest.clone()))
.build()
.unwrap();
crate::app::test_seams::put_object(usecase, &FS::new(), build_request(input, Method::PUT))
.await
.unwrap();
for (case, valid, wrong_key, condition, range) in [
("ordinary", true, false, false, false),
("missing", false, false, false, false),
("wrong", true, true, false, false),
("missing-conditional", false, false, true, false),
("missing-range", false, false, false, true),
("wrong-conditional", true, true, true, false),
("wrong-range", true, true, false, true),
("conditional", true, false, true, false),
("range", true, false, false, true),
] {
let mut headers = Vec::new();
if valid {
headers.extend([
("x-amz-server-side-encryption-customer-algorithm", "AES256"),
(
"x-amz-server-side-encryption-customer-key",
if wrong_key { wrong_encoded.as_str() } else { encoded.as_str() },
),
(
"x-amz-server-side-encryption-customer-key-md5",
if wrong_key { wrong_digest.as_str() } else { digest.as_str() },
),
]);
}
if condition {
headers.push(("if-none-match", "*"));
}
if range {
headers.push(("range", "bytes=1-3"));
}
let response = wire_get(server, &format!("/{bucket}/encrypted"), &headers, true).await;
if !valid || wrong_key {
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{case}");
assert_error_body(&response, "InvalidRequest");
let message = if wrong_key {
"The provided encryption parameters did not match the ones used originally to encrypt the object."
} else {
"The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object."
};
assert!(String::from_utf8_lossy(response.body()).contains(message));
} else {
assert_eq!(
response.status().as_u16(),
if condition {
304
} else if range {
206
} else {
200
}
);
assert_eq!(
response.body().as_ref(),
if condition {
&b""[..]
} else if range {
&b"rot"[..]
} else {
&b"protected payload"[..]
}
);
}
}
}
#[derive(Clone)]
struct CopyProbeReader {
remaining: usize,
writes: Arc<std::sync::Mutex<Vec<(usize, usize)>>>,
}
impl tokio::io::AsyncRead for CopyProbeReader {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
_: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
let length = self.remaining.min(buf.remaining());
if length > 0 {
let target = buf.initialize_unfilled_to(length);
self.writes.lock().unwrap().push((target.as_ptr() as usize, length));
target.fill(b'x');
buf.advance(length);
self.remaining -= length;
}
std::task::Poll::Ready(Ok(()))
}
}
#[tokio::test]
async fn get_reader_handoff_preserves_baseline_copy_count() {
for length in [0, 1, 4097, 131073] {
let writes = Arc::new(std::sync::Mutex::new(Vec::new()));
let reader = CopyProbeReader {
remaining: length,
writes: Arc::clone(&writes),
};
let mut body = DefaultObjectUsecase::build_reader_blob(
reader,
length as i64,
"copy-probe",
None,
4096,
GetObjectStreamStrategy::Standard,
"copy-probe-bucket",
"object",
GetObjectBodyLifecycle::disabled(),
None,
)
.await
.unwrap()
.into_s3s();
let mut chunks = Vec::new();
let mut observed_writes = 0;
let mut retained = 0;
while let Some(chunk) = body.next().await {
let chunk = chunk.unwrap();
let begin = chunk.as_ptr() as usize;
let end = begin + chunk.len();
let writes = writes.lock().unwrap();
let mut intervals: Vec<_> = writes[observed_writes..]
.iter()
.map(|(write, count)| (begin.max(*write), end.min(write + count)))
.filter(|(first, last)| first < last)
.collect();
intervals.sort_unstable();
let mut cursor = begin;
for (first, last) in intervals {
retained += last.saturating_sub(cursor.max(first));
cursor = cursor.max(last);
}
observed_writes = writes.len();
chunks.push(chunk);
}
let writes = writes.lock().unwrap();
let total: usize = chunks.iter().map(Bytes::len).sum();
assert_eq!(total, length);
assert!(chunks.iter().all(|chunk| chunk.iter().all(|byte| *byte == b'x')));
assert_eq!(
total.saturating_sub(retained),
usize::from(length > 0),
"only the established primed byte is copied"
);
assert_eq!(writes.len(), chunks.len());
}
}
#[tokio::test]
#[serial_test::serial]
async fn get_memory_handoff_retains_the_original_allocation() {
for once in ["false", "true"] {
let bytes = Bytes::from(vec![b'm'; 8193]);
let address = bytes.as_ptr() as usize;
let mut body = temp_env::with_var(ENV_RUSTFS_GET_SMALL_BODY_ONCE_ENABLE, Some(once), || {
DefaultObjectUsecase::build_memory_bytes_blob(
bytes.clone(),
bytes.len() as i64,
GET_MEMORY_BODY_SOURCE_BUFFERED_BODY,
GetObjectBodyLifecycle::disabled(),
)
})
.into_s3s();
let mut chunks = Vec::new();
while let Some(chunk) = body.next().await {
chunks.push(chunk.unwrap());
}
let retained: usize = chunks
.iter()
.map(|chunk| {
let start = chunk.as_ptr() as usize;
(start + chunk.len())
.min(address + bytes.len())
.saturating_sub(start.max(address))
})
.sum();
assert_eq!(chunks.iter().map(Bytes::len).sum::<usize>(), bytes.len());
assert_eq!(retained, bytes.len());
assert_eq!(retained, 8193, "{once}");
}
}
fn normal_etag(index: usize) -> &'static [u8] {
[b"\"abc\"".as_slice(), b"W/\"\"", b"\"a\\b\"", b"\"a\"b\"", b"\"\xc3\xa9\""][index]
}
fn assert_conditional_headers(headers: &HeaderMap, index: usize, info: &ObjectInfo) {
let etag = if index == 1 { &b"\"W/\""[..] } else { normal_etag(index) };
assert_eq!(headers[http::header::ETAG].as_bytes(), etag);
assert_eq!(headers[http::header::CACHE_CONTROL], "max-age=7");
assert_eq!(headers[http::header::CONTENT_LOCATION], "/snapshot");
assert_eq!(headers[http::header::VARY], "Origin");
assert_eq!(headers[http::header::EXPIRES], "Wed, 21 Oct 2037 07:28:00 GMT");
assert_eq!(headers["x-amz-version-id"], info.version_id.unwrap().to_string());
assert_eq!(headers[http::header::LAST_MODIFIED], info.mod_time.unwrap().format(RFC1123).unwrap());
}
fn assert_error_body(response: &http::Response<Bytes>, code: &str) {
let body = String::from_utf8_lossy(response.body());
assert!(body.contains(&format!("<Code>{code}</Code>")), "{body}");
let message = match code {
"InternalError" => Some("We encountered an internal error. Please try again."),
"NoSuchKey" => Some("The specified key does not exist."),
"AccessDenied" => Some("Access Denied"),
"PreconditionFailed" => Some("At least one of the preconditions you specified did not hold."),
"InvalidRange" => Some("The requested range is not satisfiable"),
_ => None,
};
if let Some(message) = message {
assert!(body.contains(&format!("<Message>{message}</Message>")), "{body}");
}
}
+11 -35
View File
@@ -46,7 +46,7 @@ pub(crate) struct ResponseEtag {
}
impl ResponseEtag {
fn from_stored(value: &str) -> Self {
pub(super) fn from_stored(value: &str) -> Self {
let weak = value.starts_with("W/\"");
let etag = stored_etag(value);
Self {
@@ -55,7 +55,7 @@ impl ResponseEtag {
}
}
fn from_proxy(value: &str) -> Option<Self> {
pub(super) fn from_proxy(value: &str) -> Option<Self> {
let (opaque, weak) = if let Some(rest) = value.strip_prefix("W/\"") {
(rest.strip_suffix('"')?, true)
} else if let Some(rest) = value.strip_prefix('"') {
@@ -89,30 +89,6 @@ pub(crate) struct HeadObjectFields {
pub(crate) sse_customer_key_md5: Option<String>,
}
async fn head_answer(
bucket: &str,
method: &http::Method,
headers: &HeaderMap,
output: HeadObjectOutputFacts,
) -> Answered<HeadObjectOutputFacts> {
let mut answer = Answered::new(output);
if headers.contains_key(http::header::ORIGIN)
&& let Some(cors) = crate::app::storage_api::object_usecase::head_cors_headers(bucket, method, headers).await
{
answer.headers = cors;
}
answer
}
fn complete_head(helper: OperationHelper, result: &S3Result<Answered<HeadObjectOutputFacts>>) {
let headers = HeaderMap::new();
let (status, headers) = match result {
Ok(answer) => (answer.status.unwrap_or(StatusCode::OK), &answer.headers),
Err(_) => (StatusCode::OK, &headers),
};
let _ = helper.complete_owned(result, status, headers);
}
/// Whether the caller asked for the tag set itself (`x-amz-tagging-directive: ACCESS`).
fn tagging_directive_is_access(headers: &http::HeaderMap) -> bool {
headers
@@ -331,10 +307,10 @@ impl DefaultObjectUsecase {
helper: OperationHelper,
output: HeadObjectOutputFacts,
) -> S3Result<Answered<HeadObjectOutputFacts>> {
let mut response = head_answer(bucket, req.method(), req.headers(), output).await;
let mut response = read_answer(bucket, req.method(), req.headers(), output).await;
mark_on_demand_migration_response(&mut response.headers);
let result = Ok(response);
complete_head(helper, &result);
complete_read(helper, &result);
result
}
@@ -462,7 +438,7 @@ impl DefaultObjectUsecase {
dto,
etag: Some(ResponseEtag::from_stored(&representation.etag_hex)),
};
let mut response = head_answer(bucket, req.method(), req.headers(), output).await;
let mut response = read_answer(bucket, req.method(), req.headers(), output).await;
if let Some(content_range) = content_range {
response.headers.insert(
http::header::CONTENT_RANGE,
@@ -546,7 +522,7 @@ impl DefaultObjectUsecase {
.serve_veeam_sosapi_head(&req, &bucket, &store, kind, part_number, rs.as_ref())
.await?;
let result = Ok(response);
complete_head(helper, &result);
complete_read(helper, &result);
return result;
}
@@ -590,9 +566,9 @@ impl DefaultObjectUsecase {
&& let Some(output) =
Self::proxy_head_object_to_replication_targets(&req, &bucket, &key, &opts, input.part_number).await
{
let response = head_answer(&bucket, req.method(), req.headers(), output).await;
let response = read_answer(&bucket, req.method(), req.headers(), output).await;
let result = Ok(response);
complete_head(helper.version_id(input.version_id.clone().unwrap_or_default()), &result);
complete_read(helper.version_id(input.version_id.clone().unwrap_or_default()), &result);
return result;
}
if let Some(miss) = odm_miss
@@ -764,11 +740,11 @@ impl DefaultObjectUsecase {
// NOTE ON CORS:
// Bucket-level CORS headers are intentionally applied only for object retrieval
// operations (GET/HEAD) via `wrap_response_with_cors`. Other S3 operations that
// operations (GET/HEAD) via `read_answer`. Other S3 operations that
// interact with objects (PUT/POST/DELETE/LIST, etc.) rely on the system-level
// CORS layer instead. In case both are applicable, this bucket-level CORS logic
// takes precedence for these read operations.
let mut response = head_answer(&bucket, req.method(), req.headers(), output).await;
let mut response = read_answer(&bucket, req.method(), req.headers(), output).await;
// Emit additional-checksum headers (XXHash3/64/128, SHA-512) that s3s cannot
// carry on the typed HeadObjectOutput (#1257).
@@ -878,7 +854,7 @@ impl DefaultObjectUsecase {
}
let result = Ok(response);
complete_head(helper, &result);
complete_read(helper, &result);
result
}
+3 -1
View File
@@ -123,7 +123,7 @@ use super::storage_api::object_usecase::{
RFC1123, check_preconditions, parse_object_lock_legal_hold_native, parse_object_lock_retention_native,
parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata,
validate_bucket_exists, validate_new_object_key, validate_object_key, validate_sse_headers_for_read,
validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors,
validate_sse_headers_for_write, validate_ssec_for_read,
};
use crate::app::runtime_sources::{
AppContext, current_app_context, current_notify_interface_for_context, current_object_data_cache_for_context,
@@ -212,6 +212,8 @@ pub(crate) mod cuobject;
mod delete;
mod extract;
mod get;
mod get_response;
pub(crate) use get_response::{GetObjectBody, GetObjectFields, GetObjectOutputFacts};
mod head;
#[cfg(feature = "gateway-stack")]
pub(crate) use head::ResponseEtag;
@@ -428,7 +428,7 @@ mod tests {
.expect("GET input must build");
let req = build_request(input, Method::GET);
let mut response = DefaultObjectUsecase::from_global()
.execute_get_object(req)
.execute_get_object_from_s3s(req)
.await
.expect("app-layer GET must succeed");
let mut body = response.output.body.take().expect("GET response must include a body");
+48 -66
View File
@@ -18,8 +18,7 @@ use super::*;
#[cfg(test)]
use crate::app::storage_api::s3::dto::ObjectLockRetentionMode;
use crate::app::storage_api::s3::dto::{
ChecksumAlgorithm, ObjectLockLegalHoldStatus, ObjectLockMode, Range, ServerSideEncryption, StorageClass, Timestamp,
TimestampFormat,
ChecksumAlgorithm, ObjectLockLegalHoldStatus, ObjectLockMode, ServerSideEncryption, StorageClass, Timestamp, TimestampFormat,
};
use crate::app::storage_api::s3::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Result, StdError, s3_error};
@@ -29,6 +28,30 @@ pub(super) const RUSTFS_EXPECTED_CURRENT_VERSION_ID: &str = "x-rustfs-expected-c
pub(super) type S3StdError = Box<dyn std::error::Error + Send + Sync + 'static>;
pub(super) async fn read_answer<T>(
bucket: &str,
method: &http::Method,
headers: &HeaderMap,
output: T,
) -> crate::app::Answered<T> {
let mut answer = crate::app::Answered::new(output);
if headers.contains_key(http::header::ORIGIN)
&& let Some(cors) = crate::app::storage_api::object_usecase::head_cors_headers(bucket, method, headers).await
{
answer.headers = cors;
}
answer
}
pub(super) fn complete_read<T>(helper: OperationHelper, result: &rustfs_s3_types::S3Result<crate::app::Answered<T>>) {
let headers = HeaderMap::new();
let (status, headers) = match result {
Ok(answer) => (answer.status.unwrap_or(StatusCode::OK), &answer.headers),
Err(_) => (StatusCode::OK, &headers),
};
let _ = helper.complete_owned(result, status, headers);
}
pub(crate) fn s3s_body_error_to_io(err: StdError) -> io::Error {
io::Error::other(err)
}
@@ -144,33 +167,8 @@ pub(super) fn decoded_content_length_from_headers(headers: &HeaderMap) -> S3Resu
}
}
/// Losslessly convert an s3s [`Range`] into the internal [`HTTPRangeSpec`].
///
/// Shared by GET and HEAD so both apply identical range semantics. s3s parses
/// `first`/`last` as `u64`, but its own parser already rejects any value greater
/// than `i64::MAX`, so the int branch is a checked cast that never truncates.
///
/// The suffix length, however, is an unchecked `u64`. A naive `length as i64`
/// truncates deterministically: `bytes=-18446744073709551615` wraps to `-1` and
/// is then read as "last 1 byte", and `bytes=-0` yields a 0-length 206 instead
/// of a 416. This function instead mirrors s3s [`Range::check`] semantics:
/// * a zero-length suffix is rejected with `InvalidRange` (416), matching AWS
/// S3 and MinIO;
/// * a suffix larger than `i64::MAX` is clamped to `i64::MAX`. Object sizes in
/// this system are bounded by `i64::MAX`, so such a suffix always covers the
/// whole object, and [`HTTPRangeSpec::get_length`] clamps it to the real
/// size once the object is known.
pub(super) fn range_to_http_range_spec(range: Range) -> S3Result<HTTPRangeSpec> {
use rustfs_gateway_types::ByteRange;
let range = match range {
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
Range::Int { first, last: None } => ByteRange::From { first },
Range::Suffix { length } => ByteRange::Suffix { length },
};
Ok(byte_range_to_http_range_spec(range)?)
}
/// The range rules shared by native HEAD requests and the remaining legacy adapters.
/// Converts a native range to storage's signed range representation.
/// Zero suffixes are unsatisfiable; suffixes larger than `i64::MAX` cover the whole object.
pub(super) fn byte_range_to_http_range_spec(range: rustfs_gateway_types::ByteRange) -> rustfs_s3_types::S3Result<HTTPRangeSpec> {
use rustfs_gateway_types::ByteRange;
let (start, end, suffix) = match range {
@@ -813,10 +811,6 @@ pub(super) fn response_storage_class_name(info: &ObjectInfo, metadata: &HashMap<
(effective_class != storageclass::STANDARD).then(|| effective_class.to_string())
}
pub(super) fn response_storage_class(info: &ObjectInfo, metadata: &HashMap<String, String>) -> Option<StorageClass> {
response_storage_class_name(info, metadata).map(StorageClass::from)
}
pub(super) fn response_storage_class_for_object_attributes(
info: &ObjectInfo,
metadata: &HashMap<String, String>,
@@ -1903,7 +1897,7 @@ mod tests {
user_defined: Arc::new(metadata.clone()),
..Default::default()
};
assert!(response_storage_class(&standard_info, &metadata).is_none());
assert!(response_storage_class_name(&standard_info, &metadata).is_none());
let mut metadata = HashMap::new();
metadata.insert(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string());
@@ -1913,7 +1907,7 @@ mod tests {
..Default::default()
};
assert!(
response_storage_class(&label_only_info, &metadata).is_none(),
response_storage_class_name(&label_only_info, &metadata).is_none(),
"historical STANDARD_IA labels must report the effective implicit STANDARD layout"
);
@@ -1922,25 +1916,18 @@ mod tests {
..Default::default()
};
assert_eq!(
response_storage_class(&rrs_info, &HashMap::new())
.as_ref()
.map(StorageClass::as_str),
response_storage_class_name(&rrs_info, &HashMap::new()).as_deref(),
Some(storageclass::RRS)
);
let mut transitioned_info = label_only_info;
transitioned_info.transitioned_object.tier = "WARM-TIER".to_string();
assert!(
response_storage_class(&transitioned_info, &metadata).is_none(),
response_storage_class_name(&transitioned_info, &metadata).is_none(),
"a tier name without a completed transition must not override the effective local class"
);
transitioned_info.transitioned_object.status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
assert_eq!(
response_storage_class(&transitioned_info, &metadata)
.as_ref()
.map(StorageClass::as_str),
Some("WARM-TIER")
);
assert_eq!(response_storage_class_name(&transitioned_info, &metadata).as_deref(), Some("WARM-TIER"));
let mut metadata = HashMap::new();
metadata.insert(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD.to_string());
@@ -1950,7 +1937,7 @@ mod tests {
..Default::default()
};
assert!(
response_storage_class(&standard_metadata_info, &metadata).is_none(),
response_storage_class_name(&standard_metadata_info, &metadata).is_none(),
"STANDARD must be omitted even when it only arrives via metadata fallback"
);
}
@@ -2107,7 +2094,7 @@ mod tests {
const I64_MAX_AS_U64: u64 = i64::MAX as u64;
/// The conversion itself: s3s `Range` (u64) -> internal `HTTPRangeSpec`
/// The conversion itself: native `ByteRange` (u64) -> internal `HTTPRangeSpec`
/// (i64). This directly guards the suffix truncation fix. Reverting to
/// `length as i64` regresses the zero-suffix, `i64::MAX + 1` and `u64::MAX`
/// rows below.
@@ -2115,10 +2102,10 @@ mod tests {
fn range_to_http_range_spec_is_lossless() {
// Zero-length suffix (`bytes=-0`) is unsatisfiable -> InvalidRange (416),
// never a 0-length 206.
let zero_suffix = range_to_http_range_spec(Range::Suffix { length: 0 });
let zero_suffix = byte_range_to_http_range_spec(rustfs_gateway_types::ByteRange::Suffix { length: 0 });
assert_eq!(
zero_suffix.as_ref().err().map(|e| e.code()),
Some(&S3ErrorCode::InvalidRange),
Some(&rustfs_s3_types::S3ErrorCode::InvalidRange),
"bytes=-0 must map to InvalidRange (416)"
);
@@ -2131,7 +2118,7 @@ mod tests {
(u64::MAX, i64::MAX), // was -1 under `as i64` -> read as "last 1 byte"
];
for (length, expected_start) in suffix_cases {
let spec = range_to_http_range_spec(Range::Suffix { length })
let spec = byte_range_to_http_range_spec(rustfs_gateway_types::ByteRange::Suffix { length })
.unwrap_or_else(|_| panic!("suffix {length} must convert losslessly"));
assert!(spec.is_suffix_length, "suffix {length} must stay a suffix spec");
assert_eq!(spec.start, expected_start, "suffix {length} start");
@@ -2140,20 +2127,18 @@ mod tests {
// Int ranges: s3s already rejects first/last > i64::MAX, so the checked
// cast never truncates. first-last and open-ended must not regress.
let int_first_last = range_to_http_range_spec(Range::Int {
first: 10,
last: Some(20),
})
.expect("first-last converts");
let int_first_last = byte_range_to_http_range_spec(rustfs_gateway_types::ByteRange::FromTo { first: 10, last: 20 })
.expect("first-last converts");
assert!(!int_first_last.is_suffix_length);
assert_eq!((int_first_last.start, int_first_last.end), (10, 20));
let int_open = range_to_http_range_spec(Range::Int { first: 5, last: None }).expect("open-ended converts");
let int_open =
byte_range_to_http_range_spec(rustfs_gateway_types::ByteRange::From { first: 5 }).expect("open-ended converts");
assert_eq!((int_open.start, int_open.end), (5, -1));
let int_max = range_to_http_range_spec(Range::Int {
let int_max = byte_range_to_http_range_spec(rustfs_gateway_types::ByteRange::FromTo {
first: I64_MAX_AS_U64,
last: Some(I64_MAX_AS_U64),
last: I64_MAX_AS_U64,
})
.expect("i64::MAX int converts");
assert_eq!((int_max.start, int_max.end), (i64::MAX, i64::MAX));
@@ -2173,8 +2158,8 @@ mod tests {
Unsatisfiable,
}
fn derive(range: Range, size: i64) -> Outcome {
let spec = match range_to_http_range_spec(range) {
fn derive(range: rustfs_gateway_types::ByteRange, size: i64) -> Outcome {
let spec = match byte_range_to_http_range_spec(range) {
Ok(spec) => spec,
Err(_) => return Outcome::Unsatisfiable,
};
@@ -2187,7 +2172,7 @@ mod tests {
}
}
let suffix = |length: u64| Range::Suffix { length };
let suffix = |length: u64| rustfs_gateway_types::ByteRange::Suffix { length };
// size, range, expected
let normal = 100_i64;
@@ -2212,15 +2197,12 @@ mod tests {
// Normal first-last and open-ended int ranges must not regress.
(
normal,
Range::Int {
first: 10,
last: Some(19),
},
rustfs_gateway_types::ByteRange::FromTo { first: 10, last: 19 },
Outcome::Partial(10, 10, "bytes 10-19/100".into()),
),
(
normal,
Range::Int { first: 90, last: None },
rustfs_gateway_types::ByteRange::From { first: 90 },
Outcome::Partial(90, 10, "bytes 90-99/100".into()),
),
];
+2 -3
View File
@@ -338,8 +338,8 @@ pub(crate) mod access {
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards,
set_requested_object_lock_retain_until,
};
}
@@ -1827,7 +1827,6 @@ pub(crate) mod object_usecase {
parse_object_lock_legal_hold_native, parse_object_lock_retention_native, parse_part_number_i32_to_usize,
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_new_object_key,
validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
wrap_response_with_cors,
};
#[cfg(all(any(feature = "cuobject", feature = "nixl"), target_os = "linux"))]
pub(crate) use crate::storage::storage_api::{
+13 -4
View File
@@ -14,15 +14,24 @@
use crate::app::object_usecase::DefaultObjectUsecase;
use crate::app::storage_api::s3::{GetObjectInput, GetObjectOutput, S3Request, S3Response, S3Result};
use std::future::Future;
/// Calls `DefaultObjectUsecase::execute_get_object` with today's arguments.
///
/// Owned by the GetObject batch (C). Tests outside that batch's files call this seam instead of the use-case method;
/// when batch C changes the use-case signature it changes only this body.
pub(crate) fn get_object(
pub(crate) async fn get_object(
usecase: &DefaultObjectUsecase,
req: S3Request<GetObjectInput>,
) -> impl Future<Output = S3Result<S3Response<GetObjectOutput>>> + '_ {
usecase.execute_get_object(req)
) -> S3Result<S3Response<GetObjectOutput>> {
use crate::app::object_usecase::GetObjectFields;
use crate::app::{Answered, RequestEnvelope};
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
let envelope = RequestEnvelope::from_s3s(&req);
let Answered { output, status, headers } = usecase
.execute_get_object_fields(envelope, GetObjectFields::from_s3s(req.input))
.await?;
let mut response = S3Response::new(output.into_s3s().map_err(crate::app::dto_values::seam_error)?);
response.status = status;
response.headers = headers;
Ok(response)
}
+174 -18
View File
@@ -13,19 +13,21 @@
// limitations under the License.
//! The gateway operations of migration batch C (rustfs/backlog#2765): GetObject, HeadObject and GetObjectAttributes.
//! HeadObject is served; the batch appends the remaining rows here.
//! GetObject and HeadObject are served; the batch appends the remaining rows here.
use super::{OpRow, Served, resp, row};
use crate::app::object_usecase::{DefaultObjectUsecase, HeadObjectOutputFacts, ResponseEtag};
use crate::app::object_usecase::{
DefaultObjectUsecase, GetObjectBody, GetObjectOutputFacts, HeadObjectOutputFacts, ResponseEtag,
};
use crate::app::{Answered, RequestEnvelope};
use crate::runtime_sources::AppContext;
use crate::storage_api::server::http::access::access_owned::object_read::authorize_head_object;
use crate::storage_api::server::http::access::access_owned::object_read::{authorize_get_object, authorize_head_object};
use rustfs_gateway::{Req, Resp, dto};
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::{S3ErrorCode, S3Result, s3_error};
use std::sync::Arc;
pub(super) const ROWS: &[OpRow] = &[row::<dto::HeadObject>()];
pub(super) const ROWS: &[OpRow] = &[row::<dto::GetObject>(), row::<dto::HeadObject>()];
impl Served for dto::HeadObject {
const METRIC: S3Operation = S3Operation::HeadObject;
@@ -52,14 +54,91 @@ impl Served for dto::HeadObject {
}
}
impl Served for dto::GetObject {
const METRIC: S3Operation = S3Operation::GetObject;
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
let input = request.input();
let server_ctx = envelope
.server_context()
.cloned()
.ok_or_else(|| s3_error!(InternalError, "server context is missing"))?;
Ok(authorize_get_object(
&server_ctx,
envelope,
input.bucket.as_str(),
input.key.as_str(),
input.version_id.as_deref(),
)
.await?)
}
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
get_result(
DefaultObjectUsecase::with_context(context)
.execute_get_object(envelope, request.into_input())
.await,
)
}
}
fn get_result(result: S3Result<Answered<GetObjectOutputFacts>>) -> S3Result<Resp<dto::GetObject>> {
match result {
Ok(answered) => get_response(answered),
Err(error) if is_not_modified(&error) => Ok(Resp::with_status(dto::GetObjectOutput::default(), 304)
.with_extra_headers(error.into_parts().headers.unwrap_or_default())),
Err(error) => Err(error),
}
}
fn is_not_modified(error: &rustfs_s3_types::S3Error) -> bool {
error.code() == &S3ErrorCode::NotModified && error.status_code() == Some(http::StatusCode::NOT_MODIFIED)
}
pub(super) fn get_response(answered: Answered<GetObjectOutputFacts>) -> S3Result<Resp<dto::GetObject>> {
let Answered {
mut output,
status,
mut headers,
} = answered;
insert_etag(&mut headers, output.etag)
.map_err(|_| s3_error!(InternalError, "We encountered an internal error. Please try again."))?;
output.dto.body = output
.body
.map(|body| match body {
GetObjectBody::Bytes(bytes) => Ok(rustfs_gateway_stream::ByteStream::from_bytes(bytes)),
GetObjectBody::Stream(stream) => rustfs_gateway_stream::ByteStream::new(stream)
.map_err(|_| s3_error!(InternalError, "Invalid GetObject body capabilities")),
})
.transpose()?;
// Range encoding follows use-case completion; the legacy typed answer has no override.
let status = status.or_else(|| output.dto.content_range.as_ref().map(|_| http::StatusCode::PARTIAL_CONTENT));
Ok(resp::<dto::GetObject>(Answered {
output: output.dto,
status,
headers,
}))
}
fn insert_etag(headers: &mut http::HeaderMap, etag: Option<ResponseEtag>) -> S3Result<()> {
if let Some(ResponseEtag { opaque, weak }) = etag {
let value = if weak {
format!("W/\"{}\"", opaque)
} else {
format!("\"{}\"", opaque)
};
headers.insert(
http::header::ETAG,
http::HeaderValue::from_str(&value).map_err(|_| s3_error!(InternalError, "Invalid stored ETag header"))?,
);
}
Ok(())
}
/// The use case has already recorded this conditional refusal as an error. Preserve its
/// completed response without re-parsing the stored headers through the generic error codec.
fn head_result(result: S3Result<Answered<HeadObjectOutputFacts>>) -> S3Result<Resp<dto::HeadObject>> {
match result {
Ok(answered) => head_response(answered),
Err(error)
if error.code() == &S3ErrorCode::NotModified && error.status_code() == Some(http::StatusCode::NOT_MODIFIED) =>
{
Err(error) if is_not_modified(&error) => {
let parts = error.into_parts();
Ok(Resp::with_status(dto::HeadObjectOutput::default(), 304).with_extra_headers(parts.headers.unwrap_or_default()))
}
@@ -74,17 +153,7 @@ pub(super) fn head_response(answered: Answered<HeadObjectOutputFacts>) -> S3Resu
status,
mut headers,
} = answered;
if let Some(ResponseEtag { opaque, weak }) = output.etag {
let value = if weak {
format!("W/\"{}\"", opaque)
} else {
format!("\"{}\"", opaque)
};
headers.insert(
http::header::ETAG,
http::HeaderValue::from_str(&value).map_err(|_| s3_error!(InternalError, "Invalid stored ETag header"))?,
);
}
insert_etag(&mut headers, output.etag)?;
Ok(resp::<dto::HeadObject>(Answered {
output: output.dto,
status,
@@ -96,6 +165,93 @@ pub(super) fn head_response(answered: Answered<HeadObjectOutputFacts>) -> S3Resu
mod tests {
use super::*;
#[test]
fn get_terminal_304_settles_without_a_body() {
for status in [None, Some(http::StatusCode::NOT_MODIFIED)] {
let mut error = rustfs_s3_types::S3Error::new(S3ErrorCode::NotModified);
if let Some(status) = status {
error.set_status_code(status);
}
let mut headers = http::HeaderMap::new();
headers.insert(http::header::ETAG, http::HeaderValue::from_static("\"stored\""));
error.set_headers(headers.clone());
let (answer, status, actual) = get_result(Err(error)).unwrap().into_parts();
let rustfs_gateway::Answer::Settled(output) = answer else {
panic!("a conditional GET must settle before body handoff");
};
assert_eq!(status, 304);
assert_eq!(actual, headers);
assert!(output.body.is_none());
}
}
#[test]
fn get_terminal_304_requires_both_code_and_status() {
for (code, status) in [
(S3ErrorCode::NotModified, http::StatusCode::BAD_REQUEST),
(S3ErrorCode::AccessDenied, http::StatusCode::NOT_MODIFIED),
(S3ErrorCode::InternalError, http::StatusCode::INTERNAL_SERVER_ERROR),
] {
let mut error = rustfs_s3_types::S3Error::new(code.clone());
error.set_status_code(status);
let Err(error) = get_result(Err(error)) else {
panic!("unrelated refusal stays an error");
};
assert_eq!(error.code(), &code);
assert_eq!(error.status_code(), Some(status));
}
}
#[tokio::test]
async fn get_response_handoff_retains_bytes_for_both_body_shapes() {
use http_body_util::BodyExt as _;
use rustfs_gateway_stream::ByteStream;
for streaming in [false, true] {
let bytes = bytes::Bytes::from(vec![b'x'; 8193]);
let address = bytes.as_ptr();
let body = if streaming {
GetObjectBody::Stream(Box::pin(ByteStream::from_bytes(bytes.clone())))
} else {
GetObjectBody::Bytes(bytes.clone())
};
let response = get_response(Answered::new(GetObjectOutputFacts {
dto: dto::GetObjectOutput {
content_length: Some(bytes.len() as i64),
..Default::default()
},
body: Some(body),
..Default::default()
}))
.unwrap();
let (output, status, _) = response.into_parts();
assert_eq!(status, 200);
let rustfs_gateway::Answer::Settled(output) = output else {
panic!("GET must settle its response before body handoff");
};
let mut body = output.body.unwrap().into_body();
let frame = body.frame().await.unwrap().unwrap().into_data().unwrap();
assert_eq!(frame.as_ptr(), address);
assert_eq!(frame, bytes);
assert!(body.frame().await.is_none());
}
}
#[test]
fn get_range_status_changes_only_at_the_wire_edge() {
for status in [None, Some(http::StatusCode::ACCEPTED)] {
let mut answered = Answered::new(GetObjectOutputFacts {
dto: dto::GetObjectOutput {
content_range: Some("bytes 1-3/7".to_owned()),
..Default::default()
},
..Default::default()
});
answered.status = status;
let (_, actual, _) = get_response(answered).unwrap().into_parts();
assert_eq!(actual, status.map_or(206, |status| status.as_u16()));
}
}
#[test]
fn head_terminal_304_uses_the_resolved_status() {
for status in [None, Some(http::StatusCode::NOT_MODIFIED)] {
+2 -12
View File
@@ -3835,18 +3835,8 @@ impl S3Access for FS {
///
/// This method returns `Ok(())` by default.
async fn get_object(&self, req: &mut S3Request<GetObjectInput>) -> S3Result<()> {
let bucket = req.input.bucket.clone();
let source_generation = capture_odm_read_generation(self.server_ctx(), req, &bucket).await;
let req_info = ext_req_info_mut(&mut req.extensions)?;
req_info.bucket = Some(req.input.bucket.clone());
req_info.object = Some(req.input.key.clone());
req_info.version_id = req.input.version_id.clone();
// GHSA-3ppv: a versioned read (?versionId=...) must authorize against
// s3:GetObjectVersion, not s3:GetObject.
authorize_request(req, versioned_read_action(req.input.version_id.as_deref())).await?;
req.install_odm_read_generation(source_generation);
Ok(())
let (bucket, key, version) = (req.input.bucket.clone(), req.input.key.clone(), req.input.version_id.clone());
access_owned::object_read::authorize_get_object(self.server_ctx(), req, &bucket, &key, version.as_deref()).await
}
/// Checks whether the GetObjectAcl request has accesses to the resources.
+16 -2
View File
@@ -22,12 +22,26 @@
//! Downstream: the shared shapes in `storage/access.rs`.
use crate::storage::access::{
AuthorizationRequest, S3Result, ServerContextSlot, authorize_request, capture_odm_read_generation, req_info_mut,
versioned_read_action,
AuthorizationRequest, S3Result, ServerContextSlot, authorize_object_request, authorize_request, capture_odm_read_generation,
req_info_mut, versioned_read_action,
};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_utils::http::{SUFFIX_SOURCE_REPLICATION_CHECK, get_header};
/// GET uses the same versioned read decision and ODM snapshot on both edges.
pub(crate) async fn authorize_get_object<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &mut R,
bucket: &str,
key: &str,
version_id: Option<&str>,
) -> S3Result<()> {
let source_generation = capture_odm_read_generation(server_ctx, req, bucket).await;
authorize_object_request(req, bucket, key, version_id, versioned_read_action(version_id)).await?;
req.install_odm_read_generation(source_generation);
Ok(())
}
/// Both HEAD edges retain the captured ODM generation and the additional replication decision.
pub(crate) async fn authorize_head_object<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Head authorization tests for the legacy request and owned envelope.
//! Read authorization tests for the legacy request and owned envelope.
use super::authorize_head_object;
use crate::app::RequestEnvelope;
@@ -90,3 +90,79 @@ fn head_replication_marker_requires_its_own_permission() {
}
});
}
#[test]
#[serial_test::serial]
fn get_authorization_binds_version_and_explicit_denial() {
crate::app::gating_test_env::run_large_stack_test("get-owned-authorization", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let store = context.object_store();
let slot = ServerContextSlot::new();
assert!(slot.install(Arc::clone(&context)));
let mut failures = Vec::new();
for (actions, explicit_deny, current_allowed, version_allowed) in [
(&["s3:PutObject"][..], false, false, false),
(&["s3:GetObject"][..], false, true, false),
(&["s3:GetObjectVersion"][..], false, true, true),
(&["s3:GetObject", "s3:GetObjectVersion"][..], true, false, false),
] {
let bucket = format!("get-auth-{}", uuid::Uuid::new_v4().simple());
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
let resource = format!("arn:aws:s3:::{bucket}/key");
let mut statements =
vec![serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":actions,"Resource":resource})];
if explicit_deny {
statements
.push(serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":actions,"Resource":resource}));
}
store
.update_bucket_metadata_config(
&bucket,
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(),
)
.await
.unwrap();
for (version, key) in [None, Some("null"), Some("selected-version")]
.into_iter()
.flat_map(|version| ["key", "other"].map(|key| (version, key)))
{
let mut request = S3Request {
input: (),
method: Method::GET,
uri: Uri::from_static("/bucket/key"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
request.extensions.insert(ReqInfo::default());
request.extensions.insert(Arc::clone(&slot));
let mut envelope = RequestEnvelope::from_s3s(&request);
let legacy = super::authorize_get_object(&slot, &mut request, &bucket, key, version).await;
let native = super::authorize_get_object(&slot, &mut envelope, &bucket, key, version).await;
let allowed = key == "key" && if version.is_some() { version_allowed } else { current_allowed };
for (edge, result) in [("legacy", legacy), ("native", native)] {
let label = format!("actions={actions:?} deny={explicit_deny} version={version:?} key={key} edge={edge}");
let expected = if allowed { None } else { Some(&S3ErrorCode::AccessDenied) };
if result.as_ref().err().map(|error| error.code()) != expected {
failures.push(format!("{label}: expected {expected:?}, got {result:?}"));
}
}
for (edge, info) in [
("legacy", request.extensions.get::<ReqInfo>().unwrap()),
("native", envelope.req_info().unwrap()),
] {
let actual = (info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref());
let expected = (Some(bucket.as_str()), Some(key), version);
if actual != expected {
failures.push(format!("{edge}: expected resource {expected:?}, got {actual:?}"));
}
}
}
}
assert!(failures.is_empty(), "{}", failures.join("\n"));
});
}
+1 -30
View File
@@ -35,7 +35,7 @@ use s3s::dto::{
Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
ObjectLockRetention, ObjectLockRetentionMode, QueueConfiguration, TopicConfiguration,
};
use s3s::{S3Error, S3ErrorCode, S3Response, S3Result};
use s3s::{S3Error, S3ErrorCode, S3Result};
use serde_urlencoded::from_bytes;
use std::collections::HashMap;
use std::sync::{Arc, OnceLock, RwLock};
@@ -923,13 +923,6 @@ pub(crate) async fn validate_bucket_exists(store: &super::ECStore, bucket: &str)
Ok(())
}
/// Quick check if CORS processing is needed (lightweight check for Origin header)
/// This avoids unnecessary function calls for non-CORS requests
#[inline]
pub(crate) fn needs_cors_processing(headers: &HeaderMap) -> bool {
headers.contains_key(cors::standard::ORIGIN)
}
/// Apply CORS headers to response based on bucket CORS configuration and request origin
///
/// This function:
@@ -941,9 +934,6 @@ pub(crate) fn needs_cors_processing(headers: &HeaderMap) -> bool {
/// - `None`: bucket has no CORS config (or request has no valid `Origin`)
/// - `Some(empty headers)`: bucket CORS exists but request is denied / no rule matched
/// - `Some(non-empty headers)`: bucket CORS exists and request matched
///
/// Note: This function should only be called if `needs_cors_processing()` returns true
/// to avoid unnecessary overhead for non-CORS requests.
pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, request: &cors::CorsRequest) -> Option<HeaderMap> {
use http::HeaderValue;
@@ -1115,25 +1105,6 @@ pub(crate) fn matches_origin_pattern(pattern: &str, origin: &str) -> bool {
}
}
/// Wrap S3Response with CORS grants if needed. Cache variation is applied by
/// `ConditionalCorsService` to all responses, including originless requests.
/// This function performs a lightweight check first to avoid unnecessary CORS processing
/// for non-CORS requests (requests without Origin header)
pub(crate) async fn wrap_response_with_cors<T>(
bucket: &str,
method: &http::Method,
headers: &HeaderMap,
output: T,
) -> S3Response<T> {
if needs_cors_processing(headers)
&& let Some(cors_headers) = apply_cors_headers(bucket, method, &cors::CorsRequest::from_headers(headers)).await
{
S3Response::with_headers(output, cors_headers)
} else {
S3Response::new(output)
}
}
/// Parse part number from Option<i32> to Option<usize> with validation
/// This function checks that the part number is greater than 0 and
/// converts it to usize, returning an error if invalid
+157 -1
View File
@@ -21,6 +21,10 @@
use crate::storage::ecfs::*;
use crate::storage::s3_api;
use rustfs_gateway_stream::{BoxPayloadStream, PayloadRead, StreamError, StreamErrorKind};
use std::pin::Pin;
use std::sync::{Mutex, PoisonError};
use std::task::{Context, Poll};
use tracing::instrument;
impl FS {
@@ -41,7 +45,14 @@ impl FS {
.get::<crate::storage::request_context::RequestContext>()
.map(|ctx| ctx.request_id.clone());
let usecase = s3_api::object_usecase_for(self);
let result = usecase.execute_get_object(req).await;
record_s3_op(S3Operation::GetObject);
let result = through_gateway_seam_answered(
req,
|input| Ok(s3_api::GetObjectFields::from_s3s(input)),
s3_api::GetObjectOutputFacts::into_s3s,
|envelope, input| usecase.execute_get_object_fields(envelope, input),
)
.await;
if let Err(error) = &result {
log_object_internal_error("GetObject", request_id.as_deref(), error);
}
@@ -106,3 +117,148 @@ impl s3_api::HeadObjectOutputFacts {
Ok(output)
}
}
impl s3_api::GetObjectFields {
pub(crate) fn from_s3s(input: GetObjectInput) -> Self {
use rustfs_gateway_types::ByteRange;
Self {
bucket: input.bucket,
key: input.key,
version_id: input.version_id,
part_number: input.part_number,
range: input.range.map(|range| match range {
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
Range::Int { first, last: None } => ByteRange::From { first },
Range::Suffix { length } => ByteRange::Suffix { length },
}),
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
}
}
}
impl s3_api::GetObjectOutputFacts {
pub(crate) fn into_s3s(self) -> Result<GetObjectOutput, rustfs_gateway_types::compat::ConversionError> {
let mut output = rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::get_object::output_to_s3s(self.dto)?;
output.e_tag = self.etag.map(|etag| {
if etag.weak {
ETag::Weak(etag.opaque)
} else {
ETag::Strong(etag.opaque)
}
});
output.body = self.body.map(s3_api::GetObjectBody::into_s3s);
Ok(output)
}
}
impl s3_api::GetObjectBody {
pub(crate) fn into_s3s(self) -> StreamingBlob {
match self {
Self::Bytes(bytes) => StreamingBlob::from_bytes(bytes),
Self::Stream(payload) => StreamingBlob::new(LegacyGetStream {
payload: Mutex::new(payload),
done: false,
}),
}
}
}
// The legacy body requires Sync. Polling uses exclusive access, not a mutex lock.
struct LegacyGetStream {
payload: Mutex<BoxPayloadStream>,
done: bool,
}
impl futures::Stream for LegacyGetStream {
type Item = Result<bytes::Bytes, Box<dyn std::error::Error + Send + Sync>>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
if this.done {
return Poll::Ready(None);
}
let event = std::task::ready!(
this.payload
.get_mut()
.unwrap_or_else(PoisonError::into_inner)
.as_mut()
.poll_read(cx)
);
Poll::Ready(match event {
Ok(PayloadRead::Chunk(bytes)) => Some(Ok(bytes)),
Ok(PayloadRead::Eof { trailers }) => {
this.done = true;
if trailers.is_empty() {
None
} else {
Some(Err(Box::new(rustfs_gateway_types::compat::ConversionError {
field: "body",
reason: "legacy GetObject cannot encode response trailers",
})))
}
}
Err(error) => {
this.done = true;
Some(Err(legacy_get_body_error(error)))
}
})
}
}
fn legacy_get_body_error(error: StreamError) -> Box<dyn std::error::Error + Send + Sync> {
if matches!(error.kind(), StreamErrorKind::Upstream(_)) {
let StreamErrorKind::Upstream(source) = error.into_kind() else {
unreachable!("the owned error kind was checked above")
};
source
} else {
Box::new(error)
}
}
impl s3s::stream::ByteStream for LegacyGetStream {
fn remaining_length(&self) -> s3s::stream::RemainingLength {
match self
.payload
.lock()
.unwrap_or_else(PoisonError::into_inner)
.len_hint()
.and_then(|length| usize::try_from(length).ok())
{
Some(length) => s3s::stream::RemainingLength::new_exact(length),
None => s3s::stream::RemainingLength::unknown(),
}
}
}
#[cfg(test)]
mod tests {
use super::{StreamError, legacy_get_body_error};
#[test]
fn get_body_preserves_upstream_cause_and_native_framing_errors() {
let error = legacy_get_body_error(StreamError::upstream(Box::new(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"body ended early",
))));
let source = error
.downcast_ref::<std::io::Error>()
.expect("the original cause stays at the legacy boundary");
assert_eq!(source.kind(), std::io::ErrorKind::UnexpectedEof);
for error in [
StreamError::incomplete_body(),
StreamError::length_mismatch(9, 4),
StreamError::polled_after_eof(),
] {
let error = error.with_bytes_before_error(4);
let message = error.to_string();
let error = legacy_get_body_error(error);
let source = error
.downcast_ref::<StreamError>()
.expect("native framing errors retain their type");
assert_eq!(source.bytes_before_error(), 4);
assert_eq!(source.to_string(), message);
}
}
}
+3 -1
View File
@@ -23,7 +23,9 @@ pub(crate) use crate::app::object::reject_veeam_sosapi_mutation;
pub(crate) use crate::app::object::shared::mark_on_demand_migration_list_local_only;
pub(crate) use crate::app::object::{PutObjectWriteInput, PutObjectWriteOutput, legacy_put_body};
pub(crate) use crate::app::object_usecase::DefaultObjectUsecase;
pub(crate) use crate::app::object_usecase::{HeadObjectFields, HeadObjectOutputFacts};
pub(crate) use crate::app::object_usecase::{
GetObjectBody, GetObjectFields, GetObjectOutputFacts, HeadObjectFields, HeadObjectOutputFacts,
};
pub(crate) mod bucket;
pub(crate) mod common;
+3 -4
View File
@@ -141,7 +141,7 @@ pub(crate) use super::ecfs_extend::{
parse_object_lock_legal_hold_native, parse_object_lock_retention_native, parse_part_number_i32_to_usize,
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled_state,
validate_list_object_unordered_with_delimiter, validate_new_object_key, validate_object_key, wrap_response_with_cors,
validate_list_object_unordered_with_delimiter, validate_new_object_key, validate_object_key,
};
pub(crate) use super::sse::{
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
@@ -170,9 +170,8 @@ pub(crate) mod access_consumer {
authorize_copy_object_tags, authorize_request, authorize_request_core, bucket_config_mutation_incarnation,
delete_object_authorize_action, has_bypass_governance_header, load_bucket_generation_from_store,
log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization,
prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, replication_proxy_allowed,
replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards,
set_requested_object_lock_retain_until,
prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, replication_request_authorized,
req_info_mut, req_info_ref, retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
};
pub(crate) use super::super::denial_audit::DenialAuditor;
pub(crate) use super::super::helper::{AuditRequestView, DenialScope};
@@ -12,14 +12,29 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! HeadObject wire checks and refusal-matrix row for migration batch C (rustfs/backlog#2767).
//! Object read wire checks and refusal-matrix rows for migration batch C (rustfs/backlog#2767).
pub(crate) const MATRIX: &[super::MatrixRow] = &[super::MatrixRow {
operation: "HeadObject",
metric: rustfs_s3_ops::S3Operation::HeadObject,
has_error_body: false,
call: head_object,
}];
pub(crate) const MATRIX: &[super::MatrixRow] = &[
super::MatrixRow {
operation: "HeadObject",
metric: rustfs_s3_ops::S3Operation::HeadObject,
has_error_body: false,
call: head_object,
},
super::MatrixRow {
operation: "GetObject",
metric: rustfs_s3_ops::S3Operation::GetObject,
has_error_body: true,
call: get_object,
},
];
fn get_object<'a>(client: &'a aws_sdk_s3::Client, _bucket: &'a str) -> futures::future::BoxFuture<'a, super::Outcome> {
Box::pin(async move {
super::put_object_in_the_store(MATRIX_BUCKET, "get-matrix").await;
super::outcome(client.get_object().bucket(MATRIX_BUCKET).key("get-matrix").send().await)
})
}
pub(super) const MATRIX_BUCKET: &str = "object-read-matrix";
@@ -13,11 +13,11 @@
// limitations under the License.
const GET_OBJECT: &str = include_str!("../src/app/object/get.rs");
const PUT_OBJECT: &str = include_str!("../src/app/object/put.rs");
const PUT_OBJECT: &str = include_str!("../src/storage/legacy_edge/object_write.rs");
const SELECT_OBJECT: &str = include_str!("../src/app/object/mod.rs");
fn assert_async_method_is_measured(source: &str, method: &str) {
let method_marker = format!("pub async fn {method}");
let method_marker = format!("async fn {method}(");
let method_start = source
.find(&method_marker)
.unwrap_or_else(|| panic!("missing method {method}"));
@@ -41,7 +41,11 @@ fn assert_async_method_is_measured(source: &str, method: &str) {
#[test]
fn object_request_profiling_is_attached_to_the_matching_operation() {
assert_async_method_is_measured(GET_OBJECT, "execute_get_object");
assert_async_method_is_measured(GET_OBJECT, "execute_get_object_fields");
assert!(
GET_OBJECT.contains("label = \"rustfs::app::object::get::DefaultObjectUsecase::execute_get_object\""),
"the native entry must retain the established profiling label"
);
assert_async_method_is_measured(PUT_OBJECT, "execute_put_object");
assert_async_method_is_measured(SELECT_OBJECT, "execute_select_object_content");
}
+1 -1
View File
@@ -1091,7 +1091,7 @@ trace_hot_spans=(
# directly (backlog#1821). Sets and SetDisks no longer paginate; ECStore is
# the only listing entry point.
"rustfs/src/app/bucket_usecase.rs:execute_list_objects_v2"
"rustfs/src/app/object:execute_get_object"
"rustfs/src/app/object:execute_get_object_fields"
)
# A hot span in a file that is due for a mechanical split (rustfs/backlog#2765)
+32 -1
View File
@@ -55,6 +55,7 @@ SEAMS = {
"execute_get_object": Seam(
batch="C",
seam_file="rustfs/src/app/test_seams/get_object.rs",
entry_method="execute_get_object_fields",
owners=(
"rustfs/src/app/object/get.rs",
"rustfs/src/app/object/head.rs",
@@ -392,7 +393,7 @@ def main(argv: list[str]) -> int:
SEAM_BODY = {
"execute_get_object": "pub(crate) fn get_object(u: &U, r: R) -> F { u.execute_get_object(r) }\n",
"execute_get_object": "pub(crate) fn get_object(u: &U, r: R) -> F { u.execute_get_object_fields(r) }\n",
"execute_head_object": "pub(crate) fn head_object(u: &U, r: R) -> F { u.execute_head_object_fields(r) }\n",
"execute_list_objects_v2": "pub(crate) fn list_objects_v2(u: &U, r: R) -> F { u.execute_list_objects_v2(r) }\n",
"execute_put_object": "pub(crate) fn put_object(u: &U, f: &Fs, r: R) -> F { u.execute_put_object(f, r) }\n",
@@ -432,6 +433,36 @@ class SelfTest(unittest.TestCase):
def run_check(self) -> list[str]:
return check(self.root, sorted(p for p in self.files if p.endswith(".rs")))
def test_external_get_fields_call_fails(self):
self.write("rustfs/src/app/storage.rs", "#[cfg(test)] fn t() { u.execute_get_object_fields(r); }")
self.assertRegex(" ".join(self.run_check()), "test calls execute_get_object_fields directly")
def test_get_fields_seam_missing_call_fails(self):
self.write(SEAMS["execute_get_object"].seam_file, "fn get_object() {}")
self.assertRegex(" ".join(self.run_check()), "must call execute_get_object_fields exactly once, found 0")
def test_get_fields_seam_duplicate_call_fails(self):
self.write(SEAMS["execute_get_object"].seam_file, "fn get_object() { u.execute_get_object_fields(r); u.execute_get_object_fields(r); }")
self.assertRegex(" ".join(self.run_check()), "must call execute_get_object_fields exactly once, found 2")
def test_get_seam_wrapper_instead_of_fields_fails(self):
self.write(SEAMS["execute_get_object"].seam_file, "fn get_object() { u.execute_get_object(r); }")
self.assertRegex(" ".join(self.run_check()), "must call execute_get_object_fields exactly once, found 0")
def test_get_seam_both_entries_fails(self):
self.write(SEAMS["execute_get_object"].seam_file, "fn get_object() { u.execute_get_object_fields(r); u.execute_get_object(r); }")
self.assertRegex(" ".join(self.run_check()), "must not call execute_get_object")
def test_commented_and_quoted_get_fields_calls_do_not_count(self):
self.write(SEAMS["execute_get_object"].seam_file, 'fn get_object() { let s = "u.execute_get_object_fields(r)"; /* u.execute_get_object_fields(r); */ }')
self.assertRegex(" ".join(self.run_check()), "must call execute_get_object_fields exactly once, found 0")
def test_get_owners_can_call_both_entries(self):
self.write("rustfs/src/app/object/get.rs", "#[cfg(test)] fn t() { u.execute_get_object(r); u.execute_get_object_fields(r); }")
self.assertEqual(self.run_check(), [])
# -- negative cases: each must fail --
def test_external_head_fields_call_fails(self):
self.write("rustfs/src/app/storage.rs", "#[cfg(test)] fn t() { u.execute_head_object_fields(r); }")
self.assertRegex(" ".join(self.run_check()), "test calls execute_head_object_fields directly")
+19
View File
@@ -301,6 +301,25 @@ checked=$((checked + 1))
if ! (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
fail "logging guardrails failed with the TRACE hot span in $LISTING: $(tail -3 "$LOG_OUT")"
fi
GET_SOURCE="rustfs/src/app/object/get.rs"
cp "$FX/$GET_SOURCE" "$FX/.get-source.saved"
"$PYTHON" - "$FX/$GET_SOURCE" <<'PY'
import sys
from pathlib import Path
path = Path(sys.argv[1])
text = path.read_text()
marker = '#[instrument(name = "execute_get_object", level = "trace", skip(self, envelope, input))]'
if text.count(marker) != 1:
sys.exit("split probe: native GET must carry exactly one TRACE span")
path.write_text(text.replace(marker, marker.replace('"trace"', '"debug"')))
PY
checked=$((checked + 1))
if (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
fail "logging guardrails passed after the native GET hot span lost TRACE"
elif ! grep -qF "execute_get_object_fields must remain TRACE-only" "$LOG_OUT"; then
fail "native GET logging guardrails failed for another reason: $(tail -3 "$LOG_OUT")"
fi
mv "$FX/.get-source.saved" "$FX/$GET_SOURCE"
"$PYTHON" - "$FX/$LISTING" <<'PY'
import sys
from pathlib import Path