mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
feat(gateway): serve PutObject through the owned write path
Register PutObject and share write authorization with legacy PUT and
browser POST. Carry authenticated trailer declarations and captured
frames into the owned write, preserve typed eager-read errors, and keep
the table-catalog guard tied to the actual write call.
Verified on RustFS 0567078f5 with gateway d75b6ce on macOS. All 1022
affected tests passed with a 4 MiB test stack, along with both
all-target Clippy configurations, script tests, test wiring, and
prepush checks. A separate gateway wire run stored and read an empty
object using the rebuilt server binary.
Fourteen single-fault checks covered trailer selection and capture,
typed producer failures, tag policies, the shared write grants, and
POST policy projection. The reserved-prefix source guard also rejected
a fixture whose owned write call was replaced.
This commit is contained in:
@@ -474,7 +474,9 @@ where
|
||||
let mut buffer = pool
|
||||
.try_acquire_buffer(size)
|
||||
.unwrap_or_else(|| rustfs_io_core::PooledBuffer::unpooled(size));
|
||||
super::put::read_small_put_body_into(body, &mut *buffer, size).await?;
|
||||
super::put::read_small_put_body_into(body, &mut *buffer, size)
|
||||
.await
|
||||
.map_err(S3Error::from)?;
|
||||
Ok(ExtractStagedBody {
|
||||
reader: Some(super::put::PooledBufferReader::new(buffer, size)),
|
||||
})
|
||||
@@ -2094,7 +2096,7 @@ impl DefaultObjectUsecase {
|
||||
let Some(body) = body else {
|
||||
return Err(Into::into(s3_error!(IncompleteBody)));
|
||||
};
|
||||
let (chunks, _) = super::native_body::BodyChunks::new(body);
|
||||
let (chunks, _) = super::native_body::BodyChunks::with_trailers(body, input.frame_trailers);
|
||||
let body: super::request_body::UploadChunks = Box::new(chunks);
|
||||
let (body, progress_control) = if let Some(control) = envelope.body_read_control() {
|
||||
control.activate_upload(
|
||||
|
||||
@@ -337,7 +337,7 @@ impl DefaultObjectUsecase {
|
||||
},
|
||||
};
|
||||
let committed = self
|
||||
.put_object_core(write, super::native_body::from_io_stream(body), start_time)
|
||||
.put_object_core(write, super::native_body::from_io_stream(body), None, start_time)
|
||||
.await
|
||||
.map_err(|error| ApiError {
|
||||
code: error.code().clone().into(),
|
||||
|
||||
@@ -93,7 +93,7 @@ impl PayloadStream for LegacyPayload {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FrameTrailers(Mutex<Option<HeaderMap>>);
|
||||
pub(crate) struct FrameTrailers(Mutex<Option<HeaderMap>>);
|
||||
|
||||
impl TrailerSource for FrameTrailers {
|
||||
fn lookup(&self, name: &str) -> TrailerValue {
|
||||
@@ -115,8 +115,13 @@ pub(super) struct BodyChunks {
|
||||
}
|
||||
|
||||
impl BodyChunks {
|
||||
#[cfg(test)]
|
||||
pub(super) fn new(body: Body) -> (Self, SharedTrailerSource) {
|
||||
let trailers = Arc::new(FrameTrailers::default());
|
||||
Self::with_trailers(body, None)
|
||||
}
|
||||
|
||||
pub(super) fn with_trailers(body: Body, trailers: Option<Arc<FrameTrailers>>) -> (Self, SharedTrailerSource) {
|
||||
let trailers = trailers.unwrap_or_default();
|
||||
(
|
||||
Self {
|
||||
body: Mutex::new(body),
|
||||
|
||||
@@ -541,7 +541,7 @@ fn select_put_path_with_concurrency(
|
||||
/// where the allocation cost is negligible (≤4KiB memcpy).
|
||||
const POOL_BYPASS_MAX_SIZE: usize = 4 * 1024;
|
||||
|
||||
pub(super) async fn read_small_put_body_into<R, B>(body: &mut R, buf: &mut B, size: usize) -> S3Result<()>
|
||||
pub(super) async fn read_small_put_body_into<R, B>(body: &mut R, buf: &mut B, size: usize) -> rustfs_s3_types::S3Result<()>
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
B: bytes::BufMut,
|
||||
@@ -554,7 +554,7 @@ where
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
if read == 0 {
|
||||
return Err(s3_error!(IncompleteBody));
|
||||
return Err(rustfs_s3_types::s3_error!(IncompleteBody));
|
||||
}
|
||||
filled += read;
|
||||
}
|
||||
@@ -564,13 +564,17 @@ where
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
if extra_read != 0 {
|
||||
return Err(s3_error!(UnexpectedContent));
|
||||
return Err(rustfs_s3_types::s3_error!(UnexpectedContent));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_small_put_body_exact_pooled<R>(mut body: R, size: usize, pool: &BytesPool) -> S3Result<PooledBuffer>
|
||||
async fn read_small_put_body_exact_pooled<R>(
|
||||
mut body: R,
|
||||
size: usize,
|
||||
pool: &BytesPool,
|
||||
) -> rustfs_s3_types::S3Result<PooledBuffer>
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
@@ -585,7 +589,7 @@ where
|
||||
/// Read small PUT body into a directly-allocated buffer, bypassing BytesPool.
|
||||
/// Used for objects ≤4KiB where pool contention under high concurrency
|
||||
/// outweighs the allocation cost.
|
||||
async fn read_small_put_body_exact_direct<R>(mut body: R, size: usize) -> S3Result<std::io::Cursor<Vec<u8>>>
|
||||
async fn read_small_put_body_exact_direct<R>(mut body: R, size: usize) -> rustfs_s3_types::S3Result<std::io::Cursor<Vec<u8>>>
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
@@ -594,7 +598,7 @@ where
|
||||
Ok(std::io::Cursor::new(buf))
|
||||
}
|
||||
|
||||
async fn read_zero_copy_put_body_exact<S, E>(mut body: S, size: usize) -> S3Result<ChunkedBytesReader>
|
||||
async fn read_zero_copy_put_body_exact<S, E>(mut body: S, size: usize) -> rustfs_s3_types::S3Result<ChunkedBytesReader>
|
||||
where
|
||||
S: futures::Stream<Item = std::result::Result<Bytes, E>> + Unpin,
|
||||
E: Into<StdError>,
|
||||
@@ -604,14 +608,14 @@ where
|
||||
|
||||
while filled < size {
|
||||
let Some(chunk) = body.next().await else {
|
||||
return Err(s3_error!(IncompleteBody));
|
||||
return Err(rustfs_s3_types::s3_error!(IncompleteBody));
|
||||
};
|
||||
let chunk = chunk.map_err(|err| ApiError::from(s3s_body_error_to_io(err.into())))?;
|
||||
if chunk.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if filled.saturating_add(chunk.len()) > size {
|
||||
return Err(s3_error!(UnexpectedContent));
|
||||
return Err(rustfs_s3_types::s3_error!(UnexpectedContent));
|
||||
}
|
||||
|
||||
rustfs_io_metrics::record_zero_copy_buffer_operation("put_chunk", chunk.len());
|
||||
@@ -622,7 +626,7 @@ where
|
||||
while let Some(chunk) = body.next().await {
|
||||
let chunk = chunk.map_err(|err| ApiError::from(s3s_body_error_to_io(err.into())))?;
|
||||
if !chunk.is_empty() {
|
||||
return Err(s3_error!(UnexpectedContent));
|
||||
return Err(rustfs_s3_types::s3_error!(UnexpectedContent));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1387,7 +1391,7 @@ impl DefaultObjectUsecase {
|
||||
not(all(any(feature = "cuobject", feature = "nixl"), target_os = "linux")),
|
||||
allow(unused_mut, reason = "mutated only by the cuObject RDMA branch")
|
||||
)]
|
||||
let mut committed = self.put_object_core(write, body, start_time).await?;
|
||||
let mut committed = self.put_object_core(write, body, input.frame_trailers, start_time).await?;
|
||||
#[cfg(not(all(any(feature = "cuobject", feature = "nixl"), target_os = "linux")))]
|
||||
let _ = rdma_put;
|
||||
#[cfg(all(any(feature = "cuobject", feature = "nixl"), target_os = "linux"))]
|
||||
@@ -1461,6 +1465,7 @@ impl DefaultObjectUsecase {
|
||||
&self,
|
||||
write: PutObjectWriteRequest<'_>,
|
||||
body: NativeBody,
|
||||
frame_trailers: Option<Arc<super::native_body::FrameTrailers>>,
|
||||
start_time: Instant,
|
||||
) -> rustfs_s3_types::S3Result<PutObjectCommitted> {
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode};
|
||||
@@ -1486,7 +1491,7 @@ impl DefaultObjectUsecase {
|
||||
preserve_etag,
|
||||
mut origin,
|
||||
} = write;
|
||||
let (chunks, _) = super::native_body::BodyChunks::new(body);
|
||||
let (chunks, _) = super::native_body::BodyChunks::with_trailers(body, frame_trailers);
|
||||
let body: UploadChunks = match max_content_length {
|
||||
Some(limit) => Box::new(MaxContentLengthStream {
|
||||
inner: chunks,
|
||||
@@ -2442,6 +2447,7 @@ mod tests {
|
||||
use crate::app::storage_api::s3::dto::{GetObjectInput, HeadObjectInput, ObjectLockRetentionMode};
|
||||
use futures::StreamExt;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, Method};
|
||||
use rustfs_s3_types::S3ErrorCode as OwnedS3ErrorCode;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
@@ -3650,6 +3656,55 @@ mod tests {
|
||||
assert_eq!(error.code().as_str(), "XAmzContentSHA256Mismatch");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn eager_put_readers_preserve_typed_producer_failure() {
|
||||
use rustfs_gateway_stream::StreamError;
|
||||
use rustfs_io_core::BytesPoolConfig;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("fixture producer failure")]
|
||||
struct ProducerFailure;
|
||||
|
||||
fn producer_error() -> StreamError {
|
||||
StreamError::upstream(Box::new(ProducerFailure))
|
||||
}
|
||||
|
||||
fn assert_producer_failure<T>(result: Result<T, impl Into<rustfs_s3_types::S3Error>>, path: &str) {
|
||||
let Err(error) = result else {
|
||||
panic!("{path} accepted a producer error");
|
||||
};
|
||||
let error = error.into();
|
||||
assert_eq!(error.code().as_str(), "InternalError", "{path}");
|
||||
let mut cause = std::error::Error::source(&error);
|
||||
let mut stream = false;
|
||||
let mut producer = false;
|
||||
for _ in 0..8 {
|
||||
let Some(error) = cause else { break };
|
||||
stream |= error.is::<StreamError>();
|
||||
producer |= error.is::<ProducerFailure>();
|
||||
cause = error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.and_then(std::io::Error::get_ref)
|
||||
.map(|inner| inner as &dyn std::error::Error)
|
||||
.or_else(|| error.source());
|
||||
}
|
||||
assert!(stream, "{path} lost the typed stream error");
|
||||
assert!(producer, "{path} lost the typed producer failure");
|
||||
}
|
||||
|
||||
let stream_reader = || {
|
||||
StreamReader::new(futures::stream::iter([
|
||||
Ok(Bytes::from_static(b"a")),
|
||||
Err(std::io::Error::other(producer_error())),
|
||||
]))
|
||||
};
|
||||
assert_producer_failure(read_small_put_body_exact_direct(stream_reader(), 2).await, "direct");
|
||||
let pool = BytesPool::with_config(BytesPoolConfig::default());
|
||||
assert_producer_failure(read_small_put_body_exact_pooled(stream_reader(), 2, &pool).await, "pooled");
|
||||
let zero_copy = futures::stream::iter([Ok(Bytes::from_static(b"a")), Err(producer_error())]);
|
||||
assert_producer_failure(read_zero_copy_put_body_exact(zero_copy, 2).await, "zero-copy");
|
||||
}
|
||||
|
||||
struct FragmentedBody {
|
||||
data: std::io::Cursor<Vec<u8>>,
|
||||
}
|
||||
@@ -3835,7 +3890,7 @@ mod tests {
|
||||
assert!(pool.try_acquire_buffer(8192).is_none());
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
assert_eq!(owner.await.err().expect("trickle must expire").code(), &S3ErrorCode::RequestTimeout);
|
||||
assert_eq!(owner.await.err().expect("trickle must expire").code(), &OwnedS3ErrorCode::RequestTimeout);
|
||||
let healthy = read_small_put_body_exact_pooled(std::io::Cursor::new(vec![1; 8192]), 8192, &pool)
|
||||
.await
|
||||
.expect("timed-out buffer must be reusable");
|
||||
@@ -3879,7 +3934,7 @@ mod tests {
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
|
||||
assert_eq!(err.code(), &OwnedS3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3892,7 +3947,7 @@ mod tests {
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::UnexpectedContent);
|
||||
assert_eq!(err.code(), &OwnedS3ErrorCode::UnexpectedContent);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3911,12 +3966,12 @@ mod tests {
|
||||
let short = read_small_put_body_exact_direct(std::io::Cursor::new(b"hell".to_vec()), 5)
|
||||
.await
|
||||
.expect_err("short direct body should fail");
|
||||
assert_eq!(short.code(), &S3ErrorCode::IncompleteBody);
|
||||
assert_eq!(short.code(), &OwnedS3ErrorCode::IncompleteBody);
|
||||
|
||||
let extra = read_small_put_body_exact_direct(std::io::Cursor::new(b"hello!".to_vec()), 5)
|
||||
.await
|
||||
.expect_err("extra direct body should fail");
|
||||
assert_eq!(extra.code(), &S3ErrorCode::UnexpectedContent);
|
||||
assert_eq!(extra.code(), &OwnedS3ErrorCode::UnexpectedContent);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3958,7 +4013,7 @@ mod tests {
|
||||
let extra = read_small_put_body_exact_direct(std::io::Cursor::new(vec![1u8]), 0)
|
||||
.await
|
||||
.expect_err("non-empty body declared as empty should fail");
|
||||
assert_eq!(extra.code(), &S3ErrorCode::UnexpectedContent);
|
||||
assert_eq!(extra.code(), &OwnedS3ErrorCode::UnexpectedContent);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3983,7 +4038,7 @@ mod tests {
|
||||
.await
|
||||
.expect_err("a partial body followed by an I/O error must fail");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(err.code(), &OwnedS3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4011,7 +4066,7 @@ mod tests {
|
||||
.await
|
||||
.expect_err("a fragmented body longer than declared must fail");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::UnexpectedContent);
|
||||
assert_eq!(err.code(), &OwnedS3ErrorCode::UnexpectedContent);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4069,7 +4124,7 @@ mod tests {
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::UnexpectedContent);
|
||||
assert_eq!(err.code(), &OwnedS3ErrorCode::UnexpectedContent);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -25,6 +25,7 @@ pub(crate) struct PutObjectWriteInput {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) key: String,
|
||||
pub(crate) body: Option<Body>,
|
||||
pub(crate) frame_trailers: Option<std::sync::Arc<super::native_body::FrameTrailers>>,
|
||||
pub(crate) trailing_headers: Option<SharedTrailerSource>,
|
||||
pub(crate) content_length: Option<i64>,
|
||||
pub(crate) metadata: Option<HashMap<String, String>>,
|
||||
@@ -73,3 +74,13 @@ pub(crate) struct PutObjectWriteOutput {
|
||||
pub(crate) headers: http::HeaderMap,
|
||||
pub(crate) status: Option<http::StatusCode>,
|
||||
}
|
||||
|
||||
impl PutObjectWriteInput {
|
||||
/// Gateway frames publish to the same source that checksum readers observe.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn capture_frame_trailers(&mut self) {
|
||||
let trailers = std::sync::Arc::new(super::native_body::FrameTrailers::default());
|
||||
self.frame_trailers = Some(trailers.clone());
|
||||
self.trailing_headers = Some(trailers);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,175 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch D (rustfs/backlog#2765): PutObject, UploadPart and PostObject.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
//! PutObject gateway registration and native request/response projection.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
use super::{OpRow, Served, row};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::object_usecase::{DefaultObjectUsecase, PutObjectWriteInput, PutObjectWriteOutput};
|
||||
use crate::runtime_sources::AppContext;
|
||||
use crate::storage_api::server::http::access::access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
use rustfs_gateway_types::{ChecksumAlgorithm, Timestamp};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{S3Result, s3_error};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) const ROWS: &[OpRow] = &[row::<dto::PutObject>()];
|
||||
|
||||
impl Served for dto::PutObject {
|
||||
const METRIC: S3Operation = S3Operation::PutObject;
|
||||
|
||||
fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> impl std::future::Future<Output = S3Result<()>> + Send {
|
||||
let input = request.input();
|
||||
let bucket = input.bucket.as_str().to_owned();
|
||||
let key = input.key.as_str().to_owned();
|
||||
let retain_until = retain_until(input.object_lock_retain_until_date.as_ref());
|
||||
let version_id = request_version_id(envelope);
|
||||
let specific_etag = input.if_match.as_deref().is_some_and(|value| value.trim() != "*");
|
||||
let legal_hold = input.object_lock_legal_hold_status.is_some();
|
||||
let retention = input.object_lock_mode.is_some() || input.object_lock_retain_until_date.is_some();
|
||||
async move {
|
||||
let server = envelope
|
||||
.server_context()
|
||||
.cloned()
|
||||
.ok_or_else(|| s3_error!(InternalError, "server context is not initialized"))?;
|
||||
let facts = PutObjectAuthorization {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
version_id: version_id.as_deref(),
|
||||
retain_until: retain_until?,
|
||||
specific_etag,
|
||||
legal_hold,
|
||||
retention,
|
||||
post: false,
|
||||
};
|
||||
Ok(authorize_put_object(envelope, &server, facts, |_| Ok(())).await?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let mut input = native_input(request.into_input())?;
|
||||
input.version_id = request_version_id(&envelope);
|
||||
if let Some(declaration) = envelope.headers().get("x-amz-trailer").filter(|value| !value.is_empty()) {
|
||||
// The gateway verifies the declared trailer independently of the SDK algorithm.
|
||||
let algorithm = declaration
|
||||
.to_str()
|
||||
.ok()
|
||||
.and_then(|name| ChecksumAlgorithm::from_header_name(name.trim_matches([' ', '\t'])))
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "Invalid checksum trailer declaration"))?;
|
||||
input.checksum_algorithm = Some(algorithm.wire_name().to_owned());
|
||||
input.capture_frame_trailers();
|
||||
}
|
||||
let output = DefaultObjectUsecase::with_context(context)
|
||||
.execute_put_object_owned(envelope, input)
|
||||
.await?;
|
||||
native_output(output)
|
||||
}
|
||||
}
|
||||
|
||||
// PutObject's destination version is a RustFS replication extension, absent from the AWS DTO.
|
||||
fn request_version_id(envelope: &RequestEnvelope) -> Option<String> {
|
||||
url::form_urlencoded::parse(envelope.uri().query()?.as_bytes())
|
||||
.find(|(name, _)| name == "versionId")
|
||||
.map(|(_, value)| value.into_owned())
|
||||
}
|
||||
|
||||
fn retain_until(value: Option<&Timestamp>) -> S3Result<Option<time::OffsetDateTime>> {
|
||||
value
|
||||
.map(|value| {
|
||||
time::OffsetDateTime::from_unix_timestamp_nanos(
|
||||
i128::from(value.secs()) * 1_000_000_000 + i128::from(value.subsec_nanos()),
|
||||
)
|
||||
.map_err(|_| s3_error!(InvalidArgument, "Invalid retain-until date"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn native_input(input: dto::PutObjectInput) -> S3Result<PutObjectWriteInput> {
|
||||
let checksum = |algorithm| {
|
||||
input
|
||||
.checksum_spec
|
||||
.as_ref()
|
||||
.filter(|value| value.algorithm() == algorithm)
|
||||
.map(|value| value.render_base64().to_owned())
|
||||
};
|
||||
Ok(PutObjectWriteInput {
|
||||
checksum_crc32: checksum(ChecksumAlgorithm::Crc32),
|
||||
checksum_crc32c: checksum(ChecksumAlgorithm::Crc32c),
|
||||
checksum_sha1: checksum(ChecksumAlgorithm::Sha1),
|
||||
checksum_sha256: checksum(ChecksumAlgorithm::Sha256),
|
||||
checksum_crc64nvme: checksum(ChecksumAlgorithm::Crc64Nvme),
|
||||
bucket: input.bucket.as_str().to_owned(),
|
||||
key: input.key.as_str().to_owned(),
|
||||
body: input.body.map(|body| body.into_body()),
|
||||
trailing_headers: None,
|
||||
frame_trailers: None,
|
||||
content_length: Some(input.content_length),
|
||||
metadata: Some(input.metadata.into_iter().collect()),
|
||||
specific_if_none_match: input.if_none_match.as_deref().is_some_and(|value| value.trim() != "*"),
|
||||
object_lock_retain_until_date: retain_until(input.object_lock_retain_until_date.as_ref())?,
|
||||
cache_control: input.cache_control,
|
||||
content_disposition: input.content_disposition,
|
||||
content_encoding: input.content_encoding,
|
||||
content_language: input.content_language,
|
||||
content_type: input.content_type,
|
||||
expires: input.expires.map(|value| value.as_str().to_owned()),
|
||||
tagging: input.tagging,
|
||||
version_id: None,
|
||||
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key.map(|value| value.expose_secret().to_owned()),
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
ssekms_key_id: input.ssekms_key_id,
|
||||
content_md5: input.content_md5,
|
||||
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
|
||||
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
|
||||
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
|
||||
website_redirect_location: input.website_redirect_location,
|
||||
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
|
||||
})
|
||||
}
|
||||
|
||||
fn native_output(output: PutObjectWriteOutput) -> S3Result<Resp<dto::PutObject>> {
|
||||
let mut headers = output.headers;
|
||||
for (name, value) in [
|
||||
("x-amz-checksum-crc32", output.checksum_crc32),
|
||||
("x-amz-checksum-crc32c", output.checksum_crc32c),
|
||||
("x-amz-checksum-sha1", output.checksum_sha1),
|
||||
("x-amz-checksum-sha256", output.checksum_sha256),
|
||||
("x-amz-checksum-crc64nvme", output.checksum_crc64nvme),
|
||||
] {
|
||||
if let Some(value) = value {
|
||||
headers.insert(
|
||||
name,
|
||||
value
|
||||
.parse()
|
||||
.map_err(|_| s3_error!(InternalError, "Invalid stored checksum"))?,
|
||||
);
|
||||
}
|
||||
}
|
||||
let dto = dto::PutObjectOutput {
|
||||
e_tag: crate::app::dto_values::stored_etag(
|
||||
output
|
||||
.e_tag
|
||||
.as_deref()
|
||||
.ok_or_else(|| s3_error!(InternalError, "PutObject completed without an ETag"))?,
|
||||
),
|
||||
version_id: output.version_id,
|
||||
expiration: output.expiration.map(Into::into),
|
||||
server_side_encryption: output.server_side_encryption.map(dto::ServerSideEncryption::custom),
|
||||
sse_customer_algorithm: output.sse_customer_algorithm,
|
||||
sse_customer_key_md5: output.sse_customer_key_md5,
|
||||
ssekms_key_id: output.ssekms_key_id,
|
||||
..Default::default()
|
||||
};
|
||||
Ok(match output.status {
|
||||
Some(status) => Resp::with_status(dto, status.as_u16()),
|
||||
None => Resp::new(dto),
|
||||
}
|
||||
.with_extra_headers(headers))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "object_write_test.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! PutObject edge authorization and native frame tests against real storage.
|
||||
|
||||
use super::*;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::server::http::access::test_support::{
|
||||
AuthorizationRequest, BucketOperations as _, ETag, ETagCondition, FS, MakeBucketOptions, ObjectOperations as _,
|
||||
ObjectOptions, PutObjectInput as LegacyPutObjectInput, ReqInfo, S3Access as _, S3Request, get_bucket_metadata,
|
||||
set_bucket_metadata,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use rustfs_gateway::{Limits, MetaView, SseConfig, TargetKind, TransportSecurity, WireRequest};
|
||||
use rustfs_gateway_stream::{ByteStream, PayloadCaps, PayloadRead, PayloadStream, StreamError, TrailingHeaders};
|
||||
use serial_test::serial;
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
fn request(input: dto::PutObjectInput) -> Req<dto::PutObject> {
|
||||
let wire = WireRequest::accept(
|
||||
http::Request::builder()
|
||||
.uri("/")
|
||||
.header("host", "localhost")
|
||||
.body(Bytes::new())
|
||||
.unwrap(),
|
||||
&Limits::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let meta = MetaView::of(&wire, TargetKind::Service).unwrap();
|
||||
let proof = rustfs_gateway::enforce_sse(&meta, TransportSecurity::Encrypted, &SseConfig::strict()).unwrap();
|
||||
Req::new(input, proof)
|
||||
}
|
||||
|
||||
async fn fixture(actions: &[&str]) -> (Arc<ServerContextSlot>, String) {
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(crate::app::gating_test_env::shared_gating_ambient().await));
|
||||
let bucket = format!("put-gateway-{}", uuid::Uuid::new_v4());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
if !actions.is_empty() {
|
||||
set_policy(
|
||||
&bucket,
|
||||
serde_json::json!({
|
||||
"Effect": "Allow",
|
||||
"Principal": {"AWS": "*"},
|
||||
"Action": actions,
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/allowed"),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
(server, bucket)
|
||||
}
|
||||
|
||||
async fn set_policy(bucket: &str, statement: serde_json::Value) {
|
||||
let policy = serde_json::json!({"Version": "2012-10-17", "Statement": [statement]}).to_string();
|
||||
let mut metadata = (*get_bucket_metadata(bucket).await.unwrap()).clone();
|
||||
metadata.policy_config = Some(serde_json::from_str(&policy).unwrap());
|
||||
metadata.policy_config_json = policy.into_bytes();
|
||||
set_bucket_metadata(bucket.to_owned(), metadata).await.unwrap();
|
||||
}
|
||||
|
||||
fn envelope(server: &Arc<ServerContextSlot>, headers: HeaderMap) -> RequestEnvelope {
|
||||
let mut legacy = build_request((), Method::PUT);
|
||||
legacy.extensions.insert(Arc::clone(server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
legacy.headers = headers;
|
||||
RequestEnvelope::from_s3s(&legacy)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn put_authorization_checks_action_resource_and_write_privileges_on_both_edges() {
|
||||
let cases = [
|
||||
(vec!["s3:PutObject"], "allowed", None, true),
|
||||
(vec![], "allowed", None, false),
|
||||
(vec!["s3:PutObject"], "denied", None, false),
|
||||
(vec!["s3:GetObject"], "allowed", None, false),
|
||||
(vec!["s3:PutObject"], "allowed", Some(("if-match", "\"tag\"")), false),
|
||||
(vec!["s3:PutObject"], "allowed", Some(("dto-if-match", "\"tag\"")), false),
|
||||
(vec!["s3:PutObject"], "allowed", Some(("x-amz-replication-status", "REPLICA")), false),
|
||||
(vec!["s3:PutObject"], "allowed", Some(("x-amz-object-lock-legal-hold", "ON")), false),
|
||||
(vec!["s3:PutObject"], "allowed", Some(("x-amz-object-lock-mode", "GOVERNANCE")), false),
|
||||
(
|
||||
vec!["s3:PutObject"],
|
||||
"allowed",
|
||||
Some(("x-amz-object-lock-retain-until-date", "2030-01-01T00:00:00Z")),
|
||||
false,
|
||||
),
|
||||
(vec!["s3:PutObject", "s3:GetObject"], "allowed", Some(("if-match", "\"tag\"")), true),
|
||||
(
|
||||
vec!["s3:PutObject", "s3:PutObjectLegalHold"],
|
||||
"allowed",
|
||||
Some(("x-amz-object-lock-legal-hold", "ON")),
|
||||
true,
|
||||
),
|
||||
(
|
||||
vec!["s3:PutObject", "s3:PutObjectRetention"],
|
||||
"allowed",
|
||||
Some(("x-amz-object-lock-mode", "GOVERNANCE")),
|
||||
true,
|
||||
),
|
||||
];
|
||||
for (actions, key, extra, allowed) in cases {
|
||||
let (server, bucket) = fixture(&actions).await;
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some((name, value)) = extra {
|
||||
headers.insert(name, value.parse().unwrap());
|
||||
}
|
||||
let retain = headers
|
||||
.get("x-amz-object-lock-retain-until-date")
|
||||
.map(|_| time::OffsetDateTime::from_unix_timestamp(1_893_456_000).unwrap());
|
||||
let input = dto::PutObjectInput {
|
||||
object_lock_retain_until_date: retain.map(crate::app::dto_values::timestamp),
|
||||
bucket: rustfs_gateway_types::BucketName::new(bucket.clone()).unwrap(),
|
||||
key: rustfs_gateway_types::ObjectKey::new(key).unwrap(),
|
||||
if_match: headers
|
||||
.get("if-match")
|
||||
.or_else(|| headers.get("dto-if-match"))
|
||||
.map(|value| value.to_str().unwrap().to_owned()),
|
||||
object_lock_legal_hold_status: headers
|
||||
.get("x-amz-object-lock-legal-hold")
|
||||
.map(|_| dto::ObjectLockLegalHoldStatus::custom("ON")),
|
||||
object_lock_mode: headers
|
||||
.get("x-amz-object-lock-mode")
|
||||
.map(|_| dto::ObjectLockMode::custom("GOVERNANCE")),
|
||||
..Default::default()
|
||||
};
|
||||
let mut native = envelope(&server, headers.clone());
|
||||
let native_result = <dto::PutObject as Served>::authorize(&mut native, &request(input)).await;
|
||||
let mut legacy = build_request(
|
||||
LegacyPutObjectInput {
|
||||
object_lock_retain_until_date: retain.map(Into::into),
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_owned(),
|
||||
if_match: headers
|
||||
.get("if-match")
|
||||
.or_else(|| headers.get("dto-if-match"))
|
||||
.map(|_| ETagCondition::ETag(ETag::Strong("tag".to_owned()))),
|
||||
object_lock_legal_hold_status: headers.get("x-amz-object-lock-legal-hold").map(|_| "ON".to_owned().into()),
|
||||
object_lock_mode: headers.get("x-amz-object-lock-mode").map(|_| "GOVERNANCE".to_owned().into()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::PUT,
|
||||
);
|
||||
legacy.headers = headers;
|
||||
legacy.extensions.insert(Arc::clone(&server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
let legacy_result = FS::with_server_ctx(Arc::clone(&server)).put_object(&mut legacy).await;
|
||||
for result in [
|
||||
native_result.map_err(|error| error.code().as_str().to_owned()),
|
||||
legacy_result.map_err(|error| error.code().as_str().to_owned()),
|
||||
] {
|
||||
assert_eq!(
|
||||
result,
|
||||
if allowed { Ok(()) } else { Err("AccessDenied".to_owned()) },
|
||||
"{actions:?} {key} {extra:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(AuthorizationRequest::requested_retain_until(&native).map(|value| value.0), retain);
|
||||
let info = native.req_info().unwrap();
|
||||
assert_eq!(info.bucket.as_deref(), Some(bucket.as_str()));
|
||||
assert_eq!(info.object.as_deref(), Some(key));
|
||||
if allowed {
|
||||
assert!(native.bucket_generation().is_some());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn put_authorization_honors_request_object_tag_policy_on_both_edges() {
|
||||
let (server, bucket) = fixture(&[]).await;
|
||||
set_policy(
|
||||
&bucket,
|
||||
serde_json::json!({
|
||||
"Effect": "Allow",
|
||||
"Principal": {"AWS": "*"},
|
||||
"Action": "s3:PutObject",
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/allowed"),
|
||||
"Condition": {"StringEquals": {"s3:RequestObjectTag/visibility": "private"}},
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
for (tagging, allowed) in [
|
||||
(Some("visibility=private"), true),
|
||||
(Some("visibility=public"), false),
|
||||
(None, false),
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(value) = tagging {
|
||||
headers.insert("x-amz-tagging", HeaderValue::from_static(value));
|
||||
}
|
||||
let mut native = envelope(&server, headers.clone());
|
||||
let request = request(dto::PutObjectInput {
|
||||
bucket: rustfs_gateway_types::BucketName::new(bucket.clone()).unwrap(),
|
||||
key: rustfs_gateway_types::ObjectKey::new("allowed").unwrap(),
|
||||
tagging: tagging.map(str::to_owned),
|
||||
..Default::default()
|
||||
});
|
||||
let native = <dto::PutObject as Served>::authorize(&mut native, &request)
|
||||
.await
|
||||
.err()
|
||||
.map(|error| error.code().as_str().to_owned());
|
||||
let mut legacy = build_request(
|
||||
LegacyPutObjectInput {
|
||||
bucket: bucket.clone(),
|
||||
key: "allowed".to_owned(),
|
||||
tagging: tagging.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
Method::PUT,
|
||||
);
|
||||
legacy.headers = headers;
|
||||
legacy.extensions.insert(Arc::clone(&server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
let legacy = FS::with_server_ctx(Arc::clone(&server))
|
||||
.put_object(&mut legacy)
|
||||
.await
|
||||
.err()
|
||||
.map(|error| error.code().as_str().to_owned());
|
||||
let expected = (!allowed).then_some("AccessDenied");
|
||||
assert_eq!(legacy.as_deref(), expected, "legacy edge with {tagging:?}");
|
||||
assert_eq!(native.as_deref(), expected, "gateway edge with {tagging:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("fixture producer failure")]
|
||||
struct FixtureProducerFailure;
|
||||
|
||||
struct Frames(VecDeque<Result<PayloadRead, StreamError>>);
|
||||
impl PayloadStream for Frames {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<PayloadRead, StreamError>> {
|
||||
Poll::Ready(self.0.pop_front().unwrap_or_else(|| Err(StreamError::polled_after_eof())))
|
||||
}
|
||||
fn caps(&self) -> PayloadCaps {
|
||||
PayloadCaps::PUSH
|
||||
}
|
||||
fn len_hint(&self) -> Option<u64> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn native_put_frames_reach_checksum_validation_and_storage() {
|
||||
let (server, bucket) = fixture(&["s3:PutObject"]).await;
|
||||
for (case, length, checksum, producer_error, expected) in [
|
||||
("valid", 3, "NSRBwg==", false, None),
|
||||
("bad-checksum", 3, "AAAAAA==", false, Some("BadDigest")),
|
||||
("short", 4, "NSRBwg==", false, Some("IncompleteBody")),
|
||||
("producer", 3, "NSRBwg==", true, Some("InternalError")),
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-trailer", HeaderValue::from_static("x-amz-checksum-crc32"));
|
||||
headers.insert("x-amz-sdk-checksum-algorithm", HeaderValue::from_static("CRC32"));
|
||||
headers.insert("x-amz-decoded-content-length", length.to_string().parse().unwrap());
|
||||
let mut trailers = HeaderMap::new();
|
||||
trailers.insert("x-amz-checksum-crc32", checksum.parse().unwrap());
|
||||
let end = if producer_error {
|
||||
Err(StreamError::upstream(Box::new(FixtureProducerFailure)))
|
||||
} else {
|
||||
Ok(PayloadRead::Eof {
|
||||
trailers: TrailingHeaders::from_header_map(trailers),
|
||||
})
|
||||
};
|
||||
let body = ByteStream::new(Box::pin(Frames(VecDeque::from([
|
||||
Ok(PayloadRead::Chunk(Bytes::from_static(b"abc"))),
|
||||
end,
|
||||
]))))
|
||||
.unwrap();
|
||||
let req = request(dto::PutObjectInput {
|
||||
bucket: rustfs_gateway_types::BucketName::new(bucket.clone()).unwrap(),
|
||||
key: rustfs_gateway_types::ObjectKey::new("allowed").unwrap(),
|
||||
body: Some(body),
|
||||
content_length: length,
|
||||
checksum_algorithm: Some(dto::ChecksumAlgorithm::custom("CRC32")),
|
||||
..Default::default()
|
||||
});
|
||||
let mut envelope = envelope(&server, headers);
|
||||
<dto::PutObject as Served>::authorize(&mut envelope, &req).await.unwrap();
|
||||
let result = <dto::PutObject as Served>::execute(server.app_context(), envelope, req).await;
|
||||
match expected {
|
||||
None => {
|
||||
let response = result.unwrap();
|
||||
assert_eq!(
|
||||
response.output().unwrap().e_tag,
|
||||
crate::app::dto_values::stored_etag("900150983cd24fb0d6963f7d28e17f72")
|
||||
);
|
||||
let (_, status, headers) = response.into_parts();
|
||||
assert_eq!(status, 200, "{case}");
|
||||
assert_eq!(headers["x-amz-checksum-crc32"], checksum);
|
||||
let stored = server
|
||||
.object_store()
|
||||
.unwrap()
|
||||
.get_object_info(&bucket, "allowed", &ObjectOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored.size, 3);
|
||||
assert_eq!(stored.etag.as_deref(), Some("900150983cd24fb0d6963f7d28e17f72"));
|
||||
server
|
||||
.object_store()
|
||||
.unwrap()
|
||||
.delete_object(&bucket, "allowed", ObjectOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
Some(code) => {
|
||||
let error = match result {
|
||||
Ok(_) => panic!("{case} must refuse"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code().as_str(), code, "{case}: {error}");
|
||||
if producer_error {
|
||||
let mut source = std::error::Error::source(&error);
|
||||
let mut native_error = false;
|
||||
let mut producer_error = false;
|
||||
for _ in 0..8 {
|
||||
let Some(error) = source else { break };
|
||||
native_error |= error.is::<StreamError>();
|
||||
producer_error |= error.is::<FixtureProducerFailure>();
|
||||
source = error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.and_then(std::io::Error::get_ref)
|
||||
.map(|inner| inner as &dyn std::error::Error)
|
||||
.or_else(|| error.source());
|
||||
}
|
||||
assert!(native_error, "the native producer error must remain in the cause chain");
|
||||
assert!(producer_error, "the concrete producer failure must remain in the cause chain");
|
||||
}
|
||||
assert!(
|
||||
server
|
||||
.object_store()
|
||||
.unwrap()
|
||||
.get_object_info(&bucket, "allowed", &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"{case} must not publish"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn put_source_version_selects_versioned_read_permission() {
|
||||
const VERSION: &str = "0194e0f1-0000-7000-8000-000000000000";
|
||||
for query in [false, true] {
|
||||
for (read, allowed) in [("s3:GetObject", false), ("s3:GetObjectVersion", true)] {
|
||||
let (server, bucket) = fixture(&["s3:PutObject", read, "s3:ReplicateObject"]).await;
|
||||
let mut headers = HeaderMap::new();
|
||||
let mut legacy = build_request((), Method::PUT);
|
||||
legacy.extensions.insert(Arc::clone(&server));
|
||||
legacy.extensions.insert(ReqInfo::default());
|
||||
if query {
|
||||
legacy.uri = format!("/?versionId={VERSION}").parse().unwrap();
|
||||
} else {
|
||||
headers.insert("x-minio-source-version-id", HeaderValue::from_static(VERSION));
|
||||
}
|
||||
legacy.headers = headers;
|
||||
let mut envelope = RequestEnvelope::from_s3s(&legacy);
|
||||
let req = request(dto::PutObjectInput {
|
||||
bucket: rustfs_gateway_types::BucketName::new(bucket).unwrap(),
|
||||
key: rustfs_gateway_types::ObjectKey::new("allowed").unwrap(),
|
||||
if_match: Some("\"tag\"".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
let result = <dto::PutObject as Served>::authorize(&mut envelope, &req).await;
|
||||
assert_eq!(
|
||||
result.map_err(|error| error.code().as_str().to_owned()),
|
||||
if allowed { Ok(()) } else { Err("AccessDenied".to_owned()) }
|
||||
);
|
||||
assert_eq!(envelope.req_info().unwrap().version_id.as_deref(), Some(VERSION));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_put_projection_preserves_unvalidated_write_fields() {
|
||||
let input = dto::PutObjectInput {
|
||||
bucket: rustfs_gateway_types::BucketName::new("put-fields").unwrap(),
|
||||
key: rustfs_gateway_types::ObjectKey::new("key").unwrap(),
|
||||
storage_class: Some(dto::StorageClass::custom("INVALID")),
|
||||
server_side_encryption: Some(dto::ServerSideEncryption::custom("INVALID")),
|
||||
object_lock_mode: Some(dto::ObjectLockMode::custom("INVALID")),
|
||||
object_lock_legal_hold_status: Some(dto::ObjectLockLegalHoldStatus::custom("INVALID")),
|
||||
tagging: Some("classification=public".to_owned()),
|
||||
metadata: [("purpose".to_owned(), "projection".to_owned())].into_iter().collect(),
|
||||
checksum_spec: Some(rustfs_gateway_types::ChecksumSpec::parse_header("x-amz-checksum-crc32", "NSRBwg==").unwrap()),
|
||||
..Default::default()
|
||||
};
|
||||
let input = native_input(input).unwrap();
|
||||
assert_eq!(input.storage_class.as_deref(), Some("INVALID"));
|
||||
assert_eq!(input.server_side_encryption.as_deref(), Some("INVALID"));
|
||||
assert_eq!(input.object_lock_mode.as_deref(), Some("INVALID"));
|
||||
assert_eq!(input.object_lock_legal_hold_status.as_deref(), Some("INVALID"));
|
||||
assert_eq!(input.tagging.as_deref(), Some("classification=public"));
|
||||
assert_eq!(input.metadata.unwrap().get("purpose").map(String::as_str), Some("projection"));
|
||||
assert_eq!(input.checksum_crc32.as_deref(), Some("NSRBwg=="));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn legacy_post_projection_keeps_early_refusal_order() {
|
||||
for (offset, invalid, actions, expected) in [
|
||||
(true, true, vec![], Some("NotImplemented")),
|
||||
(false, true, vec![], Some("InvalidArgument")),
|
||||
(false, false, vec![], Some("AccessDenied")),
|
||||
(false, false, vec!["s3:PutObject"], None),
|
||||
] {
|
||||
let (server, bucket) = fixture(&actions).await;
|
||||
let mut req = build_request(
|
||||
LegacyPutObjectInput {
|
||||
bucket,
|
||||
key: "allowed".to_owned(),
|
||||
storage_class: Some(if invalid { "BAD\nFIELD" } else { "STANDARD" }.to_owned().into()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::POST,
|
||||
);
|
||||
req.extensions.insert(Arc::clone(&server));
|
||||
req.extensions.insert(ReqInfo::default());
|
||||
if offset {
|
||||
req.headers.insert("x-amz-write-offset-bytes", HeaderValue::from_static("0"));
|
||||
}
|
||||
let result = FS::with_server_ctx(server).put_object(&mut req).await;
|
||||
assert_eq!(result.err().map(|error| error.code().as_str().to_owned()), expected.map(str::to_owned));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_put_response_refuses_a_missing_etag() {
|
||||
let error = match native_output(PutObjectWriteOutput::default()) {
|
||||
Ok(_) => panic!("an absent ETag must not become an empty tag"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code().as_str(), "InternalError");
|
||||
}
|
||||
@@ -4242,52 +4242,24 @@ impl S3Access for FS {
|
||||
/// This method returns `Ok(())` by default.
|
||||
async fn put_object(&self, req: &mut S3Request<PutObjectInput>) -> S3Result<()> {
|
||||
crate::hp_guard!("S3Access::put_object");
|
||||
async {
|
||||
let req_info = ext_req_info_mut(&mut req.extensions)?;
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
req_info.object = Some(req.input.key.clone());
|
||||
req_info.version_id = super::options::destination_version_id(req.input.version_id.as_deref(), &req.headers);
|
||||
|
||||
if has_write_offset_bytes_header(&req.headers) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::NotImplemented,
|
||||
ApiError::error_code_to_message(&S3ErrorCode::NotImplemented),
|
||||
));
|
||||
}
|
||||
|
||||
let bucket = req.input.bucket.clone();
|
||||
// Snapshot before authorization, but preserve AccessDenied precedence
|
||||
// by exposing any bucket-state error only after authorization succeeds.
|
||||
let bucket_generation = load_bucket_generation(self.server_ctx(), req, &bucket).await;
|
||||
#[cfg(test)]
|
||||
let bucket_generation = if PUT_GENERATION_READ_QUORUM_FAILURE.try_with(|_| ()).is_ok() {
|
||||
Err(ApiError::from(StorageError::ErasureReadQuorum).into())
|
||||
} else {
|
||||
bucket_generation
|
||||
};
|
||||
let requested_retain_until = req.input.object_lock_retain_until_date.clone();
|
||||
set_requested_object_lock_retain_until(req, requested_retain_until.map(OffsetDateTime::from));
|
||||
set_post_object_policy_headers(req)?;
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?;
|
||||
let specific_etag = req.input.if_match.as_ref().is_some_and(|etag| !etag.is_any());
|
||||
authorize_write_etag_condition(req, specific_etag).await?;
|
||||
req.install_bucket_generation(bucket_generation?);
|
||||
|
||||
if req.method != http::Method::POST {
|
||||
authorize_replication_only_put_headers(req).await?;
|
||||
}
|
||||
|
||||
if legal_hold_write_requested(req.input.object_lock_legal_hold_status.as_ref()) {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?;
|
||||
}
|
||||
|
||||
if retention_write_requested(req.input.object_lock_mode.as_ref(), req.input.object_lock_retain_until_date.as_ref()) {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
.await
|
||||
use access_owned::object_write::{PutObjectAuthorization, authorize_put_object};
|
||||
let bucket = req.input.bucket.clone();
|
||||
let key = req.input.key.clone();
|
||||
let version_id = req.input.version_id.clone();
|
||||
let input = PutObjectAuthorization {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
version_id: version_id.as_deref(),
|
||||
retain_until: req.input.object_lock_retain_until_date.clone().map(OffsetDateTime::from),
|
||||
specific_etag: req.input.if_match.as_ref().is_some_and(|etag| !etag.is_any()),
|
||||
legal_hold: legal_hold_write_requested(req.input.object_lock_legal_hold_status.as_ref()),
|
||||
retention: retention_write_requested(
|
||||
req.input.object_lock_mode.as_ref(),
|
||||
req.input.object_lock_retain_until_date.as_ref(),
|
||||
),
|
||||
post: req.method == http::Method::POST,
|
||||
};
|
||||
authorize_put_object(req, self.server_ctx(), input, set_post_object_policy_headers).await
|
||||
}
|
||||
|
||||
/// Checks whether the PutObjectAcl request has accesses to the resources.
|
||||
|
||||
@@ -20,3 +20,64 @@
|
||||
//! policy evaluation itself (`authorize_request`), or the operation's use case.
|
||||
//! Upstream: the legacy hook in `storage/access.rs` and the batch's `server/gateway_handlers/object_write.rs`.
|
||||
//! Downstream: the shared shapes in `storage/access.rs`.
|
||||
|
||||
use super::super::{
|
||||
Action, ApiError, AuthorizationRequest, S3Action, S3Error, S3ErrorCode, S3Result, ServerContextSlot,
|
||||
authorize_replication_only_put_headers, authorize_request, authorize_write_etag_condition, has_write_offset_bytes_header,
|
||||
load_bucket_generation, missing_req_info_error, set_requested_object_lock_retain_until,
|
||||
};
|
||||
|
||||
/// Request facts needed by both PutObject authorization edges.
|
||||
pub(crate) struct PutObjectAuthorization<'a> {
|
||||
pub(crate) bucket: &'a str,
|
||||
pub(crate) key: &'a str,
|
||||
pub(crate) version_id: Option<&'a str>,
|
||||
pub(crate) retain_until: Option<time::OffsetDateTime>,
|
||||
pub(crate) specific_etag: bool,
|
||||
pub(crate) legal_hold: bool,
|
||||
pub(crate) retention: bool,
|
||||
pub(crate) post: bool,
|
||||
}
|
||||
|
||||
/// Preserves write refusal ordering and captures the generation authorized for publication.
|
||||
pub(crate) async fn authorize_put_object<R: AuthorizationRequest + ?Sized>(
|
||||
req: &mut R,
|
||||
server: &ServerContextSlot,
|
||||
input: PutObjectAuthorization<'_>,
|
||||
prepare_policy: impl FnOnce(&mut R) -> S3Result<()> + Send,
|
||||
) -> S3Result<()> {
|
||||
let version = crate::storage::options::destination_version_id(input.version_id, req.headers());
|
||||
let info = req.req_info_mut().ok_or_else(missing_req_info_error)?;
|
||||
info.bucket = Some(input.bucket.to_owned());
|
||||
info.object = Some(input.key.to_owned());
|
||||
info.version_id = version;
|
||||
if has_write_offset_bytes_header(req.headers()) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::NotImplemented,
|
||||
ApiError::error_code_to_message(&S3ErrorCode::NotImplemented),
|
||||
));
|
||||
}
|
||||
// Capture first, but let authorization refusals precede bucket-state failures.
|
||||
let generation = load_bucket_generation(server, req, input.bucket).await;
|
||||
#[cfg(test)]
|
||||
let generation = if super::super::PUT_GENERATION_READ_QUORUM_FAILURE.try_with(|_| ()).is_ok() {
|
||||
Err(ApiError::from(super::super::StorageError::ErasureReadQuorum).into())
|
||||
} else {
|
||||
generation
|
||||
};
|
||||
set_requested_object_lock_retain_until(req, input.retain_until);
|
||||
prepare_policy(req)?;
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?;
|
||||
authorize_write_etag_condition(req, input.specific_etag).await?;
|
||||
req.install_bucket_generation(generation?);
|
||||
if !input.post {
|
||||
authorize_replication_only_put_headers(req).await?;
|
||||
}
|
||||
if input.legal_hold {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?;
|
||||
}
|
||||
if input.retention {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ impl PutObjectWriteInput {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
body: input.body.map(s3_api::legacy_put_body),
|
||||
frame_trailers: None,
|
||||
trailing_headers: crate::app::trailer_adapter::trailer_source(trailing_headers),
|
||||
content_length: input.content_length,
|
||||
metadata: input.metadata,
|
||||
|
||||
@@ -241,6 +241,21 @@ pub(crate) mod server {
|
||||
AccessMarkers, VerifiedSignature, admit_operation, authorize_bucket_config_mutation, authorize_bucket_request,
|
||||
authorize_object_request, resolve_req_info,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{AuthorizationRequest, ReqInfo};
|
||||
pub(crate) use crate::storage::storage_api::contract::bucket::{BucketOperations, MakeBucketOptions};
|
||||
pub(crate) use crate::storage::storage_api::contract::object::ObjectOperations;
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
FS, StorageObjectOptions as ObjectOptions, get_bucket_metadata, set_bucket_metadata,
|
||||
};
|
||||
pub(crate) use s3s::{
|
||||
S3Request,
|
||||
access::S3Access,
|
||||
dto::{ETag, ETagCondition, PutObjectInput},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
|
||||
@@ -14361,6 +14361,8 @@ fn table_metadata_pointer_json_round_trips() {
|
||||
assert_eq!(pointer.metadata_location, location);
|
||||
}
|
||||
|
||||
const PUT_OBJECT_MUTATION_BOUNDARY: &str = "self.put_object_core(write, body, input.frame_trailers, start_time).await?;";
|
||||
|
||||
fn assert_object_mutation_reserved_prefix_guards(delete: &str, legacy_delete: &str) {
|
||||
assert_object_mutation_reserved_prefix_sources(
|
||||
delete,
|
||||
@@ -14450,7 +14452,7 @@ fn assert_object_mutation_reserved_prefix_sources(delete: &str, legacy_delete: &
|
||||
put,
|
||||
"fn execute_put_object_inner(",
|
||||
"validate_new_object_key(&key, request_method_name)?;",
|
||||
"self.put_object_core(write, body, start_time).await?;",
|
||||
PUT_OBJECT_MUTATION_BOUNDARY,
|
||||
),
|
||||
(
|
||||
copy,
|
||||
@@ -16609,6 +16611,7 @@ fn put_object_reserved_prefix_guard_rejects_broken_native_delegation() {
|
||||
"self.unchecked_put(envelope, input)",
|
||||
),
|
||||
("put", "validate_new_object_key(&key, request_method_name)?;", "unchecked_key();"),
|
||||
("put", PUT_OBJECT_MUTATION_BOUNDARY, "unchecked_put_core();"),
|
||||
(
|
||||
"put",
|
||||
"validate_table_catalog_object_mutation(&bucket, &key).await?;",
|
||||
@@ -16636,7 +16639,7 @@ fn put_object_reserved_prefix_guard_rejects_broken_native_delegation() {
|
||||
if surface == "extract" { &broken } else { extract },
|
||||
);
|
||||
});
|
||||
let expected = if from.starts_with("validate_") {
|
||||
let expected = if from.starts_with("validate_") || from == PUT_OBJECT_MUTATION_BOUNDARY {
|
||||
format!("missing or out-of-order object mutation guard: {from}")
|
||||
} else {
|
||||
let signature = match surface {
|
||||
|
||||
@@ -12,9 +12,26 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch D (rustfs/backlog#2765). None is served yet.
|
||||
//! PutObject refusal rows and legacy/gateway wire behavior. UploadPart remains unserved.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[super::MatrixRow {
|
||||
operation: "PutObject",
|
||||
metric: rustfs_s3_ops::S3Operation::PutObject,
|
||||
has_error_body: true,
|
||||
call: |client, bucket| {
|
||||
Box::pin(async move {
|
||||
super::outcome(
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("put-matrix")
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"matrix"))
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
})
|
||||
},
|
||||
}];
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -149,8 +166,34 @@ impl WireClient<'_> {
|
||||
.map(|(name, value)| format!("{name}:{}\n", value.trim()))
|
||||
.collect();
|
||||
let signed_headers = headers.keys().cloned().collect::<Vec<_>>().join(";");
|
||||
let (canonical_path, query) = path.split_once('?').unwrap_or((path, ""));
|
||||
let encode = |raw: &str| {
|
||||
percent_encoding::percent_decode_str(raw)
|
||||
.map(|byte| {
|
||||
if byte.is_ascii_alphanumeric() || b"-_.~".contains(&byte) {
|
||||
char::from(byte).to_string()
|
||||
} else {
|
||||
format!("%{byte:02X}")
|
||||
}
|
||||
})
|
||||
.collect::<String>()
|
||||
};
|
||||
let mut pairs = query
|
||||
.split('&')
|
||||
.filter(|pair| !pair.is_empty())
|
||||
.map(|pair| {
|
||||
let (name, value) = pair.split_once('=').unwrap_or((pair, ""));
|
||||
(encode(name), encode(value))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
pairs.sort();
|
||||
let canonical_query = pairs
|
||||
.into_iter()
|
||||
.map(|(name, value)| format!("{name}={value}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("&");
|
||||
let canonical = format!(
|
||||
"{method}\n{path}\n\n{canonical_headers}\n{signed_headers}\n{}",
|
||||
"{method}\n{canonical_path}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{}",
|
||||
headers["x-amz-content-sha256"]
|
||||
);
|
||||
let message = format!("AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{}", hex(&Sha256::digest(canonical.as_bytes())));
|
||||
@@ -320,3 +363,272 @@ fn legacy_owned_post_keeps_implicit_status_and_redirect_selection() {
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_put_wire_stores_bytes_and_refuses_corruption_and_wrong_resources() {
|
||||
super::with_stack(super::GATEWAY, |server| async move {
|
||||
const BUCKET: &str = "dg-put-wire";
|
||||
const OTHER: &str = "dg-put-other";
|
||||
const USER: &str = "dgputscoped";
|
||||
for bucket in [BUCKET, OTHER] {
|
||||
super::create_bucket_in_the_store(bucket).await;
|
||||
}
|
||||
super::create_user_allowed_on_one_bucket(USER, "s3:PutObject", &format!("{BUCKET}/allowed")).await;
|
||||
let endpoint = server.endpoint();
|
||||
let client = WireClient {
|
||||
endpoint: &endpoint,
|
||||
access_key: USER,
|
||||
secret_key: super::USER_SECRET_KEY,
|
||||
};
|
||||
let answer = client
|
||||
.request(
|
||||
"PUT",
|
||||
&format!("/{BUCKET}/allowed"),
|
||||
b"hello",
|
||||
&[("x-amz-checksum-crc32", "NhCmhg==")],
|
||||
Some(5),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(answer.status, 200, "{}", answer.body);
|
||||
assert_eq!(answer.headers.get("x-amz-checksum-crc32").map(String::as_str), Some("NhCmhg=="));
|
||||
assert_eq!(
|
||||
answer.headers.get("etag").map(String::as_str),
|
||||
Some("\"5d41402abc4b2a76b9719d911017c592\"")
|
||||
);
|
||||
for path in [format!("/{BUCKET}/denied"), format!("/{OTHER}/allowed")] {
|
||||
let answer = client.request("PUT", &path, b"hello", &[], Some(5)).await;
|
||||
assert_eq!(answer.status, 403, "{path}: {}", answer.body);
|
||||
assert!(answer.body.contains("<Code>AccessDenied</Code>"));
|
||||
}
|
||||
let root = WireClient {
|
||||
endpoint: &endpoint,
|
||||
access_key: server.access_key(),
|
||||
secret_key: server.secret_key(),
|
||||
};
|
||||
for path in [format!("/{BUCKET}/denied"), format!("/{OTHER}/allowed")] {
|
||||
assert_eq!(root.request("HEAD", &path, b"", &[], Some(0)).await.status, 404);
|
||||
}
|
||||
for (key, content, checksum, declared, expected) in [
|
||||
("wrong-crc", b"hello".as_slice(), "AAAAAA==", 5, "BadDigest"),
|
||||
("short", b"abc".as_slice(), "NSRBwg==", 5, "IncompleteBody"),
|
||||
] {
|
||||
let answer = root
|
||||
.request(
|
||||
"PUT",
|
||||
&format!("/{BUCKET}/{key}"),
|
||||
content,
|
||||
&[("x-amz-checksum-crc32", checksum)],
|
||||
Some(declared),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(answer.status, 400, "{key}: {}", answer.body);
|
||||
assert!(answer.body.contains(&format!("<Code>{expected}</Code>")), "{key}: {}", answer.body);
|
||||
assert_eq!(
|
||||
root.request("HEAD", &format!("/{BUCKET}/{key}"), b"", &[], Some(0))
|
||||
.await
|
||||
.status,
|
||||
404
|
||||
);
|
||||
}
|
||||
let checksums = [
|
||||
("x-amz-checksum-crc32", "NhCmhg=="),
|
||||
("x-amz-checksum-sha256", "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="),
|
||||
];
|
||||
let path = format!("/{BUCKET}/multiple-checksums");
|
||||
let answer = root.request("PUT", &path, b"hello", &checksums, Some(5)).await;
|
||||
assert_eq!(answer.status, 400, "{}", answer.body);
|
||||
assert!(answer.body.contains("<Code>InvalidRequest</Code>"), "{}", answer.body);
|
||||
assert_eq!(root.request("HEAD", &path, b"", &[], Some(0)).await.status, 404);
|
||||
|
||||
let sha512 = base64_simd::STANDARD.encode_to_string(sha2::Sha512::digest(b"hello"));
|
||||
let answer = root
|
||||
.request(
|
||||
"PUT",
|
||||
&format!("/{BUCKET}/sha512"),
|
||||
b"hello",
|
||||
&[("x-amz-checksum-sha512", &sha512)],
|
||||
Some(5),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(answer.status, 200, "{}", answer.body);
|
||||
assert_eq!(answer.headers.get("x-amz-checksum-sha512"), Some(&sha512));
|
||||
|
||||
for (key, extra) in [
|
||||
("plain-chunked", vec![]),
|
||||
("decoded-chunked", vec![("x-amz-decoded-content-length", "5")]),
|
||||
] {
|
||||
let path = format!("/{BUCKET}/{key}");
|
||||
let answer = root.request("PUT", &path, b"hello", &extra, None).await;
|
||||
assert_eq!(answer.status, 411, "{key}: {}", answer.body);
|
||||
assert!(answer.body.contains("<Code>MissingContentLength</Code>"), "{}", answer.body);
|
||||
assert_eq!(root.request("HEAD", &path, b"", &[], Some(0)).await.status, 404);
|
||||
}
|
||||
let context = rustfs::app::context::get_global_app_context().unwrap();
|
||||
let store = context.object_store();
|
||||
use rustfs_storage_api::ObjectOperations as _;
|
||||
let object = store
|
||||
.get_object_info(BUCKET, "allowed", &rustfs_ecstore::api::object::ObjectOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(object.size, 5);
|
||||
assert_eq!(object.etag.as_deref(), Some("5d41402abc4b2a76b9719d911017c592"));
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
const CRC32_TRAILER_HEADERS: [(&str, &str); 4] = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", "5"),
|
||||
("x-amz-trailer", "x-amz-checksum-crc32"),
|
||||
("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER"),
|
||||
];
|
||||
const VALID_CRC32_TRAILER: &[u8] = b"5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n\r\n";
|
||||
|
||||
async fn gateway_crc32_trailer_control(server: super::RustFSServer, sdk_algorithm: Option<&str>) {
|
||||
const BUCKET: &str = "dg-put-trailer";
|
||||
super::create_bucket_in_the_store(BUCKET).await;
|
||||
let endpoint = server.endpoint();
|
||||
let client = WireClient {
|
||||
endpoint: &endpoint,
|
||||
access_key: server.access_key(),
|
||||
secret_key: server.secret_key(),
|
||||
};
|
||||
let mut headers = CRC32_TRAILER_HEADERS.to_vec();
|
||||
if let Some(algorithm) = sdk_algorithm {
|
||||
headers.push(("x-amz-sdk-checksum-algorithm", algorithm));
|
||||
}
|
||||
let answer = client
|
||||
.request("PUT", &format!("/{BUCKET}/object"), VALID_CRC32_TRAILER, &headers, None)
|
||||
.await;
|
||||
assert_eq!(answer.status, 200, "{}", answer.body);
|
||||
assert_eq!(
|
||||
answer.headers.get("x-amz-checksum-crc32").map(String::as_str),
|
||||
Some("NhCmhg=="),
|
||||
"the response must echo the declared CRC32 trailer"
|
||||
);
|
||||
assert_eq!(
|
||||
answer.headers.get("etag").map(String::as_str),
|
||||
Some("\"5d41402abc4b2a76b9719d911017c592\"")
|
||||
);
|
||||
let stored = super::s3_client(&endpoint, server.access_key(), server.secret_key())
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored.body.collect().await.unwrap().into_bytes().as_ref(), b"hello");
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_put_crc32_trailer_without_sdk_algorithm() {
|
||||
super::with_stack(super::GATEWAY, |server| gateway_crc32_trailer_control(server, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_put_crc32_trailer_with_matching_sdk_algorithm() {
|
||||
super::with_stack(super::GATEWAY, |server| gateway_crc32_trailer_control(server, Some("CRC32")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_put_crc32_trailer_overrides_mismatching_sdk_algorithm() {
|
||||
super::with_stack(super::GATEWAY, |server| gateway_crc32_trailer_control(server, Some("SHA256")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_put_refuses_invalid_crc32_trailers_without_publishing() {
|
||||
super::with_stack(super::GATEWAY, |server| async move {
|
||||
const BUCKET: &str = "dg-put-trailer-refusal";
|
||||
super::create_bucket_in_the_store(BUCKET).await;
|
||||
let endpoint = server.endpoint();
|
||||
let client = WireClient {
|
||||
endpoint: &endpoint,
|
||||
access_key: server.access_key(),
|
||||
secret_key: server.secret_key(),
|
||||
};
|
||||
for (key, body, expected) in [
|
||||
(
|
||||
"bad-trailer",
|
||||
b"5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n".as_slice(),
|
||||
"BadDigest",
|
||||
),
|
||||
("missing-trailer", b"5\r\nhello\r\n0\r\n\r\n".as_slice(), "BadDigest"),
|
||||
(
|
||||
"short-trailer-body",
|
||||
b"3\r\nabc\r\n0\r\nx-amz-checksum-crc32:NSRBwg==\r\n\r\n".as_slice(),
|
||||
"IncompleteBody",
|
||||
),
|
||||
("conflict", VALID_CRC32_TRAILER, "InvalidRequest"),
|
||||
] {
|
||||
let mut headers = CRC32_TRAILER_HEADERS.to_vec();
|
||||
if key == "conflict" {
|
||||
headers.push(("x-amz-checksum-crc32", "NhCmhg=="));
|
||||
}
|
||||
let path = format!("/{BUCKET}/{key}");
|
||||
let answer = client.request("PUT", &path, body, &headers, None).await;
|
||||
assert_eq!(answer.status, 400, "{key}: {}", answer.body);
|
||||
assert!(answer.body.contains(&format!("<Code>{expected}</Code>")), "{key}: {}", answer.body);
|
||||
assert_eq!(client.request("HEAD", &path, b"", &[], Some(0)).await.status, 404, "{key}");
|
||||
}
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn destination_version_queries(server: super::RustFSServer) {
|
||||
let endpoint = server.endpoint();
|
||||
let client = WireClient {
|
||||
endpoint: &endpoint,
|
||||
access_key: server.access_key(),
|
||||
secret_key: server.secret_key(),
|
||||
};
|
||||
const BUCKET: &str = "dg-version-query";
|
||||
super::create_bucket_in_the_store(BUCKET).await;
|
||||
for version in ["null", "invalid", ""] {
|
||||
let answer = client
|
||||
.query_put(&format!("/{BUCKET}/version"), &format!("versionId={version}"), &[])
|
||||
.await;
|
||||
assert_eq!(answer.status, 400, "{version}: {}", answer.body);
|
||||
assert!(answer.body.contains("<Code>InvalidArgument</Code>"), "{}", answer.body);
|
||||
assert!(
|
||||
answer
|
||||
.body
|
||||
.contains("A destination versionId requires replication authorization"),
|
||||
"{}",
|
||||
answer.body
|
||||
);
|
||||
}
|
||||
let answer = client
|
||||
.query_put(
|
||||
&format!("/{BUCKET}/version"),
|
||||
"versionId=null",
|
||||
&[("x-minio-source-replication-request", "true")],
|
||||
)
|
||||
.await;
|
||||
assert_eq!(answer.status, 200, "{}", answer.body);
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn legacy_put_destination_version_query_answers() {
|
||||
super::with_stack(super::LEGACY, destination_version_queries);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_put_destination_version_query_answers() {
|
||||
super::with_stack(super::GATEWAY, destination_version_queries);
|
||||
}
|
||||
|
||||
impl WireClient<'_> {
|
||||
async fn query_put(&self, path: &str, query: &str, extra: &[(&str, &str)]) -> WireAnswer {
|
||||
self.request("PUT", &format!("{path}?{query}"), b"hello", extra, Some(5))
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user