refactor(protocols): use owned multipart lifecycle paths

Route CreateMultipartUpload, CompleteMultipartUpload and AbortMultipartUpload through the installed server's multipart use case. Keep the measured protocol projections and operation counts.

The upload-id scalar compatibility helper remains in Complete and Abort. UploadPart, UploadPartCopy, PostObject and persisted-owner work are outside this change.

Validation: 11 new tests; the runtime contract has 8 refusal cases and 2 successful flows. The restored selection passed 59/59 on d3abf8023. Both Clippy modes, fmt, script-tests, test wiring and prepush-quick passed. Non-test diff: +231/-42. The original client baseline ran on a98bd9028.

Effective mutations (38), grouped by the test that failed:
- multipart_create_projection_matches_original_fields_and_times: create-input-name, create-input-metadata, create-input-tagging, create-input-storage, create-input-sse, create-input-kms, create-input-md5, create-input-mode, create-input-hold, create-input-date, timestamp.
- multipart_refusals_match_original_client_errors: complete-error-message.
- complete_input_preserves_part_fields: legacy-part-checksum.
- multipart_create_projection_preserves_refusals: create-input-refusal.
- multipart_create_output_matches_original_for_stored_names_and_empty_values: create-output-name, create-output-strings.
- multipart_complete_projection_matches_original_parts_and_conditions: complete-condition, complete-parts, complete-part-number, complete-part-etag, complete-part-checksum, complete-condition-refusal.
- multipart_completion_projection_preserves_original_escaped_tags: complete-output-escape, complete-output-weak.
- multipart_completion_projection_preserves_header_only_field_behavior: complete-output-headerdrop, complete-output-header-validation.
- protocol_multipart_lifecycle_stores_parts_and_aborts: count-create, count-complete, count-abort, complete-location, complete-storage.
- multipart_uses_bound_store_with_a_foreign_ambient_context: create-storage, abort-storage, bound-store.
- multipart_envelopes_preserve_original_request_facts: request-facts.
- protocol_policy_denials_and_request_conditions: permission-create, permission-complete, permission-abort.

The final restored run had no leaky summary; an earlier control marked one pure projection test leaky. The linker reported the non-failing __eh_frame warning. The ambient fault's test exited 100 with target=false/ambient=true; its wrapper exited 1 because it expected generic assertion wording.

Not verified: CI, network protocol suites and the released-reference comparison.
This commit is contained in:
overtrue
2026-11-16 19:09:59 +08:00
parent d3abf80237
commit f68ef7d5f6
6 changed files with 925 additions and 49 deletions
+35 -42
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod multipart;
mod owned;
use crate::runtime_sources::current_action_credentials;
@@ -927,13 +928,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
let bucket = input.bucket.as_str().to_owned();
let key = input.key.as_str().to_owned();
let uri = build_object_uri(&bucket, &key, &[("uploads", None)])?;
let context = gateway_request_context(Method::POST, &uri, HeaderMap::new());
let wire = RequestWire::of(&context);
let input = gateway_s3s_compat::generated::ops::create_multipart_upload::input_to_s3s(input, &wire)
.map_err(|e| conversion_error("CreateMultipartUpload input", e))?;
let req = Self::create_request(
input,
let input = multipart::create_multipart_input(input)?;
let envelope = Self::create_envelope(
Method::POST,
uri,
RequestParams {
@@ -941,13 +937,12 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
object: Some(key),
credentials,
},
)?;
match self.fs.create_multipart_upload(req).await {
Ok(response) => gateway_s3s_compat::generated::ops::create_multipart_upload::output_from_s3s(response.output)
.map_err(|e| conversion_error("CreateMultipartUpload output", e)),
Err(e) => Err(e.into()),
}
);
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::CreateMultipartUpload);
self.owned_multipart_usecase()
.execute_create_multipart_upload(envelope, input, None)
.await
.and_then(multipart::create_multipart_output)
}
async fn upload_part(
@@ -1024,13 +1019,11 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
let bucket = input.bucket.as_str().to_owned();
let key = input.key.as_str().to_owned();
let input = gateway_s3s_compat::generated::ops::complete_multipart_upload::input_to_s3s(input)
.map_err(|e| conversion_error("CompleteMultipartUpload input", e))?;
let upload_id = input.upload_id.clone();
// This scalar projection remains until the compatibility seam retires.
let upload_id = gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
let input = multipart::complete_multipart_input(input)?;
let uri = build_object_uri(&bucket, &key, &[("uploadId", Some(upload_id.as_str()))])?;
let req = Self::create_request(
input,
let envelope = Self::create_envelope(
Method::POST,
uri,
RequestParams {
@@ -1038,13 +1031,13 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
object: Some(key),
credentials,
},
)?;
match self.fs.complete_multipart_upload(req).await {
Ok(response) => gateway_s3s_compat::generated::ops::complete_multipart_upload::output_from_s3s(response.output)
.map_err(|e| conversion_error("CompleteMultipartUpload output", e)),
Err(e) => Err(e.into()),
}
);
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::CompleteMultipartUpload);
self.owned_multipart_usecase()
.execute_complete_multipart_upload(envelope, input)
.await
.map_err(multipart::complete_multipart_error)
.and_then(multipart::complete_multipart_output)
}
async fn abort_multipart_upload(
@@ -1064,27 +1057,27 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
let bucket = input.bucket.as_str().to_owned();
let key = input.key.as_str().to_owned();
let input = gateway_s3s_compat::generated::ops::abort_multipart_upload::input_to_s3s(input)
.map_err(|e| conversion_error("AbortMultipartUpload input", e))?;
let upload_id = input.upload_id.clone();
multipart::abort_multipart_input(&input)?;
// This scalar projection remains until the compatibility seam retires.
let upload_id = gateway_s3s_compat::leaf::upload_id_to_s3s(&input.upload_id);
let uri = build_object_uri(&bucket, &key, &[("uploadId", Some(upload_id.as_str()))])?;
let req = Self::create_request(
input,
let envelope = Self::create_envelope(
Method::DELETE,
uri,
RequestParams {
bucket: Some(bucket),
object: Some(key),
bucket: Some(bucket.clone()),
object: Some(key.clone()),
credentials,
},
)?;
match self.fs.abort_multipart_upload(req).await {
Ok(response) => gateway_s3s_compat::generated::ops::abort_multipart_upload::output_from_s3s(response.output)
.map_err(|e| conversion_error("AbortMultipartUpload output", e)),
Err(e) => Err(e.into()),
}
);
let target = crate::app::multipart_usecase::reject_abort_capabilities(&envelope).map(|()| {
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::AbortMultipartUpload);
crate::app::multipart_usecase::AbortMultipartUploadTarget::Legacy { bucket, key, upload_id }
});
self.owned_multipart_usecase()
.execute_abort_multipart_upload(envelope, target)
.await?;
Ok(gateway_dto::AbortMultipartUploadOutput::default())
}
async fn upload_part_copy(
+200
View File
@@ -0,0 +1,200 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Legacy-compatible multipart projections for protocol storage clients.
//! The shared multipart use cases own storage and validation; protocol sessions own authorization.
use super::ProtocolStorageClient;
use rustfs_gateway_types::{ETag, Timestamp, dto};
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
impl ProtocolStorageClient {
pub(super) fn owned_multipart_usecase(&self) -> crate::app::multipart_usecase::DefaultMultipartUsecase {
crate::app::multipart_usecase::DefaultMultipartUsecase::with_context(self.fs.server_ctx().app_context())
}
}
fn nonempty(value: Option<String>) -> Option<String> {
value.filter(|value| !value.is_empty())
}
fn multipart_timestamp(value: Timestamp, operation: &str, field: &'static str) -> S3Result<time::OffsetDateTime> {
let nanos = i128::from(value.subsec_nanos());
time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(value.secs()) * 1_000_000_000 + nanos)
.map_err(|_| super::conversion_error(operation, format!("{field}: an instant outside what the s3s shape can hold")))
}
pub(super) fn create_multipart_input(
input: dto::CreateMultipartUploadInput,
) -> S3Result<crate::app::multipart_usecase::CreateMultipartUploadFields> {
for (field, present) in [
(
"object_lock_event_hold",
input
.object_lock_event_hold
.as_ref()
.is_some_and(|value| !value.as_str().is_empty()),
),
(
"object_lock_event_hold_duration_days",
input.object_lock_event_hold_duration_days.is_some(),
),
(
"object_lock_event_hold_duration_years",
input.object_lock_event_hold_duration_years.is_some(),
),
] {
if present {
return Err(super::conversion_error(
"CreateMultipartUpload input",
format!("{field}: an Object Lock event hold, which no pinned s3s shape holds and RustFS does not store"),
));
}
}
Ok(crate::app::multipart_usecase::CreateMultipartUploadFields {
bucket: input.bucket.as_str().to_owned(),
key: input.key.as_str().to_owned(),
metadata: (!input.metadata.is_empty()).then(|| input.metadata.into_iter().collect()),
tagging: nonempty(input.tagging),
storage_class: nonempty(input.storage_class.map(|value| value.as_str().to_owned())),
server_side_encryption: nonempty(input.server_side_encryption.map(|value| value.as_str().to_owned())),
sse_customer_algorithm: nonempty(input.sse_customer_algorithm),
sse_customer_key: nonempty(input.sse_customer_key.map(|value| value.expose_secret().to_owned())),
sse_customer_key_md5: nonempty(input.sse_customer_key_md5),
ssekms_key_id: nonempty(input.ssekms_key_id),
object_lock_legal_hold_status: nonempty(input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned())),
object_lock_mode: nonempty(input.object_lock_mode.map(|value| value.as_str().to_owned())),
object_lock_retain_until_date: input
.object_lock_retain_until_date
.map(|value| multipart_timestamp(value, "CreateMultipartUpload input", "object_lock_retain_until_date"))
.transpose()?,
})
}
pub(super) fn create_multipart_output(
output: crate::app::multipart_usecase::CreateMultipartUploadResult,
) -> S3Result<dto::CreateMultipartUploadOutput> {
Ok(dto::CreateMultipartUploadOutput {
bucket: crate::app::dto_values::stored_bucket_name(&output.bucket)?,
key: crate::app::dto_values::stored_object_key(&output.key)?,
upload_id: output.upload_id,
server_side_encryption: output.server_side_encryption.map(dto::ServerSideEncryption::custom),
sse_customer_algorithm: output.sse_customer_algorithm,
sse_customer_key_md5: output.sse_customer_key_md5,
ssekms_key_id: output.ssekms_key_id,
checksum_algorithm: output.checksum_algorithm.map(dto::ChecksumAlgorithm::custom),
checksum_type: output.checksum_type.map(dto::ChecksumType::custom),
..Default::default()
})
}
fn complete_condition(value: Option<String>, field: &'static str) -> S3Result<Option<String>> {
let Some(value) = nonempty(value) else { return Ok(None) };
if value == "*" {
return Ok(Some(value));
}
let quoted = value.strip_prefix("W/").unwrap_or(&value);
let valid = if quoted.starts_with('"') && quoted.ends_with('"') && quoted.len() >= 2 {
quoted.bytes().all(|byte| byte == b'\t' || (b' '..=b'~').contains(&byte))
} else {
value.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
};
if !valid {
return Err(super::conversion_error(
"CompleteMultipartUpload input",
format!("{field}: not an entity-tag condition the s3s input can hold"),
));
}
Ok(Some(if quoted.starts_with('"') {
value
} else {
format!("\"{value}\"")
}))
}
pub(super) fn complete_multipart_input(
mut input: dto::CompleteMultipartUploadInput,
) -> S3Result<dto::CompleteMultipartUploadInput> {
input.if_match = complete_condition(input.if_match, "if_match")?;
input.if_none_match = complete_condition(input.if_none_match, "if_none_match")?;
input.checksum_spec = None;
input.checksum_type = input.checksum_type.filter(|value| !value.as_str().is_empty());
input.sse_customer_algorithm = nonempty(input.sse_customer_algorithm);
input.sse_customer_key = input.sse_customer_key.filter(|value| !value.expose_secret().is_empty());
input.sse_customer_key_md5 = nonempty(input.sse_customer_key_md5);
for part in &mut input.multipart_upload.parts {
part.e_tag = part
.e_tag
.take()
.map(|value| ETag::from_stored(value.opaque_tag().to_owned()));
}
Ok(input)
}
pub(super) fn complete_multipart_output(
mut output: dto::CompleteMultipartUploadOutput,
) -> S3Result<dto::CompleteMultipartUploadOutput> {
for (field, value) in [
("x-amz-checksum-md5", output.checksum_header_md5.take()),
("x-amz-checksum-sha512", output.checksum_header_sha512.take()),
("x-amz-checksum-xxhash3", output.checksum_header_xxhash3.take()),
("x-amz-checksum-xxhash64", output.checksum_header_xxhash64.take()),
("x-amz-checksum-xxhash128", output.checksum_header_xxhash128.take()),
("x-amz-server-side-encryption-customer-algorithm", output.sse_customer_algorithm.take()),
("x-amz-server-side-encryption-customer-key-md5", output.sse_customer_key_md5.take()),
] {
if let Some(value) = value {
http::HeaderValue::from_str(&value).map_err(|_| {
crate::app::dto_values::seam_error(rustfs_gateway_types::compat::ConversionError {
field,
reason: "not a header value",
})
})?;
}
}
output.e_tag = output
.e_tag
.map(|value| {
let escaped = quick_xml::escape::escape(value.opaque_tag()).into_owned();
if value.is_weak() {
ETag::new_weak(escaped)
} else {
ETag::new(escaped)
}
.map_err(|_| {
super::conversion_error("CompleteMultipartUpload output", "e_tag: not an entity tag the gateway can write")
})
})
.transpose()?;
Ok(output)
}
pub(super) fn complete_multipart_error(mut error: S3Error) -> S3Error {
if error.message().is_none() && matches!(error.code(), S3ErrorCode::PreconditionFailed) {
error.set_message("At least one of the preconditions you specified did not hold.");
}
error
}
pub(super) fn abort_multipart_input(input: &dto::AbortMultipartUploadInput) -> S3Result<()> {
input
.if_match_initiated_time
.map(|value| multipart_timestamp(value, "AbortMultipartUpload input", "if_match_initiated_time"))
.transpose()?;
Ok(())
}
#[cfg(test)]
#[path = "multipart_test.rs"]
mod tests;
@@ -0,0 +1,483 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Protocol multipart compatibility probes against the original conversion and storage edge.
//! These tests do not define storage or authorization rules.
use super::super::{RequestParams, S3, gateway_s3s_compat as compat};
use super::*;
use http::{HeaderMap, Method};
use rustfs_gateway_types::compat::s3s_5761ddfe::leaf::RequestWire;
use rustfs_protocols::common::client::s3::StorageBackend;
use rustfs_s3_types::{S3Error, S3ErrorCode};
fn create_input() -> dto::CreateMultipartUploadInput {
dto::CreateMultipartUploadInput {
bucket: rustfs_gateway_types::BucketName::new("multipart-fields").expect("bucket"),
key: rustfs_gateway_types::ObjectKey::new("a ?#+%.txt").expect("key"),
..Default::default()
}
}
fn original_create(
input: dto::CreateMultipartUploadInput,
) -> S3Result<crate::app::multipart_usecase::CreateMultipartUploadFields> {
let uri = super::super::build_object_uri(input.bucket.as_str(), input.key.as_str(), &[("uploads", None)])?;
let context = super::super::gateway_request_context(Method::POST, &uri, HeaderMap::new());
let input = compat::generated::ops::create_multipart_upload::input_to_s3s(input, &RequestWire::of(&context))
.map_err(|error| super::super::conversion_error("CreateMultipartUpload input", error))?;
Ok(crate::app::multipart_usecase::CreateMultipartUploadFields::from_legacy(input))
}
#[test]
fn multipart_create_projection_matches_original_fields_and_times() {
for empty in [false, true] {
for seconds in [1_800_000_000, -1] {
let make = || {
let mut input = create_input();
input.tagging = Some(if empty { "" } else { "label=value" }.into());
input.storage_class = Some(dto::StorageClass::custom(if empty { "" } else { "STANDARD" }));
input.object_lock_mode = Some(dto::ObjectLockMode::custom(if empty { "" } else { "GOVERNANCE" }));
input.object_lock_legal_hold_status = Some(dto::ObjectLockLegalHoldStatus::custom(if empty { "" } else { "ON" }));
input.sse_customer_algorithm = Some(if empty { "" } else { "AES256" }.into());
input.ssekms_key_id = Some(if empty { "" } else { "key-label" }.into());
input.sse_customer_key_md5 = Some(if empty { "" } else { "digest" }.into());
input.metadata.insert("label".into(), "value".into());
input.object_lock_retain_until_date = Some(Timestamp::from_secs_nanos(seconds, 123_456_789).expect("timestamp"));
input
};
let expected = original_create(make()).expect("original input");
let actual = create_multipart_input(make()).expect("owned input");
assert_eq!(
(actual.bucket, actual.key, actual.metadata, actual.tagging, actual.storage_class),
(expected.bucket, expected.key, expected.metadata, expected.tagging, expected.storage_class)
);
assert_eq!(
(
actual.server_side_encryption,
actual.sse_customer_algorithm,
actual.sse_customer_key_md5,
actual.ssekms_key_id
),
(
expected.server_side_encryption,
expected.sse_customer_algorithm,
expected.sse_customer_key_md5,
expected.ssekms_key_id
)
);
assert_eq!(
(
actual.object_lock_mode,
actual.object_lock_legal_hold_status,
actual.object_lock_retain_until_date
),
(
expected.object_lock_mode,
expected.object_lock_legal_hold_status,
expected.object_lock_retain_until_date
)
);
}
}
}
fn error_facts(error: S3Error) -> (String, Option<String>, Option<http::StatusCode>, Option<HeaderMap>) {
(
error.code().as_str().to_owned(),
error.message().map(str::to_owned),
error.status_code(),
error.headers().cloned(),
)
}
#[test]
fn multipart_create_projection_preserves_refusals() {
for field in 0..4 {
let make = || {
let mut input = create_input();
match field {
0 => input.object_lock_event_hold = Some(dto::ObjectLockEventHold::custom("ON")),
1 => input.object_lock_event_hold_duration_days = Some(0),
2 => input.object_lock_event_hold_duration_years = Some(-1),
_ => input.object_lock_retain_until_date = Some(Timestamp::from_secs(i64::MAX)),
}
input
};
let expected = original_create(make()).err().expect("original refusal");
let actual = create_multipart_input(make()).err().expect("owned refusal");
assert_eq!(error_facts(actual), error_facts(expected), "field {field}");
}
}
fn complete_input() -> dto::CompleteMultipartUploadInput {
dto::CompleteMultipartUploadInput {
bucket: rustfs_gateway_types::BucketName::new("multipart-fields").expect("bucket"),
key: rustfs_gateway_types::ObjectKey::new("key").expect("key"),
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire("missing-upload"),
..Default::default()
}
}
#[test]
fn multipart_complete_projection_matches_original_parts_and_conditions() {
for condition in [
None,
Some(""),
Some("*"),
Some("abc-1"),
Some("\"abc\""),
Some("W/\"abc\""),
Some("bad tag"),
Some("\"one\", \"two\""),
Some("W/\"one\", W/\"two\""),
] {
let make = || {
let mut input = complete_input();
input.if_match = condition.map(str::to_owned);
input.if_none_match = condition.map(str::to_owned);
input.sse_customer_algorithm = Some(String::new());
input.multipart_upload.parts.push(dto::CompletedPart {
part_number: -1,
e_tag: Some(ETag::new_weak("a&b").expect("weak tag")),
checksum_crc32: Some(String::new()),
..Default::default()
});
input
};
let expected = compat::generated::ops::complete_multipart_upload::input_to_s3s(make())
.map_err(|error| super::super::conversion_error("CompleteMultipartUpload input", error))
.and_then(|input| {
compat::generated::ops::complete_multipart_upload::input_from_s3s(input)
.map_err(crate::app::dto_values::seam_error)
});
let actual = complete_multipart_input(make());
match (actual, expected) {
(Ok(actual), Ok(expected)) => {
assert_eq!(
(actual.if_match, actual.if_none_match, actual.sse_customer_algorithm),
(expected.if_match, expected.if_none_match, expected.sse_customer_algorithm)
);
let snapshot = |parts: dto::CompletedMultipartUpload| {
parts
.parts
.into_iter()
.map(|part| (part.part_number, part.e_tag.map(|tag| tag.opaque_tag().to_owned()), part.checksum_crc32))
.collect::<Vec<_>>()
};
assert_eq!(snapshot(actual.multipart_upload), snapshot(expected.multipart_upload));
assert!(actual.checksum_spec.is_none());
}
(Err(actual), Err(expected)) => assert_eq!(error_facts(actual), error_facts(expected)),
_ => panic!("owned and original conversion outcomes differ"),
}
}
}
#[test]
fn multipart_completion_projection_preserves_original_escaped_tags() {
for (opaque, weak) in [("abc", false), ("a&b<c", false), ("a\"b", true)] {
let input = dto::CompleteMultipartUploadOutput {
e_tag: Some(if weak { ETag::new_weak(opaque) } else { ETag::new(opaque) }.expect("tag")),
..Default::default()
};
let mut legacy =
compat::generated::ops::complete_multipart_upload::output_to_s3s(input.clone()).expect("original output");
let stored = if weak { format!("W/\"{opaque}\"") } else { opaque.to_owned() };
legacy.e_tag = Some(crate::app::storage_api::object_utils::to_s3s_xml_etag(&stored));
let expected = compat::generated::ops::complete_multipart_upload::output_from_s3s(legacy).expect("original answer");
let actual = complete_multipart_output(input).expect("owned answer");
let tag = |value: Option<ETag>| value.map(|value| (value.opaque_tag().to_owned(), value.is_weak()));
assert_eq!(tag(actual.e_tag), tag(expected.e_tag));
}
}
#[test]
#[serial_test::serial]
fn multipart_refusals_match_original_client_errors() {
crate::app::gating_test_env::run_large_stack_test("protocol-multipart-original-errors", || async {
let bucket = "multipart-original-errors";
let (_temp, root, backend, _) = crate::protocols::mainflow_test::object_client(bucket).await;
for (part, condition) in [
(None, None),
(Some(0), None),
(Some(-1), None),
(Some(10001), None),
(Some(1), None),
(Some(1), Some("bad tag")),
(Some(1), Some("missing")),
] {
let make = || {
let mut input = complete_input();
input.bucket = rustfs_gateway_types::BucketName::new(bucket).expect("bucket");
input.key = rustfs_gateway_types::ObjectKey::new("directory/a ?#+%.txt").expect("key");
input.if_match = condition.map(str::to_owned);
if let Some(part_number) = part {
input.multipart_upload.parts.push(dto::CompletedPart {
part_number,
..Default::default()
});
}
input
};
let expected = match compat::generated::ops::complete_multipart_upload::input_to_s3s(make()) {
Err(error) => super::super::conversion_error("CompleteMultipartUpload input", error),
Ok(legacy) => {
let uri =
super::super::build_object_uri(bucket, "directory/a ?#+%.txt", &[("uploadId", Some("missing-upload"))])
.expect("URI");
let request = ProtocolStorageClient::create_request(
legacy,
Method::POST,
uri,
RequestParams {
bucket: Some(bucket.into()),
object: Some("directory/a ?#+%.txt".into()),
credentials: &root,
},
)
.expect("request");
backend
.fs
.complete_multipart_upload(request)
.await
.err()
.map(S3Error::from)
.expect("original refusal")
}
};
let actual = backend
.complete_multipart_upload(make(), &root)
.await
.expect_err("owned refusal");
assert_eq!(error_facts(actual), error_facts(expected), "part {part:?}, condition {condition:?}");
}
let error = backend
.abort_multipart_upload(
dto::AbortMultipartUploadInput {
bucket: rustfs_gateway_types::BucketName::new(bucket).expect("bucket"),
key: rustfs_gateway_types::ObjectKey::new("key").expect("key"),
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire("missing-upload"),
..Default::default()
},
&root,
)
.await
.expect_err("missing upload");
assert_eq!(error.code(), &S3ErrorCode::NoSuchUpload);
});
}
#[test]
fn multipart_completion_projection_preserves_header_only_field_behavior() {
let output = dto::CompleteMultipartUploadOutput {
checksum_header_xxhash3: Some("AAAAAAAAAAA=".into()),
sse_customer_algorithm: Some("AES256".into()),
sse_customer_key_md5: Some("digest".into()),
..Default::default()
};
let output = complete_multipart_output(output).expect("header fields remain valid");
assert!(output.checksum_header_xxhash3.is_none());
assert!(output.sse_customer_algorithm.is_none());
assert!(output.sse_customer_key_md5.is_none());
let error = complete_multipart_output(dto::CompleteMultipartUploadOutput {
sse_customer_algorithm: Some("bad\nvalue".into()),
..Default::default()
})
.expect_err("invalid original response header");
assert_eq!(error.code(), &S3ErrorCode::InternalError);
assert!(
error
.message()
.expect("message")
.contains("x-amz-server-side-encryption-customer-algorithm: not a header value")
);
}
#[test]
fn multipart_envelopes_preserve_original_request_facts() {
let credentials = rustfs_credentials::Credentials {
access_key: "multipart-facts".into(),
secret_key: "test-secret".into(),
..Default::default()
};
for (method, query) in [
(Method::POST, vec![("uploads", None)]),
(Method::POST, vec![("uploadId", Some("session/+?=&%"))]),
(Method::DELETE, vec![("uploadId", Some("session/+?=&%"))]),
] {
let uri = super::super::build_object_uri("multipart-fields", "a ?#+%.txt", &query).expect("URI");
let params = || RequestParams {
bucket: Some("multipart-fields".into()),
object: Some("a ?#+%.txt".into()),
credentials: &credentials,
};
let request = ProtocolStorageClient::create_request((), method.clone(), uri.clone(), params()).expect("original request");
let envelope = ProtocolStorageClient::create_envelope(method, uri, params());
assert_eq!(
(envelope.method(), envelope.uri(), envelope.headers()),
(&request.method, &request.uri, &request.headers)
);
let original = request.extensions.get::<super::super::ReqInfo>().expect("original facts");
let owned = envelope.req_info().expect("owned facts");
assert_eq!(
(
&owned.bucket,
&owned.object,
&owned.version_id,
owned.is_owner,
owned.keystone_session_present
),
(
&original.bucket,
&original.object,
&original.version_id,
original.is_owner,
original.keystone_session_present
)
);
assert_eq!(
owned.cred.as_ref().map(|value| &value.access_key),
original.cred.as_ref().map(|value| &value.access_key)
);
assert!(envelope.server_context().is_none());
}
}
#[test]
#[serial_test::serial]
fn multipart_uses_bound_store_with_a_foreign_ambient_context() {
crate::app::gating_test_env::run_large_stack_test("protocol-multipart-bound-store", || async {
use crate::app::storage_api::test::contract::multipart::MultipartOperations;
use crate::storage_api::protocols::client::StorageObjectOptions;
let bucket = "multipart-bound-store";
let (_foreign_temp, root, foreign_client, _) = crate::protocols::mainflow_test::object_client(bucket).await;
let foreign = foreign_client.fs.server_ctx().app_context().expect("foreign context");
let (_temp, _, backend, _) = crate::protocols::mainflow_test::object_client(bucket).await;
let bound = backend.fs.server_ctx().app_context().expect("bound context");
let ambient = crate::runtime_sources::current_app_context().expect("published foreign context");
assert!(std::sync::Arc::ptr_eq(&ambient, &foreign));
assert!(!std::sync::Arc::ptr_eq(&ambient, &bound));
for legacy in [true, false] {
let mut input = create_input();
input.bucket = rustfs_gateway_types::BucketName::new(bucket).expect("bucket");
let key = input.key.as_str().to_owned();
let params = || RequestParams {
bucket: Some(bucket.to_owned()),
object: Some(key.clone()),
credentials: &root,
};
let created = if legacy {
let uri = super::super::build_object_uri(bucket, &key, &[("uploads", None)]).expect("URI");
let context = super::super::gateway_request_context(Method::POST, &uri, HeaderMap::new());
let input = compat::generated::ops::create_multipart_upload::input_to_s3s(input, &RequestWire::of(&context))
.expect("original input");
let request = ProtocolStorageClient::create_request(input, Method::POST, uri, params()).expect("request");
let output = backend
.fs
.create_multipart_upload(request)
.await
.expect("original bound create")
.output;
compat::generated::ops::create_multipart_upload::output_from_s3s(output).expect("original output")
} else {
backend
.create_multipart_upload(input, &root)
.await
.expect("owned bound create")
};
let options = StorageObjectOptions::default();
let target_lookup = bound
.object_store()
.get_multipart_info(bucket, &key, &created.upload_id, &options)
.await;
let ambient_lookup = foreign
.object_store()
.get_multipart_info(bucket, &key, &created.upload_id, &options)
.await;
assert!(
target_lookup.is_ok() && ambient_lookup.is_err(),
"legacy={legacy}: target lookup succeeded={}, ambient lookup succeeded={}",
target_lookup.is_ok(),
ambient_lookup.is_ok()
);
let input = dto::AbortMultipartUploadInput {
bucket: created.bucket,
key: created.key,
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire(&created.upload_id),
..Default::default()
};
if legacy {
let uri = super::super::build_object_uri(bucket, &key, &[("uploadId", Some(&created.upload_id))]).expect("URI");
let input = compat::generated::ops::abort_multipart_upload::input_to_s3s(input).expect("original input");
let request = ProtocolStorageClient::create_request(input, Method::DELETE, uri, params()).expect("request");
backend
.fs
.abort_multipart_upload(request)
.await
.expect("original bound abort");
} else {
backend.abort_multipart_upload(input, &root).await.expect("owned bound abort");
}
assert!(
bound
.object_store()
.get_multipart_info(bucket, &key, &created.upload_id, &options)
.await
.is_err()
);
}
});
}
#[test]
fn multipart_create_output_matches_original_for_stored_names_and_empty_values() {
for name in ["multipart-fields", "a-s3alias"] {
for value in ["", "value"] {
let legacy = super::super::CreateMultipartUploadOutput {
bucket: Some(name.into()),
key: Some("key".into()),
upload_id: Some(value.into()),
sse_customer_algorithm: Some(value.into()),
sse_customer_key_md5: Some(value.into()),
ssekms_key_id: Some(value.into()),
..Default::default()
};
let expected = compat::generated::ops::create_multipart_upload::output_from_s3s(legacy)
.map_err(|error| super::super::conversion_error("CreateMultipartUpload output", error));
let actual = create_multipart_output(crate::app::multipart_usecase::CreateMultipartUploadResult {
bucket: name.into(),
key: "key".into(),
upload_id: value.into(),
sse_customer_algorithm: Some(value.into()),
sse_customer_key_md5: Some(value.into()),
ssekms_key_id: Some(value.into()),
server_side_encryption: None,
checksum_algorithm: None,
checksum_type: None,
});
let snapshot = |output: dto::CreateMultipartUploadOutput| {
(
output.bucket.as_str().to_owned(),
output.key.as_str().to_owned(),
output.upload_id,
output.sse_customer_algorithm,
output.sse_customer_key_md5,
output.ssekms_key_id,
)
};
assert_eq!(actual.map(snapshot).map_err(error_facts), expected.map(snapshot).map_err(error_facts));
}
}
}
+182 -1
View File
@@ -28,7 +28,7 @@ use rustfs_s3_types::S3ErrorCode;
const KEY: &str = "directory/a ?#+%.txt";
async fn object_client(bucket: &str) -> (tempfile::TempDir, Credentials, ProtocolStorageClient, String) {
pub(super) async fn object_client(bucket: &str) -> (tempfile::TempDir, Credentials, ProtocolStorageClient, String) {
let root = Credentials {
access_key: "protocol-mainflow-root".into(),
secret_key: "protocol-mainflow-secret".into(),
@@ -47,6 +47,7 @@ async fn object_client(bucket: &str) -> (tempfile::TempDir, Credentials, Protoco
.await
.expect("seed object");
let slot = ServerContextSlot::new();
crate::runtime_sources::publish_test_app_context(context.clone());
assert!(slot.install(context));
(
temp,
@@ -196,3 +197,183 @@ fn protocol_mainflow_refuses_missing_objects_buckets_and_unsatisfied_ranges() {
);
});
}
fn multipart_count(operation: rustfs_s3_ops::S3Operation) -> u64 {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
.find(|row| row.op == operation.as_str())
.map_or(0, |row| row.total)
}
fn multipart_create(bucket: &str, key: &str) -> rustfs_gateway_types::dto::CreateMultipartUploadInput {
rustfs_gateway_types::dto::CreateMultipartUploadInput {
bucket: rustfs_gateway_types::BucketName::new(bucket).expect("bucket"),
key: rustfs_gateway_types::ObjectKey::new(key).expect("key"),
..Default::default()
}
}
fn multipart_complete(bucket: &str, key: &str, upload: &str) -> rustfs_gateway_types::dto::CompleteMultipartUploadInput {
rustfs_gateway_types::dto::CompleteMultipartUploadInput {
bucket: rustfs_gateway_types::BucketName::new(bucket).expect("bucket"),
key: rustfs_gateway_types::ObjectKey::new(key).expect("key"),
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire(upload),
..Default::default()
}
}
#[test]
#[serial_test::serial]
fn protocol_multipart_lifecycle_stores_parts_and_aborts() {
crate::app::gating_test_env::run_large_stack_test("protocol-multipart-lifecycle", || async {
use rustfs_gateway_types::dto;
let bucket = "protocol-multipart-lifecycle";
let (_temp, root, backend, _) = object_client(bucket).await;
let create_count = multipart_count(rustfs_s3_ops::S3Operation::CreateMultipartUpload);
let complete_count = multipart_count(rustfs_s3_ops::S3Operation::CompleteMultipartUpload);
let created = backend
.create_multipart_upload(multipart_create(bucket, KEY), &root)
.await
.expect("create upload");
assert_eq!(multipart_count(rustfs_s3_ops::S3Operation::CreateMultipartUpload), create_count + 1);
assert_eq!(created.bucket.as_str(), bucket);
assert_eq!(created.key.as_str(), KEY);
let uploaded = backend
.upload_part(
dto::UploadPartInput {
bucket: created.bucket.clone(),
key: created.key.clone(),
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire(&created.upload_id),
part_number: 1,
content_length: 14,
body: Some(rustfs_gateway_stream::ByteStream::from_bytes(bytes::Bytes::from_static(
b"multipart-data",
))),
..Default::default()
},
&root,
)
.await
.expect("stage part");
let mut complete = multipart_complete(bucket, KEY, &created.upload_id);
complete.multipart_upload.parts.push(dto::CompletedPart {
part_number: 1,
e_tag: Some(uploaded.e_tag),
..Default::default()
});
let completed = backend
.complete_multipart_upload(complete, &root)
.await
.expect("complete upload");
assert_eq!(multipart_count(rustfs_s3_ops::S3Operation::CompleteMultipartUpload), complete_count + 1);
assert_eq!(completed.bucket.as_ref().map(|value| value.as_str()), Some(bucket));
assert_eq!(completed.key.as_ref().map(|value| value.as_str()), Some(KEY));
assert_eq!(
completed.location.as_deref(),
Some(format!("/{bucket}/directory/a%20%3F%23%2B%25.txt").as_str())
);
let read = backend
.get_object(bucket, KEY, &root, None)
.await
.expect("read completed object");
assert_eq!(
read.body
.expect("body")
.into_body()
.collect()
.await
.expect("read bytes")
.to_bytes()
.as_ref(),
b"multipart-data"
);
let created = backend
.create_multipart_upload(multipart_create(bucket, "aborted"), &root)
.await
.expect("create aborted upload");
let abort = || dto::AbortMultipartUploadInput {
bucket: created.bucket.clone(),
key: created.key.clone(),
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire(&created.upload_id),
..Default::default()
};
let abort_count = multipart_count(rustfs_s3_ops::S3Operation::AbortMultipartUpload);
backend.abort_multipart_upload(abort(), &root).await.expect("abort upload");
assert_eq!(multipart_count(rustfs_s3_ops::S3Operation::AbortMultipartUpload), abort_count + 1);
let error = backend
.upload_part(
dto::UploadPartInput {
bucket: created.bucket,
key: created.key,
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire(&created.upload_id),
part_number: 1,
content_length: 0,
body: Some(rustfs_gateway_stream::ByteStream::from_bytes(bytes::Bytes::new())),
..Default::default()
},
&root,
)
.await
.expect_err("aborted session is absent");
assert_eq!(error.code(), &S3ErrorCode::NoSuchUpload);
});
}
#[test]
#[serial_test::serial]
fn protocol_multipart_lifecycle_refuses_invalid_parts_and_missing_uploads() {
crate::app::gating_test_env::run_large_stack_test("protocol-multipart-refusals", || async {
use rustfs_gateway_types::dto;
let bucket = "protocol-multipart-refusals";
let (_temp, root, backend, _) = object_client(bucket).await;
let created = backend
.create_multipart_upload(multipart_create(bucket, KEY), &root)
.await
.expect("create upload");
for (label, number, tag, expected) in [
("empty", None, None, S3ErrorCode::InvalidRequest),
("zero", Some(0), None, S3ErrorCode::InvalidPart),
("negative", Some(-1), None, S3ErrorCode::InvalidPart),
("too-large", Some(10001), None, S3ErrorCode::InvalidPart),
("missing-tag", Some(1), None, S3ErrorCode::InvalidPart),
("wrong-tag", Some(1), Some("wrong"), S3ErrorCode::InvalidPart),
] {
let mut input = multipart_complete(bucket, KEY, &created.upload_id);
if let Some(part_number) = number {
input.multipart_upload.parts.push(dto::CompletedPart {
part_number,
e_tag: tag.map(rustfs_gateway_types::ETag::from_stored),
..Default::default()
});
}
let error = backend.complete_multipart_upload(input, &root).await.expect_err(label);
assert_eq!(error.code(), &expected, "{label}: {error}");
}
let error = backend
.abort_multipart_upload(
dto::AbortMultipartUploadInput {
bucket: created.bucket,
key: created.key,
upload_id: rustfs_gateway_types::UploadIdClaim::from_wire("missing-upload"),
..Default::default()
},
&root,
)
.await
.expect_err("missing abort upload");
assert_eq!(error.code(), &S3ErrorCode::NoSuchUpload);
let mut input = multipart_complete(bucket, KEY, "missing-upload");
input.multipart_upload.parts.push(dto::CompletedPart {
part_number: 1,
e_tag: Some(rustfs_gateway_types::ETag::from_stored("wrong")),
..Default::default()
});
let error = backend
.complete_multipart_upload(input, &root)
.await
.expect_err("missing complete upload");
assert_eq!(error.code(), &S3ErrorCode::NoSuchUpload);
});
}
+23 -6
View File
@@ -187,7 +187,7 @@ fn protocol_policy_denials_and_request_conditions() {
.expect("load user")
.0
.expect("user exists");
let allow = json!({"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket","s3:DeleteBucket","s3:DeleteBucket"],
let allow = json!({"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket","s3:DeleteBucket","s3:DeleteBucket","s3:AbortMultipartUpload"],
"Resource":[format!("arn:aws:s3:::{bucket}"),format!("arn:aws:s3:::{bucket}/*")]});
iam.set_policy(
"protocol-policy",
@@ -215,10 +215,24 @@ fn protocol_policy_denials_and_request_conditions() {
})
.collect();
for session in &sessions {
backend
.authorize_operation(session, &S3Action::GetObject, bucket, Some("secret.txt"))
.await
.expect("IAM allow works");
for action in [
S3Action::GetObject,
S3Action::CreateMultipartUpload,
S3Action::CompleteMultipartUpload,
S3Action::AbortMultipartUpload,
] {
let result = backend
.authorize_operation(session, &action, bucket, Some("secret.txt"))
.await;
if session.protocol == Protocol::WebDav && action != S3Action::GetObject {
assert!(
matches!(result, Err(AuthorizationError::AccessDenied)),
"WebDAV does not expose multipart operations"
);
} else {
result.expect("IAM allow works");
}
}
for action in [S3Action::HeadBucket, S3Action::DeleteBucket] {
backend
.authorize_operation(session, &action, bucket, None)
@@ -227,7 +241,7 @@ fn protocol_policy_denials_and_request_conditions() {
}
}
let bucket_policy = json!({"Version":"2012-10-17","Statement":[{
"Effect":"Deny","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket","s3:DeleteBucket","s3:DeleteBucket"],
"Effect":"Deny","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket","s3:DeleteBucket","s3:DeleteBucket","s3:AbortMultipartUpload"],
"Resource":[format!("arn:aws:s3:::{bucket}"),format!("arn:aws:s3:::{bucket}/*")],
"Condition":{"StringLike":{"aws:username":"*"}}
}]});
@@ -252,6 +266,9 @@ fn protocol_policy_denials_and_request_conditions() {
S3Action::HeadObject,
S3Action::PutObject,
S3Action::DeleteObject,
S3Action::CreateMultipartUpload,
S3Action::CompleteMultipartUpload,
S3Action::AbortMultipartUpload,
S3Action::HeadBucket,
S3Action::DeleteBucket,
] {
@@ -37,6 +37,7 @@ fn complete_input_preserves_part_fields() {
CompletedPart {
part_number: None,
e_tag: Some(ETag::Strong(String::new())),
checksum_crc32: Some(String::new()),
..Default::default()
},
CompletedPart {
@@ -60,6 +61,7 @@ fn complete_input_preserves_part_fields() {
.map(|part| (part.part_number, part.e_tag.as_ref().map(|tag| tag.opaque_tag())))
.collect();
assert_eq!(read, [(0, Some("")), (2, Some("abc")), (3, None)]);
assert_eq!(converted.parts[0].checksum_crc32.as_deref(), Some(""));
}
/// Several per-algorithm checksum members, and one of the wrong width, do not stop the request: