mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
feat(gateway): serve CreateMultipartUpload through the owned path
Share authorization between both CreateMultipartUpload edges. Keep the decoded legacy fields intact, preserve the replication destination version and apply the captured bucket incarnation before creating the upload. Leave checksum, encryption and retention metadata on the same storage path. Verify the frozen legacy behavior on2c2af51abefore the migration and exercise both stacks with 66 refusal cases and 22 success controls. Read back sessions, version, retention and encryption metadata. Fault probes made authorization tests fail on both edges when an action or the resource binding was lost. Storage tests detected a different destination version and unexpected sessions when generation, quota, KMS or SSE checks were removed. The wire and helper tests failed for early name rejection, lost timestamp precision, metadata, checksum, SSE response, the operation count and unexpected notification. On77da6339, 185 scoped tests passed with a 32 MiB test stack. The default stack run passed 184 tests and reproduced the existing overflow in completion_absent_binding_refusal_order_baseline. Both all-target Clippy variants, test wiring, script tests and prepush passed. After integration with97a7639f, gateway all-target check and prepush passed. Whole-workspace CI and rio-v2 were outside the local verification.
This commit is contained in:
@@ -84,8 +84,7 @@ use crate::app::object_data_cache::{
|
||||
invalidate_object_data_cache_before_mutation,
|
||||
};
|
||||
use crate::app::object_usecase::{
|
||||
apply_quota_admission, build_put_like_object_lock_metadata, map_quota_check_outcome, s3s_body_error_to_io,
|
||||
validate_existing_object_lock_for_write,
|
||||
apply_quota_admission, map_quota_check_outcome, s3s_body_error_to_io, validate_existing_object_lock_for_write,
|
||||
};
|
||||
use crate::app::runtime_sources::{
|
||||
AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context,
|
||||
@@ -130,6 +129,8 @@ use uuid::Uuid;
|
||||
mod abort;
|
||||
mod complete;
|
||||
mod create;
|
||||
mod create_values;
|
||||
pub(crate) use create_values::{CreateMultipartUploadFields, CreateMultipartUploadResult};
|
||||
mod list;
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
@@ -538,7 +539,7 @@ mod tests {
|
||||
.build()
|
||||
.expect("valid create request DTO");
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload(build_request(create, Method::POST))
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
|
||||
.await
|
||||
.expect_err("reserved key must be rejected before accessing storage");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
@@ -1026,7 +1026,7 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
let upload_id = usecase
|
||||
.execute_create_multipart_upload(create_request)
|
||||
.execute_create_multipart_upload_from_s3s(create_request)
|
||||
.await
|
||||
.expect("authorized REPLICA-only create should succeed")
|
||||
.output
|
||||
@@ -1124,7 +1124,7 @@ mod tests {
|
||||
.headers
|
||||
.insert("x-amz-checksum-type", HeaderValue::from_static("COMPOSITE"));
|
||||
let upload_id = usecase
|
||||
.execute_create_multipart_upload(create)
|
||||
.execute_create_multipart_upload_from_s3s(create)
|
||||
.await
|
||||
.expect("SSE-C create")
|
||||
.output
|
||||
@@ -1514,7 +1514,7 @@ mod tests {
|
||||
.headers
|
||||
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
|
||||
let first_upload_id = usecase
|
||||
.execute_create_multipart_upload(initial_create_req)
|
||||
.execute_create_multipart_upload_from_s3s(initial_create_req)
|
||||
.await
|
||||
.expect("initial conditional create multipart upload should succeed")
|
||||
.output
|
||||
@@ -1561,13 +1561,13 @@ mod tests {
|
||||
.headers
|
||||
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
|
||||
let create_err = usecase
|
||||
.execute_create_multipart_upload(second_create_req)
|
||||
.execute_create_multipart_upload_from_s3s(second_create_req)
|
||||
.await
|
||||
.expect_err("create multipart upload with If-None-Match: * on existing key must fail");
|
||||
assert_eq!(create_err.code(), &crate::app::storage_api::s3::S3ErrorCode::PreconditionFailed);
|
||||
|
||||
let second_upload_id = usecase
|
||||
.execute_create_multipart_upload(build_request(create_input, Method::POST))
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create_input, Method::POST))
|
||||
.await
|
||||
.expect("unconditional create multipart upload should succeed")
|
||||
.output
|
||||
@@ -1770,7 +1770,7 @@ mod tests {
|
||||
.headers
|
||||
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
|
||||
let dm_upload_id = usecase
|
||||
.execute_create_multipart_upload(dm_create_req)
|
||||
.execute_create_multipart_upload_from_s3s(dm_create_req)
|
||||
.await
|
||||
.expect("If-None-Match: * create must succeed past a delete marker")
|
||||
.output
|
||||
|
||||
@@ -15,16 +15,15 @@
|
||||
//! CreateMultipartUpload.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
ChecksumAlgorithm, ChecksumType, CreateMultipartUploadInput, CreateMultipartUploadOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::object::shared::build_put_like_object_lock_metadata_native;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
|
||||
fn create_multipart_upload_metadata(
|
||||
input_metadata: Option<HashMap<String, String>>,
|
||||
headers: &HeaderMap,
|
||||
tagging: Option<String>,
|
||||
storage_class: Option<&crate::app::storage_api::s3::dto::StorageClass>,
|
||||
storage_class: Option<&str>,
|
||||
) -> HashMap<String, String> {
|
||||
let mut metadata = input_metadata.unwrap_or_default();
|
||||
namespace_reserved_user_metadata(&mut metadata);
|
||||
@@ -34,7 +33,7 @@ fn create_multipart_upload_metadata(
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
|
||||
}
|
||||
if let Some(storage_class) = storage_class {
|
||||
metadata.insert(AMZ_STORAGE_CLASS.to_owned(), storage_class.as_str().to_owned());
|
||||
metadata.insert(AMZ_STORAGE_CLASS.to_owned(), storage_class.to_owned());
|
||||
}
|
||||
|
||||
metadata
|
||||
@@ -63,32 +62,37 @@ fn should_advertise_session_compression(multipart_enabled: bool, ciphertext_pass
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, envelope, input, query_version_id))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub async fn execute_create_multipart_upload(
|
||||
pub(crate) async fn execute_create_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
validate_new_object_key(&req.input.key, "CreateMultipartUpload")?;
|
||||
envelope: RequestEnvelope,
|
||||
input: CreateMultipartUploadFields,
|
||||
query_version_id: Option<String>,
|
||||
) -> S3Result<CreateMultipartUploadResult> {
|
||||
validate_new_object_key(&input.key, "CreateMultipartUpload")?;
|
||||
let headers = envelope.headers();
|
||||
let multipart_max_total_object_size = parse_presigned_multipart_max_total_object_size(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
headers,
|
||||
envelope.uri().query(),
|
||||
envelope.verified_sigv4().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
headers,
|
||||
envelope.uri().query(),
|
||||
envelope.verified_presigned().is_some(),
|
||||
)?;
|
||||
let helper =
|
||||
OperationHelper::new(&req, EventName::ObjectCreatedCreateMultipartUpload, S3Operation::CreateMultipartUpload)
|
||||
.suppress_event();
|
||||
let replication_authorized = replication_request_authorized(&req);
|
||||
let CreateMultipartUploadInput {
|
||||
let helper = OperationHelper::from_view(
|
||||
&envelope.audit_view(),
|
||||
EventName::ObjectCreatedCreateMultipartUpload,
|
||||
S3Operation::CreateMultipartUpload,
|
||||
)
|
||||
.suppress_event();
|
||||
let replication_authorized = replication_request_authorized(&envelope);
|
||||
let CreateMultipartUploadFields {
|
||||
bucket,
|
||||
key,
|
||||
tagging,
|
||||
version_id,
|
||||
storage_class,
|
||||
server_side_encryption,
|
||||
sse_customer_algorithm,
|
||||
@@ -99,24 +103,21 @@ impl DefaultMultipartUsecase {
|
||||
object_lock_mode,
|
||||
object_lock_retain_until_date,
|
||||
metadata: input_metadata,
|
||||
..
|
||||
} = req.input.clone();
|
||||
} = input;
|
||||
|
||||
let server_side_encryption = server_side_encryption
|
||||
.map(|value| value.as_str().to_owned())
|
||||
.or(extract_server_side_encryption_from_headers(&req.headers)?);
|
||||
let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(headers)?);
|
||||
let ssekms_key_id = ssekms_key_id.or_else(|| {
|
||||
req.headers
|
||||
headers
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(ToOwned::to_owned)
|
||||
});
|
||||
|
||||
// Validate storage class if provided
|
||||
if let Some(ref storage_class) = storage_class
|
||||
&& !is_valid_storage_class(storage_class.as_str())
|
||||
if let Some(storage_class) = storage_class.as_deref()
|
||||
&& !is_valid_storage_class(storage_class)
|
||||
{
|
||||
return Err(s3_error!(InvalidStorageClass));
|
||||
return Err(s3_error!(InvalidStorageClass, "The storage class you specified is not valid."));
|
||||
}
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
@@ -128,11 +129,11 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
validate_archive_content_encoding(
|
||||
&key,
|
||||
req.headers.get("content-type").and_then(|value| value.to_str().ok()),
|
||||
req.headers.get("content-encoding").and_then(|value| value.to_str().ok()),
|
||||
headers.get("content-type").and_then(|value| value.to_str().ok()),
|
||||
headers.get("content-encoding").and_then(|value| value.to_str().ok()),
|
||||
)?;
|
||||
|
||||
let mut metadata = create_multipart_upload_metadata(input_metadata, &req.headers, tagging, storage_class.as_ref());
|
||||
let mut metadata = create_multipart_upload_metadata(input_metadata, headers, tagging, storage_class.as_deref());
|
||||
if let Some(limit) = multipart_max_total_object_size {
|
||||
insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, limit.to_string());
|
||||
}
|
||||
@@ -142,9 +143,9 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
let has_explicit_object_lock_retention = object_lock_mode.is_some()
|
||||
|| object_lock_retain_until_date.is_some()
|
||||
|| has_replication_retention_update(&req.headers, replication_authorized);
|
||||
|| has_replication_retention_update(headers, replication_authorized);
|
||||
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
|
||||
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata(
|
||||
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata_native(
|
||||
&bucket,
|
||||
&object_lock_config_state,
|
||||
object_lock_legal_hold_status,
|
||||
@@ -160,20 +161,20 @@ impl DefaultMultipartUsecase {
|
||||
has_explicit_object_lock_retention,
|
||||
)?;
|
||||
let (header_sse_customer_algorithm, header_sse_customer_key, header_sse_customer_key_md5) =
|
||||
extract_ssec_params_from_headers(&req.headers)?;
|
||||
extract_ssec_params_from_headers(headers)?;
|
||||
let sse_customer_algorithm = sse_customer_algorithm.or(header_sse_customer_algorithm);
|
||||
let sse_customer_key = sse_customer_key.or(header_sse_customer_key);
|
||||
let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5);
|
||||
|
||||
// The session data key is generated here, so this is where a multipart upload is held
|
||||
// to the KMS key it names. Parts and completion also authorize their own caller.
|
||||
let session_principal = SseKmsPrincipal::from_request(&req);
|
||||
let session_principal = SseKmsPrincipal::from_request(&envelope);
|
||||
let encryption_request = PrepareEncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption,
|
||||
ssekms_key_id,
|
||||
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
|
||||
ssekms_context: extract_ssekms_context_from_headers(headers)?,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
@@ -184,8 +185,8 @@ impl DefaultMultipartUsecase {
|
||||
// session DEK is prepared; a session marker tells UploadPart to store
|
||||
// the ciphertext verbatim instead of recovering encryption material.
|
||||
let ciphertext_passthrough = replication_authorized
|
||||
&& get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
|
||||
&& rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some();
|
||||
&& get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
|
||||
&& rustfs_utils::http::ssec_transport_to_stored_metadata(headers).is_some();
|
||||
if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() {
|
||||
let check_result = map_quota_check_outcome(
|
||||
&bucket,
|
||||
@@ -228,7 +229,7 @@ impl DefaultMultipartUsecase {
|
||||
if should_advertise_session_compression(
|
||||
is_multipart_disk_compression_enabled(),
|
||||
ciphertext_passthrough,
|
||||
is_disk_compressible(&req.headers, &key),
|
||||
is_disk_compressible(headers, &key),
|
||||
) {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
@@ -239,10 +240,10 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
let mt2 = metadata.clone();
|
||||
let mut opts: ObjectOptions =
|
||||
put_opts_with_replication_authorization(&bucket, &key, version_id, &req.headers, metadata, replication_authorized)
|
||||
put_opts_with_replication_authorization(&bucket, &key, query_version_id, headers, metadata, replication_authorized)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
apply_bucket_generation_guard(&envelope, &bucket, &mut opts)?;
|
||||
|
||||
let dsc =
|
||||
must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone())
|
||||
@@ -261,7 +262,7 @@ impl DefaultMultipartUsecase {
|
||||
&& super::super::object::shared::object_lock_checks_required_for_state(&object_lock_config_state)
|
||||
{
|
||||
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
@@ -277,7 +278,7 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
let checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers);
|
||||
let checksum_type = rustfs_rio::ChecksumType::from_header(headers);
|
||||
if checksum_type.is(rustfs_rio::ChecksumType::INVALID) {
|
||||
return Err(s3_error!(InvalidArgument, "Invalid checksum type"));
|
||||
} else if checksum_type.is_set() && !checksum_type.is(rustfs_rio::ChecksumType::TRAILING) {
|
||||
@@ -299,21 +300,20 @@ impl DefaultMultipartUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let output = CreateMultipartUploadOutput {
|
||||
bucket: Some(bucket),
|
||||
key: Some(key),
|
||||
upload_id: Some(upload_id),
|
||||
server_side_encryption: effective_sse.map(Into::into),
|
||||
let output = CreateMultipartUploadResult {
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
server_side_encryption: effective_sse,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id: effective_kms_key_id,
|
||||
checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from),
|
||||
checksum_type: checksum_type.map(ChecksumType::from),
|
||||
..Default::default()
|
||||
checksum_algorithm: checksum_algo,
|
||||
checksum_type,
|
||||
};
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
let result = Ok(output);
|
||||
let _ = helper.complete_owned(&result, http::StatusCode::OK, &HeaderMap::new());
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -322,6 +322,51 @@ impl DefaultMultipartUsecase {
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::{S3ErrorCode, dto::CreateMultipartUploadInput};
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_multipart_baseline_legacy_direct() {
|
||||
use crate::app::storage_api::s3::dto::Timestamp;
|
||||
|
||||
let date = time::Date::from_calendar_date(-1, time::Month::January, 1)
|
||||
.unwrap()
|
||||
.midnight()
|
||||
.assume_utc();
|
||||
let long_key = "k".repeat(1025);
|
||||
let invalid_class = ("InvalidStorageClass", "The storage class you specified is not valid.");
|
||||
for (bucket, key, invalid, date, expected) in [
|
||||
("ab", "object", true, None, invalid_class),
|
||||
("a/b", "object", true, None, invalid_class),
|
||||
("bucket", "", true, None, ("InvalidArgument", "Invalid object key")),
|
||||
(
|
||||
"bucket",
|
||||
"a\0b",
|
||||
true,
|
||||
None,
|
||||
("InvalidArgument", "Object key contains invalid control characters: \"a\\0b\""),
|
||||
),
|
||||
("bucket", long_key.as_str(), true, None, invalid_class),
|
||||
("bucket", "object", false, Some(date), ("InternalError", "Not init")),
|
||||
("bucket", "object", true, Some(date), invalid_class),
|
||||
] {
|
||||
let input = CreateMultipartUploadInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.storage_class(invalid.then(|| StorageClass::from("unknown-class".to_owned())))
|
||||
.object_lock_retain_until_date(date.map(Timestamp::from))
|
||||
.build()
|
||||
.expect("direct fixture DTO");
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(build_request(input, Method::POST))
|
||||
.await
|
||||
.expect_err("direct fixture does not create an upload");
|
||||
assert_eq!(
|
||||
(error.code().as_str(), error.message()),
|
||||
(expected.0, Some(expected.1)),
|
||||
"{bucket:?}/{key:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_compression_is_advertised_only_for_non_passthrough_compressible_uploads() {
|
||||
@@ -358,18 +403,16 @@ mod tests {
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = make_usecase().execute_create_multipart_upload(req).await.unwrap_err();
|
||||
let err = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(req)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_upload_metadata_persists_dto_storage_class_without_raw_header() {
|
||||
let metadata = create_multipart_upload_metadata(
|
||||
None,
|
||||
&HeaderMap::new(),
|
||||
None,
|
||||
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
|
||||
);
|
||||
let metadata = create_multipart_upload_metadata(None, &HeaderMap::new(), None, Some("REDUCED_REDUNDANCY"));
|
||||
|
||||
assert_eq!(metadata.get(AMZ_STORAGE_CLASS), Some(&"REDUCED_REDUNDANCY".to_string()));
|
||||
}
|
||||
@@ -388,7 +431,7 @@ mod tests {
|
||||
Some(input_metadata),
|
||||
&headers,
|
||||
Some("project=rustfs".to_string()),
|
||||
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
|
||||
Some("REDUCED_REDUNDANCY"),
|
||||
);
|
||||
|
||||
assert_eq!(metadata.get("content-type"), Some(&"application/octet-stream".to_string()));
|
||||
@@ -401,10 +444,10 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() {
|
||||
let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
|
||||
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let input = CreateMultipartUploadInput::builder()
|
||||
.bucket(bucket)
|
||||
.bucket(bucket.clone())
|
||||
.key("object".to_string())
|
||||
.build()
|
||||
.expect("create multipart request should build");
|
||||
@@ -418,10 +461,200 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_create_multipart_upload(request)
|
||||
.await
|
||||
.expect_err("quota-enabled ciphertext multipart replication should fail before upload creation");
|
||||
let result = usecase.execute_create_multipart_upload_from_s3s(request).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list durable uploads")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
let err = result.expect_err("quota-enabled ciphertext multipart replication must be refused");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some("SSE-C ciphertext replication is unavailable for quota-enabled buckets")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn create_multipart_checks_kms_permission_before_creating_a_session() {
|
||||
crate::app::gating_test_env::run_large_stack_test("create-multipart-kms-policy", || async {
|
||||
use crate::app::storage_api::s3::dto::ServerSideEncryption;
|
||||
use crate::app::storage_api::test::{
|
||||
ReqInfo,
|
||||
sse::{RecordingKmsKeyAuthorizer, TestAuthorizationHooks},
|
||||
};
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
|
||||
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("create-multipart-kms", 16_384).await;
|
||||
let context = crate::app::runtime_sources::current_app_context().expect("test app context");
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(context.clone()));
|
||||
let key_id = "multipart-test-key";
|
||||
let kms = context.kms().handle();
|
||||
kms.reconfigure(rustfs_kms::config::KmsConfig::static_kms(
|
||||
key_id.to_owned(),
|
||||
base64_simd::STANDARD.encode_to_string([1_u8; 32]),
|
||||
))
|
||||
.await
|
||||
.expect("configure test KMS");
|
||||
for allowed in [false, true] {
|
||||
let input = CreateMultipartUploadInput {
|
||||
bucket: bucket.clone(),
|
||||
key: "object".to_owned(),
|
||||
server_side_encryption: Some(ServerSideEncryption::from_static("aws:kms")),
|
||||
ssekms_key_id: Some(key_id.to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let authorizer = RecordingKmsKeyAuthorizer::new(allowed);
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials {
|
||||
access_key: "test-kms-writer".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
req.extensions.insert(TestAuthorizationHooks {
|
||||
enforced: true,
|
||||
authorizer: authorizer.clone(),
|
||||
});
|
||||
let result = usecase.execute_create_multipart_upload_from_s3s(req).await;
|
||||
let uploads = store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list uploads after KMS decision")
|
||||
.uploads;
|
||||
assert_eq!(uploads.len(), usize::from(allowed));
|
||||
assert_eq!(authorizer.calls(), [(KmsAction::GenerateDataKeyAction, key_id.to_owned())]);
|
||||
if allowed {
|
||||
let output = result.expect("KMS authorizes the selected key").output;
|
||||
assert_eq!(output.upload_id.as_deref(), Some(uploads[0].upload_id.as_str()));
|
||||
} else {
|
||||
let error = result.expect_err("KMS denied the selected key");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
}
|
||||
kms.stop().await.expect("stop test KMS");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_preserves_legacy_date_precision() {
|
||||
use time::{Date, Month, Time};
|
||||
|
||||
let date = Date::from_calendar_date(-1, Month::January, 1)
|
||||
.unwrap()
|
||||
.with_time(Time::from_hms_nano(0, 0, 0, 123_456_789).unwrap())
|
||||
.assume_utc();
|
||||
let input = CreateMultipartUploadInput {
|
||||
bucket: "sthree-multipart".to_owned(),
|
||||
key: "object".to_owned(),
|
||||
object_lock_retain_until_date: Some(date.into()),
|
||||
..Default::default()
|
||||
};
|
||||
let fields = CreateMultipartUploadFields::from_legacy(input);
|
||||
assert_eq!(fields.object_lock_retain_until_date, Some(date));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn create_multipart_generation_fences_the_durable_upload() {
|
||||
crate::app::gating_test_env::run_large_stack_test("create-multipart-generation", || async {
|
||||
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
|
||||
use crate::app::storage_api::test::contract::bucket::{
|
||||
BucketOperations as _, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
|
||||
let context = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(context.clone()));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let bucket = format!("create-multipart-{}", Uuid::new_v4().simple());
|
||||
let other = format!("create-multipart-{}", Uuid::new_v4().simple());
|
||||
let request = |bucket: &str| {
|
||||
let input = CreateMultipartUploadInput {
|
||||
bucket: bucket.to_owned(),
|
||||
key: "object".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut request = build_request(input, Method::POST);
|
||||
request.uri = format!("/{bucket}/object?uploads").parse().expect("fixture URI");
|
||||
request.extensions.insert(slot.clone());
|
||||
request
|
||||
};
|
||||
for name in [&bucket, &other] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
|
||||
let mut stale = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &stale, &bucket)
|
||||
.await
|
||||
.expect("capture original incarnation");
|
||||
stale.install_bucket_generation(guard);
|
||||
let original = store.bucket_incarnation_id(&bucket).await.expect("old bucket generation");
|
||||
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
let current = store.bucket_incarnation_id(&bucket).await.expect("new bucket generation");
|
||||
assert_ne!(original, current);
|
||||
|
||||
let missing = request(&bucket);
|
||||
let mut wrong = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &wrong, &other)
|
||||
.await
|
||||
.expect("capture the other bucket");
|
||||
wrong.install_bucket_generation(guard);
|
||||
for (request, code) in [
|
||||
(stale, S3ErrorCode::NoSuchBucket),
|
||||
(missing, S3ErrorCode::InternalError),
|
||||
(wrong, S3ErrorCode::InternalError),
|
||||
] {
|
||||
let result = usecase.execute_create_multipart_upload_from_s3s(request).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list uploads in the recreated bucket")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
let error = result.expect_err("creation requires the target incarnation");
|
||||
assert_eq!(error.code(), &code);
|
||||
}
|
||||
|
||||
let probe = crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe::install(&bucket);
|
||||
let mut allowed = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &allowed, &bucket)
|
||||
.await
|
||||
.expect("capture current incarnation");
|
||||
allowed.install_bucket_generation(guard);
|
||||
let response = usecase
|
||||
.execute_create_multipart_upload_from_s3s(allowed)
|
||||
.await
|
||||
.expect("a current guard allows creation");
|
||||
let uploads = store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list durable uploads")
|
||||
.uploads;
|
||||
assert_eq!(uploads.len(), 1);
|
||||
assert_eq!(Some(uploads[0].upload_id.as_str()), response.output.upload_id.as_deref());
|
||||
assert_eq!(probe.audits.lock().unwrap().len(), 1);
|
||||
assert!(probe.events.lock().unwrap().is_empty());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Values consumed and produced by multipart initiation.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Keeps the decoded values in their original form until the use case validates them.
|
||||
// The customer key must never appear in diagnostics.
|
||||
pub(crate) struct CreateMultipartUploadFields {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) key: String,
|
||||
pub(crate) metadata: Option<HashMap<String, String>>,
|
||||
pub(crate) tagging: Option<String>,
|
||||
pub(crate) storage_class: Option<String>,
|
||||
pub(crate) server_side_encryption: Option<String>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
pub(crate) ssekms_key_id: Option<String>,
|
||||
pub(crate) object_lock_legal_hold_status: Option<String>,
|
||||
pub(crate) object_lock_mode: Option<String>,
|
||||
pub(crate) object_lock_retain_until_date: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
pub(crate) struct CreateMultipartUploadResult {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) key: String,
|
||||
pub(crate) upload_id: String,
|
||||
pub(crate) server_side_encryption: Option<String>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
pub(crate) ssekms_key_id: Option<String>,
|
||||
pub(crate) checksum_algorithm: Option<String>,
|
||||
pub(crate) checksum_type: Option<String>,
|
||||
}
|
||||
@@ -15,13 +15,18 @@
|
||||
//! The gateway operations of migration batch B2 (rustfs/backlog#2766): object listings, multipart
|
||||
//! listings, and multipart create and abort.
|
||||
|
||||
use super::{OpRow, Served, row};
|
||||
use super::{
|
||||
OpRow, Served,
|
||||
object_write::{request_version_id, retain_until},
|
||||
row,
|
||||
};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::multipart_usecase::DefaultMultipartUsecase;
|
||||
use crate::app::multipart_usecase::{CreateMultipartUploadFields, DefaultMultipartUsecase};
|
||||
use crate::runtime_sources::{AppContext, ServerContextSlot};
|
||||
use crate::storage_api::server::http::access::access_owned::listing_multipart::{
|
||||
authorize_bucket_listing, authorize_list_multipart_uploads, authorize_list_objects_v2,
|
||||
CreateMultipartAuthorization, authorize_bucket_listing, authorize_create_multipart_upload, authorize_list_multipart_uploads,
|
||||
authorize_list_objects_v2,
|
||||
};
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
use rustfs_policy::policy::action::S3Action;
|
||||
@@ -34,6 +39,7 @@ pub(super) const ROWS: &[OpRow] = &[
|
||||
row::<dto::ListObjects>(),
|
||||
row::<dto::ListObjectVersions>(),
|
||||
row::<dto::ListMultipartUploads>(),
|
||||
row::<dto::CreateMultipartUpload>(),
|
||||
];
|
||||
|
||||
impl Served for dto::ListObjectsV2 {
|
||||
@@ -98,6 +104,60 @@ impl Served for dto::ListMultipartUploads {
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::CreateMultipartUpload {
|
||||
const METRIC: S3Operation = S3Operation::CreateMultipartUpload;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
let server = server_context(envelope)?;
|
||||
let input = request.input();
|
||||
let version = request_version_id(envelope);
|
||||
let facts = CreateMultipartAuthorization {
|
||||
bucket: input.bucket.as_str(),
|
||||
key: input.key.as_str(),
|
||||
query_version: version.as_deref(),
|
||||
retain_until: retain_until(input.object_lock_retain_until_date.as_ref())?,
|
||||
legal_hold_present: input.object_lock_legal_hold_status.is_some(),
|
||||
retention_present: input.object_lock_mode.is_some() || input.object_lock_retain_until_date.is_some(),
|
||||
};
|
||||
authorize_create_multipart_upload(&server, envelope, facts).await
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let version = request_version_id(&envelope);
|
||||
let input = request.into_input();
|
||||
let fields = CreateMultipartUploadFields {
|
||||
bucket: input.bucket.as_str().to_owned(),
|
||||
key: input.key.as_str().to_owned(),
|
||||
metadata: Some(input.metadata.into_iter().collect()),
|
||||
tagging: input.tagging,
|
||||
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
|
||||
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key.map(|value| value.expose_secret().to_owned()),
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
ssekms_key_id: input.ssekms_key_id,
|
||||
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
|
||||
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
|
||||
object_lock_retain_until_date: retain_until(input.object_lock_retain_until_date.as_ref())?,
|
||||
};
|
||||
let output = DefaultMultipartUsecase::with_context(context)
|
||||
.execute_create_multipart_upload(envelope, fields, version)
|
||||
.await?;
|
||||
Ok(Resp::new(dto::CreateMultipartUploadOutput {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: output.upload_id,
|
||||
server_side_encryption: output.server_side_encryption.map(dto::ServerSideEncryption::custom),
|
||||
sse_customer_algorithm: output.sse_customer_algorithm,
|
||||
sse_customer_key_md5: output.sse_customer_key_md5,
|
||||
ssekms_key_id: output.ssekms_key_id,
|
||||
checksum_algorithm: output.checksum_algorithm.map(dto::ChecksumAlgorithm::custom),
|
||||
checksum_type: output.checksum_type.map(dto::ChecksumType::custom),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// The server context a generation guard is read through; the gateway envelope always has one.
|
||||
fn server_context(envelope: &RequestEnvelope) -> S3Result<Arc<ServerContextSlot>> {
|
||||
envelope
|
||||
|
||||
@@ -180,14 +180,14 @@ fn native_part_output(output: UploadPartWriteOutput) -> S3Result<Resp<dto::Uploa
|
||||
Ok(Resp::new(dto).with_extra_headers(headers))
|
||||
}
|
||||
|
||||
// PutObject's destination version is a RustFS replication extension, absent from the AWS DTO.
|
||||
fn request_version_id(envelope: &RequestEnvelope) -> Option<String> {
|
||||
// The destination version is a RustFS replication extension absent from the AWS DTOs.
|
||||
pub(super) fn request_version_id(envelope: &RequestEnvelope) -> Option<String> {
|
||||
url::form_urlencoded::parse(envelope.uri().query()?.as_bytes())
|
||||
.find(|(name, _)| name == "versionId")
|
||||
.map(|(_, value)| value.into_owned())
|
||||
}
|
||||
|
||||
fn retain_until(value: Option<&Timestamp>) -> S3Result<Option<time::OffsetDateTime>> {
|
||||
pub(super) fn retain_until(value: Option<&Timestamp>) -> S3Result<Option<time::OffsetDateTime>> {
|
||||
value
|
||||
.map(|value| {
|
||||
time::OffsetDateTime::from_unix_timestamp_nanos(
|
||||
|
||||
@@ -3441,28 +3441,21 @@ impl S3Access for FS {
|
||||
|
||||
/// Checks whether the CreateMultipartUpload request has accesses to the resources.
|
||||
async fn create_multipart_upload(&self, req: &mut S3Request<CreateMultipartUploadInput>) -> S3Result<()> {
|
||||
use access_owned::listing_multipart::{CreateMultipartAuthorization, authorize_create_multipart_upload};
|
||||
|
||||
let bucket = req.input.bucket.clone();
|
||||
let bucket_generation = load_bucket_generation(self.server_ctx(), req, &bucket).await;
|
||||
let req_info = ext_req_info_mut(&mut req.extensions)?;
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
req_info.object = Some(req.input.key.clone());
|
||||
req_info.version_id = super::options::destination_version_id(req.input.version_id.as_deref(), &req.headers);
|
||||
|
||||
let requested_retain_until = req.input.object_lock_retain_until_date.clone();
|
||||
set_requested_object_lock_retain_until(req, requested_retain_until.map(OffsetDateTime::from));
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?;
|
||||
|
||||
authorize_replication_only_put_headers(req).await?;
|
||||
|
||||
if legal_hold_write_requested(req.input.object_lock_legal_hold_status.as_ref()) {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?;
|
||||
}
|
||||
|
||||
if retention_write_requested(req.input.object_lock_mode.as_ref(), req.input.object_lock_retain_until_date.as_ref()) {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?;
|
||||
}
|
||||
|
||||
req.install_bucket_generation(bucket_generation?);
|
||||
let key = req.input.key.clone();
|
||||
let query_version = req.input.version_id.clone();
|
||||
let retain_until = req.input.object_lock_retain_until_date.clone().map(OffsetDateTime::from);
|
||||
let facts = CreateMultipartAuthorization {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
query_version: query_version.as_deref(),
|
||||
retain_until,
|
||||
legal_hold_present: req.input.object_lock_legal_hold_status.is_some(),
|
||||
retention_present: req.input.object_lock_mode.is_some() || retain_until.is_some(),
|
||||
};
|
||||
authorize_create_multipart_upload(self.server_ctx(), req, facts).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
|
||||
// The table snapshot reads the access module's private table-catalog state.
|
||||
use super::super::*;
|
||||
use crate::storage::options::destination_version_id;
|
||||
use rustfs_s3_types::S3Result;
|
||||
use rustfs_utils::http::SUFFIX_INCLUDE_DELETED;
|
||||
|
||||
@@ -73,6 +74,43 @@ pub(crate) async fn authorize_list_multipart_uploads<R: AuthorizationRequest + ?
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decoded request facts used by both multipart initiation access hooks.
|
||||
pub(crate) struct CreateMultipartAuthorization<'a> {
|
||||
pub(crate) bucket: &'a str,
|
||||
pub(crate) key: &'a str,
|
||||
pub(crate) query_version: Option<&'a str>,
|
||||
pub(crate) retain_until: Option<OffsetDateTime>,
|
||||
pub(crate) legal_hold_present: bool,
|
||||
pub(crate) retention_present: bool,
|
||||
}
|
||||
|
||||
/// Binds authorization to the requested object and captures its bucket incarnation before
|
||||
/// checking write, replication and Object Lock permissions.
|
||||
pub(crate) async fn authorize_create_multipart_upload<R: AuthorizationRequest + ?Sized>(
|
||||
server_ctx: &ServerContextSlot,
|
||||
req: &mut R,
|
||||
facts: CreateMultipartAuthorization<'_>,
|
||||
) -> S3Result<()> {
|
||||
let bucket_generation = load_bucket_generation(server_ctx, req, facts.bucket).await;
|
||||
let version = destination_version_id(facts.query_version, req.headers());
|
||||
let req_info = req_info_mut(req)?;
|
||||
req_info.bucket = Some(facts.bucket.to_owned());
|
||||
req_info.object = Some(facts.key.to_owned());
|
||||
req_info.version_id = version;
|
||||
|
||||
set_requested_object_lock_retain_until(req, facts.retain_until);
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?;
|
||||
authorize_replication_only_put_headers(req).await?;
|
||||
if facts.legal_hold_present {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?;
|
||||
}
|
||||
if facts.retention_present {
|
||||
authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?;
|
||||
}
|
||||
req.install_bucket_generation(bucket_generation?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The authorization of ListObjectsV2: the object listing's, with the bucket generation an
|
||||
/// on-demand migration merge binds to captured before the decision and handed on after an allow.
|
||||
///
|
||||
@@ -231,3 +269,7 @@ async fn prepare_table_data_plane_list_access<R: AuthorizationRequest + ?Sized>(
|
||||
req.install_table_list_access(access);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "listing_multipart_test.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Multipart initiation policy decisions take precedence over generation-read failures.
|
||||
|
||||
use super::{CreateMultipartAuthorization, authorize_create_multipart_upload};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::runtime_sources::{AppContext, ServerContextSlot};
|
||||
use crate::storage::access::tests::{TestKms, UnreadyIam, build_request, ensure_req_info};
|
||||
use crate::storage::access::{
|
||||
AuthorizationRequest, BucketGenerationGuard, CreateMultipartUploadInput, FS, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
S3Access as _,
|
||||
};
|
||||
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use http::{HeaderValue, Method};
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn create_multipart_authorization_surfaces_generation_errors_after_policy_checks() {
|
||||
crate::app::gating_test_env::run_large_stack_test("create-multipart-authorization-order", || async {
|
||||
const ACTIONS: &[&str] = &[
|
||||
"s3:PutObject",
|
||||
"s3:ReplicateObject",
|
||||
"s3:PutObjectLegalHold",
|
||||
"s3:PutObjectRetention",
|
||||
];
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let slot = ServerContextSlot::new();
|
||||
let fs = FS::with_server_ctx(slot.clone());
|
||||
assert!(slot.install(Arc::new(AppContext::new(store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)))));
|
||||
let bucket = format!("create-auth-order-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create test bucket");
|
||||
for excluded in ACTIONS.iter().copied().map(Some).chain([None]) {
|
||||
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[{
|
||||
"Effect":"Allow","Principal":{"AWS":"*"},
|
||||
"Action":ACTIONS.iter().copied().filter(|action| Some(*action) != excluded).collect::<Vec<_>>(),
|
||||
"Resource":format!("arn:aws:s3:::{bucket}/object")
|
||||
}]});
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&policy).unwrap(),
|
||||
)
|
||||
.await
|
||||
.expect("publish policy");
|
||||
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(1);
|
||||
let mut legacy = build_request(
|
||||
CreateMultipartUploadInput {
|
||||
bucket: bucket.clone(),
|
||||
key: "object".to_owned(),
|
||||
object_lock_legal_hold_status: Some(ObjectLockLegalHoldStatus::from_static("OFF")),
|
||||
object_lock_mode: Some(ObjectLockMode::from_static("GOVERNANCE")),
|
||||
object_lock_retain_until_date: Some(retain_until.into()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::POST,
|
||||
);
|
||||
legacy
|
||||
.headers
|
||||
.insert("x-rustfs-source-replication-request", HeaderValue::from_static("true"));
|
||||
legacy.extensions.insert(slot.clone());
|
||||
ensure_req_info(&mut legacy);
|
||||
legacy.install_bucket_generation(BucketGenerationGuard {
|
||||
bucket: "other-bucket".to_owned(),
|
||||
incarnation_id: uuid::Uuid::new_v4(),
|
||||
});
|
||||
let mut owned = RequestEnvelope::from_s3s(&legacy);
|
||||
let legacy = fs
|
||||
.create_multipart_upload(&mut legacy)
|
||||
.await
|
||||
.expect_err("policy or generation refusal");
|
||||
let owned = authorize_create_multipart_upload(
|
||||
&slot,
|
||||
&mut owned,
|
||||
CreateMultipartAuthorization {
|
||||
bucket: &bucket,
|
||||
key: "object",
|
||||
query_version: None,
|
||||
retain_until: Some(retain_until),
|
||||
legal_hold_present: true,
|
||||
retention_present: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("policy or generation refusal");
|
||||
let expected = if excluded.is_some() { "AccessDenied" } else { "InternalError" };
|
||||
assert_eq!(legacy.code().as_str(), expected, "legacy {excluded:?}");
|
||||
assert_eq!(owned.code().as_str(), expected, "owned {excluded:?}");
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -16,8 +16,8 @@
|
||||
//! AbortMultipartUpload: batch B2 of the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. A migrated
|
||||
//! operation converts its request into the gateway's input, runs the owned use case and converts
|
||||
//! the answer back, then applies what the legacy encoder needs and the gateway's encoder does
|
||||
//! operation projects the request into owned values, runs the use case and converts the answer
|
||||
//! back, then applies what the legacy encoder needs and the gateway's encoder does
|
||||
//! itself: `encoding-type=url` (legacy RustFS's members and per-segment rule, the one the gateway
|
||||
//! applies under its RustFS profile) and XML escaping of entity tags. The batch that moves these
|
||||
//! operations onto the gateway owns this file and is the only one that edits it.
|
||||
@@ -25,7 +25,9 @@
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api::bucket::encode_list_output_value;
|
||||
use crate::storage::s3_api::{self, mark_on_demand_migration_list_local_only};
|
||||
use crate::storage::s3_api::{
|
||||
self, CreateMultipartUploadFields, DefaultMultipartUsecase, mark_on_demand_migration_list_local_only,
|
||||
};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::{
|
||||
list_multipart_uploads, list_object_versions, list_objects, list_objects_v2,
|
||||
};
|
||||
@@ -33,6 +35,51 @@ use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl CreateMultipartUploadFields {
|
||||
pub(crate) fn from_legacy(input: CreateMultipartUploadInput) -> Self {
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
metadata: input.metadata,
|
||||
tagging: input.tagging,
|
||||
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
|
||||
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
ssekms_key_id: input.ssekms_key_id,
|
||||
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
|
||||
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
|
||||
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
pub(crate) async fn execute_create_multipart_upload_from_s3s(
|
||||
&self,
|
||||
mut req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let query_version = req.input.version_id.take();
|
||||
let output = self
|
||||
.execute_create_multipart_upload(envelope, CreateMultipartUploadFields::from_legacy(req.input), query_version)
|
||||
.await?;
|
||||
Ok(S3Response::new(CreateMultipartUploadOutput {
|
||||
bucket: Some(output.bucket),
|
||||
key: Some(output.key),
|
||||
upload_id: Some(output.upload_id),
|
||||
server_side_encryption: output.server_side_encryption.map(Into::into),
|
||||
sse_customer_algorithm: output.sse_customer_algorithm,
|
||||
sse_customer_key_md5: output.sse_customer_key_md5,
|
||||
ssekms_key_id: output.ssekms_key_id,
|
||||
checksum_algorithm: output.checksum_algorithm.map(Into::into),
|
||||
checksum_type: output.checksum_type.map(Into::into),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "abort_multipart_upload", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn abort_multipart_upload_edge(
|
||||
@@ -49,8 +96,10 @@ impl FS {
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
crate::hp_guard!("S3::create_multipart_upload");
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
usecase.execute_create_multipart_upload(req).await
|
||||
record_s3_op(S3Operation::CreateMultipartUpload);
|
||||
s3_api::multipart_usecase_for(self)
|
||||
.execute_create_multipart_upload_from_s3s(req)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn list_multipart_uploads_edge(
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
pub(crate) use crate::app::bucket_usecase::{PreparedLifecycleConfiguration, validate_lifecycle_rule_status};
|
||||
pub(crate) use crate::app::multipart_usecase::{
|
||||
DefaultMultipartUsecase, UploadPartTarget, UploadPartWriteInput, UploadPartWriteOutput,
|
||||
CreateMultipartUploadFields, DefaultMultipartUsecase, UploadPartTarget, UploadPartWriteInput, UploadPartWriteOutput,
|
||||
};
|
||||
pub(crate) use crate::app::object::reject_veeam_sosapi_mutation;
|
||||
pub(crate) use crate::app::object::shared::mark_on_demand_migration_list_local_only;
|
||||
|
||||
@@ -25,7 +25,17 @@ use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_storage_api::{MultipartOperations as _, ObjectIO as _};
|
||||
|
||||
mod create;
|
||||
|
||||
pub(crate) const MATRIX: &[MatrixRow] = &[
|
||||
MatrixRow {
|
||||
operation: "CreateMultipartUpload",
|
||||
metric: S3Operation::CreateMultipartUpload,
|
||||
has_error_body: true,
|
||||
call: |client, bucket| {
|
||||
Box::pin(async move { outcome(client.create_multipart_upload().bucket(bucket).key("matrix").send().await) })
|
||||
},
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "ListObjectsV2",
|
||||
metric: S3Operation::ListObjectsV2,
|
||||
|
||||
@@ -0,0 +1,764 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
|
||||
//! CreateMultipartUpload wire replies and durable session facts.
|
||||
|
||||
use super::super::{GATEWAY, LEGACY, create_bucket_in_the_store, s3_client, with_stack};
|
||||
use aws_sdk_s3::config::interceptors::AfterDeserializationInterceptorContextRef;
|
||||
use aws_sdk_s3::config::{ConfigBag, Intercept, RuntimeComponents};
|
||||
use aws_sdk_s3::error::{BoxError, ProvideErrorMetadata};
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::RustFSServer;
|
||||
use rustfs_storage_api::MultipartOperations as _;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const BUCKET: &str = "create-multipart-wire";
|
||||
const LOCK_BUCKET: &str = "create-multipart-locked";
|
||||
const RESERVED_BUCKET: &str = "sthree-multipart";
|
||||
const STORAGE_CLASS: &str = "x-amz-storage-class";
|
||||
const LEGAL_HOLD: &str = "x-amz-object-lock-legal-hold";
|
||||
const LOCK_MODE: &str = "x-amz-object-lock-mode";
|
||||
const RETAIN_UNTIL: &str = "x-amz-object-lock-retain-until-date";
|
||||
const DATE: &str = "2030-01-01T00:00:00.123456789Z";
|
||||
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
bucket: &'static str,
|
||||
key: &'static str,
|
||||
headers: &'static [(&'static str, &'static str)],
|
||||
}
|
||||
|
||||
const CASES: &[Case] = &[
|
||||
Case {
|
||||
name: "plain",
|
||||
bucket: BUCKET,
|
||||
key: "plain",
|
||||
headers: &[
|
||||
(STORAGE_CLASS, "STANDARD"),
|
||||
("content-type", "application/x-multipart-fixture"),
|
||||
("x-amz-meta-content-type", "user-content-type"),
|
||||
("x-amz-tagging", "project=rustfs"),
|
||||
("x-amz-checksum-algorithm", "CRC64NVME"),
|
||||
("x-amz-checksum-type", "FULL_OBJECT"),
|
||||
],
|
||||
},
|
||||
Case {
|
||||
name: "reserved",
|
||||
bucket: RESERVED_BUCKET,
|
||||
key: "a%252Fb%20%2B%26%3C%E9%9B%AA",
|
||||
headers: &[("x-amz-acl", "unknown-acl"), ("x-amz-request-payer", "unknown-payer")],
|
||||
},
|
||||
Case {
|
||||
name: "backslash",
|
||||
bucket: BUCKET,
|
||||
key: "a%5Cb",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "tab",
|
||||
bucket: BUCKET,
|
||||
key: "a%09%20b",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "double-slash",
|
||||
bucket: BUCKET,
|
||||
key: "dir//x",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "dot-dot",
|
||||
bucket: BUCKET,
|
||||
key: "dir/../x",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "internal-name",
|
||||
bucket: BUCKET,
|
||||
key: "xl.meta",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "control",
|
||||
bucket: BUCKET,
|
||||
key: "a%01b",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "empty",
|
||||
bucket: BUCKET,
|
||||
key: "",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "nul",
|
||||
bucket: BUCKET,
|
||||
key: "a%00b",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class")],
|
||||
},
|
||||
Case {
|
||||
name: "unknown-sse",
|
||||
bucket: BUCKET,
|
||||
key: "sse",
|
||||
headers: &[("x-amz-server-side-encryption", "unknown-sse")],
|
||||
},
|
||||
Case {
|
||||
name: "unlocked",
|
||||
bucket: BUCKET,
|
||||
key: "lock",
|
||||
headers: &[(LEGAL_HOLD, "unknown-hold")],
|
||||
},
|
||||
Case {
|
||||
name: "lock-mode-only",
|
||||
bucket: LOCK_BUCKET,
|
||||
key: "missing-date",
|
||||
headers: &[(LOCK_MODE, "unknown-mode")],
|
||||
},
|
||||
Case {
|
||||
name: "lock-mode-with-date",
|
||||
bucket: LOCK_BUCKET,
|
||||
key: "invalid-mode",
|
||||
headers: &[(LOCK_MODE, "unknown-mode"), (RETAIN_UNTIL, DATE)],
|
||||
},
|
||||
Case {
|
||||
name: "hold-with-date",
|
||||
bucket: LOCK_BUCKET,
|
||||
key: "invalid-hold",
|
||||
headers: &[(LEGAL_HOLD, "unknown-hold"), (LOCK_MODE, "GOVERNANCE"), (RETAIN_UNTIL, DATE)],
|
||||
},
|
||||
Case {
|
||||
name: "class-before-lock",
|
||||
bucket: LOCK_BUCKET,
|
||||
key: "invalid-class",
|
||||
headers: &[(STORAGE_CLASS, "unknown-class"), (LEGAL_HOLD, "unknown-hold")],
|
||||
},
|
||||
Case {
|
||||
name: "utc-date",
|
||||
bucket: LOCK_BUCKET,
|
||||
key: "utc-date",
|
||||
headers: &[(LOCK_MODE, "GOVERNANCE"), (RETAIN_UNTIL, DATE)],
|
||||
},
|
||||
Case {
|
||||
name: "offset-date",
|
||||
bucket: LOCK_BUCKET,
|
||||
key: "offset-date",
|
||||
headers: &[
|
||||
(LOCK_MODE, "GOVERNANCE"),
|
||||
(RETAIN_UNTIL, "2030-01-01T08:00:00.123456789+08:00"),
|
||||
],
|
||||
},
|
||||
Case {
|
||||
name: "invalid-checksum",
|
||||
bucket: BUCKET,
|
||||
key: "invalid-checksum",
|
||||
headers: &[("x-amz-checksum-algorithm", "SHA256"), ("x-amz-checksum-type", "FULL_OBJECT")],
|
||||
},
|
||||
];
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct SuccessWire(Arc<Mutex<Option<(u16, String)>>>);
|
||||
|
||||
impl Intercept for SuccessWire {
|
||||
fn name(&self) -> &'static str {
|
||||
"create-multipart-response"
|
||||
}
|
||||
|
||||
fn read_after_deserialization(
|
||||
&self,
|
||||
context: &AfterDeserializationInterceptorContextRef<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_config: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let response = context.response();
|
||||
let status = response.status().as_u16();
|
||||
if (200..300).contains(&status) {
|
||||
assert!(response.headers().get("x-amz-server-side-encryption-customer-key").is_none());
|
||||
let xml = std::str::from_utf8(response.body().bytes().expect("buffered CreateMultipartUpload XML"))?;
|
||||
*self.0.lock().expect("wire capture") = Some((status, xml.to_owned()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_refusal(name: &str, legacy: bool) -> Option<(u16, &'static str, &'static str)> {
|
||||
const STORAGE: (u16, &str, &str) = (400, "InvalidStorageClass", "The storage class you specified is not valid.");
|
||||
const LOCK_XML: (u16, &str, &str) = (
|
||||
400,
|
||||
"MalformedXML",
|
||||
"The XML you provided was not well-formed or did not validate against our published schema",
|
||||
);
|
||||
Some(match name {
|
||||
"plain" | "reserved" | "utc-date" | "offset-date" => return None,
|
||||
"backslash" | "tab" | "class-before-lock" => STORAGE,
|
||||
"double-slash" | "dot-dot" => (400, "InvalidArgument", "Invalid object key"),
|
||||
"internal-name" => (400, "InvalidArgument", "Object key contains a reserved storage filename"),
|
||||
"control" => (400, "InvalidArgument", "Object key contains characters unsupported by XML listings"),
|
||||
"empty" => (501, "NotImplemented", "Unknown operation"),
|
||||
"nul" if !legacy => (400, "InvalidArgument", "the name contains a NUL byte"),
|
||||
"nul" => (400, "InvalidArgument", "Object key contains invalid control characters: \"a\\0b\""),
|
||||
"unknown-sse" if !legacy => (
|
||||
400,
|
||||
"InvalidArgument",
|
||||
"x-amz-server-side-encryption must be one of AES256, aws:fsx, aws:kms or aws:kms:dsse",
|
||||
),
|
||||
"unknown-sse" => (
|
||||
400,
|
||||
"InvalidArgument",
|
||||
"The SSE algorithm specified is not supported. The valid values are AES256 or aws:kms.",
|
||||
),
|
||||
"unlocked" => (400, "InvalidRequest", "Bucket is missing ObjectLockConfiguration"),
|
||||
"lock-mode-only" => (
|
||||
400,
|
||||
"InvalidRequest",
|
||||
"x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
|
||||
),
|
||||
"lock-mode-with-date" | "hold-with-date" => LOCK_XML,
|
||||
"invalid-checksum" => (400, "InvalidArgument", "Invalid checksum type"),
|
||||
other => panic!("no measured answer for {other}"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn create_multipart_wire(server: RustFSServer, legacy: bool) {
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
for bucket in [BUCKET, RESERVED_BUCKET] {
|
||||
create_bucket_in_the_store(bucket).await;
|
||||
}
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(LOCK_BUCKET)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await
|
||||
.expect("create lock bucket");
|
||||
let context = get_global_app_context().expect("embedded context");
|
||||
let store = context.object_store();
|
||||
let mut expected = Vec::new();
|
||||
let mut observed = Vec::new();
|
||||
for case in CASES {
|
||||
if !legacy && case.name == "empty" {
|
||||
// The empty path does not dispatch CreateMultipartUpload.
|
||||
continue;
|
||||
}
|
||||
let uploads = || store.list_multipart_uploads(case.bucket, "", None, None, None, 100);
|
||||
let before = uploads().await.expect("list before request").uploads.len();
|
||||
let wire = SuccessWire::default();
|
||||
let result = client
|
||||
.create_multipart_upload()
|
||||
.bucket(case.bucket)
|
||||
.key("fixture")
|
||||
.customize()
|
||||
.interceptor(wire.clone())
|
||||
.mutate_request(move |request| {
|
||||
let (prefix, _) = request.uri().rsplit_once("/fixture").expect("SDK fixture path");
|
||||
request
|
||||
.set_uri(format!("{prefix}/{}?uploads", case.key))
|
||||
.expect("fixture request target");
|
||||
for (name, value) in case.headers {
|
||||
request.headers_mut().insert(*name, *value);
|
||||
}
|
||||
})
|
||||
.send()
|
||||
.await;
|
||||
let after = uploads().await.expect("list after request").uploads.len();
|
||||
expected.push((
|
||||
case.name,
|
||||
expected_refusal(case.name, legacy)
|
||||
.map(|(status, code, message)| (status, Some(code.to_owned()), Some(message.to_owned()))),
|
||||
));
|
||||
let output = match result {
|
||||
Ok(output) => {
|
||||
observed.push((case.name, None));
|
||||
output
|
||||
}
|
||||
Err(error) => {
|
||||
assert_eq!(after, before, "{}", case.name);
|
||||
let response = error.raw_response().expect("wire response");
|
||||
observed.push((
|
||||
case.name,
|
||||
Some((
|
||||
response.status().as_u16(),
|
||||
error.code().map(str::to_owned),
|
||||
error.message().map(str::to_owned),
|
||||
)),
|
||||
));
|
||||
for name in ["x-amz-request-id", "x-request-id"] {
|
||||
assert!(
|
||||
response.headers().get(name).is_some_and(|value| !value.is_empty()),
|
||||
"{name}: {}",
|
||||
case.name
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let (status, xml) = wire.0.lock().expect("wire capture").take().expect("success XML");
|
||||
assert_eq!(status, 200);
|
||||
assert!(xml.contains("<InitiateMultipartUploadResult") && xml.contains("</InitiateMultipartUploadResult>"));
|
||||
let key = output.key().expect("response key");
|
||||
let upload_id = output.upload_id().filter(|id| !id.is_empty()).expect("nonempty upload ID");
|
||||
assert_eq!(output.bucket(), Some(case.bucket));
|
||||
assert!(xml.contains(&format!("<UploadId>{upload_id}</UploadId>")));
|
||||
assert_eq!(after, before + 1, "{}", case.name);
|
||||
let info = store
|
||||
.get_multipart_info(case.bucket, key, upload_id, &Default::default())
|
||||
.await
|
||||
.expect("get durable upload metadata");
|
||||
let metadata = |name| info.user_defined.get(name).map(String::as_str);
|
||||
match case.name {
|
||||
"plain" => {
|
||||
assert_eq!(key, "plain");
|
||||
assert_eq!(metadata("content-type"), Some("application/x-multipart-fixture"));
|
||||
assert_eq!(metadata("x-amz-meta-content-type"), Some("user-content-type"));
|
||||
assert_eq!(metadata("x-amz-tagging"), Some("project=rustfs"));
|
||||
assert_eq!(metadata("x-rustfs-multipart-checksum"), Some("CRC64NVME"));
|
||||
assert_eq!(metadata("x-rustfs-multipart-checksum-type"), Some("FULL_OBJECT"));
|
||||
assert_eq!(output.checksum_algorithm().map(|value| value.as_str()), Some("CRC64NVME"));
|
||||
assert_eq!(output.checksum_type().map(|value| value.as_str()), Some("FULL_OBJECT"));
|
||||
}
|
||||
"reserved" => {
|
||||
assert_eq!(key, "a%2Fb +&<雪");
|
||||
assert_eq!(metadata("content-type"), Some("binary/octet-stream"));
|
||||
assert_eq!(output.checksum_algorithm(), None);
|
||||
}
|
||||
"utc-date" | "offset-date" => {
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
assert_eq!(key, case.key);
|
||||
assert_eq!(metadata(LOCK_MODE), Some("GOVERNANCE"));
|
||||
let date = metadata(RETAIN_UNTIL).expect("durable retention date");
|
||||
let requested = case.headers.iter().find(|(name, _)| *name == RETAIN_UNTIL).unwrap().1;
|
||||
if legacy {
|
||||
assert_eq!(date, requested);
|
||||
}
|
||||
let nanos = |value| OffsetDateTime::parse(value, &Rfc3339).unwrap().unix_timestamp_nanos();
|
||||
assert_eq!(nanos(date), nanos(requested));
|
||||
}
|
||||
other if expected_refusal(other, legacy).is_some() => {}
|
||||
other => panic!("no durable assertion for {other}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(observed, expected);
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_baseline_legacy_wire() {
|
||||
with_stack(LEGACY, |server| create_multipart_wire(server, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_wire_gateway() {
|
||||
with_stack(GATEWAY, |server| create_multipart_wire(server, false));
|
||||
}
|
||||
|
||||
async fn create_multipart_authorization(server: RustFSServer) {
|
||||
use super::super::{USER_SECRET_KEY, create_user_without_a_policy};
|
||||
use aws_sdk_s3::Client;
|
||||
use rustfs_iam::store::UserType;
|
||||
use rustfs_policy::policy::Policy;
|
||||
use serde_json::json;
|
||||
|
||||
const PUT_ONLY: &str = "multipart-put-only";
|
||||
const FULL_ACCESS: &str = "multipart-full-access";
|
||||
const NO_POLICY: &str = "multipart-no-policy";
|
||||
const TARGET: &str = "target";
|
||||
const VERSION: &str = "0194e0f1-0000-7000-8000-000000000000";
|
||||
const REFUSED_VERSION: &str = "0194e0f1-0000-7000-8000-000000000001";
|
||||
const REPLICATION: &str = "x-rustfs-source-replication-request";
|
||||
|
||||
let endpoint = server.endpoint();
|
||||
let root = s3_client(&endpoint, server.access_key(), server.secret_key());
|
||||
root.create_bucket()
|
||||
.bucket(LOCK_BUCKET)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await
|
||||
.expect("create the Object Lock bucket");
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
|
||||
let context = get_global_app_context().expect("embedded context");
|
||||
let store = context.object_store();
|
||||
let iam = context.iam().handle();
|
||||
for (user, actions) in [
|
||||
(PUT_ONLY, &["s3:PutObject"][..]),
|
||||
(
|
||||
FULL_ACCESS,
|
||||
&[
|
||||
"s3:PutObject",
|
||||
"s3:ReplicateObject",
|
||||
"s3:PutObjectLegalHold",
|
||||
"s3:PutObjectRetention",
|
||||
][..],
|
||||
),
|
||||
] {
|
||||
create_user_without_a_policy(user).await;
|
||||
let resource = format!("arn:aws:s3:::{LOCK_BUCKET}/{TARGET}");
|
||||
let policy = json!({"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":actions,"Resource":resource},
|
||||
{"Effect":"Deny","Action":["s3:PutObject"],"Resource":resource,
|
||||
"Condition":{"StringEquals":{"s3:VersionId":REFUSED_VERSION}}}
|
||||
]});
|
||||
let policy = Policy::parse_config(policy.to_string().as_bytes()).expect("valid test policy");
|
||||
iam.set_policy(user, policy).await.expect("store the test policy");
|
||||
iam.policy_db_set(user, UserType::Reg, false, user)
|
||||
.await
|
||||
.expect("attach the test policy");
|
||||
}
|
||||
create_user_without_a_policy(NO_POLICY).await;
|
||||
let put_only = s3_client(&endpoint, PUT_ONLY, USER_SECRET_KEY);
|
||||
let full = s3_client(&endpoint, FULL_ACCESS, USER_SECRET_KEY);
|
||||
let no_policy = s3_client(&endpoint, NO_POLICY, USER_SECRET_KEY);
|
||||
|
||||
struct Attempt<'a> {
|
||||
reason: &'static str,
|
||||
client: &'a Client,
|
||||
bucket: &'static str,
|
||||
key: &'static str,
|
||||
version: Option<&'static str>,
|
||||
headers: &'static [(&'static str, &'static str)],
|
||||
refusal: Option<(&'static str, u16)>,
|
||||
}
|
||||
|
||||
let denied = Some(("AccessDenied", 403));
|
||||
for row in [
|
||||
Attempt {
|
||||
reason: "no PutObject permission",
|
||||
client: &no_policy,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: None,
|
||||
headers: &[],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "wrong bucket",
|
||||
client: &put_only,
|
||||
bucket: BUCKET,
|
||||
key: TARGET,
|
||||
version: None,
|
||||
headers: &[],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "wrong key",
|
||||
client: &put_only,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: "other",
|
||||
version: None,
|
||||
headers: &[],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "no ReplicateObject permission",
|
||||
client: &put_only,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: None,
|
||||
headers: &[(REPLICATION, "true")],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "no PutObjectLegalHold permission",
|
||||
client: &put_only,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: None,
|
||||
headers: &[(LEGAL_HOLD, "OFF")],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "no PutObjectRetention permission",
|
||||
client: &put_only,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: None,
|
||||
headers: &[(LOCK_MODE, "GOVERNANCE"), (RETAIN_UNTIL, DATE)],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "query version overrides header on deny",
|
||||
client: &full,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: Some(REFUSED_VERSION),
|
||||
headers: &[("x-rustfs-source-version-id", VERSION)],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "MinIO header alias supplies the policy version",
|
||||
client: &full,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: None,
|
||||
headers: &[("x-minio-source-version-id", REFUSED_VERSION)],
|
||||
refusal: denied,
|
||||
},
|
||||
Attempt {
|
||||
reason: "valid scoped PutObject",
|
||||
client: &put_only,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: None,
|
||||
headers: &[],
|
||||
refusal: None,
|
||||
},
|
||||
Attempt {
|
||||
reason: "replication, legal hold and retention allowed",
|
||||
client: &full,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: Some(VERSION),
|
||||
headers: &[
|
||||
("x-minio-source-version-id", REFUSED_VERSION),
|
||||
(LEGAL_HOLD, "OFF"),
|
||||
(LOCK_MODE, "GOVERNANCE"),
|
||||
(RETAIN_UNTIL, DATE),
|
||||
],
|
||||
refusal: None,
|
||||
},
|
||||
Attempt {
|
||||
reason: "null query version overrides the header",
|
||||
client: &full,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: Some("null"),
|
||||
headers: &[("x-rustfs-source-version-id", REFUSED_VERSION)],
|
||||
refusal: None,
|
||||
},
|
||||
Attempt {
|
||||
reason: "a query version still needs replication authorization",
|
||||
client: &put_only,
|
||||
bucket: LOCK_BUCKET,
|
||||
key: TARGET,
|
||||
version: Some(VERSION),
|
||||
headers: &[],
|
||||
refusal: Some(("InvalidArgument", 400)),
|
||||
},
|
||||
] {
|
||||
let uploads = || store.list_multipart_uploads(row.bucket, "", None, None, None, 100);
|
||||
let before = uploads().await.expect("list before authorization").uploads.len();
|
||||
let result = row
|
||||
.client
|
||||
.create_multipart_upload()
|
||||
.bucket(row.bucket)
|
||||
.key(row.key)
|
||||
.customize()
|
||||
.mutate_request(move |request| {
|
||||
if let Some(version) = row.version {
|
||||
let uri = format!("{}&versionId={version}", request.uri());
|
||||
request.set_uri(uri).expect("CreateMultipartUpload with versionId");
|
||||
}
|
||||
for (name, value) in row.headers {
|
||||
request.headers_mut().insert(*name, *value);
|
||||
}
|
||||
})
|
||||
.send()
|
||||
.await;
|
||||
let after = uploads().await.expect("list after authorization").uploads.len();
|
||||
if let Some((code, status)) = row.refusal {
|
||||
let error = result.expect_err(row.reason);
|
||||
assert_eq!(super::super::refusal(&error), (Some(code.to_owned()), Some(status)), "{}", row.reason);
|
||||
assert_eq!(before, after, "{}", row.reason);
|
||||
} else {
|
||||
let output = result.expect(row.reason);
|
||||
assert_eq!(output.bucket(), Some(row.bucket), "{}", row.reason);
|
||||
assert_eq!(output.key(), Some(row.key), "{}", row.reason);
|
||||
assert_eq!(before + 1, after, "{}", row.reason);
|
||||
if row.version == Some(VERSION) {
|
||||
use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
|
||||
use rustfs_storage_api::CompletePart;
|
||||
|
||||
let upload_id = output.upload_id().expect("authorized upload ID");
|
||||
let options = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut body = PutObjReader::from_vec(b"part of the requested version".to_vec());
|
||||
let part = store
|
||||
.put_object_part(row.bucket, row.key, upload_id, 1, &mut body, &options)
|
||||
.await
|
||||
.expect("stage a part for the selected version");
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
row.bucket,
|
||||
row.key,
|
||||
upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&options,
|
||||
)
|
||||
.await
|
||||
.expect("publish the version persisted by CreateMultipartUpload");
|
||||
assert_eq!(completed.version_id.expect("versioned object").to_string(), VERSION);
|
||||
}
|
||||
}
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_authorization_legacy() {
|
||||
with_stack(LEGACY, create_multipart_authorization);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_authorization_gateway() {
|
||||
with_stack(GATEWAY, create_multipart_authorization);
|
||||
}
|
||||
|
||||
async fn create_multipart_encryption(server: RustFSServer, legacy: bool) {
|
||||
use rustfs_ecstore::api::bucket::{metadata::BUCKET_ETAG_CONFIG, metadata_sys};
|
||||
use rustfs_utils::http::{SUFFIX_ETAG_ALGORITHM, SUFFIX_PLAINTEXT_CHECKSUM, get_consistent_str};
|
||||
|
||||
let bucket = "create-multipart-encryption";
|
||||
create_bucket_in_the_store(bucket).await;
|
||||
metadata_sys::update(bucket, BUCKET_ETAG_CONFIG, br#"{"algorithm":"xxh3"}"#.to_vec())
|
||||
.await
|
||||
.expect("configure bucket XXH3 ETags");
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
let context = get_global_app_context().expect("embedded context");
|
||||
let store = context.object_store();
|
||||
let kms = context.kms().handle();
|
||||
kms.reconfigure(rustfs_kms::config::KmsConfig::static_kms(
|
||||
"multipart-test-key".to_owned(),
|
||||
base64_simd::STANDARD.encode_to_string([1_u8; 32]),
|
||||
))
|
||||
.await
|
||||
.expect("configure test KMS");
|
||||
|
||||
let create = |key: &'static str| {
|
||||
client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.checksum_algorithm(aws_sdk_s3::types::ChecksumAlgorithm::Crc64Nvme)
|
||||
.checksum_type(aws_sdk_s3::types::ChecksumType::FullObject)
|
||||
};
|
||||
let count = || {
|
||||
rustfs_io_metrics::s3_op_metrics_snapshot()
|
||||
.into_iter()
|
||||
.find(|row| row.op == rustfs_s3_ops::S3Operation::CreateMultipartUpload.as_str())
|
||||
.map_or(0, |row| row.total)
|
||||
};
|
||||
let customer_key = base64_simd::STANDARD.encode_to_string([0_u8; 32]);
|
||||
let customer_md5 = "cLyPS3KoaSFGi/joRB3OUQ==";
|
||||
let ssec = |md5: &'static str| {
|
||||
create("sse-c")
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(customer_key.clone())
|
||||
.sse_customer_key_md5(md5)
|
||||
};
|
||||
let uploads = || store.list_multipart_uploads(bucket, "", None, None, None, 100);
|
||||
let supplied_md5 = "1B2M2Y8AsgTpgAmY7PhCfg==";
|
||||
if legacy && !cfg!(feature = "rio-v2") {
|
||||
let before = count();
|
||||
let output = ssec(supplied_md5).send().await.expect("initiate with the supplied digest");
|
||||
let upload_id = output.upload_id().expect("upload ID for the supplied digest");
|
||||
let stored = uploads().await.expect("list the initiated upload").uploads;
|
||||
assert_eq!(stored.len(), 1);
|
||||
assert_eq!(stored[0].upload_id, upload_id);
|
||||
assert_eq!(stored[0].object, "sse-c");
|
||||
assert_eq!(output.sse_customer_key_md5(), Some(supplied_md5));
|
||||
assert_eq!(count(), before + 1);
|
||||
store
|
||||
.abort_multipart_upload(bucket, "sse-c", upload_id, &Default::default())
|
||||
.await
|
||||
.expect("remove the supplied-digest fixture");
|
||||
} else if !legacy {
|
||||
let before = count();
|
||||
let response = ssec(supplied_md5).send().await;
|
||||
assert!(uploads().await.expect("list after a refused digest").uploads.is_empty());
|
||||
let error = response.expect_err("the gateway refuses a nonmatching customer-key digest");
|
||||
assert_eq!(super::super::refusal(&error), (Some("InvalidArgument".to_owned()), Some(400)));
|
||||
assert_eq!(
|
||||
error.message(),
|
||||
Some("the customer-provided encryption key and its MD5 must be base64 of 32 and 16 bytes and must agree")
|
||||
);
|
||||
assert_eq!(count(), before);
|
||||
}
|
||||
|
||||
let before = count();
|
||||
let response = ssec(customer_md5)
|
||||
.server_side_encryption(aws_sdk_s3::types::ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await;
|
||||
assert!(
|
||||
uploads()
|
||||
.await
|
||||
.expect("list after conflicting encryption headers")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
let error = response.expect_err("an upload selects one encryption channel");
|
||||
assert_eq!(super::super::refusal(&error), (Some("InvalidArgument".to_owned()), Some(400)));
|
||||
let message = if legacy {
|
||||
"The SSE-C and managed server-side encryption headers cannot be used together."
|
||||
} else {
|
||||
"a request may name a server-managed encryption algorithm or a customer-provided key, not both"
|
||||
};
|
||||
assert_eq!(error.message(), Some(message));
|
||||
assert_eq!(count(), before + u64::from(legacy));
|
||||
|
||||
let before = count();
|
||||
let customer = ssec(customer_md5)
|
||||
.customize()
|
||||
.interceptor(SuccessWire::default())
|
||||
.send()
|
||||
.await
|
||||
.expect("create an SSE-C upload");
|
||||
assert_eq!(count(), before + 1);
|
||||
assert_eq!(customer.sse_customer_algorithm(), Some("AES256"));
|
||||
assert_eq!(customer.sse_customer_key_md5(), Some(customer_md5));
|
||||
let before = count();
|
||||
let managed = create("sse-kms")
|
||||
.server_side_encryption(aws_sdk_s3::types::ServerSideEncryption::AwsKms)
|
||||
.ssekms_key_id("multipart-test-key")
|
||||
.send()
|
||||
.await
|
||||
.expect("create an SSE-KMS upload");
|
||||
assert_eq!(count(), before + 1);
|
||||
assert_eq!(managed.server_side_encryption().map(|value| value.as_str()), Some("aws:kms"));
|
||||
assert_eq!(managed.ssekms_key_id(), Some("multipart-test-key"));
|
||||
|
||||
for output in [customer, managed] {
|
||||
assert_eq!(output.checksum_algorithm().map(|value| value.as_str()), Some("CRC64NVME"));
|
||||
assert_eq!(output.checksum_type().map(|value| value.as_str()), Some("FULL_OBJECT"));
|
||||
let info = store
|
||||
.get_multipart_info(
|
||||
bucket,
|
||||
output.key().expect("encrypted upload key"),
|
||||
output.upload_id().expect("encrypted upload ID"),
|
||||
&Default::default(),
|
||||
)
|
||||
.await
|
||||
.expect("read the durable encrypted upload");
|
||||
assert_eq!(get_consistent_str(&info.user_defined, SUFFIX_PLAINTEXT_CHECKSUM), Some("true"));
|
||||
assert_eq!(get_consistent_str(&info.user_defined, SUFFIX_ETAG_ALGORITHM), Some("xxh3"));
|
||||
}
|
||||
|
||||
kms.stop().await.expect("stop test KMS");
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_encryption_legacy() {
|
||||
with_stack(LEGACY, |server| create_multipart_encryption(server, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_encryption_gateway() {
|
||||
with_stack(GATEWAY, |server| create_multipart_encryption(server, false));
|
||||
}
|
||||
Reference in New Issue
Block a user