From f46ea6e14fa34377acf902b6faadf29acd5eb617 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 24 Jul 2026 19:11:36 +0800 Subject: [PATCH] fix(s3): enforce SSE-C copy key validation (#5185) --- .config/nextest.toml | 2 +- crates/e2e_test/src/kms/common.rs | 13 +- crates/e2e_test/src/kms/kms_local_test.rs | 3 + .../src/kms/multipart_encryption_test.rs | 13 +- crates/e2e_test/src/lib.rs | 3 + crates/e2e_test/src/ssec_copy_test.rs | 471 ++++++++++++++++++ crates/targets/src/net.rs | 32 +- docs/testing/e2e-suite-inventory.md | 5 +- rustfs/src/app/multipart_usecase.rs | 120 +++-- rustfs/src/app/object_usecase.rs | 19 +- rustfs/src/storage/sse.rs | 192 ++++--- 11 files changed, 749 insertions(+), 124 deletions(-) create mode 100644 crates/e2e_test/src/ssec_copy_test.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 087d90944..3aa472fec 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -197,7 +197,7 @@ test-group = 'ecstore-serial-flaky' [profile.e2e-smoke] default-filter = """ package(e2e_test) & ( - test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools)_test::|^fake_s3_target::/) + test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools)_test::|^fake_s3_target::/) | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/) | test(/^reliant::lifecycle::/) | test(/^reliant::tiering::/) diff --git a/crates/e2e_test/src/kms/common.rs b/crates/e2e_test/src/kms/common.rs index 9e0e68a82..06aee4450 100644 --- a/crates/e2e_test/src/kms/common.rs +++ b/crates/e2e_test/src/kms/common.rs @@ -657,14 +657,19 @@ pub async fn test_multipart_upload_with_config( .build(); info!("🔗 Completing multipart upload"); - let complete_output = s3_client + let mut complete_request = s3_client .complete_multipart_upload() .bucket(bucket) .key(&config.object_key) .upload_id(upload_id) - .multipart_upload(completed_multipart_upload) - .send() - .await?; + .multipart_upload(completed_multipart_upload); + if let EncryptionType::SSEC { .. } = &config.encryption_type { + complete_request = complete_request + .sse_customer_algorithm("AES256") + .sse_customer_key(sse_c_key_b64.as_ref().unwrap()) + .sse_customer_key_md5(sse_c_key_md5.as_ref().unwrap()); + } + let complete_output = complete_request.send().await?; debug!("Multipart upload finalized with ETag {:?}", complete_output.e_tag()); diff --git a/crates/e2e_test/src/kms/kms_local_test.rs b/crates/e2e_test/src/kms/kms_local_test.rs index d3a24404b..82b44a154 100644 --- a/crates/e2e_test/src/kms/kms_local_test.rs +++ b/crates/e2e_test/src/kms/kms_local_test.rs @@ -625,6 +625,9 @@ async fn test_multipart_upload_with_sse_c( .bucket(bucket) .key(object_key) .upload_id(upload_id) + .sse_customer_algorithm("AES256") + .sse_customer_key(&key_b64) + .sse_customer_key_md5(&key_md5) .multipart_upload(completed_multipart_upload) .send() .await?; diff --git a/crates/e2e_test/src/kms/multipart_encryption_test.rs b/crates/e2e_test/src/kms/multipart_encryption_test.rs index 22ab6c8d5..401f97c5b 100644 --- a/crates/e2e_test/src/kms/multipart_encryption_test.rs +++ b/crates/e2e_test/src/kms/multipart_encryption_test.rs @@ -566,14 +566,19 @@ async fn test_multipart_encryption_type( .set_parts(Some(completed_parts)) .build(); - let _complete_output = s3_client + let mut complete_request = s3_client .complete_multipart_upload() .bucket(bucket) .key(object_key) .upload_id(upload_id) - .multipart_upload(completed_multipart_upload) - .send() - .await?; + .multipart_upload(completed_multipart_upload); + if matches!(encryption_type, EncryptionType::SSEC) { + complete_request = complete_request + .sse_customer_algorithm("AES256") + .sse_customer_key(sse_c_key.as_ref().unwrap()) + .sse_customer_key_md5(sse_c_md5.as_ref().unwrap()); + } + let _complete_output = complete_request.send().await?; // Download and verify let mut get_request = s3_client.get_object().bucket(bucket).key(object_key); diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index c279cd47b..43e126464 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -200,6 +200,9 @@ mod copy_object_version_restore_test; #[cfg(test)] mod copy_object_checksum_test; +#[cfg(test)] +mod ssec_copy_test; + #[cfg(test)] mod multipart_storage_class_test; diff --git a/crates/e2e_test/src/ssec_copy_test.rs b/crates/e2e_test/src/ssec_copy_test.rs new file mode 100644 index 000000000..cf3245b6c --- /dev/null +++ b/crates/e2e_test/src/ssec_copy_test.rs @@ -0,0 +1,471 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box SSE-C CopyObject and multipart-copy regression coverage (backlog#1467). + +use crate::common::{RustFSTestEnvironment, init_logging}; +use aws_sdk_s3::config::interceptors::{BeforeDeserializationInterceptorContextRef, BeforeTransmitInterceptorContextRef}; +use aws_sdk_s3::config::{ConfigBag, Credentials, Intercept, Region, RuntimeComponents}; +use aws_sdk_s3::error::BoxError; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration}; +use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; +use base64::Engine; +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +type TestResult = Result<(), Box>; + +const SSE_CUSTOMER_ALGORITHM_HEADER: &str = "x-amz-server-side-encryption-customer-algorithm"; +const SSE_CUSTOMER_KEY_MD5_HEADER: &str = "x-amz-server-side-encryption-customer-key-md5"; + +struct CustomerKey { + raw: String, + encoded: String, + md5: String, +} + +struct InvalidSsec<'a> { + algorithm: Option<&'a str>, + key: Option<&'a str>, + md5: Option<&'a str>, +} + +#[derive(Clone, Debug, Default)] +struct ResponseHeaderCapture { + headers: Arc>>, + abort_attempts: Arc, +} + +impl ResponseHeaderCapture { + fn snapshot(&self) -> Result, BoxError> { + self.headers + .lock() + .map(|headers| headers.clone()) + .map_err(|_| std::io::Error::other("response header capture mutex was poisoned").into()) + } + + fn abort_attempts(&self) -> usize { + self.abort_attempts.load(Ordering::SeqCst) + } +} + +impl Intercept for ResponseHeaderCapture { + fn name(&self) -> &'static str { + "ssec-copy-response-header-capture" + } + + fn read_before_deserialization( + &self, + context: &BeforeDeserializationInterceptorContextRef<'_>, + _runtime_components: &RuntimeComponents, + _cfg: &mut ConfigBag, + ) -> Result<(), BoxError> { + let mut captured = self + .headers + .lock() + .map_err(|_| std::io::Error::other("response header capture mutex was poisoned"))?; + captured.clear(); + for name in [SSE_CUSTOMER_ALGORITHM_HEADER, SSE_CUSTOMER_KEY_MD5_HEADER] { + if let Some(value) = context.response().headers().get(name) { + captured.insert(name.to_owned(), value.to_owned()); + } + } + Ok(()) + } + + fn read_before_transmit( + &self, + context: &BeforeTransmitInterceptorContextRef<'_>, + _runtime_components: &RuntimeComponents, + _cfg: &mut ConfigBag, + ) -> Result<(), BoxError> { + let request = context.request(); + if request.method() == "DELETE" && request.uri().contains("uploadId=") { + self.abort_attempts.fetch_add(1, Ordering::SeqCst); + } + Ok(()) + } +} + +fn customer_key(byte: u8) -> CustomerKey { + let raw = [byte; 32]; + CustomerKey { + raw: String::from_utf8_lossy(&raw).into_owned(), + encoded: base64::engine::general_purpose::STANDARD.encode(raw), + md5: base64::engine::general_purpose::STANDARD.encode(md5::compute(raw).0), + } +} + +fn assert_secret_absent(error: &str, keys: &[&CustomerKey]) { + for key in keys { + assert!(!error.contains(&key.raw), "error exposed a raw SSE-C key"); + assert!(!error.contains(&key.encoded), "error exposed an encoded SSE-C key"); + assert!(!error.contains(&key.md5), "error exposed an SSE-C key MD5"); + } +} + +fn invalid_ssec_cases<'a>(correct_key: &'a CustomerKey, wrong_key: &'a CustomerKey) -> [InvalidSsec<'a>; 5] { + [ + InvalidSsec { + algorithm: None, + key: Some(&correct_key.encoded), + md5: Some(&correct_key.md5), + }, + InvalidSsec { + algorithm: Some("AES256"), + key: None, + md5: Some(&correct_key.md5), + }, + InvalidSsec { + algorithm: Some("AES256"), + key: Some(&correct_key.encoded), + md5: None, + }, + InvalidSsec { + algorithm: Some("AES256"), + key: Some(&wrong_key.encoded), + md5: Some(&wrong_key.md5), + }, + InvalidSsec { + algorithm: Some("AES256"), + key: Some(&correct_key.encoded), + md5: Some(&wrong_key.md5), + }, + ] +} + +#[tokio::test] +async fn copy_object_rotates_ssec_key_and_drops_source_encryption_metadata() -> TestResult { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(Vec::new()).await?; + let client = env.create_s3_client(); + let bucket = "ssec-copy-object"; + let source = "source.bin"; + let plaintext_copy = "plaintext-copy.bin"; + let rotated_copy = "rotated-copy.bin"; + let source_key = customer_key(0x41); + let destination_key = customer_key(0x42); + let wrong_key = customer_key(0x43); + let body = b"backlog-1467 versioned SSE-C copy payload"; + + env.create_test_bucket(bucket).await?; + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + let put = client + .put_object() + .bucket(bucket) + .key(source) + .sse_customer_algorithm("AES256") + .sse_customer_key(&source_key.encoded) + .sse_customer_key_md5(&source_key.md5) + .body(ByteStream::from_static(body)) + .send() + .await?; + let source_version = put.version_id().ok_or("versioned PUT returned no version ID")?; + let copy_source = format!("{bucket}/{source}?versionId={source_version}"); + + let plaintext = client + .copy_object() + .bucket(bucket) + .key(plaintext_copy) + .copy_source(©_source) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(&source_key.encoded) + .copy_source_sse_customer_key_md5(&source_key.md5) + .send() + .await?; + assert_eq!(plaintext.copy_source_version_id(), Some(source_version)); + let plaintext_body = client + .get_object() + .bucket(bucket) + .key(plaintext_copy) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(plaintext_body.as_ref(), body); + + let rotated = client + .copy_object() + .bucket(bucket) + .key(rotated_copy) + .copy_source(©_source) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(&source_key.encoded) + .copy_source_sse_customer_key_md5(&source_key.md5) + .sse_customer_algorithm("AES256") + .sse_customer_key(&destination_key.encoded) + .sse_customer_key_md5(&destination_key.md5) + .send() + .await?; + assert_eq!(rotated.sse_customer_algorithm(), Some("AES256")); + assert_eq!(rotated.sse_customer_key_md5(), Some(destination_key.md5.as_str())); + + let wrong_key_error = client + .get_object() + .bucket(bucket) + .key(rotated_copy) + .sse_customer_algorithm("AES256") + .sse_customer_key(&source_key.encoded) + .sse_customer_key_md5(&source_key.md5) + .send() + .await + .expect_err("the source key must not read a copy encrypted with the destination key"); + assert_secret_absent(&format!("{wrong_key_error:?}"), &[&source_key, &destination_key]); + + let rotated_body = client + .get_object() + .bucket(bucket) + .key(rotated_copy) + .sse_customer_algorithm("AES256") + .sse_customer_key(&destination_key.encoded) + .sse_customer_key_md5(&destination_key.md5) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(rotated_body.as_ref(), body); + + for (case_index, case) in invalid_ssec_cases(&source_key, &wrong_key).iter().enumerate() { + let failed_target = format!("failed-copy-{case_index}.bin"); + let mut request = client + .copy_object() + .bucket(bucket) + .key(&failed_target) + .copy_source(©_source); + if let Some(algorithm) = case.algorithm { + request = request.copy_source_sse_customer_algorithm(algorithm); + } + if let Some(key) = case.key { + request = request.copy_source_sse_customer_key(key); + } + if let Some(md5) = case.md5 { + request = request.copy_source_sse_customer_key_md5(md5); + } + let error = request + .send() + .await + .expect_err("invalid source SSE-C parameters must reject CopyObject"); + assert_secret_absent(&format!("{error:?}"), &[&source_key, &wrong_key]); + assert!( + client.head_object().bucket(bucket).key(&failed_target).send().await.is_err(), + "a rejected CopyObject must not create its target" + ); + } + env.stop_server(); + Ok(()) +} + +#[tokio::test] +async fn multipart_copy_requires_keys_on_every_stage_and_abort_leaves_no_object() -> TestResult { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(Vec::new()).await?; + let response_headers = ResponseHeaderCapture::default(); + let credentials = Credentials::new(&env.access_key, &env.secret_key, None, None, "ssec-copy-e2e"); + let config = aws_sdk_s3::Config::builder() + .credentials_provider(credentials) + .region(Region::new("us-east-1")) + .endpoint_url(&env.url) + .force_path_style(true) + .behavior_version_latest() + .http_client(SmithyHttpClientBuilder::new().build_http()) + .interceptor(response_headers.clone()) + .build(); + let client = aws_sdk_s3::Client::from_conf(config); + let bucket = "ssec-multipart-copy"; + let source = "source.bin"; + let destination = "destination.bin"; + let aborted_destination = "aborted.bin"; + let source_key = customer_key(0x51); + let destination_key = customer_key(0x52); + let wrong_key = customer_key(0x53); + let part_size = 5 * 1024 * 1024; + let body: Vec = (0..part_size * 2).map(|index| (index % 251) as u8).collect(); + + env.create_test_bucket(bucket).await?; + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + let source_put = client + .put_object() + .bucket(bucket) + .key(source) + .sse_customer_algorithm("AES256") + .sse_customer_key(&source_key.encoded) + .sse_customer_key_md5(&source_key.md5) + .body(ByteStream::from(body.clone())) + .send() + .await?; + let source_version = source_put + .version_id() + .ok_or("versioned multipart-copy source returned no version ID")?; + + let create = client + .create_multipart_upload() + .bucket(bucket) + .key(destination) + .sse_customer_algorithm("AES256") + .sse_customer_key(&destination_key.encoded) + .sse_customer_key_md5(&destination_key.md5) + .send() + .await?; + assert_eq!(create.sse_customer_algorithm(), Some("AES256")); + assert_eq!(create.sse_customer_key_md5(), Some(destination_key.md5.as_str())); + let upload_id = create.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?; + let mut completed = Vec::new(); + for part_number in 1..=2 { + let first = (part_number - 1) * part_size; + let last = part_number * part_size - 1; + let copied = client + .upload_part_copy() + .bucket(bucket) + .key(destination) + .upload_id(upload_id) + .part_number(part_number) + .copy_source(format!("{bucket}/{source}")) + .copy_source_range(format!("bytes={first}-{last}")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(&source_key.encoded) + .copy_source_sse_customer_key_md5(&source_key.md5) + .sse_customer_algorithm("AES256") + .sse_customer_key(&destination_key.encoded) + .sse_customer_key_md5(&destination_key.md5) + .send() + .await?; + assert_eq!( + copied.copy_source_version_id(), + Some(source_version), + "UploadPartCopy must return the actual latest source version" + ); + let etag = copied + .copy_part_result() + .and_then(|result| result.e_tag()) + .ok_or("UploadPartCopy returned no ETag")?; + completed.push(CompletedPart::builder().part_number(part_number).e_tag(etag).build()); + } + client + .complete_multipart_upload() + .bucket(bucket) + .key(destination) + .upload_id(upload_id) + .sse_customer_algorithm("AES256") + .sse_customer_key(&destination_key.encoded) + .sse_customer_key_md5(&destination_key.md5) + .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build()) + .send() + .await?; + let completed_headers = response_headers.snapshot()?; + assert_eq!(completed_headers.get(SSE_CUSTOMER_ALGORITHM_HEADER).map(String::as_str), Some("AES256")); + assert_eq!( + completed_headers.get(SSE_CUSTOMER_KEY_MD5_HEADER).map(String::as_str), + Some(destination_key.md5.as_str()) + ); + let downloaded = client + .get_object() + .bucket(bucket) + .key(destination) + .sse_customer_algorithm("AES256") + .sse_customer_key(&destination_key.encoded) + .sse_customer_key_md5(&destination_key.md5) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(downloaded.as_ref(), body.as_slice()); + + for (case_index, case) in invalid_ssec_cases(&destination_key, &wrong_key).iter().enumerate() { + let failed_target = format!("{aborted_destination}-{case_index}"); + let failed_create = client + .create_multipart_upload() + .bucket(bucket) + .key(&failed_target) + .sse_customer_algorithm("AES256") + .sse_customer_key(&destination_key.encoded) + .sse_customer_key_md5(&destination_key.md5) + .send() + .await?; + let failed_upload_id = failed_create + .upload_id() + .ok_or("CreateMultipartUpload returned no upload ID")?; + let mut request = client + .upload_part_copy() + .bucket(bucket) + .key(&failed_target) + .upload_id(failed_upload_id) + .part_number(1) + .copy_source(format!("{bucket}/{source}")) + .copy_source_sse_customer_algorithm("AES256") + .copy_source_sse_customer_key(&source_key.encoded) + .copy_source_sse_customer_key_md5(&source_key.md5); + if let Some(algorithm) = case.algorithm { + request = request.sse_customer_algorithm(algorithm); + } + if let Some(key) = case.key { + request = request.sse_customer_key(key); + } + if let Some(md5) = case.md5 { + request = request.sse_customer_key_md5(md5); + } + let error = request + .send() + .await + .expect_err("invalid destination SSE-C parameters must reject UploadPartCopy"); + assert_secret_absent(&format!("{error:?}"), &[&source_key, &destination_key, &wrong_key]); + + let abort_attempts_before = response_headers.abort_attempts(); + client + .abort_multipart_upload() + .bucket(bucket) + .key(&failed_target) + .upload_id(failed_upload_id) + .send() + .await?; + assert_eq!( + response_headers.abort_attempts(), + abort_attempts_before + 1, + "each failed multipart copy must issue exactly one wire-level abort attempt" + ); + assert!( + client.head_object().bucket(bucket).key(&failed_target).send().await.is_err(), + "an aborted failed multipart copy must leave no completed object" + ); + } + env.stop_server(); + Ok(()) +} diff --git a/crates/targets/src/net.rs b/crates/targets/src/net.rs index cdeaad7af..812e78772 100644 --- a/crates/targets/src/net.rs +++ b/crates/targets/src/net.rs @@ -33,6 +33,10 @@ const SENSITIVE_HEADERS: &[&str] = &[ "authorization", "x-amz-security-token", "x-amz-content-sha256", + "x-amz-server-side-encryption-customer-key", + "x-amz-server-side-encryption-customer-key-md5", + "x-amz-copy-source-server-side-encryption-customer-key", + "x-amz-copy-source-server-side-encryption-customer-key-md5", "cookie", "set-cookie", ]; @@ -466,6 +470,19 @@ mod tests { headers.insert("authorization", HeaderValue::from_static("AWS4-HMAC-SHA256 Credential=AKIA.../secret")); headers.insert("x-amz-security-token", HeaderValue::from_static("FQoGZXIvYXdzE.../session-token")); headers.insert("x-amz-content-sha256", HeaderValue::from_static("e3b0c44298fc1c149afbf4c8996fb924")); + headers.insert("x-amz-server-side-encryption-customer-key", HeaderValue::from_static("destination-key")); + headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_static("destination-key-md5"), + ); + headers.insert( + "x-amz-copy-source-server-side-encryption-customer-key", + HeaderValue::from_static("source-key"), + ); + headers.insert( + "x-amz-copy-source-server-side-encryption-customer-key-md5", + HeaderValue::from_static("source-key-md5"), + ); headers.insert("cookie", HeaderValue::from_static("session=abc123")); headers.insert("content-type", HeaderValue::from_static("application/octet-stream")); headers.insert("user-agent", HeaderValue::from_static("aws-cli/2.0")); @@ -473,7 +490,16 @@ mod tests { let params = extract_params_header(&headers); // Sensitive headers keep their name for correlation but never leak the value. - for name in ["authorization", "x-amz-security-token", "x-amz-content-sha256", "cookie"] { + for name in [ + "authorization", + "x-amz-security-token", + "x-amz-content-sha256", + "x-amz-server-side-encryption-customer-key", + "x-amz-server-side-encryption-customer-key-md5", + "x-amz-copy-source-server-side-encryption-customer-key", + "x-amz-copy-source-server-side-encryption-customer-key-md5", + "cookie", + ] { assert_eq!(params.get(name).map(String::as_str), Some(REDACTED_SECRET), "{name} must be redacted"); } // Non-sensitive headers are preserved verbatim. @@ -486,6 +512,10 @@ mod tests { assert!(is_sensitive_header("Authorization")); assert!(is_sensitive_header("X-Amz-Security-Token")); assert!(is_sensitive_header("X-AMZ-CONTENT-SHA256")); + assert!(is_sensitive_header("X-Amz-Server-Side-Encryption-Customer-Key")); + assert!(is_sensitive_header("X-Amz-Server-Side-Encryption-Customer-Key-MD5")); + assert!(is_sensitive_header("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key")); + assert!(is_sensitive_header("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5")); assert!(!is_sensitive_header("content-type")); assert!(!is_sensitive_header("x-amz-request-id")); } diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index 97a74f5ff..e02d48774 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -72,9 +72,10 @@ | protocols | 16 | | | quota_test | 14 | | | reliability_disk_fault_test | 3 | | -| reliant | 10 | 4 ✅ | +| reliant | 10 | 5 ✅ | | replication_extension_test | 47 | 20 ✅ +27 🌙 | | security_boundary_test | 4 | | +| ssec_copy_test | 2 | ✅ | | server_startup_failfast_test | 1 | | | snowball_auto_extract_test | 6 | | | special_chars_test | 14 | ✅ | @@ -86,4 +87,4 @@ `notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above. -**Total listed: 483 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24. +**Total listed: 485 tests across 67 modules · PR smoke subset: 129 tests / 32 modules** (30 full modules + 5 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24. diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 4049d8e0b..5b1111717 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -67,7 +67,7 @@ use crate::error::ApiError; use crate::table_catalog; use bytes::Bytes; use futures::StreamExt; -use http::{HeaderMap, Uri}; +use http::{HeaderMap, HeaderValue, Uri}; use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; @@ -389,6 +389,9 @@ impl DefaultMultipartUsecase { upload_id, if_match, if_none_match, + sse_customer_algorithm, + sse_customer_key, + sse_customer_key_md5, .. } = input; @@ -486,6 +489,18 @@ impl DefaultMultipartUsecase { .get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default()) .await .map_err(ApiError::from)?; + EncryptionRequest { + bucket: &bucket, + key: &key, + server_side_encryption: None, + ssekms_key_id: None, + ssekms_context: None, + sse_customer_algorithm, + sse_customer_key, + sse_customer_key_md5, + content_size: 0, + } + .validate_multipart_ssec(&multipart_info.user_defined)?; let cache_adapter = self.object_data_cache(); let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; @@ -493,7 +508,6 @@ impl DefaultMultipartUsecase { .user_defined .get("x-amz-server-side-encryption") .map(|s| ServerSideEncryption::from(s.clone())); - let ssekms_key_id = match server_side_encryption.as_ref() { Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => multipart_info .user_defined @@ -602,6 +616,26 @@ impl DefaultMultipartUsecase { let mut response = S3Response::new(output); crate::app::object_usecase::inject_additional_checksum_headers(&mut response.headers, &complete_extra_checksum_headers); + if let Some(algorithm) = multipart_info + .user_defined + .get("x-amz-server-side-encryption-customer-algorithm") + { + let value = HeaderValue::from_str(algorithm) + .map_err(|_| s3_error!(InternalError, "Invalid stored SSE-C algorithm metadata"))?; + response + .headers + .insert("x-amz-server-side-encryption-customer-algorithm", value); + } + if let Some(key_md5) = multipart_info + .user_defined + .get("x-amz-server-side-encryption-customer-key-md5") + { + let value = + HeaderValue::from_str(key_md5).map_err(|_| s3_error!(InternalError, "Invalid stored SSE-C key metadata"))?; + response + .headers + .insert("x-amz-server-side-encryption-customer-key-md5", value); + } let result = Ok(response); let _ = helper.complete(&result); rustfs_scanner::record_dirty_usage_bucket(&bucket); @@ -624,6 +658,7 @@ impl DefaultMultipartUsecase { storage_class, server_side_encryption, sse_customer_algorithm, + sse_customer_key, sse_customer_key_md5, ssekms_key_id, object_lock_legal_hold_status, @@ -674,7 +709,11 @@ impl DefaultMultipartUsecase { metadata.extend(object_lock_metadata); } apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?; - let (_, sse_customer_key, _) = extract_ssec_params_from_headers(&req.headers)?; + let (header_sse_customer_algorithm, header_sse_customer_key, header_sse_customer_key_md5) = + extract_ssec_params_from_headers(&req.headers)?; + let sse_customer_algorithm = sse_customer_algorithm.or(header_sse_customer_algorithm); + let sse_customer_key = sse_customer_key.or(header_sse_customer_key); + let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5); let encryption_request = PrepareEncryptionRequest { bucket: &bucket, @@ -760,6 +799,7 @@ impl DefaultMultipartUsecase { upload_id: Some(upload_id), server_side_encryption: effective_sse, sse_customer_algorithm, + sse_customer_key_md5, ssekms_key_id: effective_kms_key_id, checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from), checksum_type: checksum_type.map(ChecksumType::from), @@ -874,10 +914,9 @@ impl DefaultMultipartUsecase { } opts.want_checksum = reader.checksum(); - let has_ssec = sse_customer_algorithm.is_some(); - // When SSE-C headers are present, skip managed-encryption metadata to avoid - // false conflict: the bucket default SSE config stored in multipart metadata - // should not block a legitimate SSE-C upload part. + let has_ssec = fi + .user_defined + .contains_key("x-amz-server-side-encryption-customer-algorithm"); let (server_side_encryption, ssekms_key_id) = if has_ssec { (None, None) } else { @@ -909,7 +948,7 @@ impl DefaultMultipartUsecase { sse_customer_key_md5: sse_customer_key_md5.clone(), content_size: actual_size, } - .check_upload_part_customer_key_md5(&fi.user_defined, sse_customer_key_md5.clone())?; + .validate_multipart_ssec(&fi.user_defined)?; let (requested_sse, requested_kms_key_id) = if has_ssec { let ssec_material = sse_decryption(DecryptionRequest { bucket: &bucket, @@ -1139,6 +1178,18 @@ impl DefaultMultipartUsecase { .get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default()) .await .map_err(ApiError::from)?; + EncryptionRequest { + bucket: &bucket, + key: &key, + server_side_encryption: None, + ssekms_key_id: None, + ssekms_context: None, + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key: sse_customer_key.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: 0, + } + .validate_multipart_ssec(&mp_info.user_defined)?; let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?; src_opts.version_id = src_version_id.clone(); @@ -1161,6 +1212,14 @@ impl DefaultMultipartUsecase { .map_err(map_get_object_reader_error)?; let src_info = src_reader.object_info; + let src_stream = src_reader.stream; + let resolved_src_version_id = src_info.version_id.map(|version_id| { + if version_id == Uuid::nil() { + "null".to_string() + } else { + version_id.to_string() + } + }); if let Some(if_match) = copy_source_if_match { if let Some(ref etag) = src_info.etag { @@ -1185,10 +1244,10 @@ impl DefaultMultipartUsecase { } let (_start_offset, length) = if let Some(ref range_spec) = rs { - let validation_size = match src_info.is_compressed_ok() { - Ok((_, true)) => src_info.get_actual_size().unwrap_or(src_info.size), - _ => src_info.size, - }; + // Copy-source ranges are expressed over the logical plaintext object. + // Encrypted (and compressed) objects have a larger or smaller physical + // representation, so validating against `size` rejects valid later parts. + let validation_size = src_info.get_actual_size().unwrap_or(src_info.size); validate_copy_source_range_not_exceeds(range_spec, validation_size)?; @@ -1199,24 +1258,6 @@ impl DefaultMultipartUsecase { (0, src_info.size) }; - let h = build_ssec_read_headers( - copy_source_sse_customer_algorithm.as_ref(), - copy_source_sse_customer_key.as_ref(), - copy_source_sse_customer_key_md5.as_ref(), - ); - let get_opts = ObjectOptions { - version_id: src_opts.version_id.clone(), - versioned: src_opts.versioned, - version_suspended: src_opts.version_suspended, - ..Default::default() - }; - - let src_reader = store - .get_object_reader(&src_bucket, &src_key, rs.clone(), h, &get_opts) - .await - .map_err(map_get_object_reader_error)?; - let src_stream = src_reader.stream; - let is_disk_compressed = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); @@ -1240,7 +1281,9 @@ impl DefaultMultipartUsecase { .map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}")))) }) .transpose()?; - let has_ssec = sse_customer_algorithm.is_some(); + let has_ssec = mp_info + .user_defined + .contains_key("x-amz-server-side-encryption-customer-algorithm"); let ssekms_key_id = match server_side_encryption.as_ref() { Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info .user_defined @@ -1248,19 +1291,6 @@ impl DefaultMultipartUsecase { .map(|s| s.to_string()), _ => None, }; - EncryptionRequest { - bucket: &bucket, - key: &key, - server_side_encryption: server_side_encryption.clone(), - ssekms_key_id: ssekms_key_id.clone(), - ssekms_context: None, - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key: sse_customer_key.clone(), - sse_customer_key_md5: sse_customer_key_md5.clone(), - content_size: actual_size, - } - .check_upload_part_customer_key_md5(&mp_info.user_defined, sse_customer_key_md5.clone())?; - let (requested_sse, requested_kms_key_id, dst_user_defined) = if has_ssec { let ssec_material = sse_decryption(DecryptionRequest { bucket: &bucket, @@ -1357,7 +1387,7 @@ impl DefaultMultipartUsecase { let output = UploadPartCopyOutput { copy_part_result: Some(copy_part_result), - copy_source_version_id: src_version_id, + copy_source_version_id: resolved_src_version_id, server_side_encryption: requested_sse, ssekms_key_id: requested_kms_key_id, sse_customer_algorithm, diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index e96200dc7..f352b513f 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -5831,6 +5831,17 @@ impl DefaultObjectUsecase { { return Err(s3_error!(InvalidStorageClass)); } + let ssekms_context = extract_ssekms_context_from_headers(&req.headers)?; + validate_sse_headers_for_write( + requested_sse.as_ref(), + requested_kms_key_id.as_ref(), + ssekms_context.as_ref(), + sse_customer_algorithm.as_ref(), + sse_customer_key.as_ref(), + sse_customer_key_md5.as_ref(), + true, + )?; + let has_explicit_ssec = sse_customer_algorithm.is_some() || sse_customer_key.is_some() || sse_customer_key_md5.is_some(); // Validate both source and destination keys validate_object_key(&src_key, "COPY (source)")?; @@ -5957,6 +5968,9 @@ impl DefaultObjectUsecase { let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); let mut effective_sse = requested_sse.or_else(|| { + if has_explicit_ssec { + return None; + } bucket_sse_config.as_ref().and_then(|(config, _)| { config.rules.first().and_then(|rule| { rule.apply_server_side_encryption_by_default @@ -5970,6 +5984,9 @@ impl DefaultObjectUsecase { }) }); let mut effective_kms_key_id = requested_kms_key_id.or_else(|| { + if has_explicit_ssec { + return None; + } bucket_sse_config.as_ref().and_then(|(config, _)| { config.rules.first().and_then(|rule| { rule.apply_server_side_encryption_by_default @@ -6145,7 +6162,7 @@ impl DefaultObjectUsecase { key: &key, server_side_encryption: effective_sse.clone(), ssekms_key_id: effective_kms_key_id.clone(), - ssekms_context: extract_ssekms_context_from_headers(&req.headers)?, + ssekms_context, sse_customer_algorithm: sse_customer_algorithm.clone(), sse_customer_key, sse_customer_key_md5: sse_customer_key_md5.clone(), diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 3c438a672..5310767a1 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -414,22 +414,44 @@ pub struct EncryptionRequest<'a> { } impl EncryptionRequest<'_> { - pub fn check_upload_part_customer_key_md5( - &self, - user_defined: &HashMap, - customer_key_md5: Option, - ) -> Result<(), ApiError> { - if let Some(customer_key_md5) = customer_key_md5 { - // if customer_key_md5 is provided, check if it matches the metadata - let customer_key_md5_from_metadata = user_defined.get("x-amz-server-side-encryption-customer-key-md5"); - if let Some(customer_key_md5_from_metadata) = customer_key_md5_from_metadata - && customer_key_md5_from_metadata != customer_key_md5.as_str() - { - return Err(ApiError::from(StorageError::other("Customer key MD5 mismatch"))); - } + pub fn validate_multipart_ssec(&self, user_defined: &HashMap) -> Result<(), ApiError> { + let stored_algorithm = user_defined.get("x-amz-server-side-encryption-customer-algorithm"); + let stored_key_md5 = user_defined.get("x-amz-server-side-encryption-customer-key-md5"); + let session_uses_ssec = stored_algorithm.is_some() || stored_key_md5.is_some(); + let request_uses_ssec = + self.sse_customer_algorithm.is_some() || self.sse_customer_key.is_some() || self.sse_customer_key_md5.is_some(); + + if !session_uses_ssec { + return if request_uses_ssec { + Err(ssec_invalid_request( + "SSE-C parameters cannot be used for a multipart upload that was not initiated with SSE-C.", + )) + } else { + Ok(()) + }; } - Ok(()) + let (Some(algorithm), Some(key), Some(key_md5)) = ( + self.sse_customer_algorithm.as_ref(), + self.sse_customer_key.as_ref(), + self.sse_customer_key_md5.as_ref(), + ) else { + return Err(ssec_invalid_request( + "Missing SSE-C parameters. Algorithm, customer key and customer key MD5 are all required.", + )); + }; + + let validated = validate_ssec_params(SsecParams { + algorithm: algorithm.to_string(), + key: key.to_string(), + key_md5: key_md5.to_string(), + })?; + if stored_algorithm.map(String::as_str) != Some(validated.algorithm.as_str()) { + return Err(ssec_invalid_request( + "The provided encryption parameters did not match the multipart upload.", + )); + } + verify_ssec_key_match(&validated.key_md5, stored_key_md5) } } @@ -654,12 +676,9 @@ fn map_ssec_get_object_reader_error_message(err: &StorageError) -> Option Some( "The provided encryption parameters did not match the ones used originally to encrypt the object.".to_string(), ), - _ => detail.strip_prefix("unsupported SSE-C algorithm ").map(|algorithm| { - format!( - "Unsupported SSE-C algorithm: {}. Only {} is supported.", - algorithm, DEFAULT_SSE_ALGORITHM - ) - }), + _ => detail + .strip_prefix("unsupported SSE-C algorithm ") + .map(|_| format!("Unsupported SSE-C algorithm. Only {DEFAULT_SSE_ALGORITHM} is supported.")), } } @@ -1280,7 +1299,7 @@ pub async fn sse_prepare_encryption(request: PrepareEncryptionRequest<'_>) -> Re request.sse_customer_algorithm.as_ref(), request.sse_customer_key.as_ref(), request.sse_customer_key_md5.as_ref(), - false, + true, )?; let sse_type = prepare_sse_configuration_v2( @@ -2233,14 +2252,17 @@ pub fn is_managed_sse(server_side_encryption: &ServerSideEncryption) -> bool { matches!(server_side_encryption.as_str(), "AES256" | "aws:kms") } -/// Strip managed encryption metadata from object metadata +/// Strip source encryption metadata before constructing metadata for a copy destination. /// -/// Removes all managed SSE-related headers before returning object metadata to client. -/// This is necessary because encryption is transparent to S3 clients. +/// Encryption metadata describes the physical source representation and must never be +/// inherited by a plaintext destination or by a destination using a different key. pub fn strip_managed_encryption_metadata(metadata: &mut HashMap) { - const KEYS: [&str; 16] = [ + const KEYS: [&str; 19] = [ "x-amz-server-side-encryption", "x-amz-server-side-encryption-aws-kms-key-id", + "x-amz-server-side-encryption-customer-algorithm", + "x-amz-server-side-encryption-customer-key-md5", + SSEC_ORIGINAL_SIZE_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, "x-rustfs-encryption-tag", INTERNAL_ENCRYPTION_KEY_HEADER, @@ -2378,8 +2400,7 @@ fn normalize_managed_metadata(metadata: &HashMap) -> HashMap Result { if !SUPPORT_SSE_ALGORITHMS.contains(¶ms.algorithm.as_str()) { return Err(ssec_invalid_request(&format!( - "Unsupported SSE-C algorithm: {}. Only {} is supported.", - params.algorithm, DEFAULT_SSE_ALGORITHM + "Unsupported SSE-C algorithm. Only {DEFAULT_SSE_ALGORITHM} is supported." ))); } @@ -2397,7 +2418,6 @@ pub fn validate_ssec_params(params: SsecParams) -> Result HashMap { + let key_bytes = [key_byte; 32]; + HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ( + "x-amz-server-side-encryption-customer-key-md5".to_string(), + BASE64_STANDARD.encode(md5::compute(key_bytes).0), + ), + ]) } #[test] - fn test_upload_part_customer_key_md5_exact_match() { - let mut metadata = HashMap::new(); - let md5 = "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789+/==".to_string(); - metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), md5.clone()); + fn test_validate_multipart_ssec_exact_match() { + assert!( + multipart_ssec_request(42) + .validate_multipart_ssec(&multipart_ssec_metadata(42)) + .is_ok() + ); + } - let request = EncryptionRequest { - bucket: "bucket", - key: "object", - server_side_encryption: None, - ssekms_key_id: None, - ssekms_context: None, + #[test] + fn test_validate_multipart_ssec_rejects_wrong_key_without_leaking_it() { + let request = multipart_ssec_request(43); + let encoded_key = request.sse_customer_key.clone().expect("fixture key"); + let encoded_md5 = request.sse_customer_key_md5.clone().expect("fixture MD5"); + + let error = request + .validate_multipart_ssec(&multipart_ssec_metadata(42)) + .expect_err("wrong key must fail"); + + assert_eq!(error.code, S3ErrorCode::InvalidRequest); + assert!(!error.message.contains(&encoded_key)); + assert!(!error.message.contains(&encoded_md5)); + } + + #[test] + fn test_validate_multipart_ssec_rejects_missing_or_unexpected_parameters() { + let no_ssec = EncryptionRequest { sse_customer_algorithm: None, sse_customer_key: None, sse_customer_key_md5: None, - content_size: 1, + ..multipart_ssec_request(42) }; - - let result = request.check_upload_part_customer_key_md5(&metadata, Some(md5)); - assert!(result.is_ok()); + assert_eq!( + no_ssec + .validate_multipart_ssec(&multipart_ssec_metadata(42)) + .expect_err("SSE-C session requires all parameters") + .code, + S3ErrorCode::InvalidRequest + ); + assert_eq!( + multipart_ssec_request(42) + .validate_multipart_ssec(&HashMap::new()) + .expect_err("plaintext session rejects SSE-C parameters") + .code, + S3ErrorCode::InvalidRequest + ); + let incomplete_session = HashMap::from([( + "x-amz-server-side-encryption-customer-key-md5".to_string(), + multipart_ssec_request(42).sse_customer_key_md5.expect("fixture MD5"), + )]); + assert_eq!( + multipart_ssec_request(42) + .validate_multipart_ssec(&incomplete_session) + .expect_err("incomplete stored SSE-C metadata must fail closed") + .code, + S3ErrorCode::InvalidRequest + ); } // ============================================================================