fix(s3): enforce SSE-C copy key validation (#5185)

This commit is contained in:
cxymds
2026-07-24 19:11:36 +08:00
committed by GitHub
parent fa26b6730d
commit f46ea6e14f
11 changed files with 749 additions and 124 deletions
+75 -45
View File
@@ -67,7 +67,7 @@ use crate::error::ApiError;
use crate::table_catalog;
use bytes::Bytes;
use futures::StreamExt;
use http::{HeaderMap, Uri};
use http::{HeaderMap, HeaderValue, Uri};
use rustfs_s3_ops::S3Operation;
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
@@ -389,6 +389,9 @@ impl DefaultMultipartUsecase {
upload_id,
if_match,
if_none_match,
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
..
} = input;
@@ -486,6 +489,18 @@ impl DefaultMultipartUsecase {
.get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default())
.await
.map_err(ApiError::from)?;
EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption: None,
ssekms_key_id: None,
ssekms_context: None,
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
content_size: 0,
}
.validate_multipart_ssec(&multipart_info.user_defined)?;
let cache_adapter = self.object_data_cache();
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
@@ -493,7 +508,6 @@ impl DefaultMultipartUsecase {
.user_defined
.get("x-amz-server-side-encryption")
.map(|s| ServerSideEncryption::from(s.clone()));
let ssekms_key_id = match server_side_encryption.as_ref() {
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => multipart_info
.user_defined
@@ -602,6 +616,26 @@ impl DefaultMultipartUsecase {
let mut response = S3Response::new(output);
crate::app::object_usecase::inject_additional_checksum_headers(&mut response.headers, &complete_extra_checksum_headers);
if let Some(algorithm) = multipart_info
.user_defined
.get("x-amz-server-side-encryption-customer-algorithm")
{
let value = HeaderValue::from_str(algorithm)
.map_err(|_| s3_error!(InternalError, "Invalid stored SSE-C algorithm metadata"))?;
response
.headers
.insert("x-amz-server-side-encryption-customer-algorithm", value);
}
if let Some(key_md5) = multipart_info
.user_defined
.get("x-amz-server-side-encryption-customer-key-md5")
{
let value =
HeaderValue::from_str(key_md5).map_err(|_| s3_error!(InternalError, "Invalid stored SSE-C key metadata"))?;
response
.headers
.insert("x-amz-server-side-encryption-customer-key-md5", value);
}
let result = Ok(response);
let _ = helper.complete(&result);
rustfs_scanner::record_dirty_usage_bucket(&bucket);
@@ -624,6 +658,7 @@ impl DefaultMultipartUsecase {
storage_class,
server_side_encryption,
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
ssekms_key_id,
object_lock_legal_hold_status,
@@ -674,7 +709,11 @@ impl DefaultMultipartUsecase {
metadata.extend(object_lock_metadata);
}
apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?;
let (_, sse_customer_key, _) = extract_ssec_params_from_headers(&req.headers)?;
let (header_sse_customer_algorithm, header_sse_customer_key, header_sse_customer_key_md5) =
extract_ssec_params_from_headers(&req.headers)?;
let sse_customer_algorithm = sse_customer_algorithm.or(header_sse_customer_algorithm);
let sse_customer_key = sse_customer_key.or(header_sse_customer_key);
let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5);
let encryption_request = PrepareEncryptionRequest {
bucket: &bucket,
@@ -760,6 +799,7 @@ impl DefaultMultipartUsecase {
upload_id: Some(upload_id),
server_side_encryption: effective_sse,
sse_customer_algorithm,
sse_customer_key_md5,
ssekms_key_id: effective_kms_key_id,
checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from),
checksum_type: checksum_type.map(ChecksumType::from),
@@ -874,10 +914,9 @@ impl DefaultMultipartUsecase {
}
opts.want_checksum = reader.checksum();
let has_ssec = sse_customer_algorithm.is_some();
// When SSE-C headers are present, skip managed-encryption metadata to avoid
// false conflict: the bucket default SSE config stored in multipart metadata
// should not block a legitimate SSE-C upload part.
let has_ssec = fi
.user_defined
.contains_key("x-amz-server-side-encryption-customer-algorithm");
let (server_side_encryption, ssekms_key_id) = if has_ssec {
(None, None)
} else {
@@ -909,7 +948,7 @@ impl DefaultMultipartUsecase {
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
}
.check_upload_part_customer_key_md5(&fi.user_defined, sse_customer_key_md5.clone())?;
.validate_multipart_ssec(&fi.user_defined)?;
let (requested_sse, requested_kms_key_id) = if has_ssec {
let ssec_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
@@ -1139,6 +1178,18 @@ impl DefaultMultipartUsecase {
.get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default())
.await
.map_err(ApiError::from)?;
EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption: None,
ssekms_key_id: None,
ssekms_context: None,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: 0,
}
.validate_multipart_ssec(&mp_info.user_defined)?;
let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?;
src_opts.version_id = src_version_id.clone();
@@ -1161,6 +1212,14 @@ impl DefaultMultipartUsecase {
.map_err(map_get_object_reader_error)?;
let src_info = src_reader.object_info;
let src_stream = src_reader.stream;
let resolved_src_version_id = src_info.version_id.map(|version_id| {
if version_id == Uuid::nil() {
"null".to_string()
} else {
version_id.to_string()
}
});
if let Some(if_match) = copy_source_if_match {
if let Some(ref etag) = src_info.etag {
@@ -1185,10 +1244,10 @@ impl DefaultMultipartUsecase {
}
let (_start_offset, length) = if let Some(ref range_spec) = rs {
let validation_size = match src_info.is_compressed_ok() {
Ok((_, true)) => src_info.get_actual_size().unwrap_or(src_info.size),
_ => src_info.size,
};
// Copy-source ranges are expressed over the logical plaintext object.
// Encrypted (and compressed) objects have a larger or smaller physical
// representation, so validating against `size` rejects valid later parts.
let validation_size = src_info.get_actual_size().unwrap_or(src_info.size);
validate_copy_source_range_not_exceeds(range_spec, validation_size)?;
@@ -1199,24 +1258,6 @@ impl DefaultMultipartUsecase {
(0, src_info.size)
};
let h = build_ssec_read_headers(
copy_source_sse_customer_algorithm.as_ref(),
copy_source_sse_customer_key.as_ref(),
copy_source_sse_customer_key_md5.as_ref(),
);
let get_opts = ObjectOptions {
version_id: src_opts.version_id.clone(),
versioned: src_opts.versioned,
version_suspended: src_opts.version_suspended,
..Default::default()
};
let src_reader = store
.get_object_reader(&src_bucket, &src_key, rs.clone(), h, &get_opts)
.await
.map_err(map_get_object_reader_error)?;
let src_stream = src_reader.stream;
let is_disk_compressed =
rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
@@ -1240,7 +1281,9 @@ impl DefaultMultipartUsecase {
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}"))))
})
.transpose()?;
let has_ssec = sse_customer_algorithm.is_some();
let has_ssec = mp_info
.user_defined
.contains_key("x-amz-server-side-encryption-customer-algorithm");
let ssekms_key_id = match server_side_encryption.as_ref() {
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info
.user_defined
@@ -1248,19 +1291,6 @@ impl DefaultMultipartUsecase {
.map(|s| s.to_string()),
_ => None,
};
EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
ssekms_context: None,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
}
.check_upload_part_customer_key_md5(&mp_info.user_defined, sse_customer_key_md5.clone())?;
let (requested_sse, requested_kms_key_id, dst_user_defined) = if has_ssec {
let ssec_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
@@ -1357,7 +1387,7 @@ impl DefaultMultipartUsecase {
let output = UploadPartCopyOutput {
copy_part_result: Some(copy_part_result),
copy_source_version_id: src_version_id,
copy_source_version_id: resolved_src_version_id,
server_side_encryption: requested_sse,
ssekms_key_id: requested_kms_key_id,
sse_customer_algorithm,
+18 -1
View File
@@ -5831,6 +5831,17 @@ impl DefaultObjectUsecase {
{
return Err(s3_error!(InvalidStorageClass));
}
let ssekms_context = extract_ssekms_context_from_headers(&req.headers)?;
validate_sse_headers_for_write(
requested_sse.as_ref(),
requested_kms_key_id.as_ref(),
ssekms_context.as_ref(),
sse_customer_algorithm.as_ref(),
sse_customer_key.as_ref(),
sse_customer_key_md5.as_ref(),
true,
)?;
let has_explicit_ssec = sse_customer_algorithm.is_some() || sse_customer_key.is_some() || sse_customer_key_md5.is_some();
// Validate both source and destination keys
validate_object_key(&src_key, "COPY (source)")?;
@@ -5957,6 +5968,9 @@ impl DefaultObjectUsecase {
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
let mut effective_sse = requested_sse.or_else(|| {
if has_explicit_ssec {
return None;
}
bucket_sse_config.as_ref().and_then(|(config, _)| {
config.rules.first().and_then(|rule| {
rule.apply_server_side_encryption_by_default
@@ -5970,6 +5984,9 @@ impl DefaultObjectUsecase {
})
});
let mut effective_kms_key_id = requested_kms_key_id.or_else(|| {
if has_explicit_ssec {
return None;
}
bucket_sse_config.as_ref().and_then(|(config, _)| {
config.rules.first().and_then(|rule| {
rule.apply_server_side_encryption_by_default
@@ -6145,7 +6162,7 @@ impl DefaultObjectUsecase {
key: &key,
server_side_encryption: effective_sse.clone(),
ssekms_key_id: effective_kms_key_id.clone(),
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
ssekms_context,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
+126 -66
View File
@@ -414,22 +414,44 @@ pub struct EncryptionRequest<'a> {
}
impl EncryptionRequest<'_> {
pub fn check_upload_part_customer_key_md5(
&self,
user_defined: &HashMap<String, String>,
customer_key_md5: Option<SSECustomerKeyMD5>,
) -> Result<(), ApiError> {
if let Some(customer_key_md5) = customer_key_md5 {
// if customer_key_md5 is provided, check if it matches the metadata
let customer_key_md5_from_metadata = user_defined.get("x-amz-server-side-encryption-customer-key-md5");
if let Some(customer_key_md5_from_metadata) = customer_key_md5_from_metadata
&& customer_key_md5_from_metadata != customer_key_md5.as_str()
{
return Err(ApiError::from(StorageError::other("Customer key MD5 mismatch")));
}
pub fn validate_multipart_ssec(&self, user_defined: &HashMap<String, String>) -> Result<(), ApiError> {
let stored_algorithm = user_defined.get("x-amz-server-side-encryption-customer-algorithm");
let stored_key_md5 = user_defined.get("x-amz-server-side-encryption-customer-key-md5");
let session_uses_ssec = stored_algorithm.is_some() || stored_key_md5.is_some();
let request_uses_ssec =
self.sse_customer_algorithm.is_some() || self.sse_customer_key.is_some() || self.sse_customer_key_md5.is_some();
if !session_uses_ssec {
return if request_uses_ssec {
Err(ssec_invalid_request(
"SSE-C parameters cannot be used for a multipart upload that was not initiated with SSE-C.",
))
} else {
Ok(())
};
}
Ok(())
let (Some(algorithm), Some(key), Some(key_md5)) = (
self.sse_customer_algorithm.as_ref(),
self.sse_customer_key.as_ref(),
self.sse_customer_key_md5.as_ref(),
) else {
return Err(ssec_invalid_request(
"Missing SSE-C parameters. Algorithm, customer key and customer key MD5 are all required.",
));
};
let validated = validate_ssec_params(SsecParams {
algorithm: algorithm.to_string(),
key: key.to_string(),
key_md5: key_md5.to_string(),
})?;
if stored_algorithm.map(String::as_str) != Some(validated.algorithm.as_str()) {
return Err(ssec_invalid_request(
"The provided encryption parameters did not match the multipart upload.",
));
}
verify_ssec_key_match(&validated.key_md5, stored_key_md5)
}
}
@@ -654,12 +676,9 @@ fn map_ssec_get_object_reader_error_message(err: &StorageError) -> Option<String
"SSE-C key does not match object metadata" => Some(
"The provided encryption parameters did not match the ones used originally to encrypt the object.".to_string(),
),
_ => detail.strip_prefix("unsupported SSE-C algorithm ").map(|algorithm| {
format!(
"Unsupported SSE-C algorithm: {}. Only {} is supported.",
algorithm, DEFAULT_SSE_ALGORITHM
)
}),
_ => detail
.strip_prefix("unsupported SSE-C algorithm ")
.map(|_| format!("Unsupported SSE-C algorithm. Only {DEFAULT_SSE_ALGORITHM} is supported.")),
}
}
@@ -1280,7 +1299,7 @@ pub async fn sse_prepare_encryption(request: PrepareEncryptionRequest<'_>) -> Re
request.sse_customer_algorithm.as_ref(),
request.sse_customer_key.as_ref(),
request.sse_customer_key_md5.as_ref(),
false,
true,
)?;
let sse_type = prepare_sse_configuration_v2(
@@ -2233,14 +2252,17 @@ pub fn is_managed_sse(server_side_encryption: &ServerSideEncryption) -> bool {
matches!(server_side_encryption.as_str(), "AES256" | "aws:kms")
}
/// Strip managed encryption metadata from object metadata
/// Strip source encryption metadata before constructing metadata for a copy destination.
///
/// Removes all managed SSE-related headers before returning object metadata to client.
/// This is necessary because encryption is transparent to S3 clients.
/// Encryption metadata describes the physical source representation and must never be
/// inherited by a plaintext destination or by a destination using a different key.
pub fn strip_managed_encryption_metadata(metadata: &mut HashMap<String, String>) {
const KEYS: [&str; 16] = [
const KEYS: [&str; 19] = [
"x-amz-server-side-encryption",
"x-amz-server-side-encryption-aws-kms-key-id",
"x-amz-server-side-encryption-customer-algorithm",
"x-amz-server-side-encryption-customer-key-md5",
SSEC_ORIGINAL_SIZE_HEADER,
INTERNAL_ENCRYPTION_IV_HEADER,
"x-rustfs-encryption-tag",
INTERNAL_ENCRYPTION_KEY_HEADER,
@@ -2378,8 +2400,7 @@ fn normalize_managed_metadata(metadata: &HashMap<String, String>) -> HashMap<Str
pub fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, ApiError> {
if !SUPPORT_SSE_ALGORITHMS.contains(&params.algorithm.as_str()) {
return Err(ssec_invalid_request(&format!(
"Unsupported SSE-C algorithm: {}. Only {} is supported.",
params.algorithm, DEFAULT_SSE_ALGORITHM
"Unsupported SSE-C algorithm. Only {DEFAULT_SSE_ALGORITHM} is supported."
)));
}
@@ -2397,7 +2418,6 @@ pub fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, A
let computed_md5 = BASE64_STANDARD.encode(md5::compute(&key_bytes).0);
if computed_md5 != params.key_md5 {
error!("SSE-C key MD5 mismatch: expected '{}', got '{}'", params.key_md5, computed_md5);
return Err(ssec_invalid_request(
"The calculated MD5 hash of the key did not match the hash that was provided.",
));
@@ -2863,7 +2883,7 @@ mod tests {
}
#[tokio::test]
async fn test_sse_prepare_encryption_allows_ssec_headers_without_customer_key() {
async fn test_sse_prepare_encryption_rejects_ssec_headers_without_customer_key() {
let bucket = "test-bucket";
let key = "test-key";
let sse_key_md5 = BASE64_STANDARD.encode(md5::compute([42u8; 32]).0);
@@ -2879,14 +2899,10 @@ mod tests {
sse_customer_key_md5: Some(sse_key_md5),
};
let material = sse_prepare_encryption(request)
let error = sse_prepare_encryption(request)
.await
.expect("prepare should accept ssec headers");
assert!(material.is_some());
let metadata = encryption_material_to_metadata(&material.expect("ssec metadata should be generated"))
.expect("ssec metadata should be generated");
assert_eq!(metadata.get("x-amz-server-side-encryption").unwrap(), "AES256");
assert_eq!(metadata.get("x-amz-server-side-encryption-customer-algorithm").unwrap(), "AES256");
.expect_err("multipart preparation must require possession of the customer key");
assert_eq!(error.code, S3ErrorCode::InvalidRequest);
}
// ------------------------------------------------------------------------
@@ -3441,6 +3457,9 @@ mod tests {
fn test_strip_managed_encryption_metadata() {
let mut metadata = HashMap::new();
metadata.insert("x-amz-server-side-encryption".to_string(), "aws:kms".to_string());
metadata.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string());
metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), "source-key-md5".to_string());
metadata.insert(SSEC_ORIGINAL_SIZE_HEADER.to_string(), "123".to_string());
metadata.insert("x-rustfs-encryption-key".to_string(), "encrypted_key".to_string());
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string());
metadata.insert("content-type".to_string(), "text/plain".to_string());
@@ -3448,6 +3467,9 @@ mod tests {
strip_managed_encryption_metadata(&mut metadata);
assert!(!metadata.contains_key("x-amz-server-side-encryption"));
assert!(!metadata.contains_key("x-amz-server-side-encryption-customer-algorithm"));
assert!(!metadata.contains_key("x-amz-server-side-encryption-customer-key-md5"));
assert!(!metadata.contains_key(SSEC_ORIGINAL_SIZE_HEADER));
assert!(!metadata.contains_key("x-rustfs-encryption-key"));
assert!(!metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER));
assert!(metadata.contains_key("content-type"));
@@ -3650,51 +3672,89 @@ mod tests {
assert!(result.is_err());
}
#[test]
fn test_upload_part_customer_key_md5_comparison_is_case_sensitive() {
let mut metadata = HashMap::new();
metadata.insert(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
"AbCdEfGhIjKlMnOpQrStUvWxYz0123456789+/==".to_string(),
);
let request = EncryptionRequest {
fn multipart_ssec_request(key_byte: u8) -> EncryptionRequest<'static> {
let key_bytes = [key_byte; 32];
EncryptionRequest {
bucket: "bucket",
key: "object",
server_side_encryption: None,
ssekms_key_id: None,
ssekms_context: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
sse_customer_algorithm: Some("AES256".to_string()),
sse_customer_key: Some(BASE64_STANDARD.encode(key_bytes)),
sse_customer_key_md5: Some(BASE64_STANDARD.encode(md5::compute(key_bytes).0)),
content_size: 1,
};
}
}
let mismatch = "aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789+/==".to_string();
let result = request.check_upload_part_customer_key_md5(&metadata, Some(mismatch));
assert!(result.is_err());
fn multipart_ssec_metadata(key_byte: u8) -> HashMap<String, String> {
let key_bytes = [key_byte; 32];
HashMap::from([
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5::compute(key_bytes).0),
),
])
}
#[test]
fn test_upload_part_customer_key_md5_exact_match() {
let mut metadata = HashMap::new();
let md5 = "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789+/==".to_string();
metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), md5.clone());
fn test_validate_multipart_ssec_exact_match() {
assert!(
multipart_ssec_request(42)
.validate_multipart_ssec(&multipart_ssec_metadata(42))
.is_ok()
);
}
let request = EncryptionRequest {
bucket: "bucket",
key: "object",
server_side_encryption: None,
ssekms_key_id: None,
ssekms_context: None,
#[test]
fn test_validate_multipart_ssec_rejects_wrong_key_without_leaking_it() {
let request = multipart_ssec_request(43);
let encoded_key = request.sse_customer_key.clone().expect("fixture key");
let encoded_md5 = request.sse_customer_key_md5.clone().expect("fixture MD5");
let error = request
.validate_multipart_ssec(&multipart_ssec_metadata(42))
.expect_err("wrong key must fail");
assert_eq!(error.code, S3ErrorCode::InvalidRequest);
assert!(!error.message.contains(&encoded_key));
assert!(!error.message.contains(&encoded_md5));
}
#[test]
fn test_validate_multipart_ssec_rejects_missing_or_unexpected_parameters() {
let no_ssec = EncryptionRequest {
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
content_size: 1,
..multipart_ssec_request(42)
};
let result = request.check_upload_part_customer_key_md5(&metadata, Some(md5));
assert!(result.is_ok());
assert_eq!(
no_ssec
.validate_multipart_ssec(&multipart_ssec_metadata(42))
.expect_err("SSE-C session requires all parameters")
.code,
S3ErrorCode::InvalidRequest
);
assert_eq!(
multipart_ssec_request(42)
.validate_multipart_ssec(&HashMap::new())
.expect_err("plaintext session rejects SSE-C parameters")
.code,
S3ErrorCode::InvalidRequest
);
let incomplete_session = HashMap::from([(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
multipart_ssec_request(42).sse_customer_key_md5.expect("fixture MD5"),
)]);
assert_eq!(
multipart_ssec_request(42)
.validate_multipart_ssec(&incomplete_session)
.expect_err("incomplete stored SSE-C metadata must fail closed")
.code,
S3ErrorCode::InvalidRequest
);
}
// ============================================================================