diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 0ec379070..79a6f6864 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -217,7 +217,37 @@ pub(crate) async fn signed_s3_request( access_key: &str, secret_key: &str, ) -> Result> { - signed_s3_request_with_session_token(method, url, body, content_type, access_key, secret_key, None).await + signed_s3_request_with_headers(method, url, body, content_type, access_key, secret_key, &http::HeaderMap::new()).await +} + +pub(crate) async fn signed_s3_request_with_headers( + method: http::Method, + url: &str, + body: Option, + content_type: Option<&str>, + access_key: &str, + secret_key: &str, + extra_headers: &http::HeaderMap, +) -> Result> { + signed_s3_request_with_session_token( + method, + url, + body, + content_type, + SigningCredentials { + access_key, + secret_key, + session_token: None, + }, + extra_headers, + ) + .await +} + +struct SigningCredentials<'a> { + access_key: &'a str, + secret_key: &'a str, + session_token: Option<&'a str>, } async fn signed_s3_request_with_session_token( @@ -225,9 +255,8 @@ async fn signed_s3_request_with_session_token( url: &str, body: Option, content_type: Option<&str>, - access_key: &str, - secret_key: &str, - session_token: Option<&str>, + credentials: SigningCredentials<'_>, + extra_headers: &http::HeaderMap, ) -> Result> { let uri = url.parse::()?; let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string(); @@ -239,14 +268,17 @@ async fn signed_s3_request_with_session_token( if let Some(content_type) = content_type { request = request.header(CONTENT_TYPE, content_type); } + for (name, value) in extra_headers { + request = request.header(name, value); + } let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?; let signed = sign_v4( request.body(Body::empty())?, content_length, - access_key, - secret_key, - session_token.unwrap_or_default(), + credentials.access_key, + credentials.secret_key, + credentials.session_token.unwrap_or_default(), "us-east-1", ); @@ -283,8 +315,19 @@ pub(crate) async fn admin_request_with_session_token( ) -> Result<(StatusCode, String), Box> { let url = format!("{base_url}{path_and_query}"); let content_type = body.as_ref().map(|_| "application/json"); - let response = - signed_s3_request_with_session_token(method, &url, body, content_type, access_key, secret_key, session_token).await?; + let response = signed_s3_request_with_session_token( + method, + &url, + body, + content_type, + SigningCredentials { + access_key, + secret_key, + session_token, + }, + &http::HeaderMap::new(), + ) + .await?; let status = response.status(); let body = response.text().await?; Ok((status, body)) diff --git a/crates/e2e_test/src/kms/mod.rs b/crates/e2e_test/src/kms/mod.rs index ad5a96858..72dc41ba7 100644 --- a/crates/e2e_test/src/kms/mod.rs +++ b/crates/e2e_test/src/kms/mod.rs @@ -57,6 +57,9 @@ mod copy_object_version_restore_sse_test; #[cfg(test)] mod configured_roundtrip_test; +#[cfg(test)] +mod select_sse_response_test; + #[cfg(test)] mod kms_anonymous_enforcement_test; diff --git a/crates/e2e_test/src/kms/select_sse_response_test.rs b/crates/e2e_test/src/kms/select_sse_response_test.rs new file mode 100644 index 000000000..47d6c63c1 --- /dev/null +++ b/crates/e2e_test/src/kms/select_sse_response_test.rs @@ -0,0 +1,241 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! SelectObjectContent SSE response-header compatibility (backlog#1625). + +use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64, start_kms}; +use crate::common::signed_s3_request_with_headers; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::ServerSideEncryption; +use base64_simd::STANDARD as BASE64; +use http::{HeaderMap, Method}; +use std::error::Error; +use uuid::Uuid; + +type TestResult = Result>; + +const CSV_BODY: &[u8] = b"name\nalice\n"; +const SELECT_BODY: &str = r#" +SELECT * FROM S3Object +SQL +USE + +"#; +const KMS_CONTEXT: &str = "eyJ0ZW5hbnQiOiJzMy1zZWxlY3QifQ=="; +const SSE_ALGORITHM: &str = "x-amz-server-side-encryption"; +const SSE_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id"; +const SSE_KMS_CONTEXT: &str = "x-amz-server-side-encryption-context"; +const SSE_C_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm"; +const SSE_C_KEY: &str = "x-amz-server-side-encryption-customer-key"; +const SSE_C_KEY_MD5: &str = "x-amz-server-side-encryption-customer-key-md5"; +const LOG_FLUSH_SENTINEL: &str = "select-sse-log-flush-sentinel.csv"; + +async fn raw_select( + env: &crate::common::RustFSTestEnvironment, + bucket: &str, + object: &str, + request_headers: &HeaderMap, +) -> TestResult { + let url = format!("{}/{bucket}/{object}?select&select-type=2", env.url); + signed_s3_request_with_headers( + Method::POST, + &url, + Some(SELECT_BODY.to_string()), + Some("application/xml"), + &env.access_key, + &env.secret_key, + request_headers, + ) + .await +} + +async fn assert_success_headers(response: reqwest::Response, expected: &[(&str, &str)], absent: &[&str]) -> TestResult { + if response.status() != reqwest::StatusCode::OK { + let status = response.status(); + let url = response.url().clone(); + let body = response.text().await?; + panic!("Select request to {url} failed with {status}: {body}"); + } + for (name, value) in expected { + assert_eq!(response.headers().get(*name).and_then(|header| header.to_str().ok()), Some(*value)); + } + for name in absent { + assert!(response.headers().get(*name).is_none(), "successful Select response must omit {name}"); + } + let body = response.bytes().await?; + assert!( + body.windows(b"alice".len()).any(|window| window == b"alice"), + "successful Select response must contain a Records event with the selected row" + ); + assert!( + body.windows(b"End".len()).any(|window| window == b"End"), + "successful Select response must contain the terminal End event" + ); + Ok(()) +} + +async fn assert_pre_stream_failure(response: reqwest::Response) -> TestResult { + assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); + let body = response.text().await?; + assert!(body.contains(""), "pre-stream failure must return an S3 XML error: {body}"); + assert!( + body.contains("InvalidRequest"), + "invalid SSE-C parameters must preserve the S3 error code: {body}" + ); + Ok(()) +} + +fn put_object( + client: &aws_sdk_s3::Client, + bucket: &str, + object: &str, +) -> aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder { + client + .put_object() + .bucket(bucket) + .key(object) + .body(ByteStream::from_static(CSV_BODY)) +} + +#[tokio::test] +async fn select_projects_encryption_headers_and_rejects_invalid_sse_c_before_streaming() -> TestResult { + let mut kms = LocalKMSTestEnvironment::new().await?; + let log_path = format!("{}/server.log", kms.base_env.temp_dir); + kms.base_env.capture_log_path = Some(log_path.clone()); + kms.base_env + .start_rustfs_server_with_env(Vec::new(), &[("RUST_LOG", "s3s=debug,rustfs=info")]) + .await?; + let key_id = kms.configure_local_kms().await?; + start_kms(&kms.base_env.url, &kms.base_env.access_key, &kms.base_env.secret_key).await?; + + let client = kms.base_env.create_s3_client(); + let bucket = format!("select-sse-{}", Uuid::new_v4().simple()); + client.create_bucket().bucket(&bucket).send().await?; + + put_object(&client, &bucket, "plain.csv").send().await?; + put_object(&client, &bucket, "sse-s3.csv") + .server_side_encryption(ServerSideEncryption::Aes256) + .send() + .await?; + put_object(&client, &bucket, "sse-kms.csv") + .server_side_encryption(ServerSideEncryption::AwsKms) + .ssekms_key_id(&key_id) + .ssekms_encryption_context(KMS_CONTEXT) + .send() + .await?; + + let customer_key = "01234567890123456789012345678901"; + let customer_key_b64 = BASE64.encode_to_string(customer_key); + let customer_key_md5 = sse_customer_key_md5_base64(customer_key); + put_object(&client, &bucket, "sse-c.csv") + .sse_customer_algorithm("AES256") + .sse_customer_key(&customer_key_b64) + .sse_customer_key_md5(&customer_key_md5) + .send() + .await?; + + assert_success_headers( + raw_select(&kms.base_env, &bucket, "plain.csv", &HeaderMap::new()).await?, + &[], + &[ + SSE_ALGORITHM, + SSE_KMS_KEY_ID, + SSE_KMS_CONTEXT, + SSE_C_ALGORITHM, + SSE_C_KEY, + SSE_C_KEY_MD5, + ], + ) + .await?; + assert_success_headers( + raw_select(&kms.base_env, &bucket, "sse-s3.csv", &HeaderMap::new()).await?, + &[(SSE_ALGORITHM, "AES256")], + &[SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5], + ) + .await?; + assert_success_headers( + raw_select(&kms.base_env, &bucket, "sse-kms.csv", &HeaderMap::new()).await?, + &[ + (SSE_ALGORITHM, "aws:kms"), + (SSE_KMS_KEY_ID, &key_id), + (SSE_KMS_CONTEXT, KMS_CONTEXT), + ], + &[SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5], + ) + .await?; + + let mut sse_c_headers = HeaderMap::new(); + sse_c_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?); + sse_c_headers.insert(SSE_C_KEY, customer_key_b64.parse()?); + sse_c_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?); + assert_success_headers( + raw_select(&kms.base_env, &bucket, "sse-c.csv", &sse_c_headers).await?, + &[(SSE_C_ALGORITHM, "AES256"), (SSE_C_KEY_MD5, &customer_key_md5)], + &[SSE_ALGORITHM, SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_KEY], + ) + .await?; + + assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &HeaderMap::new()).await?).await?; + + let mut missing_algorithm_headers = HeaderMap::new(); + missing_algorithm_headers.insert(SSE_C_KEY, customer_key_b64.parse()?); + missing_algorithm_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?); + assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &missing_algorithm_headers).await?).await?; + + let mut wrong_algorithm_headers = sse_c_headers.clone(); + wrong_algorithm_headers.insert(SSE_C_ALGORITHM, "AES128".parse()?); + assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_algorithm_headers).await?).await?; + + let wrong_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999"); + let mut wrong_md5_headers = sse_c_headers.clone(); + wrong_md5_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?); + assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_md5_headers).await?).await?; + + let wrong_key = "99999999999999999999999999999999"; + let wrong_key_b64 = BASE64.encode_to_string(wrong_key); + let mut wrong_key_headers = HeaderMap::new(); + wrong_key_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?); + wrong_key_headers.insert(SSE_C_KEY, wrong_key_b64.parse()?); + wrong_key_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?); + assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_key_headers).await?).await?; + + put_object(&client, &bucket, LOG_FLUSH_SENTINEL).send().await?; + assert_success_headers( + raw_select(&kms.base_env, &bucket, LOG_FLUSH_SENTINEL, &HeaderMap::new()).await?, + &[], + &[ + SSE_ALGORITHM, + SSE_KMS_KEY_ID, + SSE_KMS_CONTEXT, + SSE_C_ALGORITHM, + SSE_C_KEY, + SSE_C_KEY_MD5, + ], + ) + .await?; + let mut logs = String::new(); + for _ in 0..100 { + logs = tokio::fs::read_to_string(&log_path).await?; + if logs.contains(LOG_FLUSH_SENTINEL) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!(logs.contains(LOG_FLUSH_SENTINEL), "timed out waiting for the log sink to flush"); + for secret in [customer_key, customer_key_b64.as_str(), wrong_key, wrong_key_b64.as_str()] { + assert!(!logs.contains(secret), "Select request logging leaked SSE-C customer key material"); + } + + Ok(()) +} diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index 3a93ebade..600a10727 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -2,7 +2,7 @@ use super::storage_api::select_object::StorageError; use super::storage_api::select_object::options::get_opts; use super::storage_api::select_object::request_context::spawn_traced; -use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read}; +use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers}; use super::storage_api::select_object::{ StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read, @@ -19,34 +19,20 @@ use datafusion::arrow::{ use datafusion::common::DataFusionError; use datafusion::physical_plan::SendableRecordBatchStream; use futures::StreamExt; -use http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::RANGE}; +use http::{HeaderMap, StatusCode, header::RANGE}; use rustfs_s3select_api::{ QueryError, SelectError, object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds}, query::{Context, Query}, }; use rustfs_s3select_query::instance::s3_select_query_timeout; -use rustfs_utils::http::headers::{ - AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, - AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, -}; -use rustfs_utils::http::object_encryption_keys::{ - INTERNAL_ENCRYPTION_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, - MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, -}; use s3s::dto::{ CSVOutput, CompressionType, ContinuationEvent, EndEvent, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput, JSONType, OutputSerialization, Progress, ProgressEvent, QuoteFields, RecordsEvent, SelectObjectContentEvent, SelectObjectContentEventStream, SelectObjectContentInput, SelectObjectContentOutput, SelectObjectContentRequest, Stats, StatsEvent, }; -use s3s::header::{ - X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, - X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, -}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; -use std::collections::HashMap; use std::sync::Arc; use tokio::sync::mpsc; use tokio::time::{Instant, timeout_at}; @@ -62,8 +48,6 @@ const BUSY_MESSAGE: &str = "The service is unavailable. Try again later."; const EMPTY_SELECT_EXPRESSION_MESSAGE: &str = "empty SQL expression"; const SLOW_DOWN_MESSAGE: &str = "Reduce your request rate."; const UNSUPPORTED_SQL_STRUCTURE_MESSAGE: &str = "We encountered an unsupported SQL structure. Check the SQL Reference."; -// No canonical owner exists for the KMS key ARN prefix; keep it local. -const SELECT_KMS_ARN_PREFIX: &str = "arn:aws:kms:"; #[derive(Clone, Debug)] struct SelectValidation { @@ -107,7 +91,7 @@ pub async fn execute_select_object_content( .map_err(|_| select_query_timeout_error(query_timeout.as_secs()))? .map_err(map_query_error_to_s3)?; let admission = db.try_reserve_query().map_err(map_query_error_to_s3)?; - let snapshot = timeout_at( + let (snapshot, response_headers) = timeout_at( query_deadline, prepare_select_object_snapshot(&req.headers, &input, read_principal.as_ref()), ) @@ -130,7 +114,10 @@ pub async fn execute_select_object_content( .clone() .try_reserve_owned() .map_err(|_| map_select_error_to_s3(&SelectError::InternalError))?; - let response = select_object_response(rx, &snapshot.object_info().user_defined, &req.headers)?; + let mut response = S3Response::new(SelectObjectContentOutput { + payload: Some(SelectObjectContentEventStream::new(ReceiverStream::new(rx))), + }); + response.headers = response_headers; spawn_traced(async move { send_select_events_until_deadline( output, @@ -147,186 +134,6 @@ pub async fn execute_select_object_content( Ok(response) } -fn select_object_response( - rx: mpsc::Receiver>, - metadata: &HashMap, - request_headers: &HeaderMap, -) -> S3Result> { - let response_headers = select_snapshot_sse_response_headers(metadata, request_headers)?; - let mut response = S3Response::new(SelectObjectContentOutput { - payload: Some(SelectObjectContentEventStream::new(ReceiverStream::new(rx))), - }); - response.headers = response_headers; - Ok(response) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum SelectSnapshotSseMode { - S3, - Kms, - Customer, -} - -fn invalid_select_snapshot_sse_metadata() -> S3Error { - S3Error::with_message( - S3ErrorCode::InternalError, - "Persisted SelectObjectContent encryption metadata is invalid.", - ) -} - -fn select_metadata_value<'a>(metadata: &'a HashMap, name: &str) -> S3Result> { - let mut values = metadata - .iter() - .filter_map(|(key, value)| key.eq_ignore_ascii_case(name).then_some(value.as_str())); - let Some(value) = values.next() else { - return Ok(None); - }; - if values.any(|candidate| candidate != value) { - return Err(invalid_select_snapshot_sse_metadata()); - } - Ok(Some(value)) -} - -fn select_snapshot_kms_key_id(metadata: &HashMap) -> S3Result> { - let values = [ - select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID)?, - select_metadata_value(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER)?, - select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?, - ]; - let mut resolved = None; - for value in values.into_iter().flatten() { - if resolved.is_some_and(|current| current != value) { - return Err(invalid_select_snapshot_sse_metadata()); - } - resolved = Some(value); - } - Ok(resolved) -} - -fn select_snapshot_sse_mode(metadata: &HashMap) -> S3Result> { - let public_mode = select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION)?; - let customer_algorithm = select_metadata_value(metadata, SSEC_ALGORITHM_HEADER)?; - let has_ssec_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)?.is_some(); - let has_s3_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)?.is_some(); - let has_kms_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)?.is_some(); - - let public_mode = match public_mode { - Some(AMZ_ENCRYPTION_AES) => Some(SelectSnapshotSseMode::S3), - Some(AMZ_ENCRYPTION_KMS) => Some(SelectSnapshotSseMode::Kms), - Some(_) => return Err(invalid_select_snapshot_sse_metadata()), - None => None, - }; - if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) { - return Err(invalid_select_snapshot_sse_metadata()); - } - - let resolved = if customer_algorithm.is_some() { - if public_mode == Some(SelectSnapshotSseMode::Kms) { - return Err(invalid_select_snapshot_sse_metadata()); - } - Some(SelectSnapshotSseMode::Customer) - } else { - public_mode - }; - let internal_modes = [ - has_ssec_marker.then_some(SelectSnapshotSseMode::Customer), - has_s3_marker.then_some(SelectSnapshotSseMode::S3), - has_kms_marker.then_some(SelectSnapshotSseMode::Kms), - ]; - for mode in internal_modes.into_iter().flatten() { - if resolved != Some(mode) { - return Err(invalid_select_snapshot_sse_metadata()); - } - } - if resolved.is_none() - && metadata - .keys() - .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) - { - return Err(invalid_select_snapshot_sse_metadata()); - } - Ok(resolved) -} - -fn insert_select_snapshot_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> S3Result<()> { - let value = HeaderValue::from_str(value).map_err(|_| invalid_select_snapshot_sse_metadata())?; - headers.insert(name, value); - Ok(()) -} - -fn select_snapshot_sse_response_headers(metadata: &HashMap, request_headers: &HeaderMap) -> S3Result { - if select_metadata_value(metadata, SSEC_KEY_HEADER)?.is_some() - || select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT)?.is_some() - { - return Err(invalid_select_snapshot_sse_metadata()); - } - let Some(mode) = select_snapshot_sse_mode(metadata)? else { - return Ok(HeaderMap::new()); - }; - let kms_key_id = select_snapshot_kms_key_id(metadata)?; - - let mut response_headers = HeaderMap::with_capacity(3); - match mode { - SelectSnapshotSseMode::S3 => { - if select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID)?.is_some() - || select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?.is_some() - || select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.is_some() - { - return Err(invalid_select_snapshot_sse_metadata()); - } - response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES)); - } - SelectSnapshotSseMode::Kms => { - if select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.is_some() { - return Err(invalid_select_snapshot_sse_metadata()); - } - let key_id = kms_key_id - .filter(|key_id| !key_id.is_empty()) - .ok_or_else(invalid_select_snapshot_sse_metadata)?; - response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS)); - if key_id.starts_with(SELECT_KMS_ARN_PREFIX) { - insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?; - } else { - insert_select_snapshot_header( - &mut response_headers, - X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, - &format!("{SELECT_KMS_ARN_PREFIX}{key_id}"), - )?; - } - if let Some(context) = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)? { - let context = HeaderValue::from_str(context).map_err(|_| invalid_select_snapshot_sse_metadata())?; - let mut validation_headers = HeaderMap::with_capacity(1); - validation_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, context.clone()); - super::storage_api::select_object::sse::extract_ssekms_context_from_headers(&validation_headers) - .map_err(|_| invalid_select_snapshot_sse_metadata())?; - response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, context); - } - } - SelectSnapshotSseMode::Customer => { - if kms_key_id.is_some() || select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?.is_some() { - return Err(invalid_select_snapshot_sse_metadata()); - } - let algorithm = request_headers - .get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM) - .and_then(|value| value.to_str().ok()) - .filter(|algorithm| *algorithm == AMZ_ENCRYPTION_AES) - .ok_or_else(invalid_select_snapshot_sse_metadata)?; - let key_md5 = request_headers - .get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5) - .and_then(|value| value.to_str().ok()) - .ok_or_else(invalid_select_snapshot_sse_metadata)?; - let stored_md5 = - select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.ok_or_else(invalid_select_snapshot_sse_metadata)?; - if stored_md5 != key_md5 { - return Err(invalid_select_snapshot_sse_metadata()); - } - insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?; - insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?; - } - } - Ok(response_headers) -} - async fn send_select_events_until_deadline( output: SendableRecordBatchStream, tx: mpsc::Sender>, @@ -641,7 +448,7 @@ async fn prepare_select_object_snapshot( headers: &http::HeaderMap, input: &SelectObjectContentInput, read_principal: Option<&SseKmsPrincipal>, -) -> S3Result { +) -> S3Result<(StorageSelectObjectSnapshot, HeaderMap)> { let opts = get_opts(&input.bucket, &input.key, None, None, headers) .await .map_err(ApiError::from)?; @@ -654,7 +461,12 @@ async fn prepare_select_object_snapshot( validate_sse_headers_for_read(&info.user_defined, headers)?; validate_ssec_for_read(&info.user_defined, input.sse_customer_key.as_ref(), input.sse_customer_key_md5.as_ref())?; authorize_sse_kms_object_read(read_principal, &info.user_defined).await?; - Ok(snapshot) + let response_headers = project_sse_read_response_headers( + &info.user_defined, + input.sse_customer_algorithm.as_ref(), + input.sse_customer_key_md5.as_ref(), + )?; + Ok((snapshot, response_headers)) } fn map_prepare_snapshot_error(err: StoragePrepareSelectObjectSnapshotError) -> S3Error { @@ -1043,155 +855,6 @@ mod tests { } } - #[test] - fn select_snapshot_sse_s3_headers_are_whitelisted() { - let metadata = HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string()), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()), - (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), - ("x-amz-meta-private".to_string(), "private-value".to_string()), - ]); - - let headers = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new()) - .expect("valid SSE-S3 snapshot metadata should project response headers"); - - assert_eq!(headers.len(), 1); - assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-S3 mode"), "AES256"); - assert!(headers.get("x-amz-meta-private").is_none()); - } - - #[test] - fn select_snapshot_sse_kms_headers_use_snapshot_metadata() { - let context = "eyJ0ZW5hbnQiOiJvbmUifQ=="; - for key_id in ["key-1", "arn:aws:kms:key-2"] { - let metadata = HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()), - (AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), key_id.to_string()), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), key_id.to_string()), - (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), key_id.to_string()), - (MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), context.to_string()), - ]); - - let headers = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new()) - .expect("valid SSE-KMS snapshot metadata should project response headers"); - let expected_key_id = if key_id.starts_with(SELECT_KMS_ARN_PREFIX) { - key_id.to_string() - } else { - format!("{SELECT_KMS_ARN_PREFIX}{key_id}") - }; - - assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-KMS mode"), "aws:kms"); - assert_eq!( - headers - .get(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID) - .expect("SSE-KMS key ID") - .to_str() - .expect("SSE-KMS key ID should be valid text"), - expected_key_id - ); - assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT).expect("SSE-KMS context"), context); - } - } - - #[test] - fn select_snapshot_sse_c_headers_never_echo_the_customer_key() { - let key_md5 = "customer-key-md5"; - let metadata = HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string()), - (SSEC_ALGORITHM_HEADER.to_string(), AMZ_ENCRYPTION_AES.to_string()), - (SSEC_KEY_MD5_HEADER.to_string(), key_md5.to_string()), - ("x-amz-meta-private".to_string(), "private-value".to_string()), - ]); - let mut request_headers = HeaderMap::new(); - request_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256")); - request_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, HeaderValue::from_static(key_md5)); - request_headers.insert( - http::HeaderName::from_static(SSEC_KEY_HEADER), - HeaderValue::from_static("must-not-be-returned"), - ); - - let headers = select_snapshot_sse_response_headers(&metadata, &request_headers) - .expect("validated SSE-C request values should project response headers"); - - assert_eq!(headers.len(), 2); - assert_eq!( - headers - .get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM) - .expect("SSE-C algorithm"), - "AES256" - ); - assert_eq!( - headers - .get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5) - .expect("SSE-C key MD5"), - key_md5 - ); - assert!(headers.get(SSEC_KEY_HEADER).is_none()); - assert!(headers.get("x-amz-meta-private").is_none()); - } - - #[test] - fn select_snapshot_sse_headers_fail_closed_on_corrupt_metadata() { - let invalid_context = "not-base64"; - let persisted_key = "must-not-leak"; - let corrupt_metadata = [ - HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_lowercase(), "AES256".to_string()), - (AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_uppercase(), "aws:kms".to_string()), - ]), - HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()), - (SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()), - ]), - HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string())]), - HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()), - (AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "key-2".to_string()), - ]), - HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()), - (AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()), - (MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), invalid_context.to_string()), - ]), - HashMap::from([ - (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_KMS.to_string()), - (AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()), - (AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), "persisted-context".to_string()), - ]), - HashMap::from([ - (SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()), - (SSEC_KEY_MD5_HEADER.to_string(), "customer-key-md5".to_string()), - (SSEC_KEY_HEADER.to_string(), persisted_key.to_string()), - ]), - ]; - - for metadata in corrupt_metadata { - let error = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new()) - .expect_err("corrupt snapshot encryption metadata must fail closed"); - assert_eq!(error.code(), &S3ErrorCode::InternalError); - assert!(!error.to_string().contains(invalid_context)); - assert!(!error.to_string().contains(persisted_key)); - } - } - - #[test] - fn select_response_projects_snapshot_sse_headers() { - let (_tx, rx) = mpsc::channel(1); - let metadata = HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string())]); - - let response = select_object_response(rx, &metadata, &HeaderMap::new()) - .expect("valid snapshot metadata should produce a Select response"); - - assert_eq!( - response - .headers - .get(X_AMZ_SERVER_SIDE_ENCRYPTION) - .expect("snapshot SSE response header"), - AMZ_ENCRYPTION_AES - ); - } - fn spawn_test_producer( output: SendableRecordBatchStream, channel_capacity: usize, diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index da3c2fb1f..2dd54126d 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -1053,8 +1053,8 @@ pub(crate) mod sse { pub(crate) use crate::storage::storage_api::sse_consumer::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, - get_buffer_size_opt_in, load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption, - validate_bucket_object_lock_enabled_state, + get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption, + sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state, }; pub(crate) use crate::storage::storage_api::sse_consumer::{ EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 7ddc56f84..5d45ca833 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -88,7 +88,7 @@ use base64_simd::STANDARD as BASE64_STANDARD; use chacha20poly1305::ChaCha20Poly1305; #[cfg(feature = "rio-v2")] use hmac::{Hmac, Mac}; -use http::{HeaderMap, HeaderValue}; +use http::{HeaderMap, HeaderName, HeaderValue}; use md5::{Digest as Md5Digest, Md5}; use rand::Rng; #[cfg(feature = "rio-v2")] @@ -160,11 +160,18 @@ use super::Error; use super::get_bucket_sse_config; use crate::error::ApiError; use rustfs_utils::http::headers::{ - AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, - AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, + AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, + AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, + SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER, }; +#[cfg(test)] +use rustfs_utils::http::headers::{AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID}; use rustfs_utils::path::path_join_buf; use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryptionByDefault}; +use s3s::header::{ + X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, + X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, +}; use std::borrow::Cow; // ============================================================================ @@ -2177,6 +2184,159 @@ pub struct SseReadResponseHeaders { pub ssekms_key_id: Option, } +fn invalid_sse_response_metadata() -> ApiError { + ApiError { + code: S3ErrorCode::InternalError, + message: "Persisted object encryption metadata is invalid.".to_string(), + source: None, + } +} + +fn insert_sse_response_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> Result<(), ApiError> { + let value = HeaderValue::from_str(value).map_err(|_| invalid_sse_response_metadata())?; + headers.insert(name, value); + Ok(()) +} + +/// Project response headers after the caller has completed its existing read +/// validation and KMS authorization path. +pub(crate) fn project_sse_read_response_headers( + metadata: &HashMap, + sse_customer_algorithm: Option<&SSECustomerAlgorithm>, + sse_customer_key_md5: Option<&SSECustomerKeyMD5>, +) -> Result { + if metadata.keys().any(|key| { + key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY) + || key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT) + }) { + return Err(invalid_sse_response_metadata()); + } + + let metadata = normalize_encryption_metadata_case(metadata).map_err(|_| invalid_sse_response_metadata())?; + let public_mode = metadata.get("x-amz-server-side-encryption").map(String::as_str); + let customer_algorithm = metadata.get(SSEC_ALGORITHM_HEADER).map(String::as_str); + + let public_mode = match public_mode { + Some(AMZ_ENCRYPTION_AES) => Some(SSEType::SseS3), + Some(AMZ_ENCRYPTION_KMS) => Some(SSEType::SseKms), + Some(_) => return Err(invalid_sse_response_metadata()), + None => None, + }; + if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) { + return Err(invalid_sse_response_metadata()); + } + + let mode = if customer_algorithm.is_some() { + if public_mode == Some(SSEType::SseKms) { + return Err(invalid_sse_response_metadata()); + } + Some(SSEType::SseC) + } else { + public_mode + }; + for marker_mode in [ + metadata + .contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER) + .then_some(SSEType::SseC), + metadata + .contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER) + .then_some(SSEType::SseS3), + metadata + .contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) + .then_some(SSEType::SseKms), + ] + .into_iter() + .flatten() + { + if mode != Some(marker_mode) { + return Err(invalid_sse_response_metadata()); + } + } + let Some(mode) = mode else { + if metadata + .keys() + .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) + { + return Err(invalid_sse_response_metadata()); + } + return Ok(HeaderMap::new()); + }; + + let mut key_id = None; + for value in [ + metadata.get("x-amz-server-side-encryption-aws-kms-key-id"), + metadata.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER), + metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER), + ] + .into_iter() + .flatten() + { + if key_id.is_some_and(|current| current != value) { + return Err(invalid_sse_response_metadata()); + } + key_id = Some(value); + } + + let normalized_managed = normalize_managed_metadata(metadata.as_ref(), Some(recode_minio_kms_context)); + let mut response_headers = HeaderMap::with_capacity(3); + match mode { + SSEType::SseS3 => { + if metadata.contains_key("x-amz-server-side-encryption-aws-kms-key-id") || metadata.contains_key(SSEC_KEY_MD5_HEADER) + { + return Err(invalid_sse_response_metadata()); + } + response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES)); + } + SSEType::SseKms => { + if metadata.contains_key(SSEC_KEY_MD5_HEADER) { + return Err(invalid_sse_response_metadata()); + } + let key_id = key_id + .map(String::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(invalid_sse_response_metadata)?; + response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS)); + insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?; + + if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) + && !normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) + { + return Err(invalid_sse_response_metadata()); + } + if let Some(context) = normalized_managed.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) { + serde_json::from_str::>(context).map_err(|_| invalid_sse_response_metadata())?; + let encoded = BASE64_STANDARD.encode_to_string(context); + insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, &encoded)?; + } + } + SSEType::SseC => { + if key_id.is_some() || normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) { + return Err(invalid_sse_response_metadata()); + } + let algorithm = sse_customer_algorithm + .map(|value| value.as_ref()) + .filter(|value| *value == AMZ_ENCRYPTION_AES) + .ok_or_else(|| { + ssec_invalid_request( + "The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.", + ) + })?; + let key_md5 = sse_customer_key_md5 + .map(|value| value.as_ref()) + .ok_or_else(|| ssec_invalid_request("Missing SSE-C customer key MD5."))?; + let stored_md5 = metadata.get(SSEC_KEY_MD5_HEADER).ok_or_else(invalid_sse_response_metadata)?; + if stored_md5 != key_md5 { + return Err(ssec_invalid_request( + "The provided encryption parameters did not match the ones used originally to encrypt the object.", + )); + } + insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?; + insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?; + } + } + Ok(response_headers) +} + /// Read-side response classification for an object whose payload the object /// layer's encryption resolver already decrypted. /// @@ -4054,10 +4214,10 @@ mod tests { build_kms_request_context, classify_sse_read_response, encode_minio_kms_context, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, - md5_base64, normalize_managed_metadata, recode_minio_kms_context, reset_sse_dek_provider, resolve_effective_kms_key_id, - resolve_stored_kms_key_id, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, - strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, - validate_ssec_params, verify_ssec_key_match, + md5_base64, normalize_managed_metadata, project_sse_read_response_headers, recode_minio_kms_context, + reset_sse_dek_provider, resolve_effective_kms_key_id, resolve_stored_kms_key_id, rewrap_object_encryption_metadata, + sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, + validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match, }; #[cfg(feature = "rio-v2")] use super::{ @@ -7724,6 +7884,166 @@ mod tests { assert_eq!(audit_tag(&tags, "kmsErrorClass").as_deref(), Some("access_denied")); } + #[test] + fn read_response_projection_uses_shared_managed_metadata_compatibility() { + let context_json = r#"{"tenant":"s3-select"}"#; + let context = BASE64_STANDARD.encode_to_string(context_json); + let key_id = "arn:aws:kms:us-east-1:123456789012:key/select"; + let kms_metadata = HashMap::from([ + (super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()), + (super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), key_id.to_string()), + (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), key_id.to_string()), + (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), key_id.to_string()), + (MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), context.clone()), + ]); + + let headers = project_sse_read_response_headers(&kms_metadata, None, None) + .expect("valid SSE-KMS metadata should project response headers"); + assert_eq!( + headers.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-KMS algorithm"), + super::AMZ_ENCRYPTION_KMS + ); + assert_eq!( + headers + .get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID) + .expect("SSE-KMS key ID"), + key_id + ); + assert_eq!( + headers + .get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT) + .expect("SSE-KMS context"), + context.as_str() + ); + + let s3_metadata = HashMap::from([ + (super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_AES.to_string()), + (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()), + (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), + ( + super::INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), + r#"{"bucket":"select"}"#.to_string(), + ), + ("x-amz-meta-private".to_string(), "private".to_string()), + ]); + let headers = project_sse_read_response_headers(&s3_metadata, None, None) + .expect("valid SSE-S3 metadata should project response headers"); + assert_eq!(headers.len(), 1); + assert_eq!( + headers.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-S3 algorithm"), + super::AMZ_ENCRYPTION_AES + ); + + let plaintext = project_sse_read_response_headers(&HashMap::new(), None, None) + .expect("plaintext metadata should not produce SSE headers"); + assert!(plaintext.is_empty()); + } + + #[test] + fn read_response_projection_returns_only_validated_ssec_values() { + let key_md5 = "customer-key-md5"; + let metadata = HashMap::from([ + ( + super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(), + super::AMZ_ENCRYPTION_AES.to_string(), + ), + (super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5.to_string(), key_md5.to_string()), + ]); + let algorithm = SSECustomerAlgorithm::from(super::AMZ_ENCRYPTION_AES); + let key_md5 = SSECustomerKeyMD5::from(key_md5); + + let headers = project_sse_read_response_headers(&metadata, Some(&algorithm), Some(&key_md5)) + .expect("validated SSE-C values should project response headers"); + assert_eq!(headers.len(), 2); + assert_eq!( + headers + .get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM) + .expect("SSE-C algorithm"), + super::AMZ_ENCRYPTION_AES + ); + assert_eq!( + headers + .get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5) + .expect("SSE-C key MD5"), + key_md5.as_str() + ); + assert!(headers.get(super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY).is_none()); + + let missing_algorithm = project_sse_read_response_headers(&metadata, None, Some(&key_md5)) + .expect_err("missing SSE-C algorithm must fail before streaming"); + assert_eq!(missing_algorithm.code, S3ErrorCode::InvalidRequest); + } + + #[test] + fn read_response_projection_rejects_unvalidated_or_sensitive_metadata() { + let invalid_context = "not-base64"; + let persisted_context = "must-not-leak-context"; + let persisted_key = "must-not-leak-key"; + let corrupt_metadata = [ + HashMap::from([ + ( + super::AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_lowercase(), + super::AMZ_ENCRYPTION_AES.to_string(), + ), + ( + super::AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_uppercase(), + super::AMZ_ENCRYPTION_KMS.to_string(), + ), + ]), + HashMap::from([ + (super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()), + ( + super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(), + super::AMZ_ENCRYPTION_AES.to_string(), + ), + ]), + HashMap::from([ + (super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()), + (super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()), + (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "key-2".to_string()), + ]), + HashMap::from([ + (super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()), + (super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()), + (MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), invalid_context.to_string()), + ]), + HashMap::from([ + (super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()), + (super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()), + (super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), persisted_context.to_string()), + ]), + HashMap::from([ + ( + super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(), + super::AMZ_ENCRYPTION_AES.to_string(), + ), + ( + super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5.to_string(), + "customer-key-md5".to_string(), + ), + ( + super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY.to_ascii_uppercase(), + persisted_key.to_string(), + ), + ]), + ]; + + for metadata in corrupt_metadata { + let error = project_sse_read_response_headers(&metadata, None, None) + .expect_err("corrupt encryption metadata must fail closed"); + assert_eq!(error.code, S3ErrorCode::InternalError); + for sensitive in [invalid_context, persisted_context, persisted_key] { + assert!(!error.to_string().contains(sensitive)); + } + } + + let marker_only_kms = + HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string())]); + let marker_error = project_sse_read_response_headers(&marker_only_kms, None, None) + .expect_err("marker-only KMS metadata must not bypass the existing authorization classifier"); + assert_eq!(marker_error.code, S3ErrorCode::InternalError); + } + #[tokio::test] async fn classification_matches_ssec_validation_and_headers() { let key = [0x42u8; 32]; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index b70db13f5..2e778c0c6 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -111,8 +111,8 @@ pub(crate) use super::ecfs_extend::{ pub(crate) use super::sse::{ DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal, authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, - rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, - validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, + project_sse_read_response_headers, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, + strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, }; pub(crate) mod access_consumer { @@ -363,8 +363,8 @@ pub(crate) mod sse_consumer { pub(crate) use super::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, - get_buffer_size_opt_in, load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption, - validate_bucket_object_lock_enabled_state, + get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption, + sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state, }; }