fix(s3select): return encryption response headers (#6819)

This commit is contained in:
GatewayJ
2026-08-29 19:46:34 +08:00
committed by GitHub
parent fa0be5d271
commit 79bd6fa862
7 changed files with 643 additions and 373 deletions
+52 -9
View File
@@ -217,7 +217,37 @@ pub(crate) async fn signed_s3_request(
access_key: &str, access_key: &str,
secret_key: &str, secret_key: &str,
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
signed_s3_request_with_session_token(method, url, body, content_type, access_key, secret_key, None).await signed_s3_request_with_headers(method, url, body, content_type, access_key, secret_key, &http::HeaderMap::new()).await
}
pub(crate) async fn signed_s3_request_with_headers(
method: http::Method,
url: &str,
body: Option<String>,
content_type: Option<&str>,
access_key: &str,
secret_key: &str,
extra_headers: &http::HeaderMap,
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
signed_s3_request_with_session_token(
method,
url,
body,
content_type,
SigningCredentials {
access_key,
secret_key,
session_token: None,
},
extra_headers,
)
.await
}
struct SigningCredentials<'a> {
access_key: &'a str,
secret_key: &'a str,
session_token: Option<&'a str>,
} }
async fn signed_s3_request_with_session_token( async fn signed_s3_request_with_session_token(
@@ -225,9 +255,8 @@ async fn signed_s3_request_with_session_token(
url: &str, url: &str,
body: Option<String>, body: Option<String>,
content_type: Option<&str>, content_type: Option<&str>,
access_key: &str, credentials: SigningCredentials<'_>,
secret_key: &str, extra_headers: &http::HeaderMap,
session_token: Option<&str>,
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?; let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string(); let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
@@ -239,14 +268,17 @@ async fn signed_s3_request_with_session_token(
if let Some(content_type) = content_type { if let Some(content_type) = content_type {
request = request.header(CONTENT_TYPE, content_type); request = request.header(CONTENT_TYPE, content_type);
} }
for (name, value) in extra_headers {
request = request.header(name, value);
}
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?; let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
let signed = sign_v4( let signed = sign_v4(
request.body(Body::empty())?, request.body(Body::empty())?,
content_length, content_length,
access_key, credentials.access_key,
secret_key, credentials.secret_key,
session_token.unwrap_or_default(), credentials.session_token.unwrap_or_default(),
"us-east-1", "us-east-1",
); );
@@ -283,8 +315,19 @@ pub(crate) async fn admin_request_with_session_token(
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> { ) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{base_url}{path_and_query}"); let url = format!("{base_url}{path_and_query}");
let content_type = body.as_ref().map(|_| "application/json"); let content_type = body.as_ref().map(|_| "application/json");
let response = let response = signed_s3_request_with_session_token(
signed_s3_request_with_session_token(method, &url, body, content_type, access_key, secret_key, session_token).await?; method,
&url,
body,
content_type,
SigningCredentials {
access_key,
secret_key,
session_token,
},
&http::HeaderMap::new(),
)
.await?;
let status = response.status(); let status = response.status();
let body = response.text().await?; let body = response.text().await?;
Ok((status, body)) Ok((status, body))
+3
View File
@@ -57,6 +57,9 @@ mod copy_object_version_restore_sse_test;
#[cfg(test)] #[cfg(test)]
mod configured_roundtrip_test; mod configured_roundtrip_test;
#[cfg(test)]
mod select_sse_response_test;
#[cfg(test)] #[cfg(test)]
mod kms_anonymous_enforcement_test; mod kms_anonymous_enforcement_test;
@@ -0,0 +1,241 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! SelectObjectContent SSE response-header compatibility (backlog#1625).
use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64, start_kms};
use crate::common::signed_s3_request_with_headers;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
use base64_simd::STANDARD as BASE64;
use http::{HeaderMap, Method};
use std::error::Error;
use uuid::Uuid;
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
const CSV_BODY: &[u8] = b"name\nalice\n";
const SELECT_BODY: &str = r#"<SelectObjectContentRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Expression>SELECT * FROM S3Object</Expression>
<ExpressionType>SQL</ExpressionType>
<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
<OutputSerialization><CSV/></OutputSerialization>
</SelectObjectContentRequest>"#;
const KMS_CONTEXT: &str = "eyJ0ZW5hbnQiOiJzMy1zZWxlY3QifQ==";
const SSE_ALGORITHM: &str = "x-amz-server-side-encryption";
const SSE_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id";
const SSE_KMS_CONTEXT: &str = "x-amz-server-side-encryption-context";
const SSE_C_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm";
const SSE_C_KEY: &str = "x-amz-server-side-encryption-customer-key";
const SSE_C_KEY_MD5: &str = "x-amz-server-side-encryption-customer-key-md5";
const LOG_FLUSH_SENTINEL: &str = "select-sse-log-flush-sentinel.csv";
async fn raw_select(
env: &crate::common::RustFSTestEnvironment,
bucket: &str,
object: &str,
request_headers: &HeaderMap,
) -> TestResult<reqwest::Response> {
let url = format!("{}/{bucket}/{object}?select&select-type=2", env.url);
signed_s3_request_with_headers(
Method::POST,
&url,
Some(SELECT_BODY.to_string()),
Some("application/xml"),
&env.access_key,
&env.secret_key,
request_headers,
)
.await
}
async fn assert_success_headers(response: reqwest::Response, expected: &[(&str, &str)], absent: &[&str]) -> TestResult {
if response.status() != reqwest::StatusCode::OK {
let status = response.status();
let url = response.url().clone();
let body = response.text().await?;
panic!("Select request to {url} failed with {status}: {body}");
}
for (name, value) in expected {
assert_eq!(response.headers().get(*name).and_then(|header| header.to_str().ok()), Some(*value));
}
for name in absent {
assert!(response.headers().get(*name).is_none(), "successful Select response must omit {name}");
}
let body = response.bytes().await?;
assert!(
body.windows(b"alice".len()).any(|window| window == b"alice"),
"successful Select response must contain a Records event with the selected row"
);
assert!(
body.windows(b"End".len()).any(|window| window == b"End"),
"successful Select response must contain the terminal End event"
);
Ok(())
}
async fn assert_pre_stream_failure(response: reqwest::Response) -> TestResult {
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
let body = response.text().await?;
assert!(body.contains("<Error>"), "pre-stream failure must return an S3 XML error: {body}");
assert!(
body.contains("<Code>InvalidRequest</Code>"),
"invalid SSE-C parameters must preserve the S3 error code: {body}"
);
Ok(())
}
fn put_object(
client: &aws_sdk_s3::Client,
bucket: &str,
object: &str,
) -> aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder {
client
.put_object()
.bucket(bucket)
.key(object)
.body(ByteStream::from_static(CSV_BODY))
}
#[tokio::test]
async fn select_projects_encryption_headers_and_rejects_invalid_sse_c_before_streaming() -> TestResult {
let mut kms = LocalKMSTestEnvironment::new().await?;
let log_path = format!("{}/server.log", kms.base_env.temp_dir);
kms.base_env.capture_log_path = Some(log_path.clone());
kms.base_env
.start_rustfs_server_with_env(Vec::new(), &[("RUST_LOG", "s3s=debug,rustfs=info")])
.await?;
let key_id = kms.configure_local_kms().await?;
start_kms(&kms.base_env.url, &kms.base_env.access_key, &kms.base_env.secret_key).await?;
let client = kms.base_env.create_s3_client();
let bucket = format!("select-sse-{}", Uuid::new_v4().simple());
client.create_bucket().bucket(&bucket).send().await?;
put_object(&client, &bucket, "plain.csv").send().await?;
put_object(&client, &bucket, "sse-s3.csv")
.server_side_encryption(ServerSideEncryption::Aes256)
.send()
.await?;
put_object(&client, &bucket, "sse-kms.csv")
.server_side_encryption(ServerSideEncryption::AwsKms)
.ssekms_key_id(&key_id)
.ssekms_encryption_context(KMS_CONTEXT)
.send()
.await?;
let customer_key = "01234567890123456789012345678901";
let customer_key_b64 = BASE64.encode_to_string(customer_key);
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
put_object(&client, &bucket, "sse-c.csv")
.sse_customer_algorithm("AES256")
.sse_customer_key(&customer_key_b64)
.sse_customer_key_md5(&customer_key_md5)
.send()
.await?;
assert_success_headers(
raw_select(&kms.base_env, &bucket, "plain.csv", &HeaderMap::new()).await?,
&[],
&[
SSE_ALGORITHM,
SSE_KMS_KEY_ID,
SSE_KMS_CONTEXT,
SSE_C_ALGORITHM,
SSE_C_KEY,
SSE_C_KEY_MD5,
],
)
.await?;
assert_success_headers(
raw_select(&kms.base_env, &bucket, "sse-s3.csv", &HeaderMap::new()).await?,
&[(SSE_ALGORITHM, "AES256")],
&[SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
)
.await?;
assert_success_headers(
raw_select(&kms.base_env, &bucket, "sse-kms.csv", &HeaderMap::new()).await?,
&[
(SSE_ALGORITHM, "aws:kms"),
(SSE_KMS_KEY_ID, &key_id),
(SSE_KMS_CONTEXT, KMS_CONTEXT),
],
&[SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
)
.await?;
let mut sse_c_headers = HeaderMap::new();
sse_c_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
sse_c_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
sse_c_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
assert_success_headers(
raw_select(&kms.base_env, &bucket, "sse-c.csv", &sse_c_headers).await?,
&[(SSE_C_ALGORITHM, "AES256"), (SSE_C_KEY_MD5, &customer_key_md5)],
&[SSE_ALGORITHM, SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_KEY],
)
.await?;
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &HeaderMap::new()).await?).await?;
let mut missing_algorithm_headers = HeaderMap::new();
missing_algorithm_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
missing_algorithm_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &missing_algorithm_headers).await?).await?;
let mut wrong_algorithm_headers = sse_c_headers.clone();
wrong_algorithm_headers.insert(SSE_C_ALGORITHM, "AES128".parse()?);
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_algorithm_headers).await?).await?;
let wrong_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
let mut wrong_md5_headers = sse_c_headers.clone();
wrong_md5_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_md5_headers).await?).await?;
let wrong_key = "99999999999999999999999999999999";
let wrong_key_b64 = BASE64.encode_to_string(wrong_key);
let mut wrong_key_headers = HeaderMap::new();
wrong_key_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
wrong_key_headers.insert(SSE_C_KEY, wrong_key_b64.parse()?);
wrong_key_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_key_headers).await?).await?;
put_object(&client, &bucket, LOG_FLUSH_SENTINEL).send().await?;
assert_success_headers(
raw_select(&kms.base_env, &bucket, LOG_FLUSH_SENTINEL, &HeaderMap::new()).await?,
&[],
&[
SSE_ALGORITHM,
SSE_KMS_KEY_ID,
SSE_KMS_CONTEXT,
SSE_C_ALGORITHM,
SSE_C_KEY,
SSE_C_KEY_MD5,
],
)
.await?;
let mut logs = String::new();
for _ in 0..100 {
logs = tokio::fs::read_to_string(&log_path).await?;
if logs.contains(LOG_FLUSH_SENTINEL) {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
assert!(logs.contains(LOG_FLUSH_SENTINEL), "timed out waiting for the log sink to flush");
for secret in [customer_key, customer_key_b64.as_str(), wrong_key, wrong_key_b64.as_str()] {
assert!(!logs.contains(secret), "Select request logging leaked SSE-C customer key material");
}
Ok(())
}
+14 -351
View File
@@ -2,7 +2,7 @@
use super::storage_api::select_object::StorageError; use super::storage_api::select_object::StorageError;
use super::storage_api::select_object::options::get_opts; use super::storage_api::select_object::options::get_opts;
use super::storage_api::select_object::request_context::spawn_traced; use super::storage_api::select_object::request_context::spawn_traced;
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read}; use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
use super::storage_api::select_object::{ use super::storage_api::select_object::{
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read, StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
validate_ssec_for_read, validate_ssec_for_read,
@@ -19,34 +19,20 @@ use datafusion::arrow::{
use datafusion::common::DataFusionError; use datafusion::common::DataFusionError;
use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::SendableRecordBatchStream;
use futures::StreamExt; use futures::StreamExt;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::RANGE}; use http::{HeaderMap, StatusCode, header::RANGE};
use rustfs_s3select_api::{ use rustfs_s3select_api::{
QueryError, SelectError, QueryError, SelectError,
object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds}, object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds},
query::{Context, Query}, query::{Context, Query},
}; };
use rustfs_s3select_query::instance::s3_select_query_timeout; use rustfs_s3select_query::instance::s3_select_query_timeout;
use rustfs_utils::http::headers::{
AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER,
};
use rustfs_utils::http::object_encryption_keys::{
INTERNAL_ENCRYPTION_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
};
use s3s::dto::{ use s3s::dto::{
CSVOutput, CompressionType, ContinuationEvent, EndEvent, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, CSVOutput, CompressionType, ContinuationEvent, EndEvent, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput,
JSONOutput, JSONType, OutputSerialization, Progress, ProgressEvent, QuoteFields, RecordsEvent, SelectObjectContentEvent, JSONOutput, JSONType, OutputSerialization, Progress, ProgressEvent, QuoteFields, RecordsEvent, SelectObjectContentEvent,
SelectObjectContentEventStream, SelectObjectContentInput, SelectObjectContentOutput, SelectObjectContentRequest, Stats, SelectObjectContentEventStream, SelectObjectContentInput, SelectObjectContentOutput, SelectObjectContentRequest, Stats,
StatsEvent, StatsEvent,
}; };
use s3s::header::{
X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT,
X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
};
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tokio::time::{Instant, timeout_at}; use tokio::time::{Instant, timeout_at};
@@ -62,8 +48,6 @@ const BUSY_MESSAGE: &str = "The service is unavailable. Try again later.";
const EMPTY_SELECT_EXPRESSION_MESSAGE: &str = "empty SQL expression"; const EMPTY_SELECT_EXPRESSION_MESSAGE: &str = "empty SQL expression";
const SLOW_DOWN_MESSAGE: &str = "Reduce your request rate."; const SLOW_DOWN_MESSAGE: &str = "Reduce your request rate.";
const UNSUPPORTED_SQL_STRUCTURE_MESSAGE: &str = "We encountered an unsupported SQL structure. Check the SQL Reference."; const UNSUPPORTED_SQL_STRUCTURE_MESSAGE: &str = "We encountered an unsupported SQL structure. Check the SQL Reference.";
// No canonical owner exists for the KMS key ARN prefix; keep it local.
const SELECT_KMS_ARN_PREFIX: &str = "arn:aws:kms:";
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
struct SelectValidation { struct SelectValidation {
@@ -107,7 +91,7 @@ pub async fn execute_select_object_content(
.map_err(|_| select_query_timeout_error(query_timeout.as_secs()))? .map_err(|_| select_query_timeout_error(query_timeout.as_secs()))?
.map_err(map_query_error_to_s3)?; .map_err(map_query_error_to_s3)?;
let admission = db.try_reserve_query().map_err(map_query_error_to_s3)?; let admission = db.try_reserve_query().map_err(map_query_error_to_s3)?;
let snapshot = timeout_at( let (snapshot, response_headers) = timeout_at(
query_deadline, query_deadline,
prepare_select_object_snapshot(&req.headers, &input, read_principal.as_ref()), prepare_select_object_snapshot(&req.headers, &input, read_principal.as_ref()),
) )
@@ -130,7 +114,10 @@ pub async fn execute_select_object_content(
.clone() .clone()
.try_reserve_owned() .try_reserve_owned()
.map_err(|_| map_select_error_to_s3(&SelectError::InternalError))?; .map_err(|_| map_select_error_to_s3(&SelectError::InternalError))?;
let response = select_object_response(rx, &snapshot.object_info().user_defined, &req.headers)?; let mut response = S3Response::new(SelectObjectContentOutput {
payload: Some(SelectObjectContentEventStream::new(ReceiverStream::new(rx))),
});
response.headers = response_headers;
spawn_traced(async move { spawn_traced(async move {
send_select_events_until_deadline( send_select_events_until_deadline(
output, output,
@@ -147,186 +134,6 @@ pub async fn execute_select_object_content(
Ok(response) Ok(response)
} }
fn select_object_response(
rx: mpsc::Receiver<S3Result<SelectObjectContentEvent>>,
metadata: &HashMap<String, String>,
request_headers: &HeaderMap,
) -> S3Result<S3Response<SelectObjectContentOutput>> {
let response_headers = select_snapshot_sse_response_headers(metadata, request_headers)?;
let mut response = S3Response::new(SelectObjectContentOutput {
payload: Some(SelectObjectContentEventStream::new(ReceiverStream::new(rx))),
});
response.headers = response_headers;
Ok(response)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SelectSnapshotSseMode {
S3,
Kms,
Customer,
}
fn invalid_select_snapshot_sse_metadata() -> S3Error {
S3Error::with_message(
S3ErrorCode::InternalError,
"Persisted SelectObjectContent encryption metadata is invalid.",
)
}
fn select_metadata_value<'a>(metadata: &'a HashMap<String, String>, name: &str) -> S3Result<Option<&'a str>> {
let mut values = metadata
.iter()
.filter_map(|(key, value)| key.eq_ignore_ascii_case(name).then_some(value.as_str()));
let Some(value) = values.next() else {
return Ok(None);
};
if values.any(|candidate| candidate != value) {
return Err(invalid_select_snapshot_sse_metadata());
}
Ok(Some(value))
}
fn select_snapshot_kms_key_id(metadata: &HashMap<String, String>) -> S3Result<Option<&str>> {
let values = [
select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID)?,
select_metadata_value(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER)?,
select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?,
];
let mut resolved = None;
for value in values.into_iter().flatten() {
if resolved.is_some_and(|current| current != value) {
return Err(invalid_select_snapshot_sse_metadata());
}
resolved = Some(value);
}
Ok(resolved)
}
fn select_snapshot_sse_mode(metadata: &HashMap<String, String>) -> S3Result<Option<SelectSnapshotSseMode>> {
let public_mode = select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION)?;
let customer_algorithm = select_metadata_value(metadata, SSEC_ALGORITHM_HEADER)?;
let has_ssec_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)?.is_some();
let has_s3_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)?.is_some();
let has_kms_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)?.is_some();
let public_mode = match public_mode {
Some(AMZ_ENCRYPTION_AES) => Some(SelectSnapshotSseMode::S3),
Some(AMZ_ENCRYPTION_KMS) => Some(SelectSnapshotSseMode::Kms),
Some(_) => return Err(invalid_select_snapshot_sse_metadata()),
None => None,
};
if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) {
return Err(invalid_select_snapshot_sse_metadata());
}
let resolved = if customer_algorithm.is_some() {
if public_mode == Some(SelectSnapshotSseMode::Kms) {
return Err(invalid_select_snapshot_sse_metadata());
}
Some(SelectSnapshotSseMode::Customer)
} else {
public_mode
};
let internal_modes = [
has_ssec_marker.then_some(SelectSnapshotSseMode::Customer),
has_s3_marker.then_some(SelectSnapshotSseMode::S3),
has_kms_marker.then_some(SelectSnapshotSseMode::Kms),
];
for mode in internal_modes.into_iter().flatten() {
if resolved != Some(mode) {
return Err(invalid_select_snapshot_sse_metadata());
}
}
if resolved.is_none()
&& metadata
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
{
return Err(invalid_select_snapshot_sse_metadata());
}
Ok(resolved)
}
fn insert_select_snapshot_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> S3Result<()> {
let value = HeaderValue::from_str(value).map_err(|_| invalid_select_snapshot_sse_metadata())?;
headers.insert(name, value);
Ok(())
}
fn select_snapshot_sse_response_headers(metadata: &HashMap<String, String>, request_headers: &HeaderMap) -> S3Result<HeaderMap> {
if select_metadata_value(metadata, SSEC_KEY_HEADER)?.is_some()
|| select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT)?.is_some()
{
return Err(invalid_select_snapshot_sse_metadata());
}
let Some(mode) = select_snapshot_sse_mode(metadata)? else {
return Ok(HeaderMap::new());
};
let kms_key_id = select_snapshot_kms_key_id(metadata)?;
let mut response_headers = HeaderMap::with_capacity(3);
match mode {
SelectSnapshotSseMode::S3 => {
if select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID)?.is_some()
|| select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?.is_some()
|| select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.is_some()
{
return Err(invalid_select_snapshot_sse_metadata());
}
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES));
}
SelectSnapshotSseMode::Kms => {
if select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.is_some() {
return Err(invalid_select_snapshot_sse_metadata());
}
let key_id = kms_key_id
.filter(|key_id| !key_id.is_empty())
.ok_or_else(invalid_select_snapshot_sse_metadata)?;
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS));
if key_id.starts_with(SELECT_KMS_ARN_PREFIX) {
insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?;
} else {
insert_select_snapshot_header(
&mut response_headers,
X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID,
&format!("{SELECT_KMS_ARN_PREFIX}{key_id}"),
)?;
}
if let Some(context) = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)? {
let context = HeaderValue::from_str(context).map_err(|_| invalid_select_snapshot_sse_metadata())?;
let mut validation_headers = HeaderMap::with_capacity(1);
validation_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, context.clone());
super::storage_api::select_object::sse::extract_ssekms_context_from_headers(&validation_headers)
.map_err(|_| invalid_select_snapshot_sse_metadata())?;
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, context);
}
}
SelectSnapshotSseMode::Customer => {
if kms_key_id.is_some() || select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?.is_some() {
return Err(invalid_select_snapshot_sse_metadata());
}
let algorithm = request_headers
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM)
.and_then(|value| value.to_str().ok())
.filter(|algorithm| *algorithm == AMZ_ENCRYPTION_AES)
.ok_or_else(invalid_select_snapshot_sse_metadata)?;
let key_md5 = request_headers
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5)
.and_then(|value| value.to_str().ok())
.ok_or_else(invalid_select_snapshot_sse_metadata)?;
let stored_md5 =
select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.ok_or_else(invalid_select_snapshot_sse_metadata)?;
if stored_md5 != key_md5 {
return Err(invalid_select_snapshot_sse_metadata());
}
insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?;
insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?;
}
}
Ok(response_headers)
}
async fn send_select_events_until_deadline<L: SelectSnapshotFence>( async fn send_select_events_until_deadline<L: SelectSnapshotFence>(
output: SendableRecordBatchStream, output: SendableRecordBatchStream,
tx: mpsc::Sender<S3Result<SelectObjectContentEvent>>, tx: mpsc::Sender<S3Result<SelectObjectContentEvent>>,
@@ -641,7 +448,7 @@ async fn prepare_select_object_snapshot(
headers: &http::HeaderMap, headers: &http::HeaderMap,
input: &SelectObjectContentInput, input: &SelectObjectContentInput,
read_principal: Option<&SseKmsPrincipal>, read_principal: Option<&SseKmsPrincipal>,
) -> S3Result<StorageSelectObjectSnapshot> { ) -> S3Result<(StorageSelectObjectSnapshot, HeaderMap)> {
let opts = get_opts(&input.bucket, &input.key, None, None, headers) let opts = get_opts(&input.bucket, &input.key, None, None, headers)
.await .await
.map_err(ApiError::from)?; .map_err(ApiError::from)?;
@@ -654,7 +461,12 @@ async fn prepare_select_object_snapshot(
validate_sse_headers_for_read(&info.user_defined, headers)?; validate_sse_headers_for_read(&info.user_defined, headers)?;
validate_ssec_for_read(&info.user_defined, input.sse_customer_key.as_ref(), input.sse_customer_key_md5.as_ref())?; validate_ssec_for_read(&info.user_defined, input.sse_customer_key.as_ref(), input.sse_customer_key_md5.as_ref())?;
authorize_sse_kms_object_read(read_principal, &info.user_defined).await?; authorize_sse_kms_object_read(read_principal, &info.user_defined).await?;
Ok(snapshot) let response_headers = project_sse_read_response_headers(
&info.user_defined,
input.sse_customer_algorithm.as_ref(),
input.sse_customer_key_md5.as_ref(),
)?;
Ok((snapshot, response_headers))
} }
fn map_prepare_snapshot_error(err: StoragePrepareSelectObjectSnapshotError) -> S3Error { fn map_prepare_snapshot_error(err: StoragePrepareSelectObjectSnapshotError) -> S3Error {
@@ -1043,155 +855,6 @@ mod tests {
} }
} }
#[test]
fn select_snapshot_sse_s3_headers_are_whitelisted() {
let metadata = HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
("x-amz-meta-private".to_string(), "private-value".to_string()),
]);
let headers = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new())
.expect("valid SSE-S3 snapshot metadata should project response headers");
assert_eq!(headers.len(), 1);
assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-S3 mode"), "AES256");
assert!(headers.get("x-amz-meta-private").is_none());
}
#[test]
fn select_snapshot_sse_kms_headers_use_snapshot_metadata() {
let context = "eyJ0ZW5hbnQiOiJvbmUifQ==";
for key_id in ["key-1", "arn:aws:kms:key-2"] {
let metadata = HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), key_id.to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), key_id.to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), key_id.to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), context.to_string()),
]);
let headers = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new())
.expect("valid SSE-KMS snapshot metadata should project response headers");
let expected_key_id = if key_id.starts_with(SELECT_KMS_ARN_PREFIX) {
key_id.to_string()
} else {
format!("{SELECT_KMS_ARN_PREFIX}{key_id}")
};
assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-KMS mode"), "aws:kms");
assert_eq!(
headers
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID)
.expect("SSE-KMS key ID")
.to_str()
.expect("SSE-KMS key ID should be valid text"),
expected_key_id
);
assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT).expect("SSE-KMS context"), context);
}
}
#[test]
fn select_snapshot_sse_c_headers_never_echo_the_customer_key() {
let key_md5 = "customer-key-md5";
let metadata = HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string()),
(SSEC_ALGORITHM_HEADER.to_string(), AMZ_ENCRYPTION_AES.to_string()),
(SSEC_KEY_MD5_HEADER.to_string(), key_md5.to_string()),
("x-amz-meta-private".to_string(), "private-value".to_string()),
]);
let mut request_headers = HeaderMap::new();
request_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256"));
request_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, HeaderValue::from_static(key_md5));
request_headers.insert(
http::HeaderName::from_static(SSEC_KEY_HEADER),
HeaderValue::from_static("must-not-be-returned"),
);
let headers = select_snapshot_sse_response_headers(&metadata, &request_headers)
.expect("validated SSE-C request values should project response headers");
assert_eq!(headers.len(), 2);
assert_eq!(
headers
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM)
.expect("SSE-C algorithm"),
"AES256"
);
assert_eq!(
headers
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5)
.expect("SSE-C key MD5"),
key_md5
);
assert!(headers.get(SSEC_KEY_HEADER).is_none());
assert!(headers.get("x-amz-meta-private").is_none());
}
#[test]
fn select_snapshot_sse_headers_fail_closed_on_corrupt_metadata() {
let invalid_context = "not-base64";
let persisted_key = "must-not-leak";
let corrupt_metadata = [
HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_lowercase(), "AES256".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_uppercase(), "aws:kms".to_string()),
]),
HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()),
]),
HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string())]),
HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "key-2".to_string()),
]),
HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), invalid_context.to_string()),
]),
HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_KMS.to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), "persisted-context".to_string()),
]),
HashMap::from([
(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()),
(SSEC_KEY_MD5_HEADER.to_string(), "customer-key-md5".to_string()),
(SSEC_KEY_HEADER.to_string(), persisted_key.to_string()),
]),
];
for metadata in corrupt_metadata {
let error = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new())
.expect_err("corrupt snapshot encryption metadata must fail closed");
assert_eq!(error.code(), &S3ErrorCode::InternalError);
assert!(!error.to_string().contains(invalid_context));
assert!(!error.to_string().contains(persisted_key));
}
}
#[test]
fn select_response_projects_snapshot_sse_headers() {
let (_tx, rx) = mpsc::channel(1);
let metadata = HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string())]);
let response = select_object_response(rx, &metadata, &HeaderMap::new())
.expect("valid snapshot metadata should produce a Select response");
assert_eq!(
response
.headers
.get(X_AMZ_SERVER_SIDE_ENCRYPTION)
.expect("snapshot SSE response header"),
AMZ_ENCRYPTION_AES
);
}
fn spawn_test_producer( fn spawn_test_producer(
output: SendableRecordBatchStream, output: SendableRecordBatchStream,
channel_capacity: usize, channel_capacity: usize,
+2 -2
View File
@@ -1053,8 +1053,8 @@ pub(crate) mod sse {
pub(crate) use crate::storage::storage_api::sse_consumer::{ pub(crate) use crate::storage::storage_api::sse_consumer::{
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
get_buffer_size_opt_in, load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption, get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption,
validate_bucket_object_lock_enabled_state, sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state,
}; };
pub(crate) use crate::storage::storage_api::sse_consumer::{ pub(crate) use crate::storage::storage_api::sse_consumer::{
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
+327 -7
View File
@@ -88,7 +88,7 @@ use base64_simd::STANDARD as BASE64_STANDARD;
use chacha20poly1305::ChaCha20Poly1305; use chacha20poly1305::ChaCha20Poly1305;
#[cfg(feature = "rio-v2")] #[cfg(feature = "rio-v2")]
use hmac::{Hmac, Mac}; use hmac::{Hmac, Mac};
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderName, HeaderValue};
use md5::{Digest as Md5Digest, Md5}; use md5::{Digest as Md5Digest, Md5};
use rand::Rng; use rand::Rng;
#[cfg(feature = "rio-v2")] #[cfg(feature = "rio-v2")]
@@ -160,11 +160,18 @@ use super::Error;
use super::get_bucket_sse_config; use super::get_bucket_sse_config;
use crate::error::ApiError; use crate::error::ApiError;
use rustfs_utils::http::headers::{ use rustfs_utils::http::headers::{
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER,
}; };
#[cfg(test)]
use rustfs_utils::http::headers::{AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID};
use rustfs_utils::path::path_join_buf; use rustfs_utils::path::path_join_buf;
use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryptionByDefault}; use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryptionByDefault};
use s3s::header::{
X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT,
X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
};
use std::borrow::Cow; use std::borrow::Cow;
// ============================================================================ // ============================================================================
@@ -2177,6 +2184,159 @@ pub struct SseReadResponseHeaders {
pub ssekms_key_id: Option<SSEKMSKeyId>, pub ssekms_key_id: Option<SSEKMSKeyId>,
} }
fn invalid_sse_response_metadata() -> ApiError {
ApiError {
code: S3ErrorCode::InternalError,
message: "Persisted object encryption metadata is invalid.".to_string(),
source: None,
}
}
fn insert_sse_response_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> Result<(), ApiError> {
let value = HeaderValue::from_str(value).map_err(|_| invalid_sse_response_metadata())?;
headers.insert(name, value);
Ok(())
}
/// Project response headers after the caller has completed its existing read
/// validation and KMS authorization path.
pub(crate) fn project_sse_read_response_headers(
metadata: &HashMap<String, String>,
sse_customer_algorithm: Option<&SSECustomerAlgorithm>,
sse_customer_key_md5: Option<&SSECustomerKeyMD5>,
) -> Result<HeaderMap, ApiError> {
if metadata.keys().any(|key| {
key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY)
|| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT)
}) {
return Err(invalid_sse_response_metadata());
}
let metadata = normalize_encryption_metadata_case(metadata).map_err(|_| invalid_sse_response_metadata())?;
let public_mode = metadata.get("x-amz-server-side-encryption").map(String::as_str);
let customer_algorithm = metadata.get(SSEC_ALGORITHM_HEADER).map(String::as_str);
let public_mode = match public_mode {
Some(AMZ_ENCRYPTION_AES) => Some(SSEType::SseS3),
Some(AMZ_ENCRYPTION_KMS) => Some(SSEType::SseKms),
Some(_) => return Err(invalid_sse_response_metadata()),
None => None,
};
if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) {
return Err(invalid_sse_response_metadata());
}
let mode = if customer_algorithm.is_some() {
if public_mode == Some(SSEType::SseKms) {
return Err(invalid_sse_response_metadata());
}
Some(SSEType::SseC)
} else {
public_mode
};
for marker_mode in [
metadata
.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)
.then_some(SSEType::SseC),
metadata
.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
.then_some(SSEType::SseS3),
metadata
.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)
.then_some(SSEType::SseKms),
]
.into_iter()
.flatten()
{
if mode != Some(marker_mode) {
return Err(invalid_sse_response_metadata());
}
}
let Some(mode) = mode else {
if metadata
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
{
return Err(invalid_sse_response_metadata());
}
return Ok(HeaderMap::new());
};
let mut key_id = None;
for value in [
metadata.get("x-amz-server-side-encryption-aws-kms-key-id"),
metadata.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER),
metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER),
]
.into_iter()
.flatten()
{
if key_id.is_some_and(|current| current != value) {
return Err(invalid_sse_response_metadata());
}
key_id = Some(value);
}
let normalized_managed = normalize_managed_metadata(metadata.as_ref(), Some(recode_minio_kms_context));
let mut response_headers = HeaderMap::with_capacity(3);
match mode {
SSEType::SseS3 => {
if metadata.contains_key("x-amz-server-side-encryption-aws-kms-key-id") || metadata.contains_key(SSEC_KEY_MD5_HEADER)
{
return Err(invalid_sse_response_metadata());
}
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES));
}
SSEType::SseKms => {
if metadata.contains_key(SSEC_KEY_MD5_HEADER) {
return Err(invalid_sse_response_metadata());
}
let key_id = key_id
.map(String::as_str)
.filter(|value| !value.is_empty())
.ok_or_else(invalid_sse_response_metadata)?;
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS));
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?;
if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
&& !normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER)
{
return Err(invalid_sse_response_metadata());
}
if let Some(context) = normalized_managed.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
serde_json::from_str::<HashMap<String, String>>(context).map_err(|_| invalid_sse_response_metadata())?;
let encoded = BASE64_STANDARD.encode_to_string(context);
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, &encoded)?;
}
}
SSEType::SseC => {
if key_id.is_some() || normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
return Err(invalid_sse_response_metadata());
}
let algorithm = sse_customer_algorithm
.map(|value| value.as_ref())
.filter(|value| *value == AMZ_ENCRYPTION_AES)
.ok_or_else(|| {
ssec_invalid_request(
"The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.",
)
})?;
let key_md5 = sse_customer_key_md5
.map(|value| value.as_ref())
.ok_or_else(|| ssec_invalid_request("Missing SSE-C customer key MD5."))?;
let stored_md5 = metadata.get(SSEC_KEY_MD5_HEADER).ok_or_else(invalid_sse_response_metadata)?;
if stored_md5 != key_md5 {
return Err(ssec_invalid_request(
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
));
}
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?;
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?;
}
}
Ok(response_headers)
}
/// Read-side response classification for an object whose payload the object /// Read-side response classification for an object whose payload the object
/// layer's encryption resolver already decrypted. /// layer's encryption resolver already decrypted.
/// ///
@@ -4054,10 +4214,10 @@ mod tests {
build_kms_request_context, classify_sse_read_response, encode_minio_kms_context, encryption_material_to_metadata, build_kms_request_context, classify_sse_read_response, encode_minio_kms_context, encryption_material_to_metadata,
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata,
md5_base64, normalize_managed_metadata, recode_minio_kms_context, reset_sse_dek_provider, resolve_effective_kms_key_id, md5_base64, normalize_managed_metadata, project_sse_read_response_headers, recode_minio_kms_context,
resolve_stored_kms_key_id, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, reset_sse_dek_provider, resolve_effective_kms_key_id, resolve_stored_kms_key_id, rewrap_object_encryption_metadata,
strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
validate_ssec_params, verify_ssec_key_match, validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match,
}; };
#[cfg(feature = "rio-v2")] #[cfg(feature = "rio-v2")]
use super::{ use super::{
@@ -7724,6 +7884,166 @@ mod tests {
assert_eq!(audit_tag(&tags, "kmsErrorClass").as_deref(), Some("access_denied")); assert_eq!(audit_tag(&tags, "kmsErrorClass").as_deref(), Some("access_denied"));
} }
#[test]
fn read_response_projection_uses_shared_managed_metadata_compatibility() {
let context_json = r#"{"tenant":"s3-select"}"#;
let context = BASE64_STANDARD.encode_to_string(context_json);
let key_id = "arn:aws:kms:us-east-1:123456789012:key/select";
let kms_metadata = HashMap::from([
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), key_id.to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), key_id.to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), key_id.to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), context.clone()),
]);
let headers = project_sse_read_response_headers(&kms_metadata, None, None)
.expect("valid SSE-KMS metadata should project response headers");
assert_eq!(
headers.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-KMS algorithm"),
super::AMZ_ENCRYPTION_KMS
);
assert_eq!(
headers
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID)
.expect("SSE-KMS key ID"),
key_id
);
assert_eq!(
headers
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT)
.expect("SSE-KMS context"),
context.as_str()
);
let s3_metadata = HashMap::from([
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_AES.to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
(
super::INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(),
r#"{"bucket":"select"}"#.to_string(),
),
("x-amz-meta-private".to_string(), "private".to_string()),
]);
let headers = project_sse_read_response_headers(&s3_metadata, None, None)
.expect("valid SSE-S3 metadata should project response headers");
assert_eq!(headers.len(), 1);
assert_eq!(
headers.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-S3 algorithm"),
super::AMZ_ENCRYPTION_AES
);
let plaintext = project_sse_read_response_headers(&HashMap::new(), None, None)
.expect("plaintext metadata should not produce SSE headers");
assert!(plaintext.is_empty());
}
#[test]
fn read_response_projection_returns_only_validated_ssec_values() {
let key_md5 = "customer-key-md5";
let metadata = HashMap::from([
(
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(),
super::AMZ_ENCRYPTION_AES.to_string(),
),
(super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5.to_string(), key_md5.to_string()),
]);
let algorithm = SSECustomerAlgorithm::from(super::AMZ_ENCRYPTION_AES);
let key_md5 = SSECustomerKeyMD5::from(key_md5);
let headers = project_sse_read_response_headers(&metadata, Some(&algorithm), Some(&key_md5))
.expect("validated SSE-C values should project response headers");
assert_eq!(headers.len(), 2);
assert_eq!(
headers
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM)
.expect("SSE-C algorithm"),
super::AMZ_ENCRYPTION_AES
);
assert_eq!(
headers
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5)
.expect("SSE-C key MD5"),
key_md5.as_str()
);
assert!(headers.get(super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY).is_none());
let missing_algorithm = project_sse_read_response_headers(&metadata, None, Some(&key_md5))
.expect_err("missing SSE-C algorithm must fail before streaming");
assert_eq!(missing_algorithm.code, S3ErrorCode::InvalidRequest);
}
#[test]
fn read_response_projection_rejects_unvalidated_or_sensitive_metadata() {
let invalid_context = "not-base64";
let persisted_context = "must-not-leak-context";
let persisted_key = "must-not-leak-key";
let corrupt_metadata = [
HashMap::from([
(
super::AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_lowercase(),
super::AMZ_ENCRYPTION_AES.to_string(),
),
(
super::AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_uppercase(),
super::AMZ_ENCRYPTION_KMS.to_string(),
),
]),
HashMap::from([
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
(
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(),
super::AMZ_ENCRYPTION_AES.to_string(),
),
]),
HashMap::from([
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "key-2".to_string()),
]),
HashMap::from([
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), invalid_context.to_string()),
]),
HashMap::from([
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), persisted_context.to_string()),
]),
HashMap::from([
(
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(),
super::AMZ_ENCRYPTION_AES.to_string(),
),
(
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5.to_string(),
"customer-key-md5".to_string(),
),
(
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY.to_ascii_uppercase(),
persisted_key.to_string(),
),
]),
];
for metadata in corrupt_metadata {
let error = project_sse_read_response_headers(&metadata, None, None)
.expect_err("corrupt encryption metadata must fail closed");
assert_eq!(error.code, S3ErrorCode::InternalError);
for sensitive in [invalid_context, persisted_context, persisted_key] {
assert!(!error.to_string().contains(sensitive));
}
}
let marker_only_kms =
HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string())]);
let marker_error = project_sse_read_response_headers(&marker_only_kms, None, None)
.expect_err("marker-only KMS metadata must not bypass the existing authorization classifier");
assert_eq!(marker_error.code, S3ErrorCode::InternalError);
}
#[tokio::test] #[tokio::test]
async fn classification_matches_ssec_validation_and_headers() { async fn classification_matches_ssec_validation_and_headers() {
let key = [0x42u8; 32]; let key = [0x42u8; 32];
+4 -4
View File
@@ -111,8 +111,8 @@ pub(crate) use super::ecfs_extend::{
pub(crate) use super::sse::{ pub(crate) use super::sse::{
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal, DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, project_sse_read_response_headers, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption,
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
}; };
pub(crate) mod access_consumer { pub(crate) mod access_consumer {
@@ -363,8 +363,8 @@ pub(crate) mod sse_consumer {
pub(crate) use super::{ pub(crate) use super::{
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers, authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
get_buffer_size_opt_in, load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption, get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption,
validate_bucket_object_lock_enabled_state, sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state,
}; };
} }