mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-01 17:58:22 +00:00
fix(s3select): return encryption response headers (#6819)
This commit is contained in:
@@ -217,7 +217,37 @@ pub(crate) async fn signed_s3_request(
|
|||||||
access_key: &str,
|
access_key: &str,
|
||||||
secret_key: &str,
|
secret_key: &str,
|
||||||
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
signed_s3_request_with_session_token(method, url, body, content_type, access_key, secret_key, None).await
|
signed_s3_request_with_headers(method, url, body, content_type, access_key, secret_key, &http::HeaderMap::new()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn signed_s3_request_with_headers(
|
||||||
|
method: http::Method,
|
||||||
|
url: &str,
|
||||||
|
body: Option<String>,
|
||||||
|
content_type: Option<&str>,
|
||||||
|
access_key: &str,
|
||||||
|
secret_key: &str,
|
||||||
|
extra_headers: &http::HeaderMap,
|
||||||
|
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
|
signed_s3_request_with_session_token(
|
||||||
|
method,
|
||||||
|
url,
|
||||||
|
body,
|
||||||
|
content_type,
|
||||||
|
SigningCredentials {
|
||||||
|
access_key,
|
||||||
|
secret_key,
|
||||||
|
session_token: None,
|
||||||
|
},
|
||||||
|
extra_headers,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
struct SigningCredentials<'a> {
|
||||||
|
access_key: &'a str,
|
||||||
|
secret_key: &'a str,
|
||||||
|
session_token: Option<&'a str>,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn signed_s3_request_with_session_token(
|
async fn signed_s3_request_with_session_token(
|
||||||
@@ -225,9 +255,8 @@ async fn signed_s3_request_with_session_token(
|
|||||||
url: &str,
|
url: &str,
|
||||||
body: Option<String>,
|
body: Option<String>,
|
||||||
content_type: Option<&str>,
|
content_type: Option<&str>,
|
||||||
access_key: &str,
|
credentials: SigningCredentials<'_>,
|
||||||
secret_key: &str,
|
extra_headers: &http::HeaderMap,
|
||||||
session_token: Option<&str>,
|
|
||||||
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let uri = url.parse::<http::Uri>()?;
|
let uri = url.parse::<http::Uri>()?;
|
||||||
let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
|
let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
|
||||||
@@ -239,14 +268,17 @@ async fn signed_s3_request_with_session_token(
|
|||||||
if let Some(content_type) = content_type {
|
if let Some(content_type) = content_type {
|
||||||
request = request.header(CONTENT_TYPE, content_type);
|
request = request.header(CONTENT_TYPE, content_type);
|
||||||
}
|
}
|
||||||
|
for (name, value) in extra_headers {
|
||||||
|
request = request.header(name, value);
|
||||||
|
}
|
||||||
|
|
||||||
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
|
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
|
||||||
let signed = sign_v4(
|
let signed = sign_v4(
|
||||||
request.body(Body::empty())?,
|
request.body(Body::empty())?,
|
||||||
content_length,
|
content_length,
|
||||||
access_key,
|
credentials.access_key,
|
||||||
secret_key,
|
credentials.secret_key,
|
||||||
session_token.unwrap_or_default(),
|
credentials.session_token.unwrap_or_default(),
|
||||||
"us-east-1",
|
"us-east-1",
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -283,8 +315,19 @@ pub(crate) async fn admin_request_with_session_token(
|
|||||||
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let url = format!("{base_url}{path_and_query}");
|
let url = format!("{base_url}{path_and_query}");
|
||||||
let content_type = body.as_ref().map(|_| "application/json");
|
let content_type = body.as_ref().map(|_| "application/json");
|
||||||
let response =
|
let response = signed_s3_request_with_session_token(
|
||||||
signed_s3_request_with_session_token(method, &url, body, content_type, access_key, secret_key, session_token).await?;
|
method,
|
||||||
|
&url,
|
||||||
|
body,
|
||||||
|
content_type,
|
||||||
|
SigningCredentials {
|
||||||
|
access_key,
|
||||||
|
secret_key,
|
||||||
|
session_token,
|
||||||
|
},
|
||||||
|
&http::HeaderMap::new(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
let status = response.status();
|
let status = response.status();
|
||||||
let body = response.text().await?;
|
let body = response.text().await?;
|
||||||
Ok((status, body))
|
Ok((status, body))
|
||||||
|
|||||||
@@ -57,6 +57,9 @@ mod copy_object_version_restore_sse_test;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod configured_roundtrip_test;
|
mod configured_roundtrip_test;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod select_sse_response_test;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod kms_anonymous_enforcement_test;
|
mod kms_anonymous_enforcement_test;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,241 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
//! SelectObjectContent SSE response-header compatibility (backlog#1625).
|
||||||
|
|
||||||
|
use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64, start_kms};
|
||||||
|
use crate::common::signed_s3_request_with_headers;
|
||||||
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use aws_sdk_s3::types::ServerSideEncryption;
|
||||||
|
use base64_simd::STANDARD as BASE64;
|
||||||
|
use http::{HeaderMap, Method};
|
||||||
|
use std::error::Error;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||||
|
|
||||||
|
const CSV_BODY: &[u8] = b"name\nalice\n";
|
||||||
|
const SELECT_BODY: &str = r#"<SelectObjectContentRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<Expression>SELECT * FROM S3Object</Expression>
|
||||||
|
<ExpressionType>SQL</ExpressionType>
|
||||||
|
<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
|
||||||
|
<OutputSerialization><CSV/></OutputSerialization>
|
||||||
|
</SelectObjectContentRequest>"#;
|
||||||
|
const KMS_CONTEXT: &str = "eyJ0ZW5hbnQiOiJzMy1zZWxlY3QifQ==";
|
||||||
|
const SSE_ALGORITHM: &str = "x-amz-server-side-encryption";
|
||||||
|
const SSE_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id";
|
||||||
|
const SSE_KMS_CONTEXT: &str = "x-amz-server-side-encryption-context";
|
||||||
|
const SSE_C_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm";
|
||||||
|
const SSE_C_KEY: &str = "x-amz-server-side-encryption-customer-key";
|
||||||
|
const SSE_C_KEY_MD5: &str = "x-amz-server-side-encryption-customer-key-md5";
|
||||||
|
const LOG_FLUSH_SENTINEL: &str = "select-sse-log-flush-sentinel.csv";
|
||||||
|
|
||||||
|
async fn raw_select(
|
||||||
|
env: &crate::common::RustFSTestEnvironment,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
request_headers: &HeaderMap,
|
||||||
|
) -> TestResult<reqwest::Response> {
|
||||||
|
let url = format!("{}/{bucket}/{object}?select&select-type=2", env.url);
|
||||||
|
signed_s3_request_with_headers(
|
||||||
|
Method::POST,
|
||||||
|
&url,
|
||||||
|
Some(SELECT_BODY.to_string()),
|
||||||
|
Some("application/xml"),
|
||||||
|
&env.access_key,
|
||||||
|
&env.secret_key,
|
||||||
|
request_headers,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_success_headers(response: reqwest::Response, expected: &[(&str, &str)], absent: &[&str]) -> TestResult {
|
||||||
|
if response.status() != reqwest::StatusCode::OK {
|
||||||
|
let status = response.status();
|
||||||
|
let url = response.url().clone();
|
||||||
|
let body = response.text().await?;
|
||||||
|
panic!("Select request to {url} failed with {status}: {body}");
|
||||||
|
}
|
||||||
|
for (name, value) in expected {
|
||||||
|
assert_eq!(response.headers().get(*name).and_then(|header| header.to_str().ok()), Some(*value));
|
||||||
|
}
|
||||||
|
for name in absent {
|
||||||
|
assert!(response.headers().get(*name).is_none(), "successful Select response must omit {name}");
|
||||||
|
}
|
||||||
|
let body = response.bytes().await?;
|
||||||
|
assert!(
|
||||||
|
body.windows(b"alice".len()).any(|window| window == b"alice"),
|
||||||
|
"successful Select response must contain a Records event with the selected row"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
body.windows(b"End".len()).any(|window| window == b"End"),
|
||||||
|
"successful Select response must contain the terminal End event"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_pre_stream_failure(response: reqwest::Response) -> TestResult {
|
||||||
|
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||||
|
let body = response.text().await?;
|
||||||
|
assert!(body.contains("<Error>"), "pre-stream failure must return an S3 XML error: {body}");
|
||||||
|
assert!(
|
||||||
|
body.contains("<Code>InvalidRequest</Code>"),
|
||||||
|
"invalid SSE-C parameters must preserve the S3 error code: {body}"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn put_object(
|
||||||
|
client: &aws_sdk_s3::Client,
|
||||||
|
bucket: &str,
|
||||||
|
object: &str,
|
||||||
|
) -> aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder {
|
||||||
|
client
|
||||||
|
.put_object()
|
||||||
|
.bucket(bucket)
|
||||||
|
.key(object)
|
||||||
|
.body(ByteStream::from_static(CSV_BODY))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn select_projects_encryption_headers_and_rejects_invalid_sse_c_before_streaming() -> TestResult {
|
||||||
|
let mut kms = LocalKMSTestEnvironment::new().await?;
|
||||||
|
let log_path = format!("{}/server.log", kms.base_env.temp_dir);
|
||||||
|
kms.base_env.capture_log_path = Some(log_path.clone());
|
||||||
|
kms.base_env
|
||||||
|
.start_rustfs_server_with_env(Vec::new(), &[("RUST_LOG", "s3s=debug,rustfs=info")])
|
||||||
|
.await?;
|
||||||
|
let key_id = kms.configure_local_kms().await?;
|
||||||
|
start_kms(&kms.base_env.url, &kms.base_env.access_key, &kms.base_env.secret_key).await?;
|
||||||
|
|
||||||
|
let client = kms.base_env.create_s3_client();
|
||||||
|
let bucket = format!("select-sse-{}", Uuid::new_v4().simple());
|
||||||
|
client.create_bucket().bucket(&bucket).send().await?;
|
||||||
|
|
||||||
|
put_object(&client, &bucket, "plain.csv").send().await?;
|
||||||
|
put_object(&client, &bucket, "sse-s3.csv")
|
||||||
|
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
put_object(&client, &bucket, "sse-kms.csv")
|
||||||
|
.server_side_encryption(ServerSideEncryption::AwsKms)
|
||||||
|
.ssekms_key_id(&key_id)
|
||||||
|
.ssekms_encryption_context(KMS_CONTEXT)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let customer_key = "01234567890123456789012345678901";
|
||||||
|
let customer_key_b64 = BASE64.encode_to_string(customer_key);
|
||||||
|
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
|
||||||
|
put_object(&client, &bucket, "sse-c.csv")
|
||||||
|
.sse_customer_algorithm("AES256")
|
||||||
|
.sse_customer_key(&customer_key_b64)
|
||||||
|
.sse_customer_key_md5(&customer_key_md5)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
assert_success_headers(
|
||||||
|
raw_select(&kms.base_env, &bucket, "plain.csv", &HeaderMap::new()).await?,
|
||||||
|
&[],
|
||||||
|
&[
|
||||||
|
SSE_ALGORITHM,
|
||||||
|
SSE_KMS_KEY_ID,
|
||||||
|
SSE_KMS_CONTEXT,
|
||||||
|
SSE_C_ALGORITHM,
|
||||||
|
SSE_C_KEY,
|
||||||
|
SSE_C_KEY_MD5,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_success_headers(
|
||||||
|
raw_select(&kms.base_env, &bucket, "sse-s3.csv", &HeaderMap::new()).await?,
|
||||||
|
&[(SSE_ALGORITHM, "AES256")],
|
||||||
|
&[SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_success_headers(
|
||||||
|
raw_select(&kms.base_env, &bucket, "sse-kms.csv", &HeaderMap::new()).await?,
|
||||||
|
&[
|
||||||
|
(SSE_ALGORITHM, "aws:kms"),
|
||||||
|
(SSE_KMS_KEY_ID, &key_id),
|
||||||
|
(SSE_KMS_CONTEXT, KMS_CONTEXT),
|
||||||
|
],
|
||||||
|
&[SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut sse_c_headers = HeaderMap::new();
|
||||||
|
sse_c_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
|
||||||
|
sse_c_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
|
||||||
|
sse_c_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
|
||||||
|
assert_success_headers(
|
||||||
|
raw_select(&kms.base_env, &bucket, "sse-c.csv", &sse_c_headers).await?,
|
||||||
|
&[(SSE_C_ALGORITHM, "AES256"), (SSE_C_KEY_MD5, &customer_key_md5)],
|
||||||
|
&[SSE_ALGORITHM, SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_KEY],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &HeaderMap::new()).await?).await?;
|
||||||
|
|
||||||
|
let mut missing_algorithm_headers = HeaderMap::new();
|
||||||
|
missing_algorithm_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
|
||||||
|
missing_algorithm_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
|
||||||
|
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &missing_algorithm_headers).await?).await?;
|
||||||
|
|
||||||
|
let mut wrong_algorithm_headers = sse_c_headers.clone();
|
||||||
|
wrong_algorithm_headers.insert(SSE_C_ALGORITHM, "AES128".parse()?);
|
||||||
|
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_algorithm_headers).await?).await?;
|
||||||
|
|
||||||
|
let wrong_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
|
||||||
|
let mut wrong_md5_headers = sse_c_headers.clone();
|
||||||
|
wrong_md5_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
|
||||||
|
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_md5_headers).await?).await?;
|
||||||
|
|
||||||
|
let wrong_key = "99999999999999999999999999999999";
|
||||||
|
let wrong_key_b64 = BASE64.encode_to_string(wrong_key);
|
||||||
|
let mut wrong_key_headers = HeaderMap::new();
|
||||||
|
wrong_key_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
|
||||||
|
wrong_key_headers.insert(SSE_C_KEY, wrong_key_b64.parse()?);
|
||||||
|
wrong_key_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
|
||||||
|
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_key_headers).await?).await?;
|
||||||
|
|
||||||
|
put_object(&client, &bucket, LOG_FLUSH_SENTINEL).send().await?;
|
||||||
|
assert_success_headers(
|
||||||
|
raw_select(&kms.base_env, &bucket, LOG_FLUSH_SENTINEL, &HeaderMap::new()).await?,
|
||||||
|
&[],
|
||||||
|
&[
|
||||||
|
SSE_ALGORITHM,
|
||||||
|
SSE_KMS_KEY_ID,
|
||||||
|
SSE_KMS_CONTEXT,
|
||||||
|
SSE_C_ALGORITHM,
|
||||||
|
SSE_C_KEY,
|
||||||
|
SSE_C_KEY_MD5,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let mut logs = String::new();
|
||||||
|
for _ in 0..100 {
|
||||||
|
logs = tokio::fs::read_to_string(&log_path).await?;
|
||||||
|
if logs.contains(LOG_FLUSH_SENTINEL) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
|
}
|
||||||
|
assert!(logs.contains(LOG_FLUSH_SENTINEL), "timed out waiting for the log sink to flush");
|
||||||
|
for secret in [customer_key, customer_key_b64.as_str(), wrong_key, wrong_key_b64.as_str()] {
|
||||||
|
assert!(!logs.contains(secret), "Select request logging leaked SSE-C customer key material");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
+14
-351
@@ -2,7 +2,7 @@
|
|||||||
use super::storage_api::select_object::StorageError;
|
use super::storage_api::select_object::StorageError;
|
||||||
use super::storage_api::select_object::options::get_opts;
|
use super::storage_api::select_object::options::get_opts;
|
||||||
use super::storage_api::select_object::request_context::spawn_traced;
|
use super::storage_api::select_object::request_context::spawn_traced;
|
||||||
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read};
|
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
|
||||||
use super::storage_api::select_object::{
|
use super::storage_api::select_object::{
|
||||||
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
|
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
|
||||||
validate_ssec_for_read,
|
validate_ssec_for_read,
|
||||||
@@ -19,34 +19,20 @@ use datafusion::arrow::{
|
|||||||
use datafusion::common::DataFusionError;
|
use datafusion::common::DataFusionError;
|
||||||
use datafusion::physical_plan::SendableRecordBatchStream;
|
use datafusion::physical_plan::SendableRecordBatchStream;
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::RANGE};
|
use http::{HeaderMap, StatusCode, header::RANGE};
|
||||||
use rustfs_s3select_api::{
|
use rustfs_s3select_api::{
|
||||||
QueryError, SelectError,
|
QueryError, SelectError,
|
||||||
object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds},
|
object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds},
|
||||||
query::{Context, Query},
|
query::{Context, Query},
|
||||||
};
|
};
|
||||||
use rustfs_s3select_query::instance::s3_select_query_timeout;
|
use rustfs_s3select_query::instance::s3_select_query_timeout;
|
||||||
use rustfs_utils::http::headers::{
|
|
||||||
AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
|
||||||
AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER,
|
|
||||||
};
|
|
||||||
use rustfs_utils::http::object_encryption_keys::{
|
|
||||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
|
||||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
|
||||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
|
||||||
};
|
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
CSVOutput, CompressionType, ContinuationEvent, EndEvent, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput,
|
CSVOutput, CompressionType, ContinuationEvent, EndEvent, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput,
|
||||||
JSONOutput, JSONType, OutputSerialization, Progress, ProgressEvent, QuoteFields, RecordsEvent, SelectObjectContentEvent,
|
JSONOutput, JSONType, OutputSerialization, Progress, ProgressEvent, QuoteFields, RecordsEvent, SelectObjectContentEvent,
|
||||||
SelectObjectContentEventStream, SelectObjectContentInput, SelectObjectContentOutput, SelectObjectContentRequest, Stats,
|
SelectObjectContentEventStream, SelectObjectContentInput, SelectObjectContentOutput, SelectObjectContentRequest, Stats,
|
||||||
StatsEvent,
|
StatsEvent,
|
||||||
};
|
};
|
||||||
use s3s::header::{
|
|
||||||
X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT,
|
|
||||||
X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
|
|
||||||
};
|
|
||||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio::time::{Instant, timeout_at};
|
use tokio::time::{Instant, timeout_at};
|
||||||
@@ -62,8 +48,6 @@ const BUSY_MESSAGE: &str = "The service is unavailable. Try again later.";
|
|||||||
const EMPTY_SELECT_EXPRESSION_MESSAGE: &str = "empty SQL expression";
|
const EMPTY_SELECT_EXPRESSION_MESSAGE: &str = "empty SQL expression";
|
||||||
const SLOW_DOWN_MESSAGE: &str = "Reduce your request rate.";
|
const SLOW_DOWN_MESSAGE: &str = "Reduce your request rate.";
|
||||||
const UNSUPPORTED_SQL_STRUCTURE_MESSAGE: &str = "We encountered an unsupported SQL structure. Check the SQL Reference.";
|
const UNSUPPORTED_SQL_STRUCTURE_MESSAGE: &str = "We encountered an unsupported SQL structure. Check the SQL Reference.";
|
||||||
// No canonical owner exists for the KMS key ARN prefix; keep it local.
|
|
||||||
const SELECT_KMS_ARN_PREFIX: &str = "arn:aws:kms:";
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
struct SelectValidation {
|
struct SelectValidation {
|
||||||
@@ -107,7 +91,7 @@ pub async fn execute_select_object_content(
|
|||||||
.map_err(|_| select_query_timeout_error(query_timeout.as_secs()))?
|
.map_err(|_| select_query_timeout_error(query_timeout.as_secs()))?
|
||||||
.map_err(map_query_error_to_s3)?;
|
.map_err(map_query_error_to_s3)?;
|
||||||
let admission = db.try_reserve_query().map_err(map_query_error_to_s3)?;
|
let admission = db.try_reserve_query().map_err(map_query_error_to_s3)?;
|
||||||
let snapshot = timeout_at(
|
let (snapshot, response_headers) = timeout_at(
|
||||||
query_deadline,
|
query_deadline,
|
||||||
prepare_select_object_snapshot(&req.headers, &input, read_principal.as_ref()),
|
prepare_select_object_snapshot(&req.headers, &input, read_principal.as_ref()),
|
||||||
)
|
)
|
||||||
@@ -130,7 +114,10 @@ pub async fn execute_select_object_content(
|
|||||||
.clone()
|
.clone()
|
||||||
.try_reserve_owned()
|
.try_reserve_owned()
|
||||||
.map_err(|_| map_select_error_to_s3(&SelectError::InternalError))?;
|
.map_err(|_| map_select_error_to_s3(&SelectError::InternalError))?;
|
||||||
let response = select_object_response(rx, &snapshot.object_info().user_defined, &req.headers)?;
|
let mut response = S3Response::new(SelectObjectContentOutput {
|
||||||
|
payload: Some(SelectObjectContentEventStream::new(ReceiverStream::new(rx))),
|
||||||
|
});
|
||||||
|
response.headers = response_headers;
|
||||||
spawn_traced(async move {
|
spawn_traced(async move {
|
||||||
send_select_events_until_deadline(
|
send_select_events_until_deadline(
|
||||||
output,
|
output,
|
||||||
@@ -147,186 +134,6 @@ pub async fn execute_select_object_content(
|
|||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn select_object_response(
|
|
||||||
rx: mpsc::Receiver<S3Result<SelectObjectContentEvent>>,
|
|
||||||
metadata: &HashMap<String, String>,
|
|
||||||
request_headers: &HeaderMap,
|
|
||||||
) -> S3Result<S3Response<SelectObjectContentOutput>> {
|
|
||||||
let response_headers = select_snapshot_sse_response_headers(metadata, request_headers)?;
|
|
||||||
let mut response = S3Response::new(SelectObjectContentOutput {
|
|
||||||
payload: Some(SelectObjectContentEventStream::new(ReceiverStream::new(rx))),
|
|
||||||
});
|
|
||||||
response.headers = response_headers;
|
|
||||||
Ok(response)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
||||||
enum SelectSnapshotSseMode {
|
|
||||||
S3,
|
|
||||||
Kms,
|
|
||||||
Customer,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn invalid_select_snapshot_sse_metadata() -> S3Error {
|
|
||||||
S3Error::with_message(
|
|
||||||
S3ErrorCode::InternalError,
|
|
||||||
"Persisted SelectObjectContent encryption metadata is invalid.",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn select_metadata_value<'a>(metadata: &'a HashMap<String, String>, name: &str) -> S3Result<Option<&'a str>> {
|
|
||||||
let mut values = metadata
|
|
||||||
.iter()
|
|
||||||
.filter_map(|(key, value)| key.eq_ignore_ascii_case(name).then_some(value.as_str()));
|
|
||||||
let Some(value) = values.next() else {
|
|
||||||
return Ok(None);
|
|
||||||
};
|
|
||||||
if values.any(|candidate| candidate != value) {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
Ok(Some(value))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn select_snapshot_kms_key_id(metadata: &HashMap<String, String>) -> S3Result<Option<&str>> {
|
|
||||||
let values = [
|
|
||||||
select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID)?,
|
|
||||||
select_metadata_value(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER)?,
|
|
||||||
select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?,
|
|
||||||
];
|
|
||||||
let mut resolved = None;
|
|
||||||
for value in values.into_iter().flatten() {
|
|
||||||
if resolved.is_some_and(|current| current != value) {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
resolved = Some(value);
|
|
||||||
}
|
|
||||||
Ok(resolved)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn select_snapshot_sse_mode(metadata: &HashMap<String, String>) -> S3Result<Option<SelectSnapshotSseMode>> {
|
|
||||||
let public_mode = select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION)?;
|
|
||||||
let customer_algorithm = select_metadata_value(metadata, SSEC_ALGORITHM_HEADER)?;
|
|
||||||
let has_ssec_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)?.is_some();
|
|
||||||
let has_s3_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)?.is_some();
|
|
||||||
let has_kms_marker = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)?.is_some();
|
|
||||||
|
|
||||||
let public_mode = match public_mode {
|
|
||||||
Some(AMZ_ENCRYPTION_AES) => Some(SelectSnapshotSseMode::S3),
|
|
||||||
Some(AMZ_ENCRYPTION_KMS) => Some(SelectSnapshotSseMode::Kms),
|
|
||||||
Some(_) => return Err(invalid_select_snapshot_sse_metadata()),
|
|
||||||
None => None,
|
|
||||||
};
|
|
||||||
if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
|
|
||||||
let resolved = if customer_algorithm.is_some() {
|
|
||||||
if public_mode == Some(SelectSnapshotSseMode::Kms) {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
Some(SelectSnapshotSseMode::Customer)
|
|
||||||
} else {
|
|
||||||
public_mode
|
|
||||||
};
|
|
||||||
let internal_modes = [
|
|
||||||
has_ssec_marker.then_some(SelectSnapshotSseMode::Customer),
|
|
||||||
has_s3_marker.then_some(SelectSnapshotSseMode::S3),
|
|
||||||
has_kms_marker.then_some(SelectSnapshotSseMode::Kms),
|
|
||||||
];
|
|
||||||
for mode in internal_modes.into_iter().flatten() {
|
|
||||||
if resolved != Some(mode) {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if resolved.is_none()
|
|
||||||
&& metadata
|
|
||||||
.keys()
|
|
||||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
|
||||||
{
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
Ok(resolved)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn insert_select_snapshot_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> S3Result<()> {
|
|
||||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_select_snapshot_sse_metadata())?;
|
|
||||||
headers.insert(name, value);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn select_snapshot_sse_response_headers(metadata: &HashMap<String, String>, request_headers: &HeaderMap) -> S3Result<HeaderMap> {
|
|
||||||
if select_metadata_value(metadata, SSEC_KEY_HEADER)?.is_some()
|
|
||||||
|| select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT)?.is_some()
|
|
||||||
{
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
let Some(mode) = select_snapshot_sse_mode(metadata)? else {
|
|
||||||
return Ok(HeaderMap::new());
|
|
||||||
};
|
|
||||||
let kms_key_id = select_snapshot_kms_key_id(metadata)?;
|
|
||||||
|
|
||||||
let mut response_headers = HeaderMap::with_capacity(3);
|
|
||||||
match mode {
|
|
||||||
SelectSnapshotSseMode::S3 => {
|
|
||||||
if select_metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID)?.is_some()
|
|
||||||
|| select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?.is_some()
|
|
||||||
|| select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.is_some()
|
|
||||||
{
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES));
|
|
||||||
}
|
|
||||||
SelectSnapshotSseMode::Kms => {
|
|
||||||
if select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.is_some() {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
let key_id = kms_key_id
|
|
||||||
.filter(|key_id| !key_id.is_empty())
|
|
||||||
.ok_or_else(invalid_select_snapshot_sse_metadata)?;
|
|
||||||
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS));
|
|
||||||
if key_id.starts_with(SELECT_KMS_ARN_PREFIX) {
|
|
||||||
insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?;
|
|
||||||
} else {
|
|
||||||
insert_select_snapshot_header(
|
|
||||||
&mut response_headers,
|
|
||||||
X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID,
|
|
||||||
&format!("{SELECT_KMS_ARN_PREFIX}{key_id}"),
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
if let Some(context) = select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)? {
|
|
||||||
let context = HeaderValue::from_str(context).map_err(|_| invalid_select_snapshot_sse_metadata())?;
|
|
||||||
let mut validation_headers = HeaderMap::with_capacity(1);
|
|
||||||
validation_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, context.clone());
|
|
||||||
super::storage_api::select_object::sse::extract_ssekms_context_from_headers(&validation_headers)
|
|
||||||
.map_err(|_| invalid_select_snapshot_sse_metadata())?;
|
|
||||||
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, context);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
SelectSnapshotSseMode::Customer => {
|
|
||||||
if kms_key_id.is_some() || select_metadata_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?.is_some() {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
let algorithm = request_headers
|
|
||||||
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM)
|
|
||||||
.and_then(|value| value.to_str().ok())
|
|
||||||
.filter(|algorithm| *algorithm == AMZ_ENCRYPTION_AES)
|
|
||||||
.ok_or_else(invalid_select_snapshot_sse_metadata)?;
|
|
||||||
let key_md5 = request_headers
|
|
||||||
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5)
|
|
||||||
.and_then(|value| value.to_str().ok())
|
|
||||||
.ok_or_else(invalid_select_snapshot_sse_metadata)?;
|
|
||||||
let stored_md5 =
|
|
||||||
select_metadata_value(metadata, SSEC_KEY_MD5_HEADER)?.ok_or_else(invalid_select_snapshot_sse_metadata)?;
|
|
||||||
if stored_md5 != key_md5 {
|
|
||||||
return Err(invalid_select_snapshot_sse_metadata());
|
|
||||||
}
|
|
||||||
insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?;
|
|
||||||
insert_select_snapshot_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(response_headers)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn send_select_events_until_deadline<L: SelectSnapshotFence>(
|
async fn send_select_events_until_deadline<L: SelectSnapshotFence>(
|
||||||
output: SendableRecordBatchStream,
|
output: SendableRecordBatchStream,
|
||||||
tx: mpsc::Sender<S3Result<SelectObjectContentEvent>>,
|
tx: mpsc::Sender<S3Result<SelectObjectContentEvent>>,
|
||||||
@@ -641,7 +448,7 @@ async fn prepare_select_object_snapshot(
|
|||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
input: &SelectObjectContentInput,
|
input: &SelectObjectContentInput,
|
||||||
read_principal: Option<&SseKmsPrincipal>,
|
read_principal: Option<&SseKmsPrincipal>,
|
||||||
) -> S3Result<StorageSelectObjectSnapshot> {
|
) -> S3Result<(StorageSelectObjectSnapshot, HeaderMap)> {
|
||||||
let opts = get_opts(&input.bucket, &input.key, None, None, headers)
|
let opts = get_opts(&input.bucket, &input.key, None, None, headers)
|
||||||
.await
|
.await
|
||||||
.map_err(ApiError::from)?;
|
.map_err(ApiError::from)?;
|
||||||
@@ -654,7 +461,12 @@ async fn prepare_select_object_snapshot(
|
|||||||
validate_sse_headers_for_read(&info.user_defined, headers)?;
|
validate_sse_headers_for_read(&info.user_defined, headers)?;
|
||||||
validate_ssec_for_read(&info.user_defined, input.sse_customer_key.as_ref(), input.sse_customer_key_md5.as_ref())?;
|
validate_ssec_for_read(&info.user_defined, input.sse_customer_key.as_ref(), input.sse_customer_key_md5.as_ref())?;
|
||||||
authorize_sse_kms_object_read(read_principal, &info.user_defined).await?;
|
authorize_sse_kms_object_read(read_principal, &info.user_defined).await?;
|
||||||
Ok(snapshot)
|
let response_headers = project_sse_read_response_headers(
|
||||||
|
&info.user_defined,
|
||||||
|
input.sse_customer_algorithm.as_ref(),
|
||||||
|
input.sse_customer_key_md5.as_ref(),
|
||||||
|
)?;
|
||||||
|
Ok((snapshot, response_headers))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_prepare_snapshot_error(err: StoragePrepareSelectObjectSnapshotError) -> S3Error {
|
fn map_prepare_snapshot_error(err: StoragePrepareSelectObjectSnapshotError) -> S3Error {
|
||||||
@@ -1043,155 +855,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn select_snapshot_sse_s3_headers_are_whitelisted() {
|
|
||||||
let metadata = HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string()),
|
|
||||||
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()),
|
|
||||||
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
|
|
||||||
("x-amz-meta-private".to_string(), "private-value".to_string()),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let headers = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new())
|
|
||||||
.expect("valid SSE-S3 snapshot metadata should project response headers");
|
|
||||||
|
|
||||||
assert_eq!(headers.len(), 1);
|
|
||||||
assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-S3 mode"), "AES256");
|
|
||||||
assert!(headers.get("x-amz-meta-private").is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn select_snapshot_sse_kms_headers_use_snapshot_metadata() {
|
|
||||||
let context = "eyJ0ZW5hbnQiOiJvbmUifQ==";
|
|
||||||
for key_id in ["key-1", "arn:aws:kms:key-2"] {
|
|
||||||
let metadata = HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), key_id.to_string()),
|
|
||||||
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), key_id.to_string()),
|
|
||||||
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), key_id.to_string()),
|
|
||||||
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), context.to_string()),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let headers = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new())
|
|
||||||
.expect("valid SSE-KMS snapshot metadata should project response headers");
|
|
||||||
let expected_key_id = if key_id.starts_with(SELECT_KMS_ARN_PREFIX) {
|
|
||||||
key_id.to_string()
|
|
||||||
} else {
|
|
||||||
format!("{SELECT_KMS_ARN_PREFIX}{key_id}")
|
|
||||||
};
|
|
||||||
|
|
||||||
assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-KMS mode"), "aws:kms");
|
|
||||||
assert_eq!(
|
|
||||||
headers
|
|
||||||
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID)
|
|
||||||
.expect("SSE-KMS key ID")
|
|
||||||
.to_str()
|
|
||||||
.expect("SSE-KMS key ID should be valid text"),
|
|
||||||
expected_key_id
|
|
||||||
);
|
|
||||||
assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT).expect("SSE-KMS context"), context);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn select_snapshot_sse_c_headers_never_echo_the_customer_key() {
|
|
||||||
let key_md5 = "customer-key-md5";
|
|
||||||
let metadata = HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string()),
|
|
||||||
(SSEC_ALGORITHM_HEADER.to_string(), AMZ_ENCRYPTION_AES.to_string()),
|
|
||||||
(SSEC_KEY_MD5_HEADER.to_string(), key_md5.to_string()),
|
|
||||||
("x-amz-meta-private".to_string(), "private-value".to_string()),
|
|
||||||
]);
|
|
||||||
let mut request_headers = HeaderMap::new();
|
|
||||||
request_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256"));
|
|
||||||
request_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, HeaderValue::from_static(key_md5));
|
|
||||||
request_headers.insert(
|
|
||||||
http::HeaderName::from_static(SSEC_KEY_HEADER),
|
|
||||||
HeaderValue::from_static("must-not-be-returned"),
|
|
||||||
);
|
|
||||||
|
|
||||||
let headers = select_snapshot_sse_response_headers(&metadata, &request_headers)
|
|
||||||
.expect("validated SSE-C request values should project response headers");
|
|
||||||
|
|
||||||
assert_eq!(headers.len(), 2);
|
|
||||||
assert_eq!(
|
|
||||||
headers
|
|
||||||
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM)
|
|
||||||
.expect("SSE-C algorithm"),
|
|
||||||
"AES256"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
headers
|
|
||||||
.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5)
|
|
||||||
.expect("SSE-C key MD5"),
|
|
||||||
key_md5
|
|
||||||
);
|
|
||||||
assert!(headers.get(SSEC_KEY_HEADER).is_none());
|
|
||||||
assert!(headers.get("x-amz-meta-private").is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn select_snapshot_sse_headers_fail_closed_on_corrupt_metadata() {
|
|
||||||
let invalid_context = "not-base64";
|
|
||||||
let persisted_key = "must-not-leak";
|
|
||||||
let corrupt_metadata = [
|
|
||||||
HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_lowercase(), "AES256".to_string()),
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_uppercase(), "aws:kms".to_string()),
|
|
||||||
]),
|
|
||||||
HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
|
|
||||||
(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()),
|
|
||||||
]),
|
|
||||||
HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string())]),
|
|
||||||
HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
|
|
||||||
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "key-2".to_string()),
|
|
||||||
]),
|
|
||||||
HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
|
|
||||||
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), invalid_context.to_string()),
|
|
||||||
]),
|
|
||||||
HashMap::from([
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_KMS.to_string()),
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
|
|
||||||
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), "persisted-context".to_string()),
|
|
||||||
]),
|
|
||||||
HashMap::from([
|
|
||||||
(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string()),
|
|
||||||
(SSEC_KEY_MD5_HEADER.to_string(), "customer-key-md5".to_string()),
|
|
||||||
(SSEC_KEY_HEADER.to_string(), persisted_key.to_string()),
|
|
||||||
]),
|
|
||||||
];
|
|
||||||
|
|
||||||
for metadata in corrupt_metadata {
|
|
||||||
let error = select_snapshot_sse_response_headers(&metadata, &HeaderMap::new())
|
|
||||||
.expect_err("corrupt snapshot encryption metadata must fail closed");
|
|
||||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
|
||||||
assert!(!error.to_string().contains(invalid_context));
|
|
||||||
assert!(!error.to_string().contains(persisted_key));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn select_response_projects_snapshot_sse_headers() {
|
|
||||||
let (_tx, rx) = mpsc::channel(1);
|
|
||||||
let metadata = HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), AMZ_ENCRYPTION_AES.to_string())]);
|
|
||||||
|
|
||||||
let response = select_object_response(rx, &metadata, &HeaderMap::new())
|
|
||||||
.expect("valid snapshot metadata should produce a Select response");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
response
|
|
||||||
.headers
|
|
||||||
.get(X_AMZ_SERVER_SIDE_ENCRYPTION)
|
|
||||||
.expect("snapshot SSE response header"),
|
|
||||||
AMZ_ENCRYPTION_AES
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn spawn_test_producer(
|
fn spawn_test_producer(
|
||||||
output: SendableRecordBatchStream,
|
output: SendableRecordBatchStream,
|
||||||
channel_capacity: usize,
|
channel_capacity: usize,
|
||||||
|
|||||||
@@ -1053,8 +1053,8 @@ pub(crate) mod sse {
|
|||||||
pub(crate) use crate::storage::storage_api::sse_consumer::{
|
pub(crate) use crate::storage::storage_api::sse_consumer::{
|
||||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
|
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
|
||||||
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
||||||
get_buffer_size_opt_in, load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption,
|
get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption,
|
||||||
validate_bucket_object_lock_enabled_state,
|
sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state,
|
||||||
};
|
};
|
||||||
pub(crate) use crate::storage::storage_api::sse_consumer::{
|
pub(crate) use crate::storage::storage_api::sse_consumer::{
|
||||||
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
|
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
|
||||||
|
|||||||
+327
-7
@@ -88,7 +88,7 @@ use base64_simd::STANDARD as BASE64_STANDARD;
|
|||||||
use chacha20poly1305::ChaCha20Poly1305;
|
use chacha20poly1305::ChaCha20Poly1305;
|
||||||
#[cfg(feature = "rio-v2")]
|
#[cfg(feature = "rio-v2")]
|
||||||
use hmac::{Hmac, Mac};
|
use hmac::{Hmac, Mac};
|
||||||
use http::{HeaderMap, HeaderValue};
|
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||||
use md5::{Digest as Md5Digest, Md5};
|
use md5::{Digest as Md5Digest, Md5};
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
#[cfg(feature = "rio-v2")]
|
#[cfg(feature = "rio-v2")]
|
||||||
@@ -160,11 +160,18 @@ use super::Error;
|
|||||||
use super::get_bucket_sse_config;
|
use super::get_bucket_sse_config;
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use rustfs_utils::http::headers::{
|
use rustfs_utils::http::headers::{
|
||||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
|
AMZ_ENCRYPTION_AES, AMZ_ENCRYPTION_KMS, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
|
||||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
||||||
|
SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER,
|
||||||
};
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::http::headers::{AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID};
|
||||||
use rustfs_utils::path::path_join_buf;
|
use rustfs_utils::path::path_join_buf;
|
||||||
use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryptionByDefault};
|
use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryptionByDefault};
|
||||||
|
use s3s::header::{
|
||||||
|
X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT,
|
||||||
|
X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
|
||||||
|
};
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -2177,6 +2184,159 @@ pub struct SseReadResponseHeaders {
|
|||||||
pub ssekms_key_id: Option<SSEKMSKeyId>,
|
pub ssekms_key_id: Option<SSEKMSKeyId>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn invalid_sse_response_metadata() -> ApiError {
|
||||||
|
ApiError {
|
||||||
|
code: S3ErrorCode::InternalError,
|
||||||
|
message: "Persisted object encryption metadata is invalid.".to_string(),
|
||||||
|
source: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert_sse_response_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> Result<(), ApiError> {
|
||||||
|
let value = HeaderValue::from_str(value).map_err(|_| invalid_sse_response_metadata())?;
|
||||||
|
headers.insert(name, value);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Project response headers after the caller has completed its existing read
|
||||||
|
/// validation and KMS authorization path.
|
||||||
|
pub(crate) fn project_sse_read_response_headers(
|
||||||
|
metadata: &HashMap<String, String>,
|
||||||
|
sse_customer_algorithm: Option<&SSECustomerAlgorithm>,
|
||||||
|
sse_customer_key_md5: Option<&SSECustomerKeyMD5>,
|
||||||
|
) -> Result<HeaderMap, ApiError> {
|
||||||
|
if metadata.keys().any(|key| {
|
||||||
|
key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY)
|
||||||
|
|| key.eq_ignore_ascii_case(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT)
|
||||||
|
}) {
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
|
||||||
|
let metadata = normalize_encryption_metadata_case(metadata).map_err(|_| invalid_sse_response_metadata())?;
|
||||||
|
let public_mode = metadata.get("x-amz-server-side-encryption").map(String::as_str);
|
||||||
|
let customer_algorithm = metadata.get(SSEC_ALGORITHM_HEADER).map(String::as_str);
|
||||||
|
|
||||||
|
let public_mode = match public_mode {
|
||||||
|
Some(AMZ_ENCRYPTION_AES) => Some(SSEType::SseS3),
|
||||||
|
Some(AMZ_ENCRYPTION_KMS) => Some(SSEType::SseKms),
|
||||||
|
Some(_) => return Err(invalid_sse_response_metadata()),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
if customer_algorithm.is_some_and(|algorithm| algorithm != AMZ_ENCRYPTION_AES) {
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mode = if customer_algorithm.is_some() {
|
||||||
|
if public_mode == Some(SSEType::SseKms) {
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
Some(SSEType::SseC)
|
||||||
|
} else {
|
||||||
|
public_mode
|
||||||
|
};
|
||||||
|
for marker_mode in [
|
||||||
|
metadata
|
||||||
|
.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)
|
||||||
|
.then_some(SSEType::SseC),
|
||||||
|
metadata
|
||||||
|
.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
|
||||||
|
.then_some(SSEType::SseS3),
|
||||||
|
metadata
|
||||||
|
.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)
|
||||||
|
.then_some(SSEType::SseKms),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
{
|
||||||
|
if mode != Some(marker_mode) {
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let Some(mode) = mode else {
|
||||||
|
if metadata
|
||||||
|
.keys()
|
||||||
|
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||||
|
{
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
return Ok(HeaderMap::new());
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut key_id = None;
|
||||||
|
for value in [
|
||||||
|
metadata.get("x-amz-server-side-encryption-aws-kms-key-id"),
|
||||||
|
metadata.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER),
|
||||||
|
metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
{
|
||||||
|
if key_id.is_some_and(|current| current != value) {
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
key_id = Some(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
let normalized_managed = normalize_managed_metadata(metadata.as_ref(), Some(recode_minio_kms_context));
|
||||||
|
let mut response_headers = HeaderMap::with_capacity(3);
|
||||||
|
match mode {
|
||||||
|
SSEType::SseS3 => {
|
||||||
|
if metadata.contains_key("x-amz-server-side-encryption-aws-kms-key-id") || metadata.contains_key(SSEC_KEY_MD5_HEADER)
|
||||||
|
{
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_AES));
|
||||||
|
}
|
||||||
|
SSEType::SseKms => {
|
||||||
|
if metadata.contains_key(SSEC_KEY_MD5_HEADER) {
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
let key_id = key_id
|
||||||
|
.map(String::as_str)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.ok_or_else(invalid_sse_response_metadata)?;
|
||||||
|
response_headers.insert(X_AMZ_SERVER_SIDE_ENCRYPTION, HeaderValue::from_static(AMZ_ENCRYPTION_KMS));
|
||||||
|
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key_id)?;
|
||||||
|
|
||||||
|
if metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
|
||||||
|
&& !normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER)
|
||||||
|
{
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
if let Some(context) = normalized_managed.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||||
|
serde_json::from_str::<HashMap<String, String>>(context).map_err(|_| invalid_sse_response_metadata())?;
|
||||||
|
let encoded = BASE64_STANDARD.encode_to_string(context);
|
||||||
|
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, &encoded)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SSEType::SseC => {
|
||||||
|
if key_id.is_some() || normalized_managed.contains_key(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||||
|
return Err(invalid_sse_response_metadata());
|
||||||
|
}
|
||||||
|
let algorithm = sse_customer_algorithm
|
||||||
|
.map(|value| value.as_ref())
|
||||||
|
.filter(|value| *value == AMZ_ENCRYPTION_AES)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ssec_invalid_request(
|
||||||
|
"The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let key_md5 = sse_customer_key_md5
|
||||||
|
.map(|value| value.as_ref())
|
||||||
|
.ok_or_else(|| ssec_invalid_request("Missing SSE-C customer key MD5."))?;
|
||||||
|
let stored_md5 = metadata.get(SSEC_KEY_MD5_HEADER).ok_or_else(invalid_sse_response_metadata)?;
|
||||||
|
if stored_md5 != key_md5 {
|
||||||
|
return Err(ssec_invalid_request(
|
||||||
|
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, algorithm)?;
|
||||||
|
insert_sse_response_header(&mut response_headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, key_md5)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(response_headers)
|
||||||
|
}
|
||||||
|
|
||||||
/// Read-side response classification for an object whose payload the object
|
/// Read-side response classification for an object whose payload the object
|
||||||
/// layer's encryption resolver already decrypted.
|
/// layer's encryption resolver already decrypted.
|
||||||
///
|
///
|
||||||
@@ -4054,10 +4214,10 @@ mod tests {
|
|||||||
build_kms_request_context, classify_sse_read_response, encode_minio_kms_context, encryption_material_to_metadata,
|
build_kms_request_context, classify_sse_read_response, encode_minio_kms_context, encryption_material_to_metadata,
|
||||||
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
||||||
generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||||
md5_base64, normalize_managed_metadata, recode_minio_kms_context, reset_sse_dek_provider, resolve_effective_kms_key_id,
|
md5_base64, normalize_managed_metadata, project_sse_read_response_headers, recode_minio_kms_context,
|
||||||
resolve_stored_kms_key_id, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption,
|
reset_sse_dek_provider, resolve_effective_kms_key_id, resolve_stored_kms_key_id, rewrap_object_encryption_metadata,
|
||||||
strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
|
||||||
validate_ssec_params, verify_ssec_key_match,
|
validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match,
|
||||||
};
|
};
|
||||||
#[cfg(feature = "rio-v2")]
|
#[cfg(feature = "rio-v2")]
|
||||||
use super::{
|
use super::{
|
||||||
@@ -7724,6 +7884,166 @@ mod tests {
|
|||||||
assert_eq!(audit_tag(&tags, "kmsErrorClass").as_deref(), Some("access_denied"));
|
assert_eq!(audit_tag(&tags, "kmsErrorClass").as_deref(), Some("access_denied"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_response_projection_uses_shared_managed_metadata_compatibility() {
|
||||||
|
let context_json = r#"{"tenant":"s3-select"}"#;
|
||||||
|
let context = BASE64_STANDARD.encode_to_string(context_json);
|
||||||
|
let key_id = "arn:aws:kms:us-east-1:123456789012:key/select";
|
||||||
|
let kms_metadata = HashMap::from([
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), key_id.to_string()),
|
||||||
|
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), key_id.to_string()),
|
||||||
|
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), key_id.to_string()),
|
||||||
|
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), context.clone()),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let headers = project_sse_read_response_headers(&kms_metadata, None, None)
|
||||||
|
.expect("valid SSE-KMS metadata should project response headers");
|
||||||
|
assert_eq!(
|
||||||
|
headers.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-KMS algorithm"),
|
||||||
|
super::AMZ_ENCRYPTION_KMS
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers
|
||||||
|
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID)
|
||||||
|
.expect("SSE-KMS key ID"),
|
||||||
|
key_id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers
|
||||||
|
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT)
|
||||||
|
.expect("SSE-KMS context"),
|
||||||
|
context.as_str()
|
||||||
|
);
|
||||||
|
|
||||||
|
let s3_metadata = HashMap::from([
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_AES.to_string()),
|
||||||
|
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()),
|
||||||
|
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
|
||||||
|
(
|
||||||
|
super::INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(),
|
||||||
|
r#"{"bucket":"select"}"#.to_string(),
|
||||||
|
),
|
||||||
|
("x-amz-meta-private".to_string(), "private".to_string()),
|
||||||
|
]);
|
||||||
|
let headers = project_sse_read_response_headers(&s3_metadata, None, None)
|
||||||
|
.expect("valid SSE-S3 metadata should project response headers");
|
||||||
|
assert_eq!(headers.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
headers.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION).expect("SSE-S3 algorithm"),
|
||||||
|
super::AMZ_ENCRYPTION_AES
|
||||||
|
);
|
||||||
|
|
||||||
|
let plaintext = project_sse_read_response_headers(&HashMap::new(), None, None)
|
||||||
|
.expect("plaintext metadata should not produce SSE headers");
|
||||||
|
assert!(plaintext.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_response_projection_returns_only_validated_ssec_values() {
|
||||||
|
let key_md5 = "customer-key-md5";
|
||||||
|
let metadata = HashMap::from([
|
||||||
|
(
|
||||||
|
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(),
|
||||||
|
super::AMZ_ENCRYPTION_AES.to_string(),
|
||||||
|
),
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5.to_string(), key_md5.to_string()),
|
||||||
|
]);
|
||||||
|
let algorithm = SSECustomerAlgorithm::from(super::AMZ_ENCRYPTION_AES);
|
||||||
|
let key_md5 = SSECustomerKeyMD5::from(key_md5);
|
||||||
|
|
||||||
|
let headers = project_sse_read_response_headers(&metadata, Some(&algorithm), Some(&key_md5))
|
||||||
|
.expect("validated SSE-C values should project response headers");
|
||||||
|
assert_eq!(headers.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
headers
|
||||||
|
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM)
|
||||||
|
.expect("SSE-C algorithm"),
|
||||||
|
super::AMZ_ENCRYPTION_AES
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers
|
||||||
|
.get(super::X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5)
|
||||||
|
.expect("SSE-C key MD5"),
|
||||||
|
key_md5.as_str()
|
||||||
|
);
|
||||||
|
assert!(headers.get(super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY).is_none());
|
||||||
|
|
||||||
|
let missing_algorithm = project_sse_read_response_headers(&metadata, None, Some(&key_md5))
|
||||||
|
.expect_err("missing SSE-C algorithm must fail before streaming");
|
||||||
|
assert_eq!(missing_algorithm.code, S3ErrorCode::InvalidRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_response_projection_rejects_unvalidated_or_sensitive_metadata() {
|
||||||
|
let invalid_context = "not-base64";
|
||||||
|
let persisted_context = "must-not-leak-context";
|
||||||
|
let persisted_key = "must-not-leak-key";
|
||||||
|
let corrupt_metadata = [
|
||||||
|
HashMap::from([
|
||||||
|
(
|
||||||
|
super::AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_lowercase(),
|
||||||
|
super::AMZ_ENCRYPTION_AES.to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
super::AMZ_SERVER_SIDE_ENCRYPTION.to_ascii_uppercase(),
|
||||||
|
super::AMZ_ENCRYPTION_KMS.to_string(),
|
||||||
|
),
|
||||||
|
]),
|
||||||
|
HashMap::from([
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
|
||||||
|
(
|
||||||
|
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(),
|
||||||
|
super::AMZ_ENCRYPTION_AES.to_string(),
|
||||||
|
),
|
||||||
|
]),
|
||||||
|
HashMap::from([
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
|
||||||
|
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "key-2".to_string()),
|
||||||
|
]),
|
||||||
|
HashMap::from([
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
|
||||||
|
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), invalid_context.to_string()),
|
||||||
|
]),
|
||||||
|
HashMap::from([
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), super::AMZ_ENCRYPTION_KMS.to_string()),
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
|
||||||
|
(super::AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), persisted_context.to_string()),
|
||||||
|
]),
|
||||||
|
HashMap::from([
|
||||||
|
(
|
||||||
|
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM.to_string(),
|
||||||
|
super::AMZ_ENCRYPTION_AES.to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5.to_string(),
|
||||||
|
"customer-key-md5".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
super::AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY.to_ascii_uppercase(),
|
||||||
|
persisted_key.to_string(),
|
||||||
|
),
|
||||||
|
]),
|
||||||
|
];
|
||||||
|
|
||||||
|
for metadata in corrupt_metadata {
|
||||||
|
let error = project_sse_read_response_headers(&metadata, None, None)
|
||||||
|
.expect_err("corrupt encryption metadata must fail closed");
|
||||||
|
assert_eq!(error.code, S3ErrorCode::InternalError);
|
||||||
|
for sensitive in [invalid_context, persisted_context, persisted_key] {
|
||||||
|
assert!(!error.to_string().contains(sensitive));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let marker_only_kms =
|
||||||
|
HashMap::from([(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string())]);
|
||||||
|
let marker_error = project_sse_read_response_headers(&marker_only_kms, None, None)
|
||||||
|
.expect_err("marker-only KMS metadata must not bypass the existing authorization classifier");
|
||||||
|
assert_eq!(marker_error.code, S3ErrorCode::InternalError);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn classification_matches_ssec_validation_and_headers() {
|
async fn classification_matches_ssec_validation_and_headers() {
|
||||||
let key = [0x42u8; 32];
|
let key = [0x42u8; 32];
|
||||||
|
|||||||
@@ -111,8 +111,8 @@ pub(crate) use super::ecfs_extend::{
|
|||||||
pub(crate) use super::sse::{
|
pub(crate) use super::sse::{
|
||||||
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
|
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
|
||||||
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
||||||
rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata,
|
project_sse_read_response_headers, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption,
|
||||||
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) mod access_consumer {
|
pub(crate) mod access_consumer {
|
||||||
@@ -363,8 +363,8 @@ pub(crate) mod sse_consumer {
|
|||||||
pub(crate) use super::{
|
pub(crate) use super::{
|
||||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
|
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
|
||||||
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
||||||
get_buffer_size_opt_in, load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption,
|
get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption,
|
||||||
validate_bucket_object_lock_enabled_state,
|
sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user