gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 18:27:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(sse): Temporarily refactored the SSE design for ECStore (#2813)

Browse Source 
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
This commit is contained in:
唐小鸭
2026-05-09 22:06:35 +08:00
committed by GitHub
parent 6275918d92
commit 09c2d15057
12 changed files with 1338 additions and 970 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/app/multipart_usecase.rs
+193 -178
View File
@@ -28,6 +28,7 @@ use crate::storage::s3_api::multipart::{
ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output,
parse_list_multipart_uploads_params, parse_list_parts_params,
};
use crate::storage::sse::{build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error};
use crate::storage::*;
use bytes::Bytes;
use futures::StreamExt;
@@ -48,7 +49,9 @@ use rustfs_ecstore::set_disk::is_valid_storage_class;
use rustfs_ecstore::store_api::{CompletePart, HTTPRangeSpec, MultipartUploadResult, ObjectIO, ObjectOptions, PutObjReader};
use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations};
use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
use rustfs_rio::{CompressReader, HashReader};
use rustfs_rio::{CompressReader, EncryptReader, HashReader};
#[cfg(test)]
use rustfs_rio::{DecryptReader, HardLimitReader, boxed_reader, wrap_reader};
use rustfs_s3_common::S3Operation;
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
@@ -68,6 +71,7 @@ use tracing::{instrument, warn};
use urlencoding::encode;
use uuid::Uuid;
#[cfg(test)]
fn merge_part_encryption_metadata(
metadata: &HashMap<String, String>,
part_metadata: &HashMap<String, String>,
@@ -77,6 +81,27 @@ fn merge_part_encryption_metadata(
merged
}
#[cfg(test)]
fn multipart_plaintext_size(parts: &[rustfs_filemeta::ObjectPartInfo], fallback: i64) -> i64 {
let total: i64 = parts
.iter()
.map(|part| {
if part.actual_size > 0 {
part.actual_size
} else {
part.size as i64
}
})
.sum();
if total > 0 { total } else { fallback }
}
#[cfg(test)]
fn multipart_part_numbers(parts: &[rustfs_filemeta::ObjectPartInfo]) -> Vec<usize> {
parts.iter().map(|part| part.number).collect()
}
/// Returns InvalidRange error if CopySourceRange end exceeds the source object size.
/// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec.
fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_size: i64) -> S3Result<()> {
@@ -532,7 +557,7 @@ impl DefaultMultipartUsecase {
let server_side_encryption = Some(material.server_side_encryption.clone());
let ssekms_key_id = material.kms_key_id.clone();
metadata.extend(material.metadata);
metadata.extend(encryption_material_to_metadata(&material));
(server_side_encryption, ssekms_key_id)
}
@@ -735,37 +760,64 @@ impl DefaultMultipartUsecase {
};
(sse, key_id)
};
let part_key = fi.user_defined.get("x-rustfs-encryption-key").cloned();
let part_nonce = fi.user_defined.get("x-rustfs-encryption-iv").cloned();
let encryption_request = EncryptionRequest {
EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
part_number: Some(part_id),
part_key,
part_nonce,
};
}
.check_upload_part_customer_key_md5(&fi.user_defined, sse_customer_key_md5.clone())?;
let (requested_sse, requested_kms_key_id) = if has_ssec {
let encryption_request = EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
};
encryption_request.check_upload_part_customer_key_md5(&fi.user_defined, sse_customer_key_md5.clone())?;
let (requested_sse, requested_kms_key_id) = match sse_encryption(encryption_request).await? {
Some(material) => {
let requested_sse = Some(material.server_side_encryption.clone());
let requested_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = material.wrap_reader(reader);
reader =
HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
(requested_sse, requested_kms_key_id)
match sse_encryption(encryption_request).await? {
Some(material) => {
let requested_sse = Some(material.server_side_encryption.clone());
let requested_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = EncryptReader::new_multipart(reader, material.key_bytes, material.base_nonce, part_id);
reader = HashReader::from_reader(
encrypted_reader,
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?;
(requested_sse, requested_kms_key_id)
}
None => (None, None),
}
None => (None, None),
} else if let Some(server_side_encryption) = server_side_encryption {
let managed_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
metadata: &fi.user_defined,
sse_customer_key: None,
sse_customer_key_md5: None,
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let encrypted_reader =
EncryptReader::new_multipart(reader, managed_material.key_bytes, managed_material.base_nonce, part_id);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
(Some(server_side_encryption), ssekms_key_id)
} else {
(None, None)
};
let mut reader = PutObjReader::new(reader);
@@ -905,6 +957,7 @@ impl DefaultMultipartUsecase {
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
copy_source_sse_customer_algorithm,
copy_source_sse_customer_key,
copy_source_sse_customer_key_md5,
..
@@ -940,7 +993,11 @@ impl DefaultMultipartUsecase {
let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?;
src_opts.version_id = src_version_id.clone();
let h = HeaderMap::new();
let h = build_ssec_read_headers(
copy_source_sse_customer_algorithm.as_ref(),
copy_source_sse_customer_key.as_ref(),
copy_source_sse_customer_key_md5.as_ref(),
);
let get_opts = ObjectOptions {
version_id: src_opts.version_id.clone(),
versioned: src_opts.versioned,
@@ -951,9 +1008,9 @@ impl DefaultMultipartUsecase {
let src_reader = store
.get_object_reader(&src_bucket, &src_key, rs.clone(), h, &get_opts)
.await
.map_err(ApiError::from)?;
.map_err(map_get_object_reader_error)?;
let mut src_info = src_reader.object_info;
let src_info = src_reader.object_info;
if let Some(if_match) = copy_source_if_match {
if let Some(ref etag) = src_info.etag {
@@ -992,7 +1049,11 @@ impl DefaultMultipartUsecase {
(0, src_info.size)
};
let h = HeaderMap::new();
let h = build_ssec_read_headers(
copy_source_sse_customer_algorithm.as_ref(),
copy_source_sse_customer_key.as_ref(),
copy_source_sse_customer_key_md5.as_ref(),
);
let get_opts = ObjectOptions {
version_id: src_opts.version_id.clone(),
versioned: src_opts.versioned,
@@ -1003,89 +1064,28 @@ impl DefaultMultipartUsecase {
let src_reader = store
.get_object_reader(&src_bucket, &src_key, rs.clone(), h, &get_opts)
.await
.map_err(ApiError::from)?;
.map_err(map_get_object_reader_error)?;
let src_stream = src_reader.stream;
let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
let src_decryption_request = DecryptionRequest {
bucket: &src_bucket,
key: &src_key,
metadata: &src_info.user_defined,
sse_customer_key: copy_source_sse_customer_key.as_ref(),
sse_customer_key_md5: copy_source_sse_customer_key_md5.as_ref(),
part_number: None,
parts: &src_info.parts,
etag: src_info.etag.as_deref(),
};
let actual_size = length;
let mut size = length;
let mut reader = match sse_decryption(src_decryption_request).await? {
Some(material) => {
if let Some(original) = material.original_size {
src_info.actual_size = original;
}
if material.is_multipart {
let (decrypted_stream, plaintext_size) =
material.wrap_reader(src_stream, size).await.map_err(ApiError::from)?;
size = plaintext_size;
if is_compressible {
let hrd = HashReader::from_reader(decrypted_stream, size, actual_size, None, None, false)
.map_err(ApiError::from)?;
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
size,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_reader(decrypted_stream, size, actual_size, None, None, false).map_err(ApiError::from)?
}
} else if is_compressible {
let hrd =
HashReader::from_stream(material.wrap_single_reader(src_stream), size, actual_size, None, None, false)
.map_err(ApiError::from)?;
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
size,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_stream(material.wrap_single_reader(src_stream), size, actual_size, None, None, false)
.map_err(ApiError::from)?
}
}
None => {
if is_compressible {
let hrd =
HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?;
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
size,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?
}
}
let mut reader = if is_compressible {
let hrd = HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?;
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
size,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?
};
let server_side_encryption = mp_info
@@ -1096,6 +1096,7 @@ impl DefaultMultipartUsecase {
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}"))))
})
.transpose()?;
let has_ssec = sse_customer_algorithm.is_some();
let ssekms_key_id = match server_side_encryption.as_ref() {
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info
.user_defined
@@ -1103,38 +1104,65 @@ impl DefaultMultipartUsecase {
.map(|s| s.to_string()),
_ => None,
};
let part_key = mp_info.user_defined.get("x-rustfs-encryption-key").cloned();
let part_nonce = mp_info.user_defined.get("x-rustfs-encryption-iv").cloned();
let encryption_request = EncryptionRequest {
EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
part_number: Some(part_id),
part_key,
part_nonce,
};
}
.check_upload_part_customer_key_md5(&mp_info.user_defined, sse_customer_key_md5.clone())?;
encryption_request.check_upload_part_customer_key_md5(&mp_info.user_defined, sse_customer_key_md5.clone())?;
let (requested_sse, requested_kms_key_id, dst_user_defined) = if has_ssec {
let encryption_request = EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
};
let (requested_sse, requested_kms_key_id, dst_user_defined) = match sse_encryption(encryption_request).await? {
Some(material) => {
let requested_sse = Some(material.server_side_encryption.clone());
let requested_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = material.wrap_reader(reader);
reader =
HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
let dst_user_defined = merge_part_encryption_metadata(&mp_info.user_defined, &material.metadata);
(requested_sse, requested_kms_key_id, dst_user_defined)
match sse_encryption(encryption_request).await? {
Some(material) => {
let requested_sse = Some(material.server_side_encryption.clone());
let requested_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = EncryptReader::new_multipart(reader, material.key_bytes, material.base_nonce, part_id);
reader = HashReader::from_reader(
encrypted_reader,
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?;
(requested_sse, requested_kms_key_id, mp_info.user_defined.clone())
}
None => (None, None, mp_info.user_defined.clone()),
}
None => (None, None, mp_info.user_defined.clone()),
} else if let Some(server_side_encryption) = server_side_encryption {
let managed_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
metadata: &mp_info.user_defined,
sse_customer_key: None,
sse_customer_key_md5: None,
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let encrypted_reader =
EncryptReader::new_multipart(reader, managed_material.key_bytes, managed_material.base_nonce, part_id);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
} else {
(None, None, mp_info.user_defined.clone())
};
if let Some(checksum_algorithm) = mp_info
@@ -1294,77 +1322,61 @@ mod tests {
.await
.expect("prepare multipart encryption")
.expect("managed multipart session material");
let session_metadata = session_material.metadata.clone();
let session_metadata = encryption_material_to_metadata(&session_material);
let session_nonce = session_metadata
.get("x-rustfs-encryption-iv")
.cloned()
.expect("session nonce metadata");
let part_key = session_metadata.get("x-rustfs-encryption-key").cloned();
let part_nonce = Some(session_nonce.clone());
let part_one_plaintext = vec![0x31; rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE + 23];
let part_two_plaintext = vec![0x32; rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE * 2 + 7];
let part_one_material = sse_encryption(EncryptionRequest {
let part_one_material = sse_decryption(DecryptionRequest {
bucket: "bucket",
key: "object",
server_side_encryption: Some(session_material.server_side_encryption.clone()),
ssekms_key_id: session_material.kms_key_id.clone(),
sse_customer_algorithm: None,
metadata: &session_metadata,
sse_customer_key: None,
sse_customer_key_md5: None,
content_size: part_one_plaintext.len() as i64,
part_number: Some(1),
part_key: part_key.clone(),
part_nonce: part_nonce.clone(),
})
.await
.expect("encrypt part one")
.expect("decrypt session one")
.expect("part one material");
let part_one_metadata = merge_part_encryption_metadata(&session_metadata, &part_one_material.metadata);
let mut encrypted_one = Vec::new();
part_one_material
.wrap_reader(Cursor::new(part_one_plaintext.clone()))
.read_to_end(&mut encrypted_one)
.await
.expect("read encrypted part one");
EncryptReader::new_multipart(
Cursor::new(part_one_plaintext.clone()),
part_one_material.key_bytes,
part_one_material.base_nonce,
1,
)
.read_to_end(&mut encrypted_one)
.await
.expect("read encrypted part one");
let part_two_material = sse_encryption(EncryptionRequest {
let part_two_material = sse_decryption(DecryptionRequest {
bucket: "bucket",
key: "object",
server_side_encryption: Some(session_material.server_side_encryption.clone()),
ssekms_key_id: session_material.kms_key_id.clone(),
sse_customer_algorithm: None,
metadata: &session_metadata,
sse_customer_key: None,
sse_customer_key_md5: None,
content_size: part_two_plaintext.len() as i64,
part_number: Some(2),
part_key,
part_nonce,
})
.await
.expect("encrypt part two")
.expect("decrypt session two")
.expect("part two material");
let part_two_metadata = merge_part_encryption_metadata(&session_metadata, &part_two_material.metadata);
let mut encrypted_two = Vec::new();
part_two_material
.wrap_reader(Cursor::new(part_two_plaintext.clone()))
.read_to_end(&mut encrypted_two)
.await
.expect("read encrypted part two");
EncryptReader::new_multipart(
Cursor::new(part_two_plaintext.clone()),
part_two_material.key_bytes,
part_two_material.base_nonce,
2,
)
.read_to_end(&mut encrypted_two)
.await
.expect("read encrypted part two");
assert_eq!(
session_metadata.get("x-rustfs-encryption-iv").map(String::as_str),
Some(session_nonce.as_str())
);
assert_ne!(
part_one_metadata.get("x-rustfs-encryption-iv"),
session_metadata.get("x-rustfs-encryption-iv")
);
assert_ne!(
part_two_metadata.get("x-rustfs-encryption-iv"),
session_metadata.get("x-rustfs-encryption-iv")
);
let parts = vec![
ObjectPartInfo {
@@ -1391,18 +1403,21 @@ mod tests {
metadata: &session_metadata,
sse_customer_key: None,
sse_customer_key_md5: None,
part_number: None,
parts: &parts,
etag: Some("multipart-etag"),
})
.await
.expect("decrypt multipart")
.expect("managed decryption material");
let (mut decrypted_reader, plaintext_size) = decryption_material
.wrap_reader(Cursor::new(encrypted_stream), -1)
.await
.expect("wrap multipart reader");
let plaintext_size = multipart_plaintext_size(&parts, -1);
let mut decrypted_reader = HardLimitReader::new(
boxed_reader(DecryptReader::new_multipart(
wrap_reader(Cursor::new(encrypted_stream)),
decryption_material.key_bytes,
decryption_material.base_nonce,
multipart_part_numbers(&parts),
)),
plaintext_size,
);
let mut decrypted = Vec::new();
decrypted_reader
rustfs/src/app/object_usecase.rs
+37 -119
View File
@@ -31,6 +31,7 @@ use crate::storage::options::{
};
use crate::storage::request_context::spawn_traced;
use crate::storage::s3_api::multipart::parse_list_parts_params;
use crate::storage::sse::{SSEType, build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error};
use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig};
use crate::storage::*;
use bytes::Bytes;
@@ -84,7 +85,7 @@ use rustfs_filemeta::{
use rustfs_io_metrics;
use rustfs_notify::EventArgsBuilder;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_rio::{CompressReader, DynReader, HashReader, wrap_reader};
use rustfs_rio::{CompressReader, DynReader, EncryptReader, HashReader, wrap_reader};
use rustfs_s3_common::S3Operation;
use rustfs_s3select_api::{
object_store::bytes_stream,
@@ -118,6 +119,7 @@ use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use std::collections::HashMap;
use std::ops::Add;
use std::path::Path;
use std::str::FromStr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
@@ -1279,7 +1281,7 @@ impl DefaultObjectUsecase {
opts: &ObjectOptions,
part_number: Option<usize>,
) -> S3Result<GetObjectPreparedRead<'a>> {
let h = HeaderMap::new();
let h = req.headers.clone();
let io_planning = Self::acquire_get_object_io_planning(manager, wrapper, timeout_config, bucket, key).await?;
let store = get_validated_store(bucket).await?;
@@ -1306,7 +1308,7 @@ impl DefaultObjectUsecase {
let reader = store
.get_object_reader(bucket, key, rs.clone(), h, opts)
.await
.map_err(ApiError::from)?;
.map_err(map_get_object_reader_error)?;
let info = reader.object_info;
@@ -1372,13 +1374,9 @@ impl DefaultObjectUsecase {
metadata: &info.user_defined,
sse_customer_key: req.input.sse_customer_key.as_ref(),
sse_customer_key_md5: req.input.sse_customer_key_md5.as_ref(),
part_number: None,
parts: &info.parts,
etag: info.etag.as_deref(),
};
let mut response_content_length = content_length;
let encrypted_stream = reader.stream;
let response_content_length = content_length;
let (
server_side_encryption,
@@ -1390,27 +1388,18 @@ impl DefaultObjectUsecase {
) = match sse_decryption(decryption_request).await? {
Some(material) => {
let server_side_encryption = Some(material.server_side_encryption.clone());
let sse_customer_algorithm = Some(material.algorithm.clone());
let sse_customer_algorithm = matches!(material.sse_type, SSEType::SseC).then_some(material.algorithm.clone());
let sse_customer_key_md5 = material.customer_key_md5.clone();
let ssekms_key_id = material.kms_key_id.clone();
let (decrypted_stream, plaintext_size) = material
.wrap_reader(encrypted_stream, content_length)
.await
.map_err(ApiError::from)?;
response_content_length = plaintext_size;
(
server_side_encryption,
sse_customer_algorithm,
sse_customer_key_md5,
ssekms_key_id,
material.kms_key_id,
true,
decrypted_stream,
wrap_reader(reader.stream),
)
}
None => (None, None, None, None, false, wrap_reader(encrypted_stream)),
None => (None, None, None, None, false, wrap_reader(reader.stream)),
};
Ok(GetObjectReadSetup {
@@ -1896,9 +1885,6 @@ impl DefaultObjectUsecase {
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
part_number: None,
part_key: None,
part_nonce: None,
};
let encryption_material = match sse_encryption(encryption_request).await {
@@ -1914,11 +1900,11 @@ impl DefaultObjectUsecase {
effective_sse = Some(material.server_side_encryption.clone());
effective_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = material.wrap_reader(reader);
let encrypted_reader = EncryptReader::new(reader, material.key_bytes, material.base_nonce);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
let encryption_metadata = material.metadata;
let encryption_metadata = encryption_material_to_metadata(&material);
metadata.extend(encryption_metadata.clone());
opts.user_defined.extend(encryption_metadata);
}
@@ -2577,6 +2563,7 @@ impl DefaultObjectUsecase {
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
copy_source_sse_customer_algorithm,
copy_source_sse_customer_key,
copy_source_sse_customer_key_md5,
metadata_directive,
@@ -2679,12 +2666,16 @@ impl DefaultObjectUsecase {
})
});
let h = HeaderMap::new();
let h = build_ssec_read_headers(
copy_source_sse_customer_algorithm.as_ref(),
copy_source_sse_customer_key.as_ref(),
copy_source_sse_customer_key_md5.as_ref(),
);
let gr = store
.get_object_reader(&src_bucket, &src_key, None, h, &src_get_opts)
.await
.map_err(ApiError::from)?;
.map_err(map_get_object_reader_error)?;
let mut src_info = gr.object_info.clone();
@@ -2716,25 +2707,6 @@ impl DefaultObjectUsecase {
src_info.metadata_only = true;
}
let decryption_request = DecryptionRequest {
bucket: &src_bucket,
key: &src_key,
metadata: &src_info.user_defined,
sse_customer_key: copy_source_sse_customer_key.as_ref(),
sse_customer_key_md5: copy_source_sse_customer_key_md5.as_ref(),
part_number: None,
parts: &src_info.parts,
etag: src_info.etag.as_deref(),
};
let decryption_material = sse_decryption(decryption_request).await?;
if let Some(material) = decryption_material.as_ref()
&& let Some(original) = material.original_size
{
src_info.actual_size = original;
}
strip_managed_encryption_metadata(&mut src_info.user_defined);
let actual_size = src_info.get_actual_size().map_err(ApiError::from)?;
@@ -2782,67 +2754,20 @@ impl DefaultObjectUsecase {
}
apply_bucket_default_lock_retention(&bucket, &mut src_info.user_defined, has_explicit_object_lock_retention).await?;
let mut reader = match decryption_material {
Some(material) => {
if material.is_multipart {
let (decrypted_stream, plaintext_size) =
material.wrap_reader(gr.stream, length).await.map_err(ApiError::from)?;
length = plaintext_size;
if should_compress {
let hrd = HashReader::from_reader(decrypted_stream, length, actual_size, None, None, false)
.map_err(ApiError::from)?;
length = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
length,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_reader(decrypted_stream, length, actual_size, None, None, false)
.map_err(ApiError::from)?
}
} else if should_compress {
let hrd =
HashReader::from_stream(material.wrap_single_reader(gr.stream), length, actual_size, None, None, false)
.map_err(ApiError::from)?;
length = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
length,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_stream(material.wrap_single_reader(gr.stream), length, actual_size, None, None, false)
.map_err(ApiError::from)?
}
}
None => {
if should_compress {
let hrd =
HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?;
length = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
length,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?
}
}
let mut reader = if should_compress {
let hrd = HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?;
length = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
length,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
} else {
HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?
};
let encryption_request = EncryptionRequest {
@@ -2854,20 +2779,17 @@ impl DefaultObjectUsecase {
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
part_number: None,
part_key: None,
part_nonce: None,
};
if let Some(material) = sse_encryption(encryption_request).await? {
effective_sse = Some(material.server_side_encryption.clone());
effective_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = material.wrap_reader(reader);
let encrypted_reader = EncryptReader::new(reader, material.key_bytes, material.base_nonce);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
src_info.user_defined.extend(material.metadata);
src_info.user_defined.extend(encryption_material_to_metadata(&material));
}
src_info.put_object_reader = Some(PutObjReader::new(reader));
@@ -4380,20 +4302,17 @@ impl DefaultObjectUsecase {
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
part_number: None,
part_key: None,
part_nonce: None,
})
.await?
{
effective_sse = Some(material.server_side_encryption.clone());
effective_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = material.wrap_reader(hrd);
let encrypted_reader = EncryptReader::new(hrd, material.key_bytes, material.base_nonce);
hrd = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
let encryption_metadata = material.metadata;
let encryption_metadata = encryption_material_to_metadata(&material);
metadata.extend(encryption_metadata.clone());
opts.user_defined.extend(encryption_metadata);
}
@@ -4509,7 +4428,6 @@ mod tests {
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
input,
rustfs/src/storage/mod.rs
+1 -1
View File
@@ -23,7 +23,7 @@ pub mod options;
pub mod request_context;
pub mod rpc;
pub(crate) mod s3_api;
mod sse;
pub(crate) mod sse;
pub mod timeout_wrapper;
pub mod tonic_service;
rustfs/src/storage/sse.rs
+226 -631
View File
File diff suppressed because it is too large Load Diff