chore(deps): migrate direct encoding deps to simd (#6690)

This commit is contained in:
houseme
2026-08-27 03:17:24 +08:00
committed by GitHub
parent 31031f2a46
commit ba7785d61d
81 changed files with 523 additions and 505 deletions
+2 -2
View File
@@ -121,10 +121,10 @@ tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
urlencoding.workspace = true
walkdir.workspace = true
base64 = { workspace = true }
base64-simd = { workspace = true }
rand = { workspace = true, features = ["serde"] }
chrono = { workspace = true, features = ["serde"] }
hex = { workspace = true }
hex-simd = { workspace = true }
md-5 = { workspace = true }
opentelemetry-proto = { workspace = true }
prost.workspace = true
+2 -3
View File
@@ -24,7 +24,6 @@ mod tests {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use sha2::Sha256;
@@ -74,12 +73,12 @@ mod tests {
let mut hasher = Md5::new();
hasher.update(body);
let digest = hasher.finalize();
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
base64_simd::STANDARD.encode_to_string(digest.as_slice())
}
fn checksum_sha256_base64(body: &[u8]) -> String {
let digest = Sha256::digest(body);
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
base64_simd::STANDARD.encode_to_string(digest.as_slice())
}
fn checksum_crc64nvme_base64(body: &[u8]) -> String {
+1 -2
View File
@@ -652,11 +652,10 @@ const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
async fn start_rustfs_with_compression_and_sse(
env: &mut RustFSTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use base64::Engine;
env.cleanup_existing_processes().await?;
let binary_path = rustfs_binary_path();
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
// Server output goes to a file inside the per-test temp dir so a failing
// run can be diagnosed from the child's logs.
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
@@ -27,8 +27,7 @@ mod tests {
VersioningConfiguration,
};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use sha2::{Digest, Sha256};
use tracing::info;
@@ -465,7 +464,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"deterministic synthetic payload for copy-object checksum #4996";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
client
.put_object()
@@ -534,7 +533,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"another deterministic payload whose source checksum must survive the copy";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
// Store the source WITH a SHA-256 checksum so it has one to preserve.
let put_src = client
@@ -614,7 +613,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"payload whose copy must be re-checksummed with a different algorithm";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
// Source is stored WITH a SHA-256 checksum.
client
+3 -3
View File
@@ -1113,7 +1113,7 @@ fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
@@ -1375,7 +1375,7 @@ impl S3 for FakeBackend {
Some(value) => value,
None => {
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
hex::encode(digest)
hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower)
}
};
let version = ObjectVersion {
@@ -1660,7 +1660,7 @@ impl S3 for FakeBackend {
}
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
let e_tag = hex::encode(digest);
let e_tag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
let mut state = lock(&self.store);
let existing_bytes = state
.uploads
@@ -28,7 +28,6 @@ use aws_sdk_s3::types::{
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus,
LifecycleRule, LifecycleRuleFilter, ServerSideEncryption, Transition, TransitionStorageClass, VersioningConfiguration,
};
use base64::Engine;
use bytes::Bytes;
use flate2::read::GzDecoder;
use http::header::{CONTENT_ENCODING, HOST};
@@ -1808,7 +1807,7 @@ async fn four_node_inline_fallback_controls() -> TestResult {
let collector = OtlpMetricCollector::start().await?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
configure_reader_metric_cluster(&mut cluster, &collector);
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key);
cluster.start().await?;
@@ -2017,7 +2016,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
let collector = OtlpMetricCollector::start().await?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
@@ -2489,7 +2488,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
hot.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
hot.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
hot.start().await?;
+7 -7
View File
@@ -27,7 +27,7 @@ use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use http::header::{CONTENT_TYPE, HOST};
use md5::{Digest as Md5Digest, Md5};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -64,7 +64,7 @@ pub fn init_logging() {
pub fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
BASE64.encode(hasher.finalize())
BASE64.encode_to_string(hasher.finalize())
}
pub fn assert_s3_error<T, E>(result: Result<T, SdkError<E>>, status: u16, code: &str, message: &str, context: &str)
@@ -365,7 +365,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
"rotated_at": serde_json::Value::Null,
"created_by": "e2e-test",
"encrypted_key_material": BASE64.encode(key_data),
"encrypted_key_material": BASE64.encode_to_string(key_data),
"nonce": Vec::<u8>::new()
});
@@ -383,7 +383,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
info!("Testing SSE-C encryption");
let test_key = "01234567890123456789012345678901"; // 32-byte key
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let test_data = b"Hello, KMS SSE-C World!";
let object_key = "test-sse-c-object";
@@ -551,8 +551,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
// Test SSE-C with wrong key for download
let test_key = "01234567890123456789012345678901";
let wrong_key = "98765432109876543210987654321098";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
let test_data = b"Test data for error scenarios";
@@ -807,7 +807,7 @@ pub async fn test_multipart_upload_with_config(
// Prepare encryption parameters
let (sse_c_key_b64, sse_c_key_md5) = match &config.encryption_type {
EncryptionType::SSEC { key, key_md5 } => {
let key_b64 = base64::engine::general_purpose::STANDARD.encode(key);
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
(Some(key_b64), Some(key_md5.clone()))
}
_ => (None, None),
@@ -177,7 +177,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
// Verify that files cannot be read with wrong keys
info!("🔒 Verify key isolation");
let wrong_key = "11111111111111111111111111111111";
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
// Try to read file encrypted with key1 using wrong key
@@ -24,7 +24,6 @@
use super::common::{LocalKMSTestEnvironment, SSE_C_KEY_MISMATCH_MESSAGE, assert_s3_error, sse_customer_key_md5_base64};
use crate::common::{TEST_BUCKET, init_logging};
use aws_sdk_s3::types::ServerSideEncryption;
use base64::Engine;
use std::sync::Arc;
use tokio::sync::Semaphore;
use tracing::{info, warn};
@@ -68,7 +67,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
// Test SSE-C with zero-byte file
info!("📤 Testing SSE-C with zero-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let object_key_c = "zero-byte-sse-c";
@@ -161,7 +160,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
// Test SSE-C with single byte
info!("📤 Testing SSE-C with single-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let object_key_c = "single-byte-sse-c";
@@ -287,7 +286,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
// Test 1: Invalid key length for SSE-C
info!("🔍 Testing invalid SSE-C key length");
let invalid_short_key = "short"; // Too short
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
let invalid_key_b64 = base64_simd::STANDARD.encode_to_string(invalid_short_key);
let invalid_key_md5 = sse_customer_key_md5_base64(invalid_short_key);
let invalid_key_result = s3_client
@@ -325,7 +324,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
// Test 2: Mismatched MD5 for SSE-C
info!("🔍 Testing mismatched MD5 for SSE-C key");
let valid_key = "01234567890123456789012345678901";
let valid_key_b64 = base64::engine::general_purpose::STANDARD.encode(valid_key);
let valid_key_b64 = base64_simd::STANDARD.encode_to_string(valid_key);
let wrong_md5 = sse_customer_key_md5_base64("98765432109876543210987654321098");
let wrong_md5_result = s3_client
@@ -465,7 +464,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
2 => {
// SSE-C
let key = format!("testkey{i:026}"); // 32-byte key
let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
let key_b64 = base64_simd::STANDARD.encode_to_string(&key);
let key_md5 = sse_customer_key_md5_base64(&key);
client
@@ -535,8 +534,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
let key1 = "key1key1key1key1key1key1key1key1"; // 32 bytes
let key2 = "key2key2key2key2key2key2key2key2"; // 32 bytes
let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
+3 -3
View File
@@ -138,8 +138,8 @@ async fn test_local_kms_key_isolation() {
// Test that different SSE-C keys create isolated encrypted objects
let key1 = "01234567890123456789012345678901";
let key2 = "98765432109876543210987654321098";
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
@@ -565,7 +565,7 @@ async fn test_multipart_upload_with_sse_c(
// SSE-C encryption key
let encryption_key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
let key_b64 = base64_simd::STANDARD.encode_to_string(encryption_key);
let key_md5 = sse_customer_key_md5_base64(encryption_key);
// Generate test data
+2 -2
View File
@@ -127,8 +127,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
let key1 = "01234567890123456789012345678901";
let key2 = "98765432109876543210987654321098";
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
@@ -497,7 +497,7 @@ async fn test_multipart_encryption_type(
// Prepare SSE-C keys when required
let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
let key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
let key_md5 = sse_customer_key_md5_base64(key);
(Some(key_b64), Some(key_md5))
} else {
+11 -12
View File
@@ -22,7 +22,6 @@ use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
};
use base64::Engine;
use chrono::{Duration as ChronoDuration, Utc};
use flate2::{Compression, write::GzEncoder};
use http::HeaderValue;
@@ -47,19 +46,19 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
"conditions": conditions,
});
base64::engine::general_purpose::STANDARD.encode(policy.to_string())
base64_simd::STANDARD.encode_to_string(policy.to_string())
}
fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
base64_simd::STANDARD.encode_to_string(hasher.finalize())
}
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
async fn create_restricted_user(
@@ -97,7 +96,7 @@ fn restricted_user_client(env: &RustFSTestEnvironment, username: &str, secret_ke
const LOCAL_SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
fn local_sse_master_key_value() -> String {
base64::engine::general_purpose::STANDARD.encode([0x42u8; 32])
base64_simd::STANDARD.encode_to_string([0x42u8; 32])
}
async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec<u8> {
@@ -1887,7 +1886,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
let object_key = "sse-c-object.txt";
let expected_body = b"anonymous-post-sse-c".to_vec();
let customer_key = "01234567890123456789012345678901";
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
let admin_client = env.create_s3_client();
@@ -1941,7 +1940,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
.bucket(bucket)
.key(object_key)
.sse_customer_algorithm("AES256")
.sse_customer_key(base64::engine::general_purpose::STANDARD.encode(customer_key))
.sse_customer_key(base64_simd::STANDARD.encode_to_string(customer_key))
.sse_customer_key_md5(customer_key_md5)
.send()
.await?;
@@ -1963,8 +1962,8 @@ async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Res
let object_key = "sse-c-mismatch-object.txt";
let policy_key = "01234567890123456789012345678901";
let request_key = "abcdefghijklmnopqrstuvwxyzABCDEF";
let policy_key_b64 = base64::engine::general_purpose::STANDARD.encode(policy_key);
let request_key_b64 = base64::engine::general_purpose::STANDARD.encode(request_key);
let policy_key_b64 = base64_simd::STANDARD.encode_to_string(policy_key);
let request_key_b64 = base64_simd::STANDARD.encode_to_string(request_key);
let admin_client = env.create_s3_client();
admin_client.create_bucket().bucket(bucket).send().await?;
@@ -3526,7 +3525,7 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
.await?;
@@ -3799,7 +3798,7 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
.await?;
@@ -3925,7 +3924,7 @@ async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box<dyn
let extracted_key = "nested/file.txt";
let expected_body = b"extract-sse-c-body".to_vec();
let customer_key = "01234567890123456789012345678901";
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
let client = env.create_s3_client();
+1 -2
View File
@@ -35,7 +35,6 @@ use crate::common::local_http_client;
use crate::common::rustfs_binary_path_with_features;
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
use anyhow::Result;
use base64::Engine;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::Client;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -64,7 +63,7 @@ fn basic_auth_header() -> String {
fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
let credentials = format!("{}:{}", access_key, secret_key);
let encoded = base64::engine::general_purpose::STANDARD.encode(credentials);
let encoded = base64_simd::STANDARD.encode_to_string(credentials);
format!("Basic {}", encoded)
}
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
VersioningConfiguration,
};
use aws_sdk_s3::{Client, Config};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use flate2::read::GzDecoder;
use futures::{Stream, StreamExt};
@@ -1244,7 +1244,7 @@ async fn wait_for_source_replication_pending_or_failed(
}
async fn wait_for_source_replication_status(client: &Client, bucket: &str, key: &str, expected: &str, ssec: bool) -> TestResult {
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let wait = async {
loop {
@@ -1339,7 +1339,7 @@ async fn assert_failed_replication_stays_absent_for(
ssec: bool,
duration: Duration,
) -> TestResult {
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let wait = async {
let deadline = tokio::time::Instant::now() + duration;
@@ -4332,7 +4332,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
let target_client = target_env.create_s3_client();
let key = "ssec-contract.txt";
let body = b"repl-17 SSE-C payload";
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
source_client
@@ -4387,7 +4387,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
);
// A wrong customer key must fail too.
let wrong_key = BASE64_STANDARD.encode("99999999999999999999999999999999");
let wrong_key = BASE64_STANDARD.encode_to_string("99999999999999999999999999999999");
let wrong_key_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
let wrong_read = target_client
.get_object()
@@ -4423,7 +4423,7 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
let source_client = source_env.create_s3_client();
let target_client = target_env.create_s3_client();
let key = "ssec-mp-contract.bin";
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let created = source_client
@@ -4568,7 +4568,7 @@ async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_header
.await?;
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let put_ssec = |key: &'static str| {
source_client
@@ -4741,7 +4741,7 @@ async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult
let source_client = source_env.create_s3_client();
let key = "ssec-heal-contract.txt";
let body = b"repl-22 ssec heal payload".to_vec();
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
// Target outage: the SSE-C write cannot replicate.
@@ -4856,7 +4856,7 @@ async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult {
// The SSE-C object exists before any replication wiring.
let key = "ssec-existing-contract.txt";
let body = b"repl-22 ssec existing-object payload".to_vec();
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
source_client
.put_object()
@@ -9152,7 +9152,7 @@ async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() ->
// the real SSE-C decryption; the plaintext fake simply ignores them).
target.take_requests();
let ssec_key = "01234567890123456789012345678901";
let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key);
let ssec_key_b64 = BASE64_STANDARD.encode_to_string(ssec_key);
let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key);
let _ = source_client
.get_object()
+2 -3
View File
@@ -21,7 +21,6 @@ use aws_sdk_s3::error::BoxError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -107,8 +106,8 @@ fn customer_key(byte: u8) -> CustomerKey {
hasher.update(raw);
CustomerKey {
raw: String::from_utf8_lossy(&raw).into_owned(),
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
encoded: base64_simd::STANDARD.encode_to_string(raw),
md5: base64_simd::STANDARD.encode_to_string(hasher.finalize()),
}
}