mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 08:27:06 +00:00
chore(deps): migrate direct encoding deps to simd (#6690)
This commit is contained in:
@@ -121,10 +121,10 @@ tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64 = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
prost.workspace = true
|
||||
|
||||
@@ -24,7 +24,6 @@ mod tests {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use sha2::Sha256;
|
||||
@@ -74,12 +73,12 @@ mod tests {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(body);
|
||||
let digest = hasher.finalize();
|
||||
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
|
||||
base64_simd::STANDARD.encode_to_string(digest.as_slice())
|
||||
}
|
||||
|
||||
fn checksum_sha256_base64(body: &[u8]) -> String {
|
||||
let digest = Sha256::digest(body);
|
||||
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
|
||||
base64_simd::STANDARD.encode_to_string(digest.as_slice())
|
||||
}
|
||||
|
||||
fn checksum_crc64nvme_base64(body: &[u8]) -> String {
|
||||
|
||||
@@ -652,11 +652,10 @@ const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
|
||||
async fn start_rustfs_with_compression_and_sse(
|
||||
env: &mut RustFSTestEnvironment,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use base64::Engine;
|
||||
env.cleanup_existing_processes().await?;
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
let master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
// Server output goes to a file inside the per-test temp dir so a failing
|
||||
// run can be diagnosed from the child's logs.
|
||||
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
|
||||
|
||||
@@ -27,8 +27,7 @@ mod tests {
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::info;
|
||||
@@ -465,7 +464,7 @@ mod tests {
|
||||
create_versioned_bucket(&client, dst_bucket).await;
|
||||
|
||||
let content = b"deterministic synthetic payload for copy-object checksum #4996";
|
||||
let expected_sha256 = BASE64.encode(Sha256::digest(content));
|
||||
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
|
||||
|
||||
client
|
||||
.put_object()
|
||||
@@ -534,7 +533,7 @@ mod tests {
|
||||
create_versioned_bucket(&client, dst_bucket).await;
|
||||
|
||||
let content = b"another deterministic payload whose source checksum must survive the copy";
|
||||
let expected_sha256 = BASE64.encode(Sha256::digest(content));
|
||||
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
|
||||
|
||||
// Store the source WITH a SHA-256 checksum so it has one to preserve.
|
||||
let put_src = client
|
||||
@@ -614,7 +613,7 @@ mod tests {
|
||||
create_versioned_bucket(&client, dst_bucket).await;
|
||||
|
||||
let content = b"payload whose copy must be re-checksummed with a different algorithm";
|
||||
let expected_sha256 = BASE64.encode(Sha256::digest(content));
|
||||
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
|
||||
|
||||
// Source is stored WITH a SHA-256 checksum.
|
||||
client
|
||||
|
||||
@@ -1113,7 +1113,7 @@ fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
|
||||
@@ -1375,7 +1375,7 @@ impl S3 for FakeBackend {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
hex::encode(digest)
|
||||
hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
};
|
||||
let version = ObjectVersion {
|
||||
@@ -1660,7 +1660,7 @@ impl S3 for FakeBackend {
|
||||
}
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
|
||||
let e_tag = hex::encode(digest);
|
||||
let e_tag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
let mut state = lock(&self.store);
|
||||
let existing_bytes = state
|
||||
.uploads
|
||||
|
||||
@@ -28,7 +28,6 @@ use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus,
|
||||
LifecycleRule, LifecycleRuleFilter, ServerSideEncryption, Transition, TransitionStorageClass, VersioningConfiguration,
|
||||
};
|
||||
use base64::Engine;
|
||||
use bytes::Bytes;
|
||||
use flate2::read::GzDecoder;
|
||||
use http::header::{CONTENT_ENCODING, HOST};
|
||||
@@ -1808,7 +1807,7 @@ async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key);
|
||||
cluster.start().await?;
|
||||
|
||||
@@ -2017,7 +2016,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
@@ -2489,7 +2488,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
||||
hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
|
||||
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
|
||||
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
hot.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
||||
hot.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
hot.start().await?;
|
||||
|
||||
@@ -27,7 +27,7 @@ use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -64,7 +64,7 @@ pub fn init_logging() {
|
||||
pub fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
BASE64.encode(hasher.finalize())
|
||||
BASE64.encode_to_string(hasher.finalize())
|
||||
}
|
||||
|
||||
pub fn assert_s3_error<T, E>(result: Result<T, SdkError<E>>, status: u16, code: &str, message: &str, context: &str)
|
||||
@@ -365,7 +365,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
|
||||
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "e2e-test",
|
||||
"encrypted_key_material": BASE64.encode(key_data),
|
||||
"encrypted_key_material": BASE64.encode_to_string(key_data),
|
||||
"nonce": Vec::<u8>::new()
|
||||
});
|
||||
|
||||
@@ -383,7 +383,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
|
||||
info!("Testing SSE-C encryption");
|
||||
|
||||
let test_key = "01234567890123456789012345678901"; // 32-byte key
|
||||
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
|
||||
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
|
||||
let test_key_md5 = sse_customer_key_md5_base64(test_key);
|
||||
let test_data = b"Hello, KMS SSE-C World!";
|
||||
let object_key = "test-sse-c-object";
|
||||
@@ -551,8 +551,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
|
||||
// Test SSE-C with wrong key for download
|
||||
let test_key = "01234567890123456789012345678901";
|
||||
let wrong_key = "98765432109876543210987654321098";
|
||||
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
|
||||
let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
|
||||
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
|
||||
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
|
||||
let test_key_md5 = sse_customer_key_md5_base64(test_key);
|
||||
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
|
||||
let test_data = b"Test data for error scenarios";
|
||||
@@ -807,7 +807,7 @@ pub async fn test_multipart_upload_with_config(
|
||||
// Prepare encryption parameters
|
||||
let (sse_c_key_b64, sse_c_key_md5) = match &config.encryption_type {
|
||||
EncryptionType::SSEC { key, key_md5 } => {
|
||||
let key_b64 = base64::engine::general_purpose::STANDARD.encode(key);
|
||||
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
|
||||
(Some(key_b64), Some(key_md5.clone()))
|
||||
}
|
||||
_ => (None, None),
|
||||
|
||||
@@ -177,7 +177,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
|
||||
// Verify that files cannot be read with wrong keys
|
||||
info!("🔒 Verify key isolation");
|
||||
let wrong_key = "11111111111111111111111111111111";
|
||||
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
|
||||
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
|
||||
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
|
||||
|
||||
// Try to read file encrypted with key1 using wrong key
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
use super::common::{LocalKMSTestEnvironment, SSE_C_KEY_MISMATCH_MESSAGE, assert_s3_error, sse_customer_key_md5_base64};
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64::Engine;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{info, warn};
|
||||
@@ -68,7 +67,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
|
||||
// Test SSE-C with zero-byte file
|
||||
info!("📤 Testing SSE-C with zero-byte file");
|
||||
let test_key = "01234567890123456789012345678901";
|
||||
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
|
||||
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
|
||||
let test_key_md5 = sse_customer_key_md5_base64(test_key);
|
||||
let object_key_c = "zero-byte-sse-c";
|
||||
|
||||
@@ -161,7 +160,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
|
||||
// Test SSE-C with single byte
|
||||
info!("📤 Testing SSE-C with single-byte file");
|
||||
let test_key = "01234567890123456789012345678901";
|
||||
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
|
||||
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
|
||||
let test_key_md5 = sse_customer_key_md5_base64(test_key);
|
||||
let object_key_c = "single-byte-sse-c";
|
||||
|
||||
@@ -287,7 +286,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
// Test 1: Invalid key length for SSE-C
|
||||
info!("🔍 Testing invalid SSE-C key length");
|
||||
let invalid_short_key = "short"; // Too short
|
||||
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
|
||||
let invalid_key_b64 = base64_simd::STANDARD.encode_to_string(invalid_short_key);
|
||||
let invalid_key_md5 = sse_customer_key_md5_base64(invalid_short_key);
|
||||
|
||||
let invalid_key_result = s3_client
|
||||
@@ -325,7 +324,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
// Test 2: Mismatched MD5 for SSE-C
|
||||
info!("🔍 Testing mismatched MD5 for SSE-C key");
|
||||
let valid_key = "01234567890123456789012345678901";
|
||||
let valid_key_b64 = base64::engine::general_purpose::STANDARD.encode(valid_key);
|
||||
let valid_key_b64 = base64_simd::STANDARD.encode_to_string(valid_key);
|
||||
let wrong_md5 = sse_customer_key_md5_base64("98765432109876543210987654321098");
|
||||
|
||||
let wrong_md5_result = s3_client
|
||||
@@ -465,7 +464,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
|
||||
2 => {
|
||||
// SSE-C
|
||||
let key = format!("testkey{i:026}"); // 32-byte key
|
||||
let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
|
||||
let key_b64 = base64_simd::STANDARD.encode_to_string(&key);
|
||||
let key_md5 = sse_customer_key_md5_base64(&key);
|
||||
|
||||
client
|
||||
@@ -535,8 +534,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
|
||||
let key1 = "key1key1key1key1key1key1key1key1"; // 32 bytes
|
||||
let key2 = "key2key2key2key2key2key2key2key2"; // 32 bytes
|
||||
|
||||
let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
|
||||
let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
|
||||
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
|
||||
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
|
||||
let key1_md5 = sse_customer_key_md5_base64(key1);
|
||||
let key2_md5 = sse_customer_key_md5_base64(key2);
|
||||
|
||||
|
||||
@@ -138,8 +138,8 @@ async fn test_local_kms_key_isolation() {
|
||||
// Test that different SSE-C keys create isolated encrypted objects
|
||||
let key1 = "01234567890123456789012345678901";
|
||||
let key2 = "98765432109876543210987654321098";
|
||||
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
|
||||
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
|
||||
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
|
||||
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
|
||||
let key1_md5 = sse_customer_key_md5_base64(key1);
|
||||
let key2_md5 = sse_customer_key_md5_base64(key2);
|
||||
|
||||
@@ -565,7 +565,7 @@ async fn test_multipart_upload_with_sse_c(
|
||||
|
||||
// SSE-C encryption key
|
||||
let encryption_key = "01234567890123456789012345678901";
|
||||
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
|
||||
let key_b64 = base64_simd::STANDARD.encode_to_string(encryption_key);
|
||||
let key_md5 = sse_customer_key_md5_base64(encryption_key);
|
||||
|
||||
// Generate test data
|
||||
|
||||
@@ -127,8 +127,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
|
||||
|
||||
let key1 = "01234567890123456789012345678901";
|
||||
let key2 = "98765432109876543210987654321098";
|
||||
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
|
||||
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
|
||||
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
|
||||
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
|
||||
let key1_md5 = sse_customer_key_md5_base64(key1);
|
||||
let key2_md5 = sse_customer_key_md5_base64(key2);
|
||||
|
||||
|
||||
@@ -497,7 +497,7 @@ async fn test_multipart_encryption_type(
|
||||
// Prepare SSE-C keys when required
|
||||
let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
|
||||
let key = "01234567890123456789012345678901";
|
||||
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
|
||||
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
|
||||
let key_md5 = sse_customer_key_md5_base64(key);
|
||||
(Some(key_b64), Some(key_md5))
|
||||
} else {
|
||||
|
||||
@@ -22,7 +22,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use base64::Engine;
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use http::HeaderValue;
|
||||
@@ -47,19 +46,19 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
"conditions": conditions,
|
||||
});
|
||||
|
||||
base64::engine::general_purpose::STANDARD.encode(policy.to_string())
|
||||
base64_simd::STANDARD.encode_to_string(policy.to_string())
|
||||
}
|
||||
|
||||
fn sse_customer_key_md5_base64(key: &str) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(key.as_bytes());
|
||||
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
|
||||
base64_simd::STANDARD.encode_to_string(hasher.finalize())
|
||||
}
|
||||
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
async fn create_restricted_user(
|
||||
@@ -97,7 +96,7 @@ fn restricted_user_client(env: &RustFSTestEnvironment, username: &str, secret_ke
|
||||
const LOCAL_SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
|
||||
fn local_sse_master_key_value() -> String {
|
||||
base64::engine::general_purpose::STANDARD.encode([0x42u8; 32])
|
||||
base64_simd::STANDARD.encode_to_string([0x42u8; 32])
|
||||
}
|
||||
|
||||
async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec<u8> {
|
||||
@@ -1887,7 +1886,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
|
||||
let object_key = "sse-c-object.txt";
|
||||
let expected_body = b"anonymous-post-sse-c".to_vec();
|
||||
let customer_key = "01234567890123456789012345678901";
|
||||
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
|
||||
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
@@ -1941,7 +1940,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(base64::engine::general_purpose::STANDARD.encode(customer_key))
|
||||
.sse_customer_key(base64_simd::STANDARD.encode_to_string(customer_key))
|
||||
.sse_customer_key_md5(customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
@@ -1963,8 +1962,8 @@ async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Res
|
||||
let object_key = "sse-c-mismatch-object.txt";
|
||||
let policy_key = "01234567890123456789012345678901";
|
||||
let request_key = "abcdefghijklmnopqrstuvwxyzABCDEF";
|
||||
let policy_key_b64 = base64::engine::general_purpose::STANDARD.encode(policy_key);
|
||||
let request_key_b64 = base64::engine::general_purpose::STANDARD.encode(request_key);
|
||||
let policy_key_b64 = base64_simd::STANDARD.encode_to_string(policy_key);
|
||||
let request_key_b64 = base64_simd::STANDARD.encode_to_string(request_key);
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
@@ -3526,7 +3525,7 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
|
||||
.await?;
|
||||
|
||||
@@ -3799,7 +3798,7 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
|
||||
.await?;
|
||||
|
||||
@@ -3925,7 +3924,7 @@ async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box<dyn
|
||||
let extracted_key = "nested/file.txt";
|
||||
let expected_body = b"extract-sse-c-body".to_vec();
|
||||
let customer_key = "01234567890123456789012345678901";
|
||||
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
|
||||
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
|
||||
|
||||
let client = env.create_s3_client();
|
||||
|
||||
@@ -35,7 +35,6 @@ use crate::common::local_http_client;
|
||||
use crate::common::rustfs_binary_path_with_features;
|
||||
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
|
||||
use anyhow::Result;
|
||||
use base64::Engine;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::Client;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -64,7 +63,7 @@ fn basic_auth_header() -> String {
|
||||
|
||||
fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
|
||||
let credentials = format!("{}:{}", access_key, secret_key);
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(credentials);
|
||||
let encoded = base64_simd::STANDARD.encode_to_string(credentials);
|
||||
format!("Basic {}", encoded)
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
use flate2::read::GzDecoder;
|
||||
use futures::{Stream, StreamExt};
|
||||
@@ -1244,7 +1244,7 @@ async fn wait_for_source_replication_pending_or_failed(
|
||||
}
|
||||
|
||||
async fn wait_for_source_replication_status(client: &Client, bucket: &str, key: &str, expected: &str, ssec: bool) -> TestResult {
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let wait = async {
|
||||
loop {
|
||||
@@ -1339,7 +1339,7 @@ async fn assert_failed_replication_stays_absent_for(
|
||||
ssec: bool,
|
||||
duration: Duration,
|
||||
) -> TestResult {
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let wait = async {
|
||||
let deadline = tokio::time::Instant::now() + duration;
|
||||
@@ -4332,7 +4332,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
|
||||
let target_client = target_env.create_s3_client();
|
||||
let key = "ssec-contract.txt";
|
||||
let body = b"repl-17 SSE-C payload";
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
|
||||
source_client
|
||||
@@ -4387,7 +4387,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
|
||||
);
|
||||
|
||||
// A wrong customer key must fail too.
|
||||
let wrong_key = BASE64_STANDARD.encode("99999999999999999999999999999999");
|
||||
let wrong_key = BASE64_STANDARD.encode_to_string("99999999999999999999999999999999");
|
||||
let wrong_key_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
|
||||
let wrong_read = target_client
|
||||
.get_object()
|
||||
@@ -4423,7 +4423,7 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
let key = "ssec-mp-contract.bin";
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
|
||||
let created = source_client
|
||||
@@ -4568,7 +4568,7 @@ async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_header
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let put_ssec = |key: &'static str| {
|
||||
source_client
|
||||
@@ -4741,7 +4741,7 @@ async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult
|
||||
let source_client = source_env.create_s3_client();
|
||||
let key = "ssec-heal-contract.txt";
|
||||
let body = b"repl-22 ssec heal payload".to_vec();
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
|
||||
// Target outage: the SSE-C write cannot replicate.
|
||||
@@ -4856,7 +4856,7 @@ async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult {
|
||||
// The SSE-C object exists before any replication wiring.
|
||||
let key = "ssec-existing-contract.txt";
|
||||
let body = b"repl-22 ssec existing-object payload".to_vec();
|
||||
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
source_client
|
||||
.put_object()
|
||||
@@ -9152,7 +9152,7 @@ async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() ->
|
||||
// the real SSE-C decryption; the plaintext fake simply ignores them).
|
||||
target.take_requests();
|
||||
let ssec_key = "01234567890123456789012345678901";
|
||||
let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key);
|
||||
let ssec_key_b64 = BASE64_STANDARD.encode_to_string(ssec_key);
|
||||
let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key);
|
||||
let _ = source_client
|
||||
.get_object()
|
||||
|
||||
@@ -21,7 +21,6 @@ use aws_sdk_s3::error::BoxError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -107,8 +106,8 @@ fn customer_key(byte: u8) -> CustomerKey {
|
||||
hasher.update(raw);
|
||||
CustomerKey {
|
||||
raw: String::from_utf8_lossy(&raw).into_owned(),
|
||||
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
|
||||
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
|
||||
encoded: base64_simd::STANDARD.encode_to_string(raw),
|
||||
md5: base64_simd::STANDARD.encode_to_string(hasher.finalize()),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -176,7 +176,6 @@ rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tokio-stream = { workspace = true, features = ["sync"] }
|
||||
base64 = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
|
||||
@@ -200,7 +200,7 @@ impl ReplicationTargetStore {
|
||||
}
|
||||
|
||||
pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) -> Result<(PutObjectOptions, bool)> {
|
||||
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT};
|
||||
|
||||
let mut meta = HashMap::new();
|
||||
@@ -252,7 +252,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
&& !checksum_data.is_empty()
|
||||
{
|
||||
if is_ssec {
|
||||
let encoded = BASE64_STANDARD.encode(checksum_data);
|
||||
let encoded = BASE64_STANDARD.encode_to_string(checksum_data);
|
||||
insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded);
|
||||
} else if object_info.is_encrypted() {
|
||||
// Encrypted checksums cannot be exposed as plaintext headers, and
|
||||
|
||||
@@ -31,8 +31,7 @@ use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_PROTOCOL_VERSION, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose;
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
@@ -523,11 +522,11 @@ fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64)
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
|
||||
mac.update(data.as_bytes());
|
||||
let result = mac.finalize();
|
||||
general_purpose::STANDARD.encode(result.into_bytes())
|
||||
base64_simd::STANDARD.encode_to_string(result.into_bytes())
|
||||
}
|
||||
|
||||
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -745,11 +744,11 @@ fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io::
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_signature_v2(&mut mac, scope);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||
@@ -792,11 +791,11 @@ fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std:
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_replay_scope(&mut mac, scope);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||
@@ -821,15 +820,15 @@ fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
|
||||
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||
}
|
||||
let proof = general_purpose::STANDARD
|
||||
.decode(proof)
|
||||
let proof = base64_simd::STANDARD
|
||||
.decode_to_vec(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
@@ -862,7 +861,7 @@ fn generate_replay_cache_capability_proof(
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_replay_cache_capability_proof(
|
||||
@@ -872,8 +871,8 @@ fn verify_replay_cache_capability_proof(
|
||||
boot_epoch: Uuid,
|
||||
proof: &str,
|
||||
) -> std::io::Result<()> {
|
||||
let proof = general_purpose::STANDARD
|
||||
.decode(proof)
|
||||
let proof = base64_simd::STANDARD
|
||||
.decode_to_vec(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC replay cache capability proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
@@ -1988,9 +1987,9 @@ mod tests {
|
||||
let method = Method::GET;
|
||||
let timestamp = 1640995200;
|
||||
let signature = generate_signature(secret, url, &method, timestamp);
|
||||
let mut tampered = general_purpose::STANDARD.decode(&signature).unwrap();
|
||||
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
|
||||
tampered[0] ^= 1;
|
||||
let tampered_signature = general_purpose::STANDARD.encode(tampered);
|
||||
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
|
||||
|
||||
assert!(verify_signature(secret, url, &method, timestamp, &signature));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
|
||||
|
||||
@@ -1407,8 +1407,7 @@ fn multipart_part_numbers(parts: &[ObjectPartInfo]) -> Vec<usize> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use md5::{Digest, Md5};
|
||||
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER};
|
||||
use std::collections::HashMap;
|
||||
@@ -1468,7 +1467,7 @@ mod tests {
|
||||
request: ReadEncryptionRequest<'_>,
|
||||
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
|
||||
if let Some(encoded) = request.metadata.get(TEST_OBJECT_KEY_HEADER) {
|
||||
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
|
||||
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
|
||||
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidMetadata, "invalid test object key")
|
||||
})?;
|
||||
let key_bytes = decoded.try_into().map_err(|_| {
|
||||
@@ -1493,7 +1492,7 @@ mod tests {
|
||||
.map_err(|_| {
|
||||
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
|
||||
})?;
|
||||
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
|
||||
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
|
||||
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
|
||||
})?;
|
||||
let key_bytes = decoded.try_into().map_err(|_| {
|
||||
@@ -1505,7 +1504,7 @@ mod tests {
|
||||
let base_nonce = request
|
||||
.metadata
|
||||
.get(TEST_NONCE_HEADER)
|
||||
.and_then(|encoded| BASE64_STANDARD.decode(encoded).ok())
|
||||
.and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok())
|
||||
.and_then(|bytes| bytes.try_into().ok())
|
||||
.unwrap_or_else(|| fixture_nonce(request.bucket, request.object));
|
||||
Ok(Some(ReadEncryptionMaterial {
|
||||
@@ -1533,7 +1532,7 @@ mod tests {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
TEST_DIRECT_KEY_HEADER,
|
||||
HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("test key header is valid"),
|
||||
HeaderValue::from_str(&BASE64_STANDARD.encode_to_string(key_bytes)).expect("test key header is valid"),
|
||||
);
|
||||
headers
|
||||
}
|
||||
@@ -2439,7 +2438,10 @@ mod tests {
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()),
|
||||
("X-Amz-Server-Side-Encryption-Iv".to_string(), "AAAAAAAAAAAAAAAA".to_string()),
|
||||
("X-Amz-Server-Side-Encryption-Key".to_string(), BASE64_STANDARD.encode([7_u8; 32])),
|
||||
(
|
||||
"X-Amz-Server-Side-Encryption-Key".to_string(),
|
||||
BASE64_STANDARD.encode_to_string([7_u8; 32]),
|
||||
),
|
||||
("x-rustfs-encryption-original-size".to_string(), "64".to_string()),
|
||||
])),
|
||||
..Default::default()
|
||||
@@ -2748,7 +2750,7 @@ mod tests {
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-key-md5".to_string(),
|
||||
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
|
||||
),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-original-size".to_string(),
|
||||
@@ -2796,11 +2798,11 @@ mod tests {
|
||||
name: object.to_string(),
|
||||
size: encrypted.len() as i64,
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
|
||||
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-key-md5".to_string(),
|
||||
BASE64_STANDARD.encode(md5_bytes(customer_key)),
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
|
||||
),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-original-size".to_string(),
|
||||
@@ -2856,7 +2858,7 @@ mod tests {
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-key-md5".to_string(),
|
||||
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
|
||||
),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-original-size".to_string(),
|
||||
@@ -3008,13 +3010,13 @@ mod tests {
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-key-md5".to_string(),
|
||||
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
|
||||
),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-original-size".to_string(),
|
||||
total_plaintext.to_string(),
|
||||
),
|
||||
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE)),
|
||||
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode_to_string(LEGACY_FIXTURE_BASE_NONCE)),
|
||||
])
|
||||
}
|
||||
|
||||
@@ -3751,11 +3753,11 @@ mod tests {
|
||||
name: object.to_string(),
|
||||
size: encrypted.len() as i64,
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
|
||||
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-key-md5".to_string(),
|
||||
BASE64_STANDARD.encode(md5_bytes(customer_key)),
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
|
||||
),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-original-size".to_string(),
|
||||
@@ -3821,7 +3823,7 @@ mod tests {
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-key-md5".to_string(),
|
||||
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
|
||||
),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-original-size".to_string(),
|
||||
@@ -3964,11 +3966,11 @@ mod tests {
|
||||
..Default::default()
|
||||
}]),
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
|
||||
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-key-md5".to_string(),
|
||||
BASE64_STANDARD.encode(md5_bytes(customer_key)),
|
||||
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
|
||||
),
|
||||
(
|
||||
"x-amz-server-side-encryption-customer-original-size".to_string(),
|
||||
|
||||
@@ -42,7 +42,7 @@ use crate::storage_api_contracts::{
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use rand::seq::SliceRandom;
|
||||
@@ -1398,11 +1398,11 @@ async fn persist_observed_list_objects_mutation(store: Option<&ECStore>, bucket:
|
||||
}
|
||||
|
||||
fn encode_persistent_list_metadata_string(value: &str) -> String {
|
||||
BASE64_STANDARD.encode(value.as_bytes())
|
||||
BASE64_STANDARD.encode_to_string(value.as_bytes())
|
||||
}
|
||||
|
||||
fn decode_persistent_list_metadata_string(value: &str) -> Option<String> {
|
||||
let bytes = BASE64_STANDARD.decode(value).ok()?;
|
||||
let bytes = BASE64_STANDARD.decode_to_vec(value).ok()?;
|
||||
String::from_utf8(bytes).ok()
|
||||
}
|
||||
|
||||
|
||||
@@ -90,7 +90,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos
|
||||
async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
crc-fast = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use base64::Engine as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
@@ -827,7 +826,7 @@ impl CheckpointManager {
|
||||
}
|
||||
|
||||
fn checkpoint_digest(checkpoint_data: &[u8]) -> String {
|
||||
base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data))
|
||||
base64_simd::STANDARD.encode_to_string(Sha256::digest(checkpoint_data))
|
||||
}
|
||||
|
||||
fn digest_path(task_id: &str) -> std::path::PathBuf {
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
use crate::{Error, Result};
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use base64_simd::URL_SAFE_NO_PAD;
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -147,7 +146,7 @@ pub(crate) fn encode_heal_token(marker: Option<&str>, version_marker: Option<&st
|
||||
// serde_json of a simple two-Option struct cannot fail; fall back to an
|
||||
// empty object rather than panicking if it somehow does.
|
||||
let json = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{}".to_vec());
|
||||
format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(json))
|
||||
format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(json))
|
||||
}
|
||||
|
||||
/// Decode an opaque heal continuation token back into `(marker, version_marker)`.
|
||||
@@ -174,7 +173,7 @@ pub(crate) fn decode_heal_token(token: &str) -> (Option<String>, Option<String>)
|
||||
return (None, None);
|
||||
};
|
||||
|
||||
let bytes = match URL_SAFE_NO_PAD.decode(encoded) {
|
||||
let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
@@ -234,7 +233,7 @@ const DISK_WALK_TOKEN_PREFIX: &str = "dw1:";
|
||||
/// enumerators can never misread each other's cursor: a `dw1:` token decodes to
|
||||
/// `(None, None)` under the B5 decoder, and a `v1:` token decodes to `None` here.
|
||||
pub(crate) fn encode_disk_walk_token(next_forward: &str) -> String {
|
||||
format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(next_forward.as_bytes()))
|
||||
format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(next_forward.as_bytes()))
|
||||
}
|
||||
|
||||
/// Decode a disk-walk continuation token back into the `next_forward` object key.
|
||||
@@ -261,7 +260,7 @@ pub(crate) fn decode_disk_walk_token(token: &str) -> Option<String> {
|
||||
return None;
|
||||
};
|
||||
|
||||
let bytes = match URL_SAFE_NO_PAD.decode(encoded) {
|
||||
let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
@@ -1513,7 +1512,6 @@ mod tests {
|
||||
decode_disk_walk_token, decode_heal_token, encode_disk_walk_token, encode_heal_token, is_transient_object_exists_error,
|
||||
is_transient_object_exists_message, next_heal_listing_token,
|
||||
};
|
||||
use base64::Engine as _;
|
||||
|
||||
#[test]
|
||||
fn next_heal_listing_token_returns_none_for_complete_page() {
|
||||
@@ -1564,7 +1562,7 @@ mod tests {
|
||||
assert_eq!(decode_heal_token("no-prefix-here"), (None, None));
|
||||
assert_eq!(decode_heal_token("v1:!!!not-base64!!!"), (None, None));
|
||||
// valid base64 of non-JSON bytes.
|
||||
let bad_json = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json"));
|
||||
let bad_json = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(b"not json"));
|
||||
assert_eq!(decode_heal_token(&bad_json), (None, None));
|
||||
// a raw v2-style list_objects_v2 token (no "v1:" prefix) resets cleanly.
|
||||
assert_eq!(decode_heal_token("some-opaque-legacy-token"), (None, None));
|
||||
@@ -1576,7 +1574,7 @@ mod tests {
|
||||
// list_object_versions returns NotImplemented for that pairing.
|
||||
// Craft a token whose JSON encodes (None, Some) directly and confirm coercion.
|
||||
let json = br#"{"m":null,"v":"orphan-version"}"#;
|
||||
let token = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json));
|
||||
let token = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(json));
|
||||
assert_eq!(decode_heal_token(&token), (None, None), "version-only marker must coerce to (None, None)");
|
||||
}
|
||||
|
||||
|
||||
@@ -50,8 +50,8 @@ aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
argon2 = { workspace = true }
|
||||
chacha20poly1305 = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
base64 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
base64-simd = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
subtle = { workspace = true }
|
||||
zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
//! Exit status is the verdict: 0 when every check held, 1 otherwise, so a
|
||||
//! scheduled drill fails its job instead of quietly filing a bad report.
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use rustfs_kms::backup::{BackupKek, DrillDataset, DrillDisaster, DrillRequest, DrillVerdict, run_local_drill};
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
@@ -100,7 +100,11 @@ fn disaster_from_env() -> Result<DrillDisaster, String> {
|
||||
|
||||
fn kek_from_env() -> Result<BackupKek, String> {
|
||||
let raw = Zeroizing::new(required(ENV_KEK)?);
|
||||
let decoded = Zeroizing::new(BASE64.decode(raw.trim()).map_err(|_| format!("{ENV_KEK} must be base64"))?);
|
||||
let decoded = Zeroizing::new(
|
||||
BASE64
|
||||
.decode_to_vec(raw.trim())
|
||||
.map_err(|_| format!("{ENV_KEK} must be base64"))?,
|
||||
);
|
||||
if decoded.len() != 32 {
|
||||
return Err(format!("{ENV_KEK} must decode to exactly 32 bytes"));
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient};
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -69,7 +69,7 @@ async fn run() -> Result<(), String> {
|
||||
.decrypt_key_material_for_export(&key_id)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref()));
|
||||
let encoded = Zeroizing::new(BASE64_STANDARD.encode_to_string(key_material.as_ref()));
|
||||
|
||||
let mut stdout = io::stdout().lock();
|
||||
writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}"))
|
||||
|
||||
@@ -305,7 +305,7 @@ pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -
|
||||
}
|
||||
|
||||
fn digest_value(value: &str) -> String {
|
||||
let digest = hex::encode(Sha256::digest(value.as_bytes()));
|
||||
let digest = hex_simd::encode_to_string(Sha256::digest(value.as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
|
||||
}
|
||||
|
||||
|
||||
@@ -848,8 +848,7 @@ mod tests {
|
||||
use aws_sdk_kms::config::{BehaviorVersion, Credentials, Region};
|
||||
use aws_smithy_http_client::test_util::{NeverClient, ReplayEvent, StaticReplayClient};
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// AWS KMS speaks awsJson1_1; every request goes to `/` on the regional
|
||||
@@ -977,8 +976,8 @@ mod tests {
|
||||
let ciphertext = b"encrypted-data-key".to_vec();
|
||||
let (http_client, backend) = scripted_backend(vec![ok_event(serde_json::json!({
|
||||
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
|
||||
"Plaintext": BASE64.encode(&plaintext),
|
||||
"CiphertextBlob": BASE64.encode(&ciphertext),
|
||||
"Plaintext": BASE64.encode_to_string(&plaintext),
|
||||
"CiphertextBlob": BASE64.encode_to_string(&ciphertext),
|
||||
}))]);
|
||||
|
||||
let response = backend
|
||||
@@ -997,7 +996,7 @@ mod tests {
|
||||
let plaintext = b"recovered-data-key".to_vec();
|
||||
let (_http, backend) = scripted_backend(vec![ok_event(serde_json::json!({
|
||||
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
|
||||
"Plaintext": BASE64.encode(&plaintext),
|
||||
"Plaintext": BASE64.encode_to_string(&plaintext),
|
||||
"EncryptionAlgorithm": "SYMMETRIC_DEFAULT",
|
||||
}))]);
|
||||
|
||||
@@ -1061,8 +1060,8 @@ mod tests {
|
||||
error_event(400, "ThrottlingException", "rate exceeded"),
|
||||
ok_event(serde_json::json!({
|
||||
"KeyId": "test-key",
|
||||
"Plaintext": BASE64.encode([1u8; 32]),
|
||||
"CiphertextBlob": BASE64.encode(b"blob"),
|
||||
"Plaintext": BASE64.encode_to_string([1u8; 32]),
|
||||
"CiphertextBlob": BASE64.encode_to_string(b"blob"),
|
||||
})),
|
||||
]);
|
||||
|
||||
|
||||
@@ -41,8 +41,7 @@ use crate::types::{
|
||||
DescribeKeyRequest, EncryptRequest, GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext,
|
||||
RewrapDataKeyRequest,
|
||||
};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use rand::RngExt as _;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
@@ -285,7 +284,7 @@ async fn static_backend_stateless_contract() {
|
||||
let key_id = "static-contract-key";
|
||||
let mut raw_key = [0u8; 32];
|
||||
rand::rng().fill(&mut raw_key[..]);
|
||||
let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key));
|
||||
let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode_to_string(raw_key));
|
||||
let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build");
|
||||
let backend: &dyn KmsBackend = &static_backend;
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ use aes_gcm::{
|
||||
};
|
||||
use argon2::{Algorithm, Argon2, Params, Version};
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
@@ -1268,7 +1268,7 @@ impl LocalKmsClient {
|
||||
}
|
||||
|
||||
let encrypted_bytes = BASE64
|
||||
.decode(&stored_key.encrypted_key_material)
|
||||
.decode_to_vec(&stored_key.encrypted_key_material)
|
||||
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?;
|
||||
|
||||
let effective_protection = if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified {
|
||||
@@ -1406,13 +1406,17 @@ impl LocalKmsClient {
|
||||
.encrypt(&nonce, key_material)
|
||||
.map_err(|e| KmsError::cryptographic_error("encrypt", e.to_string()))?;
|
||||
// Encode encrypted bytes to base64 string
|
||||
(BASE64.encode(&encrypted), nonce.to_vec(), StoredKeyProtection::EncryptedMasterKey)
|
||||
(
|
||||
BASE64.encode_to_string(&encrypted),
|
||||
nonce.to_vec(),
|
||||
StoredKeyProtection::EncryptedMasterKey,
|
||||
)
|
||||
} else {
|
||||
warn!(
|
||||
key_id = %master_key.key_id,
|
||||
"Local KMS is storing key material as plaintext-dev-only because no master key is configured"
|
||||
);
|
||||
(BASE64.encode(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly)
|
||||
(BASE64.encode_to_string(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly)
|
||||
};
|
||||
|
||||
let stored_key = StoredMasterKey {
|
||||
@@ -2648,10 +2652,10 @@ mod tests {
|
||||
|
||||
let tampered_material = {
|
||||
let mut material = BASE64
|
||||
.decode(pristine["encrypted_key_material"].as_str().expect("material is a string"))
|
||||
.decode_to_vec(pristine["encrypted_key_material"].as_str().expect("material is a string"))
|
||||
.expect("decode pristine material");
|
||||
*material.last_mut().expect("material is not empty") ^= 0x01;
|
||||
BASE64.encode(&material)
|
||||
BASE64.encode_to_string(&material)
|
||||
};
|
||||
|
||||
type PoisonCase = (&'static str, Vec<u8>, fn(&KmsError) -> bool);
|
||||
@@ -2976,7 +2980,7 @@ mod tests {
|
||||
"created_at": "2024-01-01T00:00:00+00:00",
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "legacy-test",
|
||||
"encrypted_key_material": BASE64.encode([7u8; 32]),
|
||||
"encrypted_key_material": BASE64.encode_to_string([7u8; 32]),
|
||||
"nonce": Vec::<u8>::new()
|
||||
});
|
||||
|
||||
|
||||
@@ -722,8 +722,7 @@ impl Default for BackendCapabilities {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::KmsConfig;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
|
||||
/// Backend that implements only the trait-mandated operations and relies
|
||||
/// on the default `capabilities` implementation.
|
||||
@@ -958,7 +957,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn static_backend_capabilities_golden() {
|
||||
let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode([0u8; 32]));
|
||||
let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode_to_string([0u8; 32]));
|
||||
let backend = static_kms::StaticKmsBackend::new(config)
|
||||
.await
|
||||
.expect("static backend should build");
|
||||
|
||||
@@ -434,8 +434,7 @@ mod tests {
|
||||
use crate::backends::KmsBackend as KmsBackendTrait;
|
||||
use crate::config::{BackendConfig, KmsBackend, StaticConfig};
|
||||
use crate::encryption::is_data_key_envelope;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
|
||||
/// Generate a random 32-byte key and return (key_id, raw_key).
|
||||
fn random_static_key(key_id: &str) -> (String, [u8; 32]) {
|
||||
@@ -447,7 +446,7 @@ mod tests {
|
||||
fn static_config(key_id: &str, raw_key: &[u8; 32]) -> StaticConfig {
|
||||
StaticConfig {
|
||||
key_id: key_id.to_string(),
|
||||
secret_key: BASE64.encode(raw_key),
|
||||
secret_key: BASE64.encode_to_string(raw_key),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar
|
||||
use crate::policy::{self, AttemptError, OpClass, RetryPolicy};
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose};
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -527,8 +527,8 @@ fn decode_stored_key_material(key_id: &str, encrypted_material: &str) -> Result<
|
||||
|
||||
// Mirrors `decrypt_key_material`: stored material is currently base64 without an
|
||||
// additional encryption layer.
|
||||
let key_material = general_purpose::STANDARD
|
||||
.decode(encrypted_material)
|
||||
let key_material = BASE64
|
||||
.decode_to_vec(encrypted_material)
|
||||
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?;
|
||||
|
||||
// Key material must be exactly 32 bytes for AES-256.
|
||||
@@ -693,7 +693,7 @@ impl VaultKmsClient {
|
||||
/// confidentiality. Any identity with KV read access to the key path can recover the
|
||||
/// plaintext master key.
|
||||
async fn encrypt_key_material(&self, key_material: &[u8]) -> Result<String> {
|
||||
Ok(general_purpose::STANDARD.encode(key_material))
|
||||
Ok(base64_simd::STANDARD.encode_to_string(key_material))
|
||||
}
|
||||
|
||||
/// Read the immutable material record of one key version.
|
||||
@@ -2405,7 +2405,7 @@ mod tests {
|
||||
tags: HashMap::new(),
|
||||
deletion_date: None,
|
||||
rotated_at: None,
|
||||
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
|
||||
encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]),
|
||||
baseline_version: None,
|
||||
wrap_budget_reserved: 0,
|
||||
}
|
||||
@@ -2869,21 +2869,21 @@ mod tests {
|
||||
));
|
||||
|
||||
// Truncated material: valid base64 of fewer than 32 bytes.
|
||||
let truncated = general_purpose::STANDARD.encode([0x42u8; 16]);
|
||||
let truncated = base64_simd::STANDARD.encode_to_string([0x42u8; 16]);
|
||||
assert!(matches!(
|
||||
decode_stored_key_material("poisoned", &truncated),
|
||||
Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned"
|
||||
));
|
||||
|
||||
// Oversized material: valid base64 of more than 32 bytes.
|
||||
let oversized = general_purpose::STANDARD.encode([0x42u8; 33]);
|
||||
let oversized = base64_simd::STANDARD.encode_to_string([0x42u8; 33]);
|
||||
assert!(matches!(
|
||||
decode_stored_key_material("poisoned", &oversized),
|
||||
Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned"
|
||||
));
|
||||
|
||||
// Well-formed material still decodes.
|
||||
let valid = general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
let valid = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
|
||||
assert_eq!(
|
||||
decode_stored_key_material("healthy", &valid).expect("valid material must decode"),
|
||||
vec![0x42u8; 32]
|
||||
@@ -3046,7 +3046,7 @@ mod tests {
|
||||
description: None,
|
||||
metadata: HashMap::new(),
|
||||
tags: HashMap::new(),
|
||||
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
|
||||
encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]),
|
||||
baseline_version: Some(1),
|
||||
deletion_date: None,
|
||||
rotated_at: None,
|
||||
@@ -3760,7 +3760,7 @@ mod tests {
|
||||
/// Base64 material distinct from `healthy_key_data`'s, standing in for the
|
||||
/// material a concurrent rotation committed.
|
||||
fn rotated_material() -> String {
|
||||
general_purpose::STANDARD.encode([0x43u8; 32])
|
||||
base64_simd::STANDARD.encode_to_string([0x43u8; 32])
|
||||
}
|
||||
|
||||
/// The issue's lost-update scenario: node A disables a key while node B's
|
||||
@@ -4358,7 +4358,7 @@ mod tests {
|
||||
let material_v2 = [0x43u8; 32];
|
||||
let record_v2 = VaultKeyVersionRecord {
|
||||
version: 2,
|
||||
encrypted_key_material: general_purpose::STANDARD.encode(material_v2),
|
||||
encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
// A well-formed envelope wrapped under version 2 — under a reverted
|
||||
@@ -4900,8 +4900,8 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn wired_decrypt_of_pre_versioning_envelope_adds_no_request() {
|
||||
let key_data = healthy_key_data();
|
||||
let key_material = general_purpose::STANDARD
|
||||
.decode(&key_data.encrypted_key_material)
|
||||
let key_material = BASE64
|
||||
.decode_to_vec(&key_data.encrypted_key_material)
|
||||
.expect("decode fixture material");
|
||||
let (encrypted_key, nonce) = AesDekCrypto::new()
|
||||
.encrypt(&key_material, b"dek-plaintext", &[])
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar
|
||||
use crate::policy::{self, AttemptError, OpClass, RetryPolicy};
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::Zoned;
|
||||
use moka::future::Cache;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -479,7 +479,7 @@ impl VaultTransitKmsClient {
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect();
|
||||
let serialized = serde_json::to_vec(&ordered)?;
|
||||
Ok(Some(BASE64.encode(serialized)))
|
||||
Ok(Some(BASE64.encode_to_string(serialized)))
|
||||
}
|
||||
|
||||
fn map_vault_error(key_id: &str, error: vaultrs::error::ClientError, operation: &str) -> KmsError {
|
||||
@@ -524,7 +524,7 @@ impl VaultTransitKmsClient {
|
||||
plaintext: &[u8],
|
||||
encryption_context: &HashMap<String, String>,
|
||||
) -> Result<String> {
|
||||
let plaintext_b64 = BASE64.encode(plaintext);
|
||||
let plaintext_b64 = BASE64.encode_to_string(plaintext);
|
||||
let plaintext_b64 = plaintext_b64.as_str();
|
||||
let aad = Self::canonicalize_context(encryption_context)?;
|
||||
let aad = aad.as_deref();
|
||||
@@ -568,7 +568,7 @@ impl VaultTransitKmsClient {
|
||||
.await?;
|
||||
|
||||
BASE64
|
||||
.decode(response.plaintext)
|
||||
.decode_to_vec(response.plaintext)
|
||||
.map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string()))
|
||||
}
|
||||
|
||||
@@ -3031,7 +3031,7 @@ mod tests {
|
||||
ScriptedResponse::ok(kv2_write_ack()),
|
||||
// decrypt of the pre-rotation envelope; Vault owns the transit
|
||||
// crypto, so the recovered material is the responder's to hand back.
|
||||
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
|
||||
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })),
|
||||
])
|
||||
.await;
|
||||
|
||||
@@ -3243,7 +3243,7 @@ mod tests {
|
||||
// rewrap, context-bound route: latest-version read, then decrypt,
|
||||
// then re-encrypt under the newest version.
|
||||
ScriptedResponse::ok(transit_key_read_data_up_to("wired-key", 2)),
|
||||
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
|
||||
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })),
|
||||
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v2:rewrapped" })),
|
||||
])
|
||||
.await;
|
||||
|
||||
@@ -945,7 +945,7 @@ async fn tree_digest(root: &Path) -> Result<ContentDigest> {
|
||||
lines.push(format!(
|
||||
"{relative}\u{1f}{}\u{1f}{modified}\u{1f}{}",
|
||||
metadata.len(),
|
||||
hex::encode(Sha256::digest(&content))
|
||||
hex_simd::encode_to_string(Sha256::digest(&content), hex_simd::AsciiCase::Lower)
|
||||
));
|
||||
}
|
||||
lines.sort();
|
||||
@@ -1199,7 +1199,7 @@ mod tests {
|
||||
let text = String::from_utf8(encoded.clone()).expect("evidence is utf-8");
|
||||
assert!(!text.contains(DRILL_MASTER_KEY), "the evidence must not carry the master key");
|
||||
assert!(
|
||||
!text.contains(&hex::encode([0x37u8; 32])),
|
||||
!text.contains(&hex_simd::encode_to_string([0x37u8; 32], hex_simd::AsciiCase::Lower)),
|
||||
"the evidence must not carry backup KEK material"
|
||||
);
|
||||
|
||||
|
||||
@@ -568,7 +568,10 @@ pub(crate) fn compute_master_key_verifier(master_key: &str, salt: Option<&[u8]>,
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&framing);
|
||||
hasher.update(derived.as_slice());
|
||||
Ok(format!("{prefix}{}", hex::encode(hasher.finalize())))
|
||||
Ok(format!(
|
||||
"{prefix}{}",
|
||||
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
|
||||
))
|
||||
}
|
||||
|
||||
/// The bundle-level protection label is the weakest state observed across
|
||||
|
||||
@@ -72,7 +72,7 @@ use aes_gcm::{
|
||||
Aes256Gcm, Nonce,
|
||||
aead::{Aead, KeyInit},
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::fs;
|
||||
@@ -640,7 +640,7 @@ fn decode_key_record(
|
||||
return Err(BackupError::corrupted(format!("bundled key record '{stem}' carries no key material")).into());
|
||||
}
|
||||
let material =
|
||||
Zeroizing::new(BASE64.decode(&probe.encrypted_key_material).map_err(|error| {
|
||||
Zeroizing::new(BASE64.decode_to_vec(&probe.encrypted_key_material).map_err(|error| {
|
||||
BackupError::corrupted(format!("bundled key record '{stem}' material is not valid base64: {error}"))
|
||||
})?);
|
||||
if !allowed_modes.contains(&protection_mode(probe.at_rest_protection)) {
|
||||
|
||||
@@ -69,7 +69,7 @@ impl ContentDigest {
|
||||
pub fn sha256_of(bytes: &[u8]) -> Self {
|
||||
Self {
|
||||
algorithm: DigestAlgorithm::Sha256,
|
||||
hex: hex::encode(Sha256::digest(bytes)),
|
||||
hex: hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -367,9 +367,8 @@ impl StaticConfig {
|
||||
/// Decode the base64-encoded secret key into raw bytes.
|
||||
/// Returns an error if the key is not valid base64 or is not exactly 32 bytes.
|
||||
pub fn decode_key(&self) -> Result<[u8; 32]> {
|
||||
use base64::Engine as _;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(&self.secret_key)
|
||||
let bytes = base64_simd::STANDARD
|
||||
.decode_to_vec(&self.secret_key)
|
||||
.map_err(|e| KmsError::configuration_error(format!("Static KMS secret key is not valid base64: {e}")))?;
|
||||
if bytes.len() != 32 {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
@@ -1963,9 +1962,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn static_kms_config_serialization_does_not_expose_key_material() {
|
||||
use base64::Engine as _;
|
||||
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
|
||||
let config = KmsConfig::static_kms("static-key".to_string(), encoded_key.clone());
|
||||
|
||||
let serialized = serde_json::to_string(&config).expect("static KMS config should serialize");
|
||||
@@ -2569,14 +2566,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_from_env_reads_static_secret_file_and_sets_default_key() {
|
||||
use base64::Engine as _;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for static KMS secret");
|
||||
let secret_path = temp_dir.path().join("static-kms-secret");
|
||||
// Named `*_key_b64` (not `*_secret`) so the logging-guardrails check does not
|
||||
// flag these fixture interpolations as secrets leaking into log strings.
|
||||
let file_key_b64 = base64::engine::general_purpose::STANDARD.encode([7u8; 32]);
|
||||
let env_key_b64 = base64::engine::general_purpose::STANDARD.encode([9u8; 32]);
|
||||
let file_key_b64 = base64_simd::STANDARD.encode_to_string([7u8; 32]);
|
||||
let env_key_b64 = base64_simd::STANDARD.encode_to_string([9u8; 32]);
|
||||
std::fs::write(&secret_path, format!("file-key:{file_key_b64}\n")).expect("write static KMS secret file");
|
||||
|
||||
with_vars(
|
||||
|
||||
@@ -46,8 +46,7 @@ use crate::error::{KmsError, Result};
|
||||
use aes_gcm::aead::{Aead, Payload};
|
||||
use aes_gcm::{Aes256Gcm, Key, KeyInit, Nonce};
|
||||
use argon2::{Algorithm, Argon2, Params, Version};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use rand::RngExt;
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -284,14 +283,16 @@ fn seal_value(label: &str, plaintext: &str, secret: &str) -> Result<String> {
|
||||
payload.extend_from_slice(&salt);
|
||||
payload.extend_from_slice(&nonce);
|
||||
payload.extend_from_slice(&ciphertext);
|
||||
Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode(payload)))
|
||||
Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode_to_string(payload)))
|
||||
}
|
||||
|
||||
fn open_value(label: &str, sealed: &str, secret: &str) -> Result<String> {
|
||||
let encoded = sealed
|
||||
.strip_prefix(SEALED_VALUE_PREFIX)
|
||||
.expect("caller checks the sealed prefix");
|
||||
let payload = BASE64_STANDARD.decode(encoded).map_err(|_| sealed_value_unreadable(label))?;
|
||||
let payload = BASE64_STANDARD
|
||||
.decode_to_vec(encoded)
|
||||
.map_err(|_| sealed_value_unreadable(label))?;
|
||||
if payload.len() <= LOCAL_KMS_MASTER_KEY_SALT_LEN + NONCE_LEN {
|
||||
return Err(sealed_value_unreadable(label));
|
||||
}
|
||||
|
||||
@@ -675,7 +675,6 @@ mod tests {
|
||||
use crate::error::KmsError;
|
||||
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage};
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use jiff::Zoned;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
@@ -1110,8 +1109,13 @@ mod tests {
|
||||
.await
|
||||
.expect("enable should succeed");
|
||||
|
||||
let base64 = base64::engine::general_purpose::STANDARD;
|
||||
let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)];
|
||||
let base64 = base64_simd::STANDARD;
|
||||
let encodings = |bytes: &[u8]| {
|
||||
vec![
|
||||
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower),
|
||||
base64.encode_to_string(bytes),
|
||||
]
|
||||
};
|
||||
let mut forbidden = vec![grant_token.to_string()];
|
||||
forbidden.extend(encodings(&data_key.plaintext_key));
|
||||
forbidden.extend(encodings(&decrypted.plaintext));
|
||||
|
||||
@@ -551,7 +551,6 @@ mod tests {
|
||||
ListKeysRequest, ListKeysResponse,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use std::future::Future;
|
||||
@@ -562,8 +561,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn static_backend() -> Arc<dyn KmsBackend> {
|
||||
let config =
|
||||
KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
|
||||
let config = KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x42u8; 32]));
|
||||
Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
|
||||
}
|
||||
|
||||
|
||||
+10
-14
@@ -23,7 +23,6 @@ use crate::encryption::context_aad;
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use base64::Engine;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
@@ -40,7 +39,7 @@ use zeroize::Zeroize;
|
||||
fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(input.as_ref());
|
||||
hex::encode(hasher.finalize())
|
||||
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
/// Data key for object encryption
|
||||
@@ -836,19 +835,16 @@ impl ObjectEncryptionService {
|
||||
// Internal headers for decryption
|
||||
headers.insert(
|
||||
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(&metadata.iv),
|
||||
base64_simd::STANDARD.encode_to_string(&metadata.iv),
|
||||
);
|
||||
|
||||
if let Some(ref tag) = metadata.tag {
|
||||
headers.insert(
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER.to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(tag),
|
||||
);
|
||||
headers.insert(INTERNAL_ENCRYPTION_TAG_HEADER.to_string(), base64_simd::STANDARD.encode_to_string(tag));
|
||||
}
|
||||
|
||||
headers.insert(
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key),
|
||||
base64_simd::STANDARD.encode_to_string(&metadata.encrypted_data_key),
|
||||
);
|
||||
|
||||
// Whatever the object was sealed under is what gets stored: for a
|
||||
@@ -906,14 +902,14 @@ impl ObjectEncryptionService {
|
||||
let iv = headers
|
||||
.get(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
.ok_or_else(|| KmsError::validation_error("Missing IV header"))?;
|
||||
let iv = base64::engine::general_purpose::STANDARD
|
||||
.decode(iv)
|
||||
let iv = base64_simd::STANDARD
|
||||
.decode_to_vec(iv)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?;
|
||||
|
||||
let tag = if let Some(tag_str) = headers.get(INTERNAL_ENCRYPTION_TAG_HEADER) {
|
||||
Some(
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(tag_str)
|
||||
base64_simd::STANDARD
|
||||
.decode_to_vec(tag_str)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid tag: {e}")))?,
|
||||
)
|
||||
} else {
|
||||
@@ -921,8 +917,8 @@ impl ObjectEncryptionService {
|
||||
};
|
||||
|
||||
let encrypted_data_key = if let Some(key_str) = headers.get(INTERNAL_ENCRYPTION_KEY_HEADER) {
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(key_str)
|
||||
base64_simd::STANDARD
|
||||
.decode_to_vec(key_str)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))?
|
||||
} else {
|
||||
Vec::new() // Empty for SSE-C
|
||||
|
||||
@@ -763,10 +763,10 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
|
||||
fn static_config(key_id: &str, fill: u8) -> KmsConfig {
|
||||
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
|
||||
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode_to_string([fill; 32]))
|
||||
}
|
||||
|
||||
/// End-to-end wiring check for the AWS backend: an admin configure request
|
||||
@@ -822,7 +822,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn redacted_config_omits_static_key_material() {
|
||||
let manager = KmsServiceManager::new();
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
|
||||
manager
|
||||
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
|
||||
.await
|
||||
@@ -1020,7 +1020,6 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_cannot_replace_existing_local_backend() {
|
||||
use base64::Engine as _;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
@@ -1029,7 +1028,7 @@ mod tests {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(local.clone()).await.expect("configure local KMS");
|
||||
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
|
||||
let error = manager
|
||||
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
|
||||
.await
|
||||
|
||||
@@ -451,6 +451,5 @@ async fn harness_restart_brings_the_service_back_over_the_same_state() {
|
||||
}
|
||||
|
||||
fn base64_of(bytes: &[u8]) -> String {
|
||||
use base64::Engine as _;
|
||||
base64::engine::general_purpose::STANDARD.encode(bytes)
|
||||
base64_simd::STANDARD.encode_to_string(bytes)
|
||||
}
|
||||
|
||||
@@ -667,7 +667,7 @@ async fn sse_c_round_trips_and_rejects_the_wrong_key() {
|
||||
async fn sse_c_validates_the_supplied_key_md5() {
|
||||
let (_kms, service) = service_with_key("sse-c-md5-unused").await;
|
||||
let customer_key = [0x33u8; 32];
|
||||
let correct_md5 = hex::encode(md5_of(&customer_key));
|
||||
let correct_md5 = hex_simd::encode_to_string(md5_of(&customer_key), hex_simd::AsciiCase::Lower);
|
||||
|
||||
service
|
||||
.encrypt_object_with_customer_key(BUCKET, "md5-ok.bin", payload(64).as_slice(), &customer_key, Some(&correct_md5))
|
||||
|
||||
@@ -34,8 +34,7 @@ use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use rustfs_kms::backends::BackendCapabilities;
|
||||
use rustfs_kms::{
|
||||
CreateKeyRequest, DeleteKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus,
|
||||
@@ -51,7 +50,7 @@ pub const STATIC_KEY_ID: &str = "behavior-static-key";
|
||||
/// Fixed rather than random so a failure is reproducible; it is test-only
|
||||
/// material and never leaves this crate's test binaries.
|
||||
pub fn static_secret_key() -> String {
|
||||
BASE64.encode([0x5au8; 32])
|
||||
BASE64.encode_to_string([0x5au8; 32])
|
||||
}
|
||||
|
||||
/// Which backend a harness instance is running.
|
||||
|
||||
@@ -99,14 +99,14 @@ swift = [
|
||||
"dep:md-5",
|
||||
"dep:hmac",
|
||||
"dep:sha1",
|
||||
"dep:hex",
|
||||
"dep:hex-simd",
|
||||
"dep:ipnetwork",
|
||||
"dep:rustfs-trusted-proxies",
|
||||
"dep:astral-tokio-tar",
|
||||
"dep:base64",
|
||||
"dep:base64-simd",
|
||||
"dep:async-compression",
|
||||
]
|
||||
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
|
||||
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64-simd", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
|
||||
sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"]
|
||||
|
||||
[dependencies]
|
||||
@@ -163,11 +163,11 @@ urlencoding = { workspace = true, optional = true }
|
||||
md-5 = { workspace = true, optional = true }
|
||||
hmac = { workspace = true, optional = true }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
hex = { workspace = true, optional = true }
|
||||
hex-simd = { workspace = true, optional = true }
|
||||
ipnetwork = { workspace = true, optional = true }
|
||||
rustfs-trusted-proxies = { workspace = true, optional = true }
|
||||
astral-tokio-tar = { workspace = true, optional = true }
|
||||
base64 = { workspace = true, optional = true }
|
||||
base64-simd = { workspace = true, optional = true }
|
||||
async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] }
|
||||
|
||||
# WebDAV specific dependencies (optional)
|
||||
|
||||
@@ -97,7 +97,7 @@ fn get_account_metadata_bucket_name(account: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(account.as_bytes());
|
||||
let hash_bytes = hasher.finalize();
|
||||
let hash = hex::encode(hash_bytes);
|
||||
let hash = hex_simd::encode_to_string(hash_bytes, hex_simd::AsciiCase::Lower);
|
||||
format!("swift-account-{}", &hash[0..16])
|
||||
}
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ pub fn generate_signature(
|
||||
mac.update(message.as_bytes());
|
||||
|
||||
let result = mac.finalize();
|
||||
let signature = hex::encode(result.into_bytes());
|
||||
let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower);
|
||||
|
||||
Ok(signature)
|
||||
}
|
||||
@@ -213,10 +213,10 @@ pub fn validate_formpost(path: &str, request: &FormPostRequest, key: &str) -> Sw
|
||||
// the sibling TempURL/SFTP checks. Decode the hex first so the comparison runs
|
||||
// over the raw HMAC bytes and does not leak via string length; a non-hex
|
||||
// provided signature can never match and is rejected the same way.
|
||||
let expected_bytes =
|
||||
hex::decode(&expected_sig).map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?;
|
||||
let expected_bytes = hex_simd::decode_to_vec(&expected_sig)
|
||||
.map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?;
|
||||
|
||||
let signatures_match = match hex::decode(request.signature.trim()) {
|
||||
let signatures_match = match hex_simd::decode_to_vec(request.signature.trim()) {
|
||||
Ok(provided_bytes) => super::tempurl::constant_time_compare(&provided_bytes, &expected_bytes),
|
||||
Err(_) => false,
|
||||
};
|
||||
|
||||
@@ -84,7 +84,11 @@ impl SLOManifest {
|
||||
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(etag_concat.as_bytes());
|
||||
format!("\"{}-{}\"", hex::encode(hasher.finalize()), self.segments.len())
|
||||
format!(
|
||||
"\"{}-{}\"",
|
||||
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower),
|
||||
self.segments.len()
|
||||
)
|
||||
}
|
||||
|
||||
/// Validate manifest against actual segments
|
||||
|
||||
@@ -300,7 +300,7 @@ pub fn generate_sync_signature(path: &str, key: &str) -> SwiftResult<String> {
|
||||
mac.update(path.as_bytes());
|
||||
|
||||
let result = mac.finalize();
|
||||
Ok(hex::encode(result.into_bytes()))
|
||||
Ok(hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower))
|
||||
}
|
||||
|
||||
/// Verify sync signature
|
||||
|
||||
@@ -125,7 +125,7 @@ impl TempURL {
|
||||
|
||||
// Hex-encode result
|
||||
let result = mac.finalize();
|
||||
let signature = hex::encode(result.into_bytes());
|
||||
let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower);
|
||||
|
||||
Ok(signature)
|
||||
}
|
||||
|
||||
@@ -675,8 +675,7 @@ fn fixed_body(message: impl Into<Bytes>) -> WebDavBody {
|
||||
|
||||
/// Decode base64 string
|
||||
fn base64_decode(encoded: &str) -> Result<Vec<u8>, ()> {
|
||||
use base64::Engine;
|
||||
base64::engine::general_purpose::STANDARD.decode(encoded).map_err(|_| ())
|
||||
base64_simd::STANDARD.decode_to_vec(encoded).map_err(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -56,7 +56,7 @@ fn keystone_credentials(project_id: &str) -> Credentials {
|
||||
fn account_metadata_bucket_name(account: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(account.as_bytes());
|
||||
let hash = hex::encode(hasher.finalize());
|
||||
let hash = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
|
||||
format!("swift-account-{}", &hash[0..16])
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ hotpath.workspace = true
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chacha20poly1305.workspace = true
|
||||
hex.workspace = true
|
||||
hex-simd.workspace = true
|
||||
hmac.workspace = true
|
||||
minlz.workspace = true
|
||||
pin-project-lite.workspace = true
|
||||
|
||||
@@ -731,12 +731,15 @@ fn random_stream_nonce() -> [u8; 12] {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use hex::encode as hex_encode;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE;
|
||||
|
||||
fn hex_encode(data: impl AsRef<[u8]>) -> String {
|
||||
hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decrypt_reader_can_start_from_non_zero_sequence_number() {
|
||||
let plaintext = vec![0xAB; DARE_PAYLOAD_SIZE * 2 + 19];
|
||||
|
||||
@@ -81,7 +81,7 @@ serde_json = { workspace = true, features = ["raw_value"] }
|
||||
md-5 = { workspace = true }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
base64.workspace = true
|
||||
base64-simd.workspace = true
|
||||
sha1.workspace = true
|
||||
sha2.workspace = true
|
||||
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
|
||||
|
||||
+14
-11
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::errors::ChecksumMismatch;
|
||||
use base64::{Engine as _, engine::general_purpose};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
use http::HeaderMap;
|
||||
use sha1::Sha1;
|
||||
@@ -330,7 +330,7 @@ impl Checksum {
|
||||
let mut hasher = checksum_type.hasher()?;
|
||||
hasher.write_all(data).ok()?;
|
||||
let raw = hasher.finalize();
|
||||
let encoded = general_purpose::STANDARD.encode(&raw);
|
||||
let encoded = BASE64_STANDARD.encode_to_string(&raw);
|
||||
|
||||
let checksum = Checksum {
|
||||
checksum_type,
|
||||
@@ -369,7 +369,7 @@ impl Checksum {
|
||||
value_string = value.to_string();
|
||||
}
|
||||
// let raw = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(&value_string).ok()?;
|
||||
let raw = general_purpose::STANDARD.decode(&value_string).ok()?;
|
||||
let raw = BASE64_STANDARD.decode_to_vec(&value_string).ok()?;
|
||||
|
||||
let checksum = Checksum {
|
||||
checksum_type,
|
||||
@@ -413,14 +413,14 @@ impl Checksum {
|
||||
if self.want_parts > 0 && self.want_parts != parts {
|
||||
return Err(ChecksumMismatch {
|
||||
want: format!("{}-{}", self.encoded, self.want_parts),
|
||||
got: format!("{}-{}", general_purpose::STANDARD.encode(&sum), parts),
|
||||
got: format!("{}-{}", base64_simd::STANDARD.encode_to_string(&sum), parts),
|
||||
});
|
||||
}
|
||||
|
||||
if sum != self.raw {
|
||||
return Err(ChecksumMismatch {
|
||||
want: self.encoded.clone(),
|
||||
got: general_purpose::STANDARD.encode(&sum),
|
||||
got: base64_simd::STANDARD.encode_to_string(&sum),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -569,7 +569,7 @@ impl Checksum {
|
||||
}
|
||||
}
|
||||
|
||||
self.encoded = general_purpose::STANDARD.encode(&self.raw);
|
||||
self.encoded = base64_simd::STANDARD.encode_to_string(&self.raw);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1160,7 +1160,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
|
||||
|
||||
let checksum_bytes = &buf[..length];
|
||||
buf = &buf[length..];
|
||||
let mut checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
|
||||
let mut checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
|
||||
|
||||
if checksum_type.is(ChecksumType::MULTIPART) {
|
||||
is_multipart = true;
|
||||
@@ -1190,7 +1190,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
|
||||
if part > 0 && (part as u64) <= parts_count {
|
||||
let offset = ((part - 1) as usize) * length;
|
||||
let part_checksum = &buf[offset..offset + length];
|
||||
checksum_str = general_purpose::STANDARD.encode(part_checksum);
|
||||
checksum_str = base64_simd::STANDARD.encode_to_string(part_checksum);
|
||||
}
|
||||
buf = &buf[want_len..];
|
||||
}
|
||||
@@ -1249,7 +1249,7 @@ pub fn read_part_checksums(mut buf: &[u8]) -> Vec<HashMap<String, String>> {
|
||||
|
||||
let checksum_bytes = &buf[..length];
|
||||
buf = &buf[length..];
|
||||
let checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
|
||||
let checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
|
||||
|
||||
part_checksum.insert(checksum_type.to_string(), checksum_str);
|
||||
}
|
||||
@@ -1553,7 +1553,6 @@ mod tests {
|
||||
// asserted alongside the raw hex so both the digest and its encoding are pinned.
|
||||
#[test]
|
||||
fn xxhash_sha512_regression_lock_non_empty() {
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
let data = b"The quick brown fox jumps over the lazy dog";
|
||||
|
||||
// XXH3-64(fox) = 0xce7d19a5418fb365 is the official upstream vector.
|
||||
@@ -1565,7 +1564,11 @@ mod tests {
|
||||
let got_hex: String = c.raw.iter().map(|b| format!("{b:02x}")).collect();
|
||||
assert_eq!(got_hex, want_hex, "{t:?} raw hex drifted");
|
||||
// encoded field must be the standard-base64 of raw (S3 wire form)
|
||||
assert_eq!(c.encoded, STANDARD.encode(&c.raw), "{t:?} encoded field != base64(raw)");
|
||||
assert_eq!(
|
||||
c.encoded,
|
||||
base64_simd::STANDARD.encode_to_string(&c.raw),
|
||||
"{t:?} encoded field != base64(raw)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -91,8 +91,7 @@ use crate::Sha256Hasher;
|
||||
use crate::compress_index::{Index, TryGetIndex};
|
||||
use crate::get_content_checksum;
|
||||
use crate::{DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, boxed_reader, wrap_reader};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose;
|
||||
|
||||
use http::HeaderMap;
|
||||
use pin_project_lite::pin_project;
|
||||
use s3s::TrailingHeaders;
|
||||
@@ -582,8 +581,8 @@ impl AsyncRead for HashReader {
|
||||
})
|
||||
{
|
||||
expected_content_hash.encoded = checksum_str;
|
||||
expected_content_hash.raw = general_purpose::STANDARD
|
||||
.decode(&expected_content_hash.encoded)
|
||||
expected_content_hash.raw = base64_simd::STANDARD
|
||||
.decode_to_vec(&expected_content_hash.encoded)
|
||||
.map_err(|_| std::io::Error::other("Invalid base64 checksum"))?;
|
||||
|
||||
if expected_content_hash.raw.is_empty() {
|
||||
@@ -598,7 +597,7 @@ impl AsyncRead for HashReader {
|
||||
&& !expected_content_hash.checksum_type.trailing()
|
||||
{
|
||||
expected_content_hash.raw = content_hash;
|
||||
expected_content_hash.encoded = general_purpose::STANDARD.encode(&expected_content_hash.raw);
|
||||
expected_content_hash.encoded = base64_simd::STANDARD.encode_to_string(&expected_content_hash.raw);
|
||||
} else if content_hash != expected_content_hash.raw {
|
||||
let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower);
|
||||
let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower);
|
||||
|
||||
Reference in New Issue
Block a user