chore(deps): migrate direct encoding deps to simd (#6690)

This commit is contained in:
houseme
2026-08-27 03:17:24 +08:00
committed by GitHub
parent 31031f2a46
commit ba7785d61d
81 changed files with 523 additions and 505 deletions
+2 -2
View File
@@ -121,10 +121,10 @@ tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
urlencoding.workspace = true
walkdir.workspace = true
base64 = { workspace = true }
base64-simd = { workspace = true }
rand = { workspace = true, features = ["serde"] }
chrono = { workspace = true, features = ["serde"] }
hex = { workspace = true }
hex-simd = { workspace = true }
md-5 = { workspace = true }
opentelemetry-proto = { workspace = true }
prost.workspace = true
+2 -3
View File
@@ -24,7 +24,6 @@ mod tests {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use sha2::Sha256;
@@ -74,12 +73,12 @@ mod tests {
let mut hasher = Md5::new();
hasher.update(body);
let digest = hasher.finalize();
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
base64_simd::STANDARD.encode_to_string(digest.as_slice())
}
fn checksum_sha256_base64(body: &[u8]) -> String {
let digest = Sha256::digest(body);
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
base64_simd::STANDARD.encode_to_string(digest.as_slice())
}
fn checksum_crc64nvme_base64(body: &[u8]) -> String {
+1 -2
View File
@@ -652,11 +652,10 @@ const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
async fn start_rustfs_with_compression_and_sse(
env: &mut RustFSTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use base64::Engine;
env.cleanup_existing_processes().await?;
let binary_path = rustfs_binary_path();
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
// Server output goes to a file inside the per-test temp dir so a failing
// run can be diagnosed from the child's logs.
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
@@ -27,8 +27,7 @@ mod tests {
VersioningConfiguration,
};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use sha2::{Digest, Sha256};
use tracing::info;
@@ -465,7 +464,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"deterministic synthetic payload for copy-object checksum #4996";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
client
.put_object()
@@ -534,7 +533,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"another deterministic payload whose source checksum must survive the copy";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
// Store the source WITH a SHA-256 checksum so it has one to preserve.
let put_src = client
@@ -614,7 +613,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"payload whose copy must be re-checksummed with a different algorithm";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
// Source is stored WITH a SHA-256 checksum.
client
+3 -3
View File
@@ -1113,7 +1113,7 @@ fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
@@ -1375,7 +1375,7 @@ impl S3 for FakeBackend {
Some(value) => value,
None => {
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
hex::encode(digest)
hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower)
}
};
let version = ObjectVersion {
@@ -1660,7 +1660,7 @@ impl S3 for FakeBackend {
}
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
let e_tag = hex::encode(digest);
let e_tag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
let mut state = lock(&self.store);
let existing_bytes = state
.uploads
@@ -28,7 +28,6 @@ use aws_sdk_s3::types::{
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus,
LifecycleRule, LifecycleRuleFilter, ServerSideEncryption, Transition, TransitionStorageClass, VersioningConfiguration,
};
use base64::Engine;
use bytes::Bytes;
use flate2::read::GzDecoder;
use http::header::{CONTENT_ENCODING, HOST};
@@ -1808,7 +1807,7 @@ async fn four_node_inline_fallback_controls() -> TestResult {
let collector = OtlpMetricCollector::start().await?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
configure_reader_metric_cluster(&mut cluster, &collector);
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key);
cluster.start().await?;
@@ -2017,7 +2016,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
let collector = OtlpMetricCollector::start().await?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
@@ -2489,7 +2488,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
hot.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
hot.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
hot.start().await?;
+7 -7
View File
@@ -27,7 +27,7 @@ use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use http::header::{CONTENT_TYPE, HOST};
use md5::{Digest as Md5Digest, Md5};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -64,7 +64,7 @@ pub fn init_logging() {
pub fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
BASE64.encode(hasher.finalize())
BASE64.encode_to_string(hasher.finalize())
}
pub fn assert_s3_error<T, E>(result: Result<T, SdkError<E>>, status: u16, code: &str, message: &str, context: &str)
@@ -365,7 +365,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
"rotated_at": serde_json::Value::Null,
"created_by": "e2e-test",
"encrypted_key_material": BASE64.encode(key_data),
"encrypted_key_material": BASE64.encode_to_string(key_data),
"nonce": Vec::<u8>::new()
});
@@ -383,7 +383,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
info!("Testing SSE-C encryption");
let test_key = "01234567890123456789012345678901"; // 32-byte key
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let test_data = b"Hello, KMS SSE-C World!";
let object_key = "test-sse-c-object";
@@ -551,8 +551,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
// Test SSE-C with wrong key for download
let test_key = "01234567890123456789012345678901";
let wrong_key = "98765432109876543210987654321098";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
let test_data = b"Test data for error scenarios";
@@ -807,7 +807,7 @@ pub async fn test_multipart_upload_with_config(
// Prepare encryption parameters
let (sse_c_key_b64, sse_c_key_md5) = match &config.encryption_type {
EncryptionType::SSEC { key, key_md5 } => {
let key_b64 = base64::engine::general_purpose::STANDARD.encode(key);
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
(Some(key_b64), Some(key_md5.clone()))
}
_ => (None, None),
@@ -177,7 +177,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
// Verify that files cannot be read with wrong keys
info!("🔒 Verify key isolation");
let wrong_key = "11111111111111111111111111111111";
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
// Try to read file encrypted with key1 using wrong key
@@ -24,7 +24,6 @@
use super::common::{LocalKMSTestEnvironment, SSE_C_KEY_MISMATCH_MESSAGE, assert_s3_error, sse_customer_key_md5_base64};
use crate::common::{TEST_BUCKET, init_logging};
use aws_sdk_s3::types::ServerSideEncryption;
use base64::Engine;
use std::sync::Arc;
use tokio::sync::Semaphore;
use tracing::{info, warn};
@@ -68,7 +67,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
// Test SSE-C with zero-byte file
info!("📤 Testing SSE-C with zero-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let object_key_c = "zero-byte-sse-c";
@@ -161,7 +160,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
// Test SSE-C with single byte
info!("📤 Testing SSE-C with single-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let object_key_c = "single-byte-sse-c";
@@ -287,7 +286,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
// Test 1: Invalid key length for SSE-C
info!("🔍 Testing invalid SSE-C key length");
let invalid_short_key = "short"; // Too short
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
let invalid_key_b64 = base64_simd::STANDARD.encode_to_string(invalid_short_key);
let invalid_key_md5 = sse_customer_key_md5_base64(invalid_short_key);
let invalid_key_result = s3_client
@@ -325,7 +324,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
// Test 2: Mismatched MD5 for SSE-C
info!("🔍 Testing mismatched MD5 for SSE-C key");
let valid_key = "01234567890123456789012345678901";
let valid_key_b64 = base64::engine::general_purpose::STANDARD.encode(valid_key);
let valid_key_b64 = base64_simd::STANDARD.encode_to_string(valid_key);
let wrong_md5 = sse_customer_key_md5_base64("98765432109876543210987654321098");
let wrong_md5_result = s3_client
@@ -465,7 +464,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
2 => {
// SSE-C
let key = format!("testkey{i:026}"); // 32-byte key
let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
let key_b64 = base64_simd::STANDARD.encode_to_string(&key);
let key_md5 = sse_customer_key_md5_base64(&key);
client
@@ -535,8 +534,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
let key1 = "key1key1key1key1key1key1key1key1"; // 32 bytes
let key2 = "key2key2key2key2key2key2key2key2"; // 32 bytes
let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
+3 -3
View File
@@ -138,8 +138,8 @@ async fn test_local_kms_key_isolation() {
// Test that different SSE-C keys create isolated encrypted objects
let key1 = "01234567890123456789012345678901";
let key2 = "98765432109876543210987654321098";
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
@@ -565,7 +565,7 @@ async fn test_multipart_upload_with_sse_c(
// SSE-C encryption key
let encryption_key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
let key_b64 = base64_simd::STANDARD.encode_to_string(encryption_key);
let key_md5 = sse_customer_key_md5_base64(encryption_key);
// Generate test data
+2 -2
View File
@@ -127,8 +127,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
let key1 = "01234567890123456789012345678901";
let key2 = "98765432109876543210987654321098";
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
@@ -497,7 +497,7 @@ async fn test_multipart_encryption_type(
// Prepare SSE-C keys when required
let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
let key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
let key_md5 = sse_customer_key_md5_base64(key);
(Some(key_b64), Some(key_md5))
} else {
+11 -12
View File
@@ -22,7 +22,6 @@ use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
};
use base64::Engine;
use chrono::{Duration as ChronoDuration, Utc};
use flate2::{Compression, write::GzEncoder};
use http::HeaderValue;
@@ -47,19 +46,19 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
"conditions": conditions,
});
base64::engine::general_purpose::STANDARD.encode(policy.to_string())
base64_simd::STANDARD.encode_to_string(policy.to_string())
}
fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
base64_simd::STANDARD.encode_to_string(hasher.finalize())
}
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
async fn create_restricted_user(
@@ -97,7 +96,7 @@ fn restricted_user_client(env: &RustFSTestEnvironment, username: &str, secret_ke
const LOCAL_SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
fn local_sse_master_key_value() -> String {
base64::engine::general_purpose::STANDARD.encode([0x42u8; 32])
base64_simd::STANDARD.encode_to_string([0x42u8; 32])
}
async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec<u8> {
@@ -1887,7 +1886,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
let object_key = "sse-c-object.txt";
let expected_body = b"anonymous-post-sse-c".to_vec();
let customer_key = "01234567890123456789012345678901";
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
let admin_client = env.create_s3_client();
@@ -1941,7 +1940,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
.bucket(bucket)
.key(object_key)
.sse_customer_algorithm("AES256")
.sse_customer_key(base64::engine::general_purpose::STANDARD.encode(customer_key))
.sse_customer_key(base64_simd::STANDARD.encode_to_string(customer_key))
.sse_customer_key_md5(customer_key_md5)
.send()
.await?;
@@ -1963,8 +1962,8 @@ async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Res
let object_key = "sse-c-mismatch-object.txt";
let policy_key = "01234567890123456789012345678901";
let request_key = "abcdefghijklmnopqrstuvwxyzABCDEF";
let policy_key_b64 = base64::engine::general_purpose::STANDARD.encode(policy_key);
let request_key_b64 = base64::engine::general_purpose::STANDARD.encode(request_key);
let policy_key_b64 = base64_simd::STANDARD.encode_to_string(policy_key);
let request_key_b64 = base64_simd::STANDARD.encode_to_string(request_key);
let admin_client = env.create_s3_client();
admin_client.create_bucket().bucket(bucket).send().await?;
@@ -3526,7 +3525,7 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
.await?;
@@ -3799,7 +3798,7 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
.await?;
@@ -3925,7 +3924,7 @@ async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box<dyn
let extracted_key = "nested/file.txt";
let expected_body = b"extract-sse-c-body".to_vec();
let customer_key = "01234567890123456789012345678901";
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
let client = env.create_s3_client();
+1 -2
View File
@@ -35,7 +35,6 @@ use crate::common::local_http_client;
use crate::common::rustfs_binary_path_with_features;
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
use anyhow::Result;
use base64::Engine;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::Client;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -64,7 +63,7 @@ fn basic_auth_header() -> String {
fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
let credentials = format!("{}:{}", access_key, secret_key);
let encoded = base64::engine::general_purpose::STANDARD.encode(credentials);
let encoded = base64_simd::STANDARD.encode_to_string(credentials);
format!("Basic {}", encoded)
}
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
VersioningConfiguration,
};
use aws_sdk_s3::{Client, Config};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use flate2::read::GzDecoder;
use futures::{Stream, StreamExt};
@@ -1244,7 +1244,7 @@ async fn wait_for_source_replication_pending_or_failed(
}
async fn wait_for_source_replication_status(client: &Client, bucket: &str, key: &str, expected: &str, ssec: bool) -> TestResult {
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let wait = async {
loop {
@@ -1339,7 +1339,7 @@ async fn assert_failed_replication_stays_absent_for(
ssec: bool,
duration: Duration,
) -> TestResult {
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let wait = async {
let deadline = tokio::time::Instant::now() + duration;
@@ -4332,7 +4332,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
let target_client = target_env.create_s3_client();
let key = "ssec-contract.txt";
let body = b"repl-17 SSE-C payload";
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
source_client
@@ -4387,7 +4387,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
);
// A wrong customer key must fail too.
let wrong_key = BASE64_STANDARD.encode("99999999999999999999999999999999");
let wrong_key = BASE64_STANDARD.encode_to_string("99999999999999999999999999999999");
let wrong_key_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
let wrong_read = target_client
.get_object()
@@ -4423,7 +4423,7 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
let source_client = source_env.create_s3_client();
let target_client = target_env.create_s3_client();
let key = "ssec-mp-contract.bin";
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let created = source_client
@@ -4568,7 +4568,7 @@ async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_header
.await?;
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let put_ssec = |key: &'static str| {
source_client
@@ -4741,7 +4741,7 @@ async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult
let source_client = source_env.create_s3_client();
let key = "ssec-heal-contract.txt";
let body = b"repl-22 ssec heal payload".to_vec();
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
// Target outage: the SSE-C write cannot replicate.
@@ -4856,7 +4856,7 @@ async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult {
// The SSE-C object exists before any replication wiring.
let key = "ssec-existing-contract.txt";
let body = b"repl-22 ssec existing-object payload".to_vec();
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
source_client
.put_object()
@@ -9152,7 +9152,7 @@ async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() ->
// the real SSE-C decryption; the plaintext fake simply ignores them).
target.take_requests();
let ssec_key = "01234567890123456789012345678901";
let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key);
let ssec_key_b64 = BASE64_STANDARD.encode_to_string(ssec_key);
let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key);
let _ = source_client
.get_object()
+2 -3
View File
@@ -21,7 +21,6 @@ use aws_sdk_s3::error::BoxError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -107,8 +106,8 @@ fn customer_key(byte: u8) -> CustomerKey {
hasher.update(raw);
CustomerKey {
raw: String::from_utf8_lossy(&raw).into_owned(),
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
encoded: base64_simd::STANDARD.encode_to_string(raw),
md5: base64_simd::STANDARD.encode_to_string(hasher.finalize()),
}
}
-1
View File
@@ -176,7 +176,6 @@ rmp.workspace = true
rmp-serde.workspace = true
tokio-util = { workspace = true, features = ["io", "compat"] }
tokio-stream = { workspace = true, features = ["sync"] }
base64 = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
hex-simd = { workspace = true }
@@ -200,7 +200,7 @@ impl ReplicationTargetStore {
}
pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) -> Result<(PutObjectOptions, bool)> {
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT};
let mut meta = HashMap::new();
@@ -252,7 +252,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
&& !checksum_data.is_empty()
{
if is_ssec {
let encoded = BASE64_STANDARD.encode(checksum_data);
let encoded = BASE64_STANDARD.encode_to_string(checksum_data);
insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded);
} else if object_info.is_encrypted() {
// Encrypted checksums cannot be exposed as plaintext headers, and
+15 -16
View File
@@ -31,8 +31,7 @@ use crate::storage_api_contracts::internode::{
NS_SCANNER_PROTOCOL_VERSION, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use base64::Engine as _;
use base64::engine::general_purpose;
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
@@ -523,11 +522,11 @@ fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64)
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
mac.update(data.as_bytes());
let result = mac.finalize();
general_purpose::STANDARD.encode(result.into_bytes())
base64_simd::STANDARD.encode_to_string(result.into_bytes())
}
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
@@ -745,11 +744,11 @@ fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io::
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_signature_v2(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
@@ -792,11 +791,11 @@ fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std:
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_scope(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
@@ -821,15 +820,15 @@ fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let proof = general_purpose::STANDARD
.decode(proof)
let proof = base64_simd::STANDARD
.decode_to_vec(proof)
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
@@ -862,7 +861,7 @@ fn generate_replay_cache_capability_proof(
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_replay_cache_capability_proof(
@@ -872,8 +871,8 @@ fn verify_replay_cache_capability_proof(
boot_epoch: Uuid,
proof: &str,
) -> std::io::Result<()> {
let proof = general_purpose::STANDARD
.decode(proof)
let proof = base64_simd::STANDARD
.decode_to_vec(proof)
.map_err(|_| std::io::Error::other("Invalid RPC replay cache capability proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
@@ -1988,9 +1987,9 @@ mod tests {
let method = Method::GET;
let timestamp = 1640995200;
let signature = generate_signature(secret, url, &method, timestamp);
let mut tampered = general_purpose::STANDARD.decode(&signature).unwrap();
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
tampered[0] ^= 1;
let tampered_signature = general_purpose::STANDARD.encode(tampered);
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
assert!(verify_signature(secret, url, &method, timestamp, &signature));
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
+20 -18
View File
@@ -1407,8 +1407,7 @@ fn multipart_part_numbers(parts: &[ObjectPartInfo]) -> Vec<usize> {
#[cfg(test)]
mod tests {
use super::*;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64_simd::STANDARD as BASE64_STANDARD;
use md5::{Digest, Md5};
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER};
use std::collections::HashMap;
@@ -1468,7 +1467,7 @@ mod tests {
request: ReadEncryptionRequest<'_>,
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
if let Some(encoded) = request.metadata.get(TEST_OBJECT_KEY_HEADER) {
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidMetadata, "invalid test object key")
})?;
let key_bytes = decoded.try_into().map_err(|_| {
@@ -1493,7 +1492,7 @@ mod tests {
.map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
})?;
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
})?;
let key_bytes = decoded.try_into().map_err(|_| {
@@ -1505,7 +1504,7 @@ mod tests {
let base_nonce = request
.metadata
.get(TEST_NONCE_HEADER)
.and_then(|encoded| BASE64_STANDARD.decode(encoded).ok())
.and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok())
.and_then(|bytes| bytes.try_into().ok())
.unwrap_or_else(|| fixture_nonce(request.bucket, request.object));
Ok(Some(ReadEncryptionMaterial {
@@ -1533,7 +1532,7 @@ mod tests {
let mut headers = HeaderMap::new();
headers.insert(
TEST_DIRECT_KEY_HEADER,
HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("test key header is valid"),
HeaderValue::from_str(&BASE64_STANDARD.encode_to_string(key_bytes)).expect("test key header is valid"),
);
headers
}
@@ -2439,7 +2438,10 @@ mod tests {
user_defined: Arc::new(HashMap::from([
("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()),
("X-Amz-Server-Side-Encryption-Iv".to_string(), "AAAAAAAAAAAAAAAA".to_string()),
("X-Amz-Server-Side-Encryption-Key".to_string(), BASE64_STANDARD.encode([7_u8; 32])),
(
"X-Amz-Server-Side-Encryption-Key".to_string(),
BASE64_STANDARD.encode_to_string([7_u8; 32]),
),
("x-rustfs-encryption-original-size".to_string(), "64".to_string()),
])),
..Default::default()
@@ -2748,7 +2750,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -2796,11 +2798,11 @@ mod tests {
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -2856,7 +2858,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3008,13 +3010,13 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
total_plaintext.to_string(),
),
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE)),
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode_to_string(LEGACY_FIXTURE_BASE_NONCE)),
])
}
@@ -3751,11 +3753,11 @@ mod tests {
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3821,7 +3823,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3964,11 +3966,11 @@ mod tests {
..Default::default()
}]),
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
+3 -3
View File
@@ -42,7 +42,7 @@ use crate::storage_api_contracts::{
};
use crate::store::ECStore;
use crate::store::utils::is_reserved_or_invalid_bucket;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use futures::future::join_all;
use rand::seq::SliceRandom;
@@ -1398,11 +1398,11 @@ async fn persist_observed_list_objects_mutation(store: Option<&ECStore>, bucket:
}
fn encode_persistent_list_metadata_string(value: &str) -> String {
BASE64_STANDARD.encode(value.as_bytes())
BASE64_STANDARD.encode_to_string(value.as_bytes())
}
fn decode_persistent_list_metadata_string(value: &str) -> Option<String> {
let bytes = BASE64_STANDARD.decode(value).ok()?;
let bytes = BASE64_STANDARD.decode_to_vec(value).ok()?;
String::from_utf8(bytes).ok()
}
+1 -1
View File
@@ -90,7 +90,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos
async-trait = { workspace = true }
futures = { workspace = true }
metrics = { workspace = true }
base64 = { workspace = true }
base64-simd = { workspace = true }
bytes = { workspace = true }
crc-fast = { workspace = true }
sha2 = { workspace = true }
+1 -2
View File
@@ -13,7 +13,6 @@
// limitations under the License.
use crate::{Error, Result};
use base64::Engine as _;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashSet;
@@ -827,7 +826,7 @@ impl CheckpointManager {
}
fn checkpoint_digest(checkpoint_data: &[u8]) -> String {
base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data))
base64_simd::STANDARD.encode_to_string(Sha256::digest(checkpoint_data))
}
fn digest_path(task_id: &str) -> std::path::PathBuf {
+7 -9
View File
@@ -14,8 +14,7 @@
use crate::{Error, Result};
use async_trait::async_trait;
use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use base64_simd::URL_SAFE_NO_PAD;
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
use rustfs_madmin::heal_commands::HealResultItem;
use serde::{Deserialize, Serialize};
@@ -147,7 +146,7 @@ pub(crate) fn encode_heal_token(marker: Option<&str>, version_marker: Option<&st
// serde_json of a simple two-Option struct cannot fail; fall back to an
// empty object rather than panicking if it somehow does.
let json = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{}".to_vec());
format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(json))
format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(json))
}
/// Decode an opaque heal continuation token back into `(marker, version_marker)`.
@@ -174,7 +173,7 @@ pub(crate) fn decode_heal_token(token: &str) -> (Option<String>, Option<String>)
return (None, None);
};
let bytes = match URL_SAFE_NO_PAD.decode(encoded) {
let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) {
Ok(bytes) => bytes,
Err(e) => {
warn!(
@@ -234,7 +233,7 @@ const DISK_WALK_TOKEN_PREFIX: &str = "dw1:";
/// enumerators can never misread each other's cursor: a `dw1:` token decodes to
/// `(None, None)` under the B5 decoder, and a `v1:` token decodes to `None` here.
pub(crate) fn encode_disk_walk_token(next_forward: &str) -> String {
format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(next_forward.as_bytes()))
format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(next_forward.as_bytes()))
}
/// Decode a disk-walk continuation token back into the `next_forward` object key.
@@ -261,7 +260,7 @@ pub(crate) fn decode_disk_walk_token(token: &str) -> Option<String> {
return None;
};
let bytes = match URL_SAFE_NO_PAD.decode(encoded) {
let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) {
Ok(bytes) => bytes,
Err(e) => {
warn!(
@@ -1513,7 +1512,6 @@ mod tests {
decode_disk_walk_token, decode_heal_token, encode_disk_walk_token, encode_heal_token, is_transient_object_exists_error,
is_transient_object_exists_message, next_heal_listing_token,
};
use base64::Engine as _;
#[test]
fn next_heal_listing_token_returns_none_for_complete_page() {
@@ -1564,7 +1562,7 @@ mod tests {
assert_eq!(decode_heal_token("no-prefix-here"), (None, None));
assert_eq!(decode_heal_token("v1:!!!not-base64!!!"), (None, None));
// valid base64 of non-JSON bytes.
let bad_json = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json"));
let bad_json = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(b"not json"));
assert_eq!(decode_heal_token(&bad_json), (None, None));
// a raw v2-style list_objects_v2 token (no "v1:" prefix) resets cleanly.
assert_eq!(decode_heal_token("some-opaque-legacy-token"), (None, None));
@@ -1576,7 +1574,7 @@ mod tests {
// list_object_versions returns NotImplemented for that pairing.
// Craft a token whose JSON encodes (None, Some) directly and confirm coercion.
let json = br#"{"m":null,"v":"orphan-version"}"#;
let token = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json));
let token = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(json));
assert_eq!(decode_heal_token(&token), (None, None), "version-only marker must coerce to (None, None)");
}
+2 -2
View File
@@ -50,8 +50,8 @@ aes-gcm = { workspace = true, features = ["rand_core"] }
argon2 = { workspace = true }
chacha20poly1305 = { workspace = true }
rand = { workspace = true, features = ["serde"] }
base64 = { workspace = true }
hex = { workspace = true }
base64-simd = { workspace = true }
hex-simd = { workspace = true }
sha2 = { workspace = true }
subtle = { workspace = true }
zeroize = { workspace = true, features = ["derive"] }
+6 -2
View File
@@ -23,7 +23,7 @@
//! Exit status is the verdict: 0 when every check held, 1 otherwise, so a
//! scheduled drill fails its job instead of quietly filing a bad report.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use rustfs_kms::backup::{BackupKek, DrillDataset, DrillDisaster, DrillRequest, DrillVerdict, run_local_drill};
use std::path::PathBuf;
use std::process::ExitCode;
@@ -100,7 +100,11 @@ fn disaster_from_env() -> Result<DrillDisaster, String> {
fn kek_from_env() -> Result<BackupKek, String> {
let raw = Zeroizing::new(required(ENV_KEK)?);
let decoded = Zeroizing::new(BASE64.decode(raw.trim()).map_err(|_| format!("{ENV_KEK} must be base64"))?);
let decoded = Zeroizing::new(
BASE64
.decode_to_vec(raw.trim())
.map_err(|_| format!("{ENV_KEK} must be base64"))?,
);
if decoded.len() != 32 {
return Err(format!("{ENV_KEK} must decode to exactly 32 bytes"));
}
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
@@ -69,7 +69,7 @@ async fn run() -> Result<(), String> {
.decrypt_key_material_for_export(&key_id)
.await
.map_err(|error| error.to_string())?;
let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref()));
let encoded = Zeroizing::new(BASE64_STANDARD.encode_to_string(key_material.as_ref()));
let mut stdout = io::stdout().lock();
writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}"))
+1 -1
View File
@@ -305,7 +305,7 @@ pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -
}
fn digest_value(value: &str) -> String {
let digest = hex::encode(Sha256::digest(value.as_bytes()));
let digest = hex_simd::encode_to_string(Sha256::digest(value.as_bytes()), hex_simd::AsciiCase::Lower);
format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
}
+6 -7
View File
@@ -848,8 +848,7 @@ mod tests {
use aws_sdk_kms::config::{BehaviorVersion, Credentials, Region};
use aws_smithy_http_client::test_util::{NeverClient, ReplayEvent, StaticReplayClient};
use aws_smithy_types::body::SdkBody;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use std::sync::atomic::{AtomicU64, Ordering};
/// AWS KMS speaks awsJson1_1; every request goes to `/` on the regional
@@ -977,8 +976,8 @@ mod tests {
let ciphertext = b"encrypted-data-key".to_vec();
let (http_client, backend) = scripted_backend(vec![ok_event(serde_json::json!({
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
"Plaintext": BASE64.encode(&plaintext),
"CiphertextBlob": BASE64.encode(&ciphertext),
"Plaintext": BASE64.encode_to_string(&plaintext),
"CiphertextBlob": BASE64.encode_to_string(&ciphertext),
}))]);
let response = backend
@@ -997,7 +996,7 @@ mod tests {
let plaintext = b"recovered-data-key".to_vec();
let (_http, backend) = scripted_backend(vec![ok_event(serde_json::json!({
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
"Plaintext": BASE64.encode(&plaintext),
"Plaintext": BASE64.encode_to_string(&plaintext),
"EncryptionAlgorithm": "SYMMETRIC_DEFAULT",
}))]);
@@ -1061,8 +1060,8 @@ mod tests {
error_event(400, "ThrottlingException", "rate exceeded"),
ok_event(serde_json::json!({
"KeyId": "test-key",
"Plaintext": BASE64.encode([1u8; 32]),
"CiphertextBlob": BASE64.encode(b"blob"),
"Plaintext": BASE64.encode_to_string([1u8; 32]),
"CiphertextBlob": BASE64.encode_to_string(b"blob"),
})),
]);
+2 -3
View File
@@ -41,8 +41,7 @@ use crate::types::{
DescribeKeyRequest, EncryptRequest, GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext,
RewrapDataKeyRequest,
};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use rand::RngExt as _;
use std::collections::HashMap;
use std::sync::Arc;
@@ -285,7 +284,7 @@ async fn static_backend_stateless_contract() {
let key_id = "static-contract-key";
let mut raw_key = [0u8; 32];
rand::rng().fill(&mut raw_key[..]);
let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key));
let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode_to_string(raw_key));
let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build");
let backend: &dyn KmsBackend = &static_backend;
+11 -7
View File
@@ -33,7 +33,7 @@ use aes_gcm::{
};
use argon2::{Algorithm, Argon2, Params, Version};
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use jiff::Zoned;
use rand::RngExt;
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
@@ -1268,7 +1268,7 @@ impl LocalKmsClient {
}
let encrypted_bytes = BASE64
.decode(&stored_key.encrypted_key_material)
.decode_to_vec(&stored_key.encrypted_key_material)
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?;
let effective_protection = if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified {
@@ -1406,13 +1406,17 @@ impl LocalKmsClient {
.encrypt(&nonce, key_material)
.map_err(|e| KmsError::cryptographic_error("encrypt", e.to_string()))?;
// Encode encrypted bytes to base64 string
(BASE64.encode(&encrypted), nonce.to_vec(), StoredKeyProtection::EncryptedMasterKey)
(
BASE64.encode_to_string(&encrypted),
nonce.to_vec(),
StoredKeyProtection::EncryptedMasterKey,
)
} else {
warn!(
key_id = %master_key.key_id,
"Local KMS is storing key material as plaintext-dev-only because no master key is configured"
);
(BASE64.encode(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly)
(BASE64.encode_to_string(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly)
};
let stored_key = StoredMasterKey {
@@ -2648,10 +2652,10 @@ mod tests {
let tampered_material = {
let mut material = BASE64
.decode(pristine["encrypted_key_material"].as_str().expect("material is a string"))
.decode_to_vec(pristine["encrypted_key_material"].as_str().expect("material is a string"))
.expect("decode pristine material");
*material.last_mut().expect("material is not empty") ^= 0x01;
BASE64.encode(&material)
BASE64.encode_to_string(&material)
};
type PoisonCase = (&'static str, Vec<u8>, fn(&KmsError) -> bool);
@@ -2976,7 +2980,7 @@ mod tests {
"created_at": "2024-01-01T00:00:00+00:00",
"rotated_at": serde_json::Value::Null,
"created_by": "legacy-test",
"encrypted_key_material": BASE64.encode([7u8; 32]),
"encrypted_key_material": BASE64.encode_to_string([7u8; 32]),
"nonce": Vec::<u8>::new()
});
+2 -3
View File
@@ -722,8 +722,7 @@ impl Default for BackendCapabilities {
mod tests {
use super::*;
use crate::config::KmsConfig;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
/// Backend that implements only the trait-mandated operations and relies
/// on the default `capabilities` implementation.
@@ -958,7 +957,7 @@ mod tests {
#[tokio::test]
async fn static_backend_capabilities_golden() {
let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode([0u8; 32]));
let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode_to_string([0u8; 32]));
let backend = static_kms::StaticKmsBackend::new(config)
.await
.expect("static backend should build");
+2 -3
View File
@@ -434,8 +434,7 @@ mod tests {
use crate::backends::KmsBackend as KmsBackendTrait;
use crate::config::{BackendConfig, KmsBackend, StaticConfig};
use crate::encryption::is_data_key_envelope;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
/// Generate a random 32-byte key and return (key_id, raw_key).
fn random_static_key(key_id: &str) -> (String, [u8; 32]) {
@@ -447,7 +446,7 @@ mod tests {
fn static_config(key_id: &str, raw_key: &[u8; 32]) -> StaticConfig {
StaticConfig {
key_id: key_id.to_string(),
secret_key: BASE64.encode(raw_key),
secret_key: BASE64.encode_to_string(raw_key),
}
}
+13 -13
View File
@@ -33,7 +33,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar
use crate::policy::{self, AttemptError, OpClass, RetryPolicy};
use crate::types::*;
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose};
use base64_simd::STANDARD as BASE64;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -527,8 +527,8 @@ fn decode_stored_key_material(key_id: &str, encrypted_material: &str) -> Result<
// Mirrors `decrypt_key_material`: stored material is currently base64 without an
// additional encryption layer.
let key_material = general_purpose::STANDARD
.decode(encrypted_material)
let key_material = BASE64
.decode_to_vec(encrypted_material)
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?;
// Key material must be exactly 32 bytes for AES-256.
@@ -693,7 +693,7 @@ impl VaultKmsClient {
/// confidentiality. Any identity with KV read access to the key path can recover the
/// plaintext master key.
async fn encrypt_key_material(&self, key_material: &[u8]) -> Result<String> {
Ok(general_purpose::STANDARD.encode(key_material))
Ok(base64_simd::STANDARD.encode_to_string(key_material))
}
/// Read the immutable material record of one key version.
@@ -2405,7 +2405,7 @@ mod tests {
tags: HashMap::new(),
deletion_date: None,
rotated_at: None,
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]),
baseline_version: None,
wrap_budget_reserved: 0,
}
@@ -2869,21 +2869,21 @@ mod tests {
));
// Truncated material: valid base64 of fewer than 32 bytes.
let truncated = general_purpose::STANDARD.encode([0x42u8; 16]);
let truncated = base64_simd::STANDARD.encode_to_string([0x42u8; 16]);
assert!(matches!(
decode_stored_key_material("poisoned", &truncated),
Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned"
));
// Oversized material: valid base64 of more than 32 bytes.
let oversized = general_purpose::STANDARD.encode([0x42u8; 33]);
let oversized = base64_simd::STANDARD.encode_to_string([0x42u8; 33]);
assert!(matches!(
decode_stored_key_material("poisoned", &oversized),
Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned"
));
// Well-formed material still decodes.
let valid = general_purpose::STANDARD.encode([0x42u8; 32]);
let valid = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
assert_eq!(
decode_stored_key_material("healthy", &valid).expect("valid material must decode"),
vec![0x42u8; 32]
@@ -3046,7 +3046,7 @@ mod tests {
description: None,
metadata: HashMap::new(),
tags: HashMap::new(),
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]),
baseline_version: Some(1),
deletion_date: None,
rotated_at: None,
@@ -3760,7 +3760,7 @@ mod tests {
/// Base64 material distinct from `healthy_key_data`'s, standing in for the
/// material a concurrent rotation committed.
fn rotated_material() -> String {
general_purpose::STANDARD.encode([0x43u8; 32])
base64_simd::STANDARD.encode_to_string([0x43u8; 32])
}
/// The issue's lost-update scenario: node A disables a key while node B's
@@ -4358,7 +4358,7 @@ mod tests {
let material_v2 = [0x43u8; 32];
let record_v2 = VaultKeyVersionRecord {
version: 2,
encrypted_key_material: general_purpose::STANDARD.encode(material_v2),
encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2),
created_at: Zoned::now(),
};
// A well-formed envelope wrapped under version 2 — under a reverted
@@ -4900,8 +4900,8 @@ mod tests {
#[tokio::test]
async fn wired_decrypt_of_pre_versioning_envelope_adds_no_request() {
let key_data = healthy_key_data();
let key_material = general_purpose::STANDARD
.decode(&key_data.encrypted_key_material)
let key_material = BASE64
.decode_to_vec(&key_data.encrypted_key_material)
.expect("decode fixture material");
let (encrypted_key, nonce) = AesDekCrypto::new()
.encrypt(&key_material, b"dek-plaintext", &[])
+6 -6
View File
@@ -31,7 +31,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar
use crate::policy::{self, AttemptError, OpClass, RetryPolicy};
use crate::types::*;
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use jiff::Zoned;
use moka::future::Cache;
use serde::{Deserialize, Serialize};
@@ -479,7 +479,7 @@ impl VaultTransitKmsClient {
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
let serialized = serde_json::to_vec(&ordered)?;
Ok(Some(BASE64.encode(serialized)))
Ok(Some(BASE64.encode_to_string(serialized)))
}
fn map_vault_error(key_id: &str, error: vaultrs::error::ClientError, operation: &str) -> KmsError {
@@ -524,7 +524,7 @@ impl VaultTransitKmsClient {
plaintext: &[u8],
encryption_context: &HashMap<String, String>,
) -> Result<String> {
let plaintext_b64 = BASE64.encode(plaintext);
let plaintext_b64 = BASE64.encode_to_string(plaintext);
let plaintext_b64 = plaintext_b64.as_str();
let aad = Self::canonicalize_context(encryption_context)?;
let aad = aad.as_deref();
@@ -568,7 +568,7 @@ impl VaultTransitKmsClient {
.await?;
BASE64
.decode(response.plaintext)
.decode_to_vec(response.plaintext)
.map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string()))
}
@@ -3031,7 +3031,7 @@ mod tests {
ScriptedResponse::ok(kv2_write_ack()),
// decrypt of the pre-rotation envelope; Vault owns the transit
// crypto, so the recovered material is the responder's to hand back.
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })),
])
.await;
@@ -3243,7 +3243,7 @@ mod tests {
// rewrap, context-bound route: latest-version read, then decrypt,
// then re-encrypt under the newest version.
ScriptedResponse::ok(transit_key_read_data_up_to("wired-key", 2)),
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })),
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v2:rewrapped" })),
])
.await;
+2 -2
View File
@@ -945,7 +945,7 @@ async fn tree_digest(root: &Path) -> Result<ContentDigest> {
lines.push(format!(
"{relative}\u{1f}{}\u{1f}{modified}\u{1f}{}",
metadata.len(),
hex::encode(Sha256::digest(&content))
hex_simd::encode_to_string(Sha256::digest(&content), hex_simd::AsciiCase::Lower)
));
}
lines.sort();
@@ -1199,7 +1199,7 @@ mod tests {
let text = String::from_utf8(encoded.clone()).expect("evidence is utf-8");
assert!(!text.contains(DRILL_MASTER_KEY), "the evidence must not carry the master key");
assert!(
!text.contains(&hex::encode([0x37u8; 32])),
!text.contains(&hex_simd::encode_to_string([0x37u8; 32], hex_simd::AsciiCase::Lower)),
"the evidence must not carry backup KEK material"
);
+4 -1
View File
@@ -568,7 +568,10 @@ pub(crate) fn compute_master_key_verifier(master_key: &str, salt: Option<&[u8]>,
let mut hasher = Sha256::new();
hasher.update(&framing);
hasher.update(derived.as_slice());
Ok(format!("{prefix}{}", hex::encode(hasher.finalize())))
Ok(format!(
"{prefix}{}",
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
))
}
/// The bundle-level protection label is the weakest state observed across
+2 -2
View File
@@ -72,7 +72,7 @@ use aes_gcm::{
Aes256Gcm, Nonce,
aead::{Aead, KeyInit},
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use tokio::fs;
@@ -640,7 +640,7 @@ fn decode_key_record(
return Err(BackupError::corrupted(format!("bundled key record '{stem}' carries no key material")).into());
}
let material =
Zeroizing::new(BASE64.decode(&probe.encrypted_key_material).map_err(|error| {
Zeroizing::new(BASE64.decode_to_vec(&probe.encrypted_key_material).map_err(|error| {
BackupError::corrupted(format!("bundled key record '{stem}' material is not valid base64: {error}"))
})?);
if !allowed_modes.contains(&protection_mode(probe.at_rest_protection)) {
+1 -1
View File
@@ -69,7 +69,7 @@ impl ContentDigest {
pub fn sha256_of(bytes: &[u8]) -> Self {
Self {
algorithm: DigestAlgorithm::Sha256,
hex: hex::encode(Sha256::digest(bytes)),
hex: hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower),
}
}
+5 -10
View File
@@ -367,9 +367,8 @@ impl StaticConfig {
/// Decode the base64-encoded secret key into raw bytes.
/// Returns an error if the key is not valid base64 or is not exactly 32 bytes.
pub fn decode_key(&self) -> Result<[u8; 32]> {
use base64::Engine as _;
let bytes = base64::engine::general_purpose::STANDARD
.decode(&self.secret_key)
let bytes = base64_simd::STANDARD
.decode_to_vec(&self.secret_key)
.map_err(|e| KmsError::configuration_error(format!("Static KMS secret key is not valid base64: {e}")))?;
if bytes.len() != 32 {
return Err(KmsError::configuration_error(format!(
@@ -1963,9 +1962,7 @@ mod tests {
#[test]
fn static_kms_config_serialization_does_not_expose_key_material() {
use base64::Engine as _;
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
let config = KmsConfig::static_kms("static-key".to_string(), encoded_key.clone());
let serialized = serde_json::to_string(&config).expect("static KMS config should serialize");
@@ -2569,14 +2566,12 @@ mod tests {
#[test]
fn test_from_env_reads_static_secret_file_and_sets_default_key() {
use base64::Engine as _;
let temp_dir = TempDir::new().expect("create temp dir for static KMS secret");
let secret_path = temp_dir.path().join("static-kms-secret");
// Named `*_key_b64` (not `*_secret`) so the logging-guardrails check does not
// flag these fixture interpolations as secrets leaking into log strings.
let file_key_b64 = base64::engine::general_purpose::STANDARD.encode([7u8; 32]);
let env_key_b64 = base64::engine::general_purpose::STANDARD.encode([9u8; 32]);
let file_key_b64 = base64_simd::STANDARD.encode_to_string([7u8; 32]);
let env_key_b64 = base64_simd::STANDARD.encode_to_string([9u8; 32]);
std::fs::write(&secret_path, format!("file-key:{file_key_b64}\n")).expect("write static KMS secret file");
with_vars(
+5 -4
View File
@@ -46,8 +46,7 @@ use crate::error::{KmsError, Result};
use aes_gcm::aead::{Aead, Payload};
use aes_gcm::{Aes256Gcm, Key, KeyInit, Nonce};
use argon2::{Algorithm, Argon2, Params, Version};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64_simd::STANDARD as BASE64_STANDARD;
use rand::RngExt;
use serde_json::Value;
use sha2::{Digest, Sha256};
@@ -284,14 +283,16 @@ fn seal_value(label: &str, plaintext: &str, secret: &str) -> Result<String> {
payload.extend_from_slice(&salt);
payload.extend_from_slice(&nonce);
payload.extend_from_slice(&ciphertext);
Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode(payload)))
Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode_to_string(payload)))
}
fn open_value(label: &str, sealed: &str, secret: &str) -> Result<String> {
let encoded = sealed
.strip_prefix(SEALED_VALUE_PREFIX)
.expect("caller checks the sealed prefix");
let payload = BASE64_STANDARD.decode(encoded).map_err(|_| sealed_value_unreadable(label))?;
let payload = BASE64_STANDARD
.decode_to_vec(encoded)
.map_err(|_| sealed_value_unreadable(label))?;
if payload.len() <= LOCAL_KMS_MASTER_KEY_SALT_LEN + NONCE_LEN {
return Err(sealed_value_unreadable(label));
}
+7 -3
View File
@@ -675,7 +675,6 @@ mod tests {
use crate::error::KmsError;
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage};
use async_trait::async_trait;
use base64::Engine as _;
use jiff::Zoned;
use std::collections::HashMap;
use std::sync::Mutex;
@@ -1110,8 +1109,13 @@ mod tests {
.await
.expect("enable should succeed");
let base64 = base64::engine::general_purpose::STANDARD;
let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)];
let base64 = base64_simd::STANDARD;
let encodings = |bytes: &[u8]| {
vec![
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower),
base64.encode_to_string(bytes),
]
};
let mut forbidden = vec![grant_token.to_string()];
forbidden.extend(encodings(&data_key.plaintext_key));
forbidden.extend(encodings(&decrypted.plaintext));
+1 -3
View File
@@ -551,7 +551,6 @@ mod tests {
ListKeysRequest, ListKeysResponse,
};
use async_trait::async_trait;
use base64::Engine as _;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use std::future::Future;
@@ -562,8 +561,7 @@ mod tests {
}
async fn static_backend() -> Arc<dyn KmsBackend> {
let config =
KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
let config = KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x42u8; 32]));
Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
}
+10 -14
View File
@@ -23,7 +23,6 @@ use crate::encryption::context_aad;
use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::types::*;
use base64::Engine;
use jiff::Zoned;
use md5::{Digest as Md5Digest, Md5};
use rand::random;
@@ -40,7 +39,7 @@ use zeroize::Zeroize;
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
/// Data key for object encryption
@@ -836,19 +835,16 @@ impl ObjectEncryptionService {
// Internal headers for decryption
headers.insert(
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(&metadata.iv),
base64_simd::STANDARD.encode_to_string(&metadata.iv),
);
if let Some(ref tag) = metadata.tag {
headers.insert(
INTERNAL_ENCRYPTION_TAG_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(tag),
);
headers.insert(INTERNAL_ENCRYPTION_TAG_HEADER.to_string(), base64_simd::STANDARD.encode_to_string(tag));
}
headers.insert(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key),
base64_simd::STANDARD.encode_to_string(&metadata.encrypted_data_key),
);
// Whatever the object was sealed under is what gets stored: for a
@@ -906,14 +902,14 @@ impl ObjectEncryptionService {
let iv = headers
.get(INTERNAL_ENCRYPTION_IV_HEADER)
.ok_or_else(|| KmsError::validation_error("Missing IV header"))?;
let iv = base64::engine::general_purpose::STANDARD
.decode(iv)
let iv = base64_simd::STANDARD
.decode_to_vec(iv)
.map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?;
let tag = if let Some(tag_str) = headers.get(INTERNAL_ENCRYPTION_TAG_HEADER) {
Some(
base64::engine::general_purpose::STANDARD
.decode(tag_str)
base64_simd::STANDARD
.decode_to_vec(tag_str)
.map_err(|e| KmsError::validation_error(format!("Invalid tag: {e}")))?,
)
} else {
@@ -921,8 +917,8 @@ impl ObjectEncryptionService {
};
let encrypted_data_key = if let Some(key_str) = headers.get(INTERNAL_ENCRYPTION_KEY_HEADER) {
base64::engine::general_purpose::STANDARD
.decode(key_str)
base64_simd::STANDARD
.decode_to_vec(key_str)
.map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))?
} else {
Vec::new() // Empty for SSE-C
+4 -5
View File
@@ -763,10 +763,10 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
#[cfg(test)]
mod tests {
use super::*;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
fn static_config(key_id: &str, fill: u8) -> KmsConfig {
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode_to_string([fill; 32]))
}
/// End-to-end wiring check for the AWS backend: an admin configure request
@@ -822,7 +822,7 @@ mod tests {
#[tokio::test]
async fn redacted_config_omits_static_key_material() {
let manager = KmsServiceManager::new();
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
manager
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
.await
@@ -1020,7 +1020,6 @@ mod tests {
#[tokio::test]
async fn configure_cannot_replace_existing_local_backend() {
use base64::Engine as _;
use tempfile::TempDir;
let key_dir = TempDir::new().expect("create local KMS directory");
@@ -1029,7 +1028,7 @@ mod tests {
let manager = KmsServiceManager::new();
manager.configure(local.clone()).await.expect("configure local KMS");
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
let error = manager
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
.await
+1 -2
View File
@@ -451,6 +451,5 @@ async fn harness_restart_brings_the_service_back_over_the_same_state() {
}
fn base64_of(bytes: &[u8]) -> String {
use base64::Engine as _;
base64::engine::general_purpose::STANDARD.encode(bytes)
base64_simd::STANDARD.encode_to_string(bytes)
}
+1 -1
View File
@@ -667,7 +667,7 @@ async fn sse_c_round_trips_and_rejects_the_wrong_key() {
async fn sse_c_validates_the_supplied_key_md5() {
let (_kms, service) = service_with_key("sse-c-md5-unused").await;
let customer_key = [0x33u8; 32];
let correct_md5 = hex::encode(md5_of(&customer_key));
let correct_md5 = hex_simd::encode_to_string(md5_of(&customer_key), hex_simd::AsciiCase::Lower);
service
.encrypt_object_with_customer_key(BUCKET, "md5-ok.bin", payload(64).as_slice(), &customer_key, Some(&correct_md5))
+2 -3
View File
@@ -34,8 +34,7 @@ use std::future::Future;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use rustfs_kms::backends::BackendCapabilities;
use rustfs_kms::{
CreateKeyRequest, DeleteKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus,
@@ -51,7 +50,7 @@ pub const STATIC_KEY_ID: &str = "behavior-static-key";
/// Fixed rather than random so a failure is reproducible; it is test-only
/// material and never leaves this crate's test binaries.
pub fn static_secret_key() -> String {
BASE64.encode([0x5au8; 32])
BASE64.encode_to_string([0x5au8; 32])
}
/// Which backend a harness instance is running.
+5 -5
View File
@@ -99,14 +99,14 @@ swift = [
"dep:md-5",
"dep:hmac",
"dep:sha1",
"dep:hex",
"dep:hex-simd",
"dep:ipnetwork",
"dep:rustfs-trusted-proxies",
"dep:astral-tokio-tar",
"dep:base64",
"dep:base64-simd",
"dep:async-compression",
]
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64-simd", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"]
[dependencies]
@@ -163,11 +163,11 @@ urlencoding = { workspace = true, optional = true }
md-5 = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
sha1 = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
hex-simd = { workspace = true, optional = true }
ipnetwork = { workspace = true, optional = true }
rustfs-trusted-proxies = { workspace = true, optional = true }
astral-tokio-tar = { workspace = true, optional = true }
base64 = { workspace = true, optional = true }
base64-simd = { workspace = true, optional = true }
async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] }
# WebDAV specific dependencies (optional)
+1 -1
View File
@@ -97,7 +97,7 @@ fn get_account_metadata_bucket_name(account: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(account.as_bytes());
let hash_bytes = hasher.finalize();
let hash = hex::encode(hash_bytes);
let hash = hex_simd::encode_to_string(hash_bytes, hex_simd::AsciiCase::Lower);
format!("swift-account-{}", &hash[0..16])
}
+4 -4
View File
@@ -182,7 +182,7 @@ pub fn generate_signature(
mac.update(message.as_bytes());
let result = mac.finalize();
let signature = hex::encode(result.into_bytes());
let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower);
Ok(signature)
}
@@ -213,10 +213,10 @@ pub fn validate_formpost(path: &str, request: &FormPostRequest, key: &str) -> Sw
// the sibling TempURL/SFTP checks. Decode the hex first so the comparison runs
// over the raw HMAC bytes and does not leak via string length; a non-hex
// provided signature can never match and is rejected the same way.
let expected_bytes =
hex::decode(&expected_sig).map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?;
let expected_bytes = hex_simd::decode_to_vec(&expected_sig)
.map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?;
let signatures_match = match hex::decode(request.signature.trim()) {
let signatures_match = match hex_simd::decode_to_vec(request.signature.trim()) {
Ok(provided_bytes) => super::tempurl::constant_time_compare(&provided_bytes, &expected_bytes),
Err(_) => false,
};
+5 -1
View File
@@ -84,7 +84,11 @@ impl SLOManifest {
let mut hasher = Md5::new();
hasher.update(etag_concat.as_bytes());
format!("\"{}-{}\"", hex::encode(hasher.finalize()), self.segments.len())
format!(
"\"{}-{}\"",
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower),
self.segments.len()
)
}
/// Validate manifest against actual segments
+1 -1
View File
@@ -300,7 +300,7 @@ pub fn generate_sync_signature(path: &str, key: &str) -> SwiftResult<String> {
mac.update(path.as_bytes());
let result = mac.finalize();
Ok(hex::encode(result.into_bytes()))
Ok(hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower))
}
/// Verify sync signature
+1 -1
View File
@@ -125,7 +125,7 @@ impl TempURL {
// Hex-encode result
let result = mac.finalize();
let signature = hex::encode(result.into_bytes());
let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower);
Ok(signature)
}
+1 -2
View File
@@ -675,8 +675,7 @@ fn fixed_body(message: impl Into<Bytes>) -> WebDavBody {
/// Decode base64 string
fn base64_decode(encoded: &str) -> Result<Vec<u8>, ()> {
use base64::Engine;
base64::engine::general_purpose::STANDARD.decode(encoded).map_err(|_| ())
base64_simd::STANDARD.decode_to_vec(encoded).map_err(|_| ())
}
#[cfg(test)]
@@ -56,7 +56,7 @@ fn keystone_credentials(project_id: &str) -> Credentials {
fn account_metadata_bucket_name(account: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(account.as_bytes());
let hash = hex::encode(hasher.finalize());
let hash = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
format!("swift-account-{}", &hash[0..16])
}
+1 -1
View File
@@ -57,7 +57,7 @@ hotpath.workspace = true
aes-gcm = { workspace = true, features = ["rand_core"] }
bytes = { workspace = true, features = ["serde"] }
chacha20poly1305.workspace = true
hex.workspace = true
hex-simd.workspace = true
hmac.workspace = true
minlz.workspace = true
pin-project-lite.workspace = true
+4 -1
View File
@@ -731,12 +731,15 @@ fn random_stream_nonce() -> [u8; 12] {
#[cfg(test)]
mod tests {
use super::*;
use hex::encode as hex_encode;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE;
fn hex_encode(data: impl AsRef<[u8]>) -> String {
hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
}
#[tokio::test]
async fn decrypt_reader_can_start_from_non_zero_sequence_number() {
let plaintext = vec![0xAB; DARE_PAYLOAD_SIZE * 2 + 19];
+1 -1
View File
@@ -81,7 +81,7 @@ serde_json = { workspace = true, features = ["raw_value"] }
md-5 = { workspace = true }
tracing.workspace = true
thiserror.workspace = true
base64.workspace = true
base64-simd.workspace = true
sha1.workspace = true
sha2.workspace = true
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
+14 -11
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::errors::ChecksumMismatch;
use base64::{Engine as _, engine::general_purpose};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use http::HeaderMap;
use sha1::Sha1;
@@ -330,7 +330,7 @@ impl Checksum {
let mut hasher = checksum_type.hasher()?;
hasher.write_all(data).ok()?;
let raw = hasher.finalize();
let encoded = general_purpose::STANDARD.encode(&raw);
let encoded = BASE64_STANDARD.encode_to_string(&raw);
let checksum = Checksum {
checksum_type,
@@ -369,7 +369,7 @@ impl Checksum {
value_string = value.to_string();
}
// let raw = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(&value_string).ok()?;
let raw = general_purpose::STANDARD.decode(&value_string).ok()?;
let raw = BASE64_STANDARD.decode_to_vec(&value_string).ok()?;
let checksum = Checksum {
checksum_type,
@@ -413,14 +413,14 @@ impl Checksum {
if self.want_parts > 0 && self.want_parts != parts {
return Err(ChecksumMismatch {
want: format!("{}-{}", self.encoded, self.want_parts),
got: format!("{}-{}", general_purpose::STANDARD.encode(&sum), parts),
got: format!("{}-{}", base64_simd::STANDARD.encode_to_string(&sum), parts),
});
}
if sum != self.raw {
return Err(ChecksumMismatch {
want: self.encoded.clone(),
got: general_purpose::STANDARD.encode(&sum),
got: base64_simd::STANDARD.encode_to_string(&sum),
});
}
@@ -569,7 +569,7 @@ impl Checksum {
}
}
self.encoded = general_purpose::STANDARD.encode(&self.raw);
self.encoded = base64_simd::STANDARD.encode_to_string(&self.raw);
Ok(())
}
}
@@ -1160,7 +1160,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
let checksum_bytes = &buf[..length];
buf = &buf[length..];
let mut checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
let mut checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
if checksum_type.is(ChecksumType::MULTIPART) {
is_multipart = true;
@@ -1190,7 +1190,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
if part > 0 && (part as u64) <= parts_count {
let offset = ((part - 1) as usize) * length;
let part_checksum = &buf[offset..offset + length];
checksum_str = general_purpose::STANDARD.encode(part_checksum);
checksum_str = base64_simd::STANDARD.encode_to_string(part_checksum);
}
buf = &buf[want_len..];
}
@@ -1249,7 +1249,7 @@ pub fn read_part_checksums(mut buf: &[u8]) -> Vec<HashMap<String, String>> {
let checksum_bytes = &buf[..length];
buf = &buf[length..];
let checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
let checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
part_checksum.insert(checksum_type.to_string(), checksum_str);
}
@@ -1553,7 +1553,6 @@ mod tests {
// asserted alongside the raw hex so both the digest and its encoding are pinned.
#[test]
fn xxhash_sha512_regression_lock_non_empty() {
use base64::{Engine as _, engine::general_purpose::STANDARD};
let data = b"The quick brown fox jumps over the lazy dog";
// XXH3-64(fox) = 0xce7d19a5418fb365 is the official upstream vector.
@@ -1565,7 +1564,11 @@ mod tests {
let got_hex: String = c.raw.iter().map(|b| format!("{b:02x}")).collect();
assert_eq!(got_hex, want_hex, "{t:?} raw hex drifted");
// encoded field must be the standard-base64 of raw (S3 wire form)
assert_eq!(c.encoded, STANDARD.encode(&c.raw), "{t:?} encoded field != base64(raw)");
assert_eq!(
c.encoded,
base64_simd::STANDARD.encode_to_string(&c.raw),
"{t:?} encoded field != base64(raw)"
);
}
}
+4 -5
View File
@@ -91,8 +91,7 @@ use crate::Sha256Hasher;
use crate::compress_index::{Index, TryGetIndex};
use crate::get_content_checksum;
use crate::{DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, boxed_reader, wrap_reader};
use base64::Engine;
use base64::engine::general_purpose;
use http::HeaderMap;
use pin_project_lite::pin_project;
use s3s::TrailingHeaders;
@@ -582,8 +581,8 @@ impl AsyncRead for HashReader {
})
{
expected_content_hash.encoded = checksum_str;
expected_content_hash.raw = general_purpose::STANDARD
.decode(&expected_content_hash.encoded)
expected_content_hash.raw = base64_simd::STANDARD
.decode_to_vec(&expected_content_hash.encoded)
.map_err(|_| std::io::Error::other("Invalid base64 checksum"))?;
if expected_content_hash.raw.is_empty() {
@@ -598,7 +597,7 @@ impl AsyncRead for HashReader {
&& !expected_content_hash.checksum_type.trailing()
{
expected_content_hash.raw = content_hash;
expected_content_hash.encoded = general_purpose::STANDARD.encode(&expected_content_hash.raw);
expected_content_hash.encoded = base64_simd::STANDARD.encode_to_string(&expected_content_hash.raw);
} else if content_hash != expected_content_hash.raw {
let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower);
let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower);