chore(deps): migrate direct encoding deps to simd (#6690)

This commit is contained in:
houseme
2026-08-27 03:17:24 +08:00
committed by GitHub
parent 31031f2a46
commit ba7785d61d
81 changed files with 523 additions and 505 deletions
Generated
+9 -11
View File
@@ -3868,7 +3868,7 @@ dependencies = [
"aws-sdk-s3",
"aws-sdk-sts",
"aws-smithy-http-client",
"base64 0.23.1",
"base64-simd",
"bytes",
"chrono",
"clap",
@@ -3876,7 +3876,7 @@ dependencies = [
"flatbuffers",
"flate2",
"futures",
"hex",
"hex-simd",
"hmac 0.13.0",
"hotpath",
"http 1.5.0",
@@ -9313,7 +9313,6 @@ dependencies = [
"aws-config",
"aws-sdk-s3",
"axum",
"base64 0.23.1",
"base64-simd",
"bytes",
"chacha20poly1305",
@@ -9573,7 +9572,6 @@ dependencies = [
"aws-smithy-http-client",
"aws-smithy-runtime-api",
"aws-smithy-types",
"base64 0.23.1",
"base64-simd",
"byteorder",
"bytes",
@@ -9739,7 +9737,7 @@ name = "rustfs-heal"
version = "1.0.0-rc.3"
dependencies = [
"async-trait",
"base64 0.23.1",
"base64-simd",
"bytes",
"crc-fast",
"futures",
@@ -9944,9 +9942,9 @@ dependencies = [
"aws-smithy-http-client",
"aws-smithy-runtime-api",
"aws-smithy-types",
"base64 0.23.1",
"base64-simd",
"chacha20poly1305",
"hex",
"hex-simd",
"hotpath",
"http 1.5.0",
"insta",
@@ -10251,12 +10249,12 @@ dependencies = [
"async-compression",
"async-trait",
"axum",
"base64 0.23.1",
"base64-simd",
"bytes",
"dav-server",
"futures",
"futures-util",
"hex",
"hex-simd",
"hmac 0.13.0",
"hotpath",
"http 1.5.0",
@@ -10355,7 +10353,7 @@ dependencies = [
"aes-gcm",
"arc-swap",
"axum",
"base64 0.23.1",
"base64-simd",
"bytes",
"crc-fast",
"faster-hex",
@@ -10395,7 +10393,7 @@ dependencies = [
"aes-gcm",
"bytes",
"chacha20poly1305",
"hex",
"hex-simd",
"hmac 0.13.0",
"hotpath",
"minlz",
-2
View File
@@ -243,7 +243,6 @@ aws-sdk-sts = { default-features = false, version = "1.113.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.15.0" }
aws-smithy-types = { version = "1.6.2" }
base64 = "0.23.1"
base64-simd = "0.8.0"
brotli = "8.0.4"
clap = { version = "4.6.6" }
@@ -266,7 +265,6 @@ hashbrown = { version = "0.17.1" }
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
# every authenticator app expects). Already in the graph transitively.
data-encoding = "2.11.1"
hex = "0.4.3"
hex-simd = "0.8.0"
highway = { version = "1.3.0" }
hostname = "0.4.2"
+2 -2
View File
@@ -121,10 +121,10 @@ tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
urlencoding.workspace = true
walkdir.workspace = true
base64 = { workspace = true }
base64-simd = { workspace = true }
rand = { workspace = true, features = ["serde"] }
chrono = { workspace = true, features = ["serde"] }
hex = { workspace = true }
hex-simd = { workspace = true }
md-5 = { workspace = true }
opentelemetry-proto = { workspace = true }
prost.workspace = true
+2 -3
View File
@@ -24,7 +24,6 @@ mod tests {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use sha2::Sha256;
@@ -74,12 +73,12 @@ mod tests {
let mut hasher = Md5::new();
hasher.update(body);
let digest = hasher.finalize();
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
base64_simd::STANDARD.encode_to_string(digest.as_slice())
}
fn checksum_sha256_base64(body: &[u8]) -> String {
let digest = Sha256::digest(body);
base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
base64_simd::STANDARD.encode_to_string(digest.as_slice())
}
fn checksum_crc64nvme_base64(body: &[u8]) -> String {
+1 -2
View File
@@ -652,11 +652,10 @@ const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
async fn start_rustfs_with_compression_and_sse(
env: &mut RustFSTestEnvironment,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use base64::Engine;
env.cleanup_existing_processes().await?;
let binary_path = rustfs_binary_path();
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
// Server output goes to a file inside the per-test temp dir so a failing
// run can be diagnosed from the child's logs.
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
@@ -27,8 +27,7 @@ mod tests {
VersioningConfiguration,
};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use sha2::{Digest, Sha256};
use tracing::info;
@@ -465,7 +464,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"deterministic synthetic payload for copy-object checksum #4996";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
client
.put_object()
@@ -534,7 +533,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"another deterministic payload whose source checksum must survive the copy";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
// Store the source WITH a SHA-256 checksum so it has one to preserve.
let put_src = client
@@ -614,7 +613,7 @@ mod tests {
create_versioned_bucket(&client, dst_bucket).await;
let content = b"payload whose copy must be re-checksummed with a different algorithm";
let expected_sha256 = BASE64.encode(Sha256::digest(content));
let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content));
// Source is stored WITH a SHA-256 checksum.
client
+3 -3
View File
@@ -1113,7 +1113,7 @@ fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result {
@@ -1375,7 +1375,7 @@ impl S3 for FakeBackend {
Some(value) => value,
None => {
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
hex::encode(digest)
hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower)
}
};
let version = ObjectVersion {
@@ -1660,7 +1660,7 @@ impl S3 for FakeBackend {
}
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?;
let e_tag = hex::encode(digest);
let e_tag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
let mut state = lock(&self.store);
let existing_bytes = state
.uploads
@@ -28,7 +28,6 @@ use aws_sdk_s3::types::{
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus,
LifecycleRule, LifecycleRuleFilter, ServerSideEncryption, Transition, TransitionStorageClass, VersioningConfiguration,
};
use base64::Engine;
use bytes::Bytes;
use flate2::read::GzDecoder;
use http::header::{CONTENT_ENCODING, HOST};
@@ -1808,7 +1807,7 @@ async fn four_node_inline_fallback_controls() -> TestResult {
let collector = OtlpMetricCollector::start().await?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
configure_reader_metric_cluster(&mut cluster, &collector);
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key);
cluster.start().await?;
@@ -2017,7 +2016,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
let collector = OtlpMetricCollector::start().await?;
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
@@ -2489,7 +2488,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
hot.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
hot.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
hot.start().await?;
+7 -7
View File
@@ -27,7 +27,7 @@ use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use http::header::{CONTENT_TYPE, HOST};
use md5::{Digest as Md5Digest, Md5};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -64,7 +64,7 @@ pub fn init_logging() {
pub fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
BASE64.encode(hasher.finalize())
BASE64.encode_to_string(hasher.finalize())
}
pub fn assert_s3_error<T, E>(result: Result<T, SdkError<E>>, status: u16, code: &str, message: &str, context: &str)
@@ -365,7 +365,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
"rotated_at": serde_json::Value::Null,
"created_by": "e2e-test",
"encrypted_key_material": BASE64.encode(key_data),
"encrypted_key_material": BASE64.encode_to_string(key_data),
"nonce": Vec::<u8>::new()
});
@@ -383,7 +383,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
info!("Testing SSE-C encryption");
let test_key = "01234567890123456789012345678901"; // 32-byte key
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let test_data = b"Hello, KMS SSE-C World!";
let object_key = "test-sse-c-object";
@@ -551,8 +551,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
// Test SSE-C with wrong key for download
let test_key = "01234567890123456789012345678901";
let wrong_key = "98765432109876543210987654321098";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
let test_data = b"Test data for error scenarios";
@@ -807,7 +807,7 @@ pub async fn test_multipart_upload_with_config(
// Prepare encryption parameters
let (sse_c_key_b64, sse_c_key_md5) = match &config.encryption_type {
EncryptionType::SSEC { key, key_md5 } => {
let key_b64 = base64::engine::general_purpose::STANDARD.encode(key);
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
(Some(key_b64), Some(key_md5.clone()))
}
_ => (None, None),
@@ -177,7 +177,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
// Verify that files cannot be read with wrong keys
info!("🔒 Verify key isolation");
let wrong_key = "11111111111111111111111111111111";
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
// Try to read file encrypted with key1 using wrong key
@@ -24,7 +24,6 @@
use super::common::{LocalKMSTestEnvironment, SSE_C_KEY_MISMATCH_MESSAGE, assert_s3_error, sse_customer_key_md5_base64};
use crate::common::{TEST_BUCKET, init_logging};
use aws_sdk_s3::types::ServerSideEncryption;
use base64::Engine;
use std::sync::Arc;
use tokio::sync::Semaphore;
use tracing::{info, warn};
@@ -68,7 +67,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
// Test SSE-C with zero-byte file
info!("📤 Testing SSE-C with zero-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let object_key_c = "zero-byte-sse-c";
@@ -161,7 +160,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
// Test SSE-C with single byte
info!("📤 Testing SSE-C with single-byte file");
let test_key = "01234567890123456789012345678901";
let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key);
let test_key_md5 = sse_customer_key_md5_base64(test_key);
let object_key_c = "single-byte-sse-c";
@@ -287,7 +286,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
// Test 1: Invalid key length for SSE-C
info!("🔍 Testing invalid SSE-C key length");
let invalid_short_key = "short"; // Too short
let invalid_key_b64 = base64::engine::general_purpose::STANDARD.encode(invalid_short_key);
let invalid_key_b64 = base64_simd::STANDARD.encode_to_string(invalid_short_key);
let invalid_key_md5 = sse_customer_key_md5_base64(invalid_short_key);
let invalid_key_result = s3_client
@@ -325,7 +324,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
// Test 2: Mismatched MD5 for SSE-C
info!("🔍 Testing mismatched MD5 for SSE-C key");
let valid_key = "01234567890123456789012345678901";
let valid_key_b64 = base64::engine::general_purpose::STANDARD.encode(valid_key);
let valid_key_b64 = base64_simd::STANDARD.encode_to_string(valid_key);
let wrong_md5 = sse_customer_key_md5_base64("98765432109876543210987654321098");
let wrong_md5_result = s3_client
@@ -465,7 +464,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
2 => {
// SSE-C
let key = format!("testkey{i:026}"); // 32-byte key
let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
let key_b64 = base64_simd::STANDARD.encode_to_string(&key);
let key_md5 = sse_customer_key_md5_base64(&key);
client
@@ -535,8 +534,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
let key1 = "key1key1key1key1key1key1key1key1"; // 32 bytes
let key2 = "key2key2key2key2key2key2key2key2"; // 32 bytes
let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
+3 -3
View File
@@ -138,8 +138,8 @@ async fn test_local_kms_key_isolation() {
// Test that different SSE-C keys create isolated encrypted objects
let key1 = "01234567890123456789012345678901";
let key2 = "98765432109876543210987654321098";
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
@@ -565,7 +565,7 @@ async fn test_multipart_upload_with_sse_c(
// SSE-C encryption key
let encryption_key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
let key_b64 = base64_simd::STANDARD.encode_to_string(encryption_key);
let key_md5 = sse_customer_key_md5_base64(encryption_key);
// Generate test data
+2 -2
View File
@@ -127,8 +127,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
let key1 = "01234567890123456789012345678901";
let key2 = "98765432109876543210987654321098";
let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
let key1_b64 = base64_simd::STANDARD.encode_to_string(key1);
let key2_b64 = base64_simd::STANDARD.encode_to_string(key2);
let key1_md5 = sse_customer_key_md5_base64(key1);
let key2_md5 = sse_customer_key_md5_base64(key2);
@@ -497,7 +497,7 @@ async fn test_multipart_encryption_type(
// Prepare SSE-C keys when required
let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
let key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
let key_b64 = base64_simd::STANDARD.encode_to_string(key);
let key_md5 = sse_customer_key_md5_base64(key);
(Some(key_b64), Some(key_md5))
} else {
+11 -12
View File
@@ -22,7 +22,6 @@ use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
};
use base64::Engine;
use chrono::{Duration as ChronoDuration, Utc};
use flate2::{Compression, write::GzEncoder};
use http::HeaderValue;
@@ -47,19 +46,19 @@ fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
"conditions": conditions,
});
base64::engine::general_purpose::STANDARD.encode(policy.to_string())
base64_simd::STANDARD.encode_to_string(policy.to_string())
}
fn sse_customer_key_md5_base64(key: &str) -> String {
let mut hasher = Md5::new();
hasher.update(key.as_bytes());
base64::engine::general_purpose::STANDARD.encode(hasher.finalize())
base64_simd::STANDARD.encode_to_string(hasher.finalize())
}
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
async fn create_restricted_user(
@@ -97,7 +96,7 @@ fn restricted_user_client(env: &RustFSTestEnvironment, username: &str, secret_ke
const LOCAL_SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
fn local_sse_master_key_value() -> String {
base64::engine::general_purpose::STANDARD.encode([0x42u8; 32])
base64_simd::STANDARD.encode_to_string([0x42u8; 32])
}
async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec<u8> {
@@ -1887,7 +1886,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
let object_key = "sse-c-object.txt";
let expected_body = b"anonymous-post-sse-c".to_vec();
let customer_key = "01234567890123456789012345678901";
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
let admin_client = env.create_s3_client();
@@ -1941,7 +1940,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition
.bucket(bucket)
.key(object_key)
.sse_customer_algorithm("AES256")
.sse_customer_key(base64::engine::general_purpose::STANDARD.encode(customer_key))
.sse_customer_key(base64_simd::STANDARD.encode_to_string(customer_key))
.sse_customer_key_md5(customer_key_md5)
.send()
.await?;
@@ -1963,8 +1962,8 @@ async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Res
let object_key = "sse-c-mismatch-object.txt";
let policy_key = "01234567890123456789012345678901";
let request_key = "abcdefghijklmnopqrstuvwxyzABCDEF";
let policy_key_b64 = base64::engine::general_purpose::STANDARD.encode(policy_key);
let request_key_b64 = base64::engine::general_purpose::STANDARD.encode(request_key);
let policy_key_b64 = base64_simd::STANDARD.encode_to_string(policy_key);
let request_key_b64 = base64_simd::STANDARD.encode_to_string(request_key);
let admin_client = env.create_s3_client();
admin_client.create_bucket().bucket(bucket).send().await?;
@@ -3526,7 +3525,7 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
.await?;
@@ -3799,7 +3798,7 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<(
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())])
.await?;
@@ -3925,7 +3924,7 @@ async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box<dyn
let extracted_key = "nested/file.txt";
let expected_body = b"extract-sse-c-body".to_vec();
let customer_key = "01234567890123456789012345678901";
let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key);
let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key);
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
let client = env.create_s3_client();
+1 -2
View File
@@ -35,7 +35,6 @@ use crate::common::local_http_client;
use crate::common::rustfs_binary_path_with_features;
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
use anyhow::Result;
use base64::Engine;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::Client;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -64,7 +63,7 @@ fn basic_auth_header() -> String {
fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
let credentials = format!("{}:{}", access_key, secret_key);
let encoded = base64::engine::general_purpose::STANDARD.encode(credentials);
let encoded = base64_simd::STANDARD.encode_to_string(credentials);
format!("Basic {}", encoded)
}
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
VersioningConfiguration,
};
use aws_sdk_s3::{Client, Config};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use flate2::read::GzDecoder;
use futures::{Stream, StreamExt};
@@ -1244,7 +1244,7 @@ async fn wait_for_source_replication_pending_or_failed(
}
async fn wait_for_source_replication_status(client: &Client, bucket: &str, key: &str, expected: &str, ssec: bool) -> TestResult {
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let wait = async {
loop {
@@ -1339,7 +1339,7 @@ async fn assert_failed_replication_stays_absent_for(
ssec: bool,
duration: Duration,
) -> TestResult {
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let wait = async {
let deadline = tokio::time::Instant::now() + duration;
@@ -4332,7 +4332,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
let target_client = target_env.create_s3_client();
let key = "ssec-contract.txt";
let body = b"repl-17 SSE-C payload";
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
source_client
@@ -4387,7 +4387,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
);
// A wrong customer key must fail too.
let wrong_key = BASE64_STANDARD.encode("99999999999999999999999999999999");
let wrong_key = BASE64_STANDARD.encode_to_string("99999999999999999999999999999999");
let wrong_key_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
let wrong_read = target_client
.get_object()
@@ -4423,7 +4423,7 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult {
let source_client = source_env.create_s3_client();
let target_client = target_env.create_s3_client();
let key = "ssec-mp-contract.bin";
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let created = source_client
@@ -4568,7 +4568,7 @@ async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_header
.await?;
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
let put_ssec = |key: &'static str| {
source_client
@@ -4741,7 +4741,7 @@ async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult
let source_client = source_env.create_s3_client();
let key = "ssec-heal-contract.txt";
let body = b"repl-22 ssec heal payload".to_vec();
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
// Target outage: the SSE-C write cannot replicate.
@@ -4856,7 +4856,7 @@ async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult {
// The SSE-C object exists before any replication wiring.
let key = "ssec-existing-contract.txt";
let body = b"repl-22 ssec existing-object payload".to_vec();
let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY);
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
source_client
.put_object()
@@ -9152,7 +9152,7 @@ async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() ->
// the real SSE-C decryption; the plaintext fake simply ignores them).
target.take_requests();
let ssec_key = "01234567890123456789012345678901";
let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key);
let ssec_key_b64 = BASE64_STANDARD.encode_to_string(ssec_key);
let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key);
let _ = source_client
.get_object()
+2 -3
View File
@@ -21,7 +21,6 @@ use aws_sdk_s3::error::BoxError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine;
use md5::{Digest as Md5Digest, Md5};
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -107,8 +106,8 @@ fn customer_key(byte: u8) -> CustomerKey {
hasher.update(raw);
CustomerKey {
raw: String::from_utf8_lossy(&raw).into_owned(),
encoded: base64::engine::general_purpose::STANDARD.encode(raw),
md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()),
encoded: base64_simd::STANDARD.encode_to_string(raw),
md5: base64_simd::STANDARD.encode_to_string(hasher.finalize()),
}
}
-1
View File
@@ -176,7 +176,6 @@ rmp.workspace = true
rmp-serde.workspace = true
tokio-util = { workspace = true, features = ["io", "compat"] }
tokio-stream = { workspace = true, features = ["sync"] }
base64 = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
hex-simd = { workspace = true }
@@ -200,7 +200,7 @@ impl ReplicationTargetStore {
}
pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) -> Result<(PutObjectOptions, bool)> {
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT};
let mut meta = HashMap::new();
@@ -252,7 +252,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
&& !checksum_data.is_empty()
{
if is_ssec {
let encoded = BASE64_STANDARD.encode(checksum_data);
let encoded = BASE64_STANDARD.encode_to_string(checksum_data);
insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded);
} else if object_info.is_encrypted() {
// Encrypted checksums cannot be exposed as plaintext headers, and
+15 -16
View File
@@ -31,8 +31,7 @@ use crate::storage_api_contracts::internode::{
NS_SCANNER_PROTOCOL_VERSION, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use base64::Engine as _;
use base64::engine::general_purpose;
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
@@ -523,11 +522,11 @@ fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64)
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
mac.update(data.as_bytes());
let result = mac.finalize();
general_purpose::STANDARD.encode(result.into_bytes())
base64_simd::STANDARD.encode_to_string(result.into_bytes())
}
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
@@ -745,11 +744,11 @@ fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io::
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_signature_v2(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
@@ -792,11 +791,11 @@ fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std:
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_scope(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
@@ -821,15 +820,15 @@ fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let proof = general_purpose::STANDARD
.decode(proof)
let proof = base64_simd::STANDARD
.decode_to_vec(proof)
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
@@ -862,7 +861,7 @@ fn generate_replay_cache_capability_proof(
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_replay_cache_capability_proof(
@@ -872,8 +871,8 @@ fn verify_replay_cache_capability_proof(
boot_epoch: Uuid,
proof: &str,
) -> std::io::Result<()> {
let proof = general_purpose::STANDARD
.decode(proof)
let proof = base64_simd::STANDARD
.decode_to_vec(proof)
.map_err(|_| std::io::Error::other("Invalid RPC replay cache capability proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
@@ -1988,9 +1987,9 @@ mod tests {
let method = Method::GET;
let timestamp = 1640995200;
let signature = generate_signature(secret, url, &method, timestamp);
let mut tampered = general_purpose::STANDARD.decode(&signature).unwrap();
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
tampered[0] ^= 1;
let tampered_signature = general_purpose::STANDARD.encode(tampered);
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
assert!(verify_signature(secret, url, &method, timestamp, &signature));
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
+20 -18
View File
@@ -1407,8 +1407,7 @@ fn multipart_part_numbers(parts: &[ObjectPartInfo]) -> Vec<usize> {
#[cfg(test)]
mod tests {
use super::*;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64_simd::STANDARD as BASE64_STANDARD;
use md5::{Digest, Md5};
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER};
use std::collections::HashMap;
@@ -1468,7 +1467,7 @@ mod tests {
request: ReadEncryptionRequest<'_>,
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
if let Some(encoded) = request.metadata.get(TEST_OBJECT_KEY_HEADER) {
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidMetadata, "invalid test object key")
})?;
let key_bytes = decoded.try_into().map_err(|_| {
@@ -1493,7 +1492,7 @@ mod tests {
.map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
})?;
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
})?;
let key_bytes = decoded.try_into().map_err(|_| {
@@ -1505,7 +1504,7 @@ mod tests {
let base_nonce = request
.metadata
.get(TEST_NONCE_HEADER)
.and_then(|encoded| BASE64_STANDARD.decode(encoded).ok())
.and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok())
.and_then(|bytes| bytes.try_into().ok())
.unwrap_or_else(|| fixture_nonce(request.bucket, request.object));
Ok(Some(ReadEncryptionMaterial {
@@ -1533,7 +1532,7 @@ mod tests {
let mut headers = HeaderMap::new();
headers.insert(
TEST_DIRECT_KEY_HEADER,
HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("test key header is valid"),
HeaderValue::from_str(&BASE64_STANDARD.encode_to_string(key_bytes)).expect("test key header is valid"),
);
headers
}
@@ -2439,7 +2438,10 @@ mod tests {
user_defined: Arc::new(HashMap::from([
("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()),
("X-Amz-Server-Side-Encryption-Iv".to_string(), "AAAAAAAAAAAAAAAA".to_string()),
("X-Amz-Server-Side-Encryption-Key".to_string(), BASE64_STANDARD.encode([7_u8; 32])),
(
"X-Amz-Server-Side-Encryption-Key".to_string(),
BASE64_STANDARD.encode_to_string([7_u8; 32]),
),
("x-rustfs-encryption-original-size".to_string(), "64".to_string()),
])),
..Default::default()
@@ -2748,7 +2750,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -2796,11 +2798,11 @@ mod tests {
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -2856,7 +2858,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3008,13 +3010,13 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
total_plaintext.to_string(),
),
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE)),
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode_to_string(LEGACY_FIXTURE_BASE_NONCE)),
])
}
@@ -3751,11 +3753,11 @@ mod tests {
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3821,7 +3823,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3964,11 +3966,11 @@ mod tests {
..Default::default()
}]),
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
+3 -3
View File
@@ -42,7 +42,7 @@ use crate::storage_api_contracts::{
};
use crate::store::ECStore;
use crate::store::utils::is_reserved_or_invalid_bucket;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use futures::future::join_all;
use rand::seq::SliceRandom;
@@ -1398,11 +1398,11 @@ async fn persist_observed_list_objects_mutation(store: Option<&ECStore>, bucket:
}
fn encode_persistent_list_metadata_string(value: &str) -> String {
BASE64_STANDARD.encode(value.as_bytes())
BASE64_STANDARD.encode_to_string(value.as_bytes())
}
fn decode_persistent_list_metadata_string(value: &str) -> Option<String> {
let bytes = BASE64_STANDARD.decode(value).ok()?;
let bytes = BASE64_STANDARD.decode_to_vec(value).ok()?;
String::from_utf8(bytes).ok()
}
+1 -1
View File
@@ -90,7 +90,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos
async-trait = { workspace = true }
futures = { workspace = true }
metrics = { workspace = true }
base64 = { workspace = true }
base64-simd = { workspace = true }
bytes = { workspace = true }
crc-fast = { workspace = true }
sha2 = { workspace = true }
+1 -2
View File
@@ -13,7 +13,6 @@
// limitations under the License.
use crate::{Error, Result};
use base64::Engine as _;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashSet;
@@ -827,7 +826,7 @@ impl CheckpointManager {
}
fn checkpoint_digest(checkpoint_data: &[u8]) -> String {
base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data))
base64_simd::STANDARD.encode_to_string(Sha256::digest(checkpoint_data))
}
fn digest_path(task_id: &str) -> std::path::PathBuf {
+7 -9
View File
@@ -14,8 +14,7 @@
use crate::{Error, Result};
use async_trait::async_trait;
use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use base64_simd::URL_SAFE_NO_PAD;
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
use rustfs_madmin::heal_commands::HealResultItem;
use serde::{Deserialize, Serialize};
@@ -147,7 +146,7 @@ pub(crate) fn encode_heal_token(marker: Option<&str>, version_marker: Option<&st
// serde_json of a simple two-Option struct cannot fail; fall back to an
// empty object rather than panicking if it somehow does.
let json = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{}".to_vec());
format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(json))
format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(json))
}
/// Decode an opaque heal continuation token back into `(marker, version_marker)`.
@@ -174,7 +173,7 @@ pub(crate) fn decode_heal_token(token: &str) -> (Option<String>, Option<String>)
return (None, None);
};
let bytes = match URL_SAFE_NO_PAD.decode(encoded) {
let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) {
Ok(bytes) => bytes,
Err(e) => {
warn!(
@@ -234,7 +233,7 @@ const DISK_WALK_TOKEN_PREFIX: &str = "dw1:";
/// enumerators can never misread each other's cursor: a `dw1:` token decodes to
/// `(None, None)` under the B5 decoder, and a `v1:` token decodes to `None` here.
pub(crate) fn encode_disk_walk_token(next_forward: &str) -> String {
format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(next_forward.as_bytes()))
format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(next_forward.as_bytes()))
}
/// Decode a disk-walk continuation token back into the `next_forward` object key.
@@ -261,7 +260,7 @@ pub(crate) fn decode_disk_walk_token(token: &str) -> Option<String> {
return None;
};
let bytes = match URL_SAFE_NO_PAD.decode(encoded) {
let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) {
Ok(bytes) => bytes,
Err(e) => {
warn!(
@@ -1513,7 +1512,6 @@ mod tests {
decode_disk_walk_token, decode_heal_token, encode_disk_walk_token, encode_heal_token, is_transient_object_exists_error,
is_transient_object_exists_message, next_heal_listing_token,
};
use base64::Engine as _;
#[test]
fn next_heal_listing_token_returns_none_for_complete_page() {
@@ -1564,7 +1562,7 @@ mod tests {
assert_eq!(decode_heal_token("no-prefix-here"), (None, None));
assert_eq!(decode_heal_token("v1:!!!not-base64!!!"), (None, None));
// valid base64 of non-JSON bytes.
let bad_json = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json"));
let bad_json = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(b"not json"));
assert_eq!(decode_heal_token(&bad_json), (None, None));
// a raw v2-style list_objects_v2 token (no "v1:" prefix) resets cleanly.
assert_eq!(decode_heal_token("some-opaque-legacy-token"), (None, None));
@@ -1576,7 +1574,7 @@ mod tests {
// list_object_versions returns NotImplemented for that pairing.
// Craft a token whose JSON encodes (None, Some) directly and confirm coercion.
let json = br#"{"m":null,"v":"orphan-version"}"#;
let token = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json));
let token = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(json));
assert_eq!(decode_heal_token(&token), (None, None), "version-only marker must coerce to (None, None)");
}
+2 -2
View File
@@ -50,8 +50,8 @@ aes-gcm = { workspace = true, features = ["rand_core"] }
argon2 = { workspace = true }
chacha20poly1305 = { workspace = true }
rand = { workspace = true, features = ["serde"] }
base64 = { workspace = true }
hex = { workspace = true }
base64-simd = { workspace = true }
hex-simd = { workspace = true }
sha2 = { workspace = true }
subtle = { workspace = true }
zeroize = { workspace = true, features = ["derive"] }
+6 -2
View File
@@ -23,7 +23,7 @@
//! Exit status is the verdict: 0 when every check held, 1 otherwise, so a
//! scheduled drill fails its job instead of quietly filing a bad report.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use rustfs_kms::backup::{BackupKek, DrillDataset, DrillDisaster, DrillRequest, DrillVerdict, run_local_drill};
use std::path::PathBuf;
use std::process::ExitCode;
@@ -100,7 +100,11 @@ fn disaster_from_env() -> Result<DrillDisaster, String> {
fn kek_from_env() -> Result<BackupKek, String> {
let raw = Zeroizing::new(required(ENV_KEK)?);
let decoded = Zeroizing::new(BASE64.decode(raw.trim()).map_err(|_| format!("{ENV_KEK} must be base64"))?);
let decoded = Zeroizing::new(
BASE64
.decode_to_vec(raw.trim())
.map_err(|_| format!("{ENV_KEK} must be base64"))?,
);
if decoded.len() != 32 {
return Err(format!("{ENV_KEK} must decode to exactly 32 bytes"));
}
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
@@ -69,7 +69,7 @@ async fn run() -> Result<(), String> {
.decrypt_key_material_for_export(&key_id)
.await
.map_err(|error| error.to_string())?;
let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref()));
let encoded = Zeroizing::new(BASE64_STANDARD.encode_to_string(key_material.as_ref()));
let mut stdout = io::stdout().lock();
writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}"))
+1 -1
View File
@@ -305,7 +305,7 @@ pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -
}
fn digest_value(value: &str) -> String {
let digest = hex::encode(Sha256::digest(value.as_bytes()));
let digest = hex_simd::encode_to_string(Sha256::digest(value.as_bytes()), hex_simd::AsciiCase::Lower);
format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
}
+6 -7
View File
@@ -848,8 +848,7 @@ mod tests {
use aws_sdk_kms::config::{BehaviorVersion, Credentials, Region};
use aws_smithy_http_client::test_util::{NeverClient, ReplayEvent, StaticReplayClient};
use aws_smithy_types::body::SdkBody;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use std::sync::atomic::{AtomicU64, Ordering};
/// AWS KMS speaks awsJson1_1; every request goes to `/` on the regional
@@ -977,8 +976,8 @@ mod tests {
let ciphertext = b"encrypted-data-key".to_vec();
let (http_client, backend) = scripted_backend(vec![ok_event(serde_json::json!({
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
"Plaintext": BASE64.encode(&plaintext),
"CiphertextBlob": BASE64.encode(&ciphertext),
"Plaintext": BASE64.encode_to_string(&plaintext),
"CiphertextBlob": BASE64.encode_to_string(&ciphertext),
}))]);
let response = backend
@@ -997,7 +996,7 @@ mod tests {
let plaintext = b"recovered-data-key".to_vec();
let (_http, backend) = scripted_backend(vec![ok_event(serde_json::json!({
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
"Plaintext": BASE64.encode(&plaintext),
"Plaintext": BASE64.encode_to_string(&plaintext),
"EncryptionAlgorithm": "SYMMETRIC_DEFAULT",
}))]);
@@ -1061,8 +1060,8 @@ mod tests {
error_event(400, "ThrottlingException", "rate exceeded"),
ok_event(serde_json::json!({
"KeyId": "test-key",
"Plaintext": BASE64.encode([1u8; 32]),
"CiphertextBlob": BASE64.encode(b"blob"),
"Plaintext": BASE64.encode_to_string([1u8; 32]),
"CiphertextBlob": BASE64.encode_to_string(b"blob"),
})),
]);
+2 -3
View File
@@ -41,8 +41,7 @@ use crate::types::{
DescribeKeyRequest, EncryptRequest, GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext,
RewrapDataKeyRequest,
};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use rand::RngExt as _;
use std::collections::HashMap;
use std::sync::Arc;
@@ -285,7 +284,7 @@ async fn static_backend_stateless_contract() {
let key_id = "static-contract-key";
let mut raw_key = [0u8; 32];
rand::rng().fill(&mut raw_key[..]);
let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key));
let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode_to_string(raw_key));
let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build");
let backend: &dyn KmsBackend = &static_backend;
+11 -7
View File
@@ -33,7 +33,7 @@ use aes_gcm::{
};
use argon2::{Algorithm, Argon2, Params, Version};
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use jiff::Zoned;
use rand::RngExt;
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
@@ -1268,7 +1268,7 @@ impl LocalKmsClient {
}
let encrypted_bytes = BASE64
.decode(&stored_key.encrypted_key_material)
.decode_to_vec(&stored_key.encrypted_key_material)
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?;
let effective_protection = if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified {
@@ -1406,13 +1406,17 @@ impl LocalKmsClient {
.encrypt(&nonce, key_material)
.map_err(|e| KmsError::cryptographic_error("encrypt", e.to_string()))?;
// Encode encrypted bytes to base64 string
(BASE64.encode(&encrypted), nonce.to_vec(), StoredKeyProtection::EncryptedMasterKey)
(
BASE64.encode_to_string(&encrypted),
nonce.to_vec(),
StoredKeyProtection::EncryptedMasterKey,
)
} else {
warn!(
key_id = %master_key.key_id,
"Local KMS is storing key material as plaintext-dev-only because no master key is configured"
);
(BASE64.encode(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly)
(BASE64.encode_to_string(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly)
};
let stored_key = StoredMasterKey {
@@ -2648,10 +2652,10 @@ mod tests {
let tampered_material = {
let mut material = BASE64
.decode(pristine["encrypted_key_material"].as_str().expect("material is a string"))
.decode_to_vec(pristine["encrypted_key_material"].as_str().expect("material is a string"))
.expect("decode pristine material");
*material.last_mut().expect("material is not empty") ^= 0x01;
BASE64.encode(&material)
BASE64.encode_to_string(&material)
};
type PoisonCase = (&'static str, Vec<u8>, fn(&KmsError) -> bool);
@@ -2976,7 +2980,7 @@ mod tests {
"created_at": "2024-01-01T00:00:00+00:00",
"rotated_at": serde_json::Value::Null,
"created_by": "legacy-test",
"encrypted_key_material": BASE64.encode([7u8; 32]),
"encrypted_key_material": BASE64.encode_to_string([7u8; 32]),
"nonce": Vec::<u8>::new()
});
+2 -3
View File
@@ -722,8 +722,7 @@ impl Default for BackendCapabilities {
mod tests {
use super::*;
use crate::config::KmsConfig;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
/// Backend that implements only the trait-mandated operations and relies
/// on the default `capabilities` implementation.
@@ -958,7 +957,7 @@ mod tests {
#[tokio::test]
async fn static_backend_capabilities_golden() {
let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode([0u8; 32]));
let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode_to_string([0u8; 32]));
let backend = static_kms::StaticKmsBackend::new(config)
.await
.expect("static backend should build");
+2 -3
View File
@@ -434,8 +434,7 @@ mod tests {
use crate::backends::KmsBackend as KmsBackendTrait;
use crate::config::{BackendConfig, KmsBackend, StaticConfig};
use crate::encryption::is_data_key_envelope;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
/// Generate a random 32-byte key and return (key_id, raw_key).
fn random_static_key(key_id: &str) -> (String, [u8; 32]) {
@@ -447,7 +446,7 @@ mod tests {
fn static_config(key_id: &str, raw_key: &[u8; 32]) -> StaticConfig {
StaticConfig {
key_id: key_id.to_string(),
secret_key: BASE64.encode(raw_key),
secret_key: BASE64.encode_to_string(raw_key),
}
}
+13 -13
View File
@@ -33,7 +33,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar
use crate::policy::{self, AttemptError, OpClass, RetryPolicy};
use crate::types::*;
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose};
use base64_simd::STANDARD as BASE64;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -527,8 +527,8 @@ fn decode_stored_key_material(key_id: &str, encrypted_material: &str) -> Result<
// Mirrors `decrypt_key_material`: stored material is currently base64 without an
// additional encryption layer.
let key_material = general_purpose::STANDARD
.decode(encrypted_material)
let key_material = BASE64
.decode_to_vec(encrypted_material)
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?;
// Key material must be exactly 32 bytes for AES-256.
@@ -693,7 +693,7 @@ impl VaultKmsClient {
/// confidentiality. Any identity with KV read access to the key path can recover the
/// plaintext master key.
async fn encrypt_key_material(&self, key_material: &[u8]) -> Result<String> {
Ok(general_purpose::STANDARD.encode(key_material))
Ok(base64_simd::STANDARD.encode_to_string(key_material))
}
/// Read the immutable material record of one key version.
@@ -2405,7 +2405,7 @@ mod tests {
tags: HashMap::new(),
deletion_date: None,
rotated_at: None,
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]),
baseline_version: None,
wrap_budget_reserved: 0,
}
@@ -2869,21 +2869,21 @@ mod tests {
));
// Truncated material: valid base64 of fewer than 32 bytes.
let truncated = general_purpose::STANDARD.encode([0x42u8; 16]);
let truncated = base64_simd::STANDARD.encode_to_string([0x42u8; 16]);
assert!(matches!(
decode_stored_key_material("poisoned", &truncated),
Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned"
));
// Oversized material: valid base64 of more than 32 bytes.
let oversized = general_purpose::STANDARD.encode([0x42u8; 33]);
let oversized = base64_simd::STANDARD.encode_to_string([0x42u8; 33]);
assert!(matches!(
decode_stored_key_material("poisoned", &oversized),
Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned"
));
// Well-formed material still decodes.
let valid = general_purpose::STANDARD.encode([0x42u8; 32]);
let valid = base64_simd::STANDARD.encode_to_string([0x42u8; 32]);
assert_eq!(
decode_stored_key_material("healthy", &valid).expect("valid material must decode"),
vec![0x42u8; 32]
@@ -3046,7 +3046,7 @@ mod tests {
description: None,
metadata: HashMap::new(),
tags: HashMap::new(),
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]),
baseline_version: Some(1),
deletion_date: None,
rotated_at: None,
@@ -3760,7 +3760,7 @@ mod tests {
/// Base64 material distinct from `healthy_key_data`'s, standing in for the
/// material a concurrent rotation committed.
fn rotated_material() -> String {
general_purpose::STANDARD.encode([0x43u8; 32])
base64_simd::STANDARD.encode_to_string([0x43u8; 32])
}
/// The issue's lost-update scenario: node A disables a key while node B's
@@ -4358,7 +4358,7 @@ mod tests {
let material_v2 = [0x43u8; 32];
let record_v2 = VaultKeyVersionRecord {
version: 2,
encrypted_key_material: general_purpose::STANDARD.encode(material_v2),
encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2),
created_at: Zoned::now(),
};
// A well-formed envelope wrapped under version 2 — under a reverted
@@ -4900,8 +4900,8 @@ mod tests {
#[tokio::test]
async fn wired_decrypt_of_pre_versioning_envelope_adds_no_request() {
let key_data = healthy_key_data();
let key_material = general_purpose::STANDARD
.decode(&key_data.encrypted_key_material)
let key_material = BASE64
.decode_to_vec(&key_data.encrypted_key_material)
.expect("decode fixture material");
let (encrypted_key, nonce) = AesDekCrypto::new()
.encrypt(&key_material, b"dek-plaintext", &[])
+6 -6
View File
@@ -31,7 +31,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar
use crate::policy::{self, AttemptError, OpClass, RetryPolicy};
use crate::types::*;
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use jiff::Zoned;
use moka::future::Cache;
use serde::{Deserialize, Serialize};
@@ -479,7 +479,7 @@ impl VaultTransitKmsClient {
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
let serialized = serde_json::to_vec(&ordered)?;
Ok(Some(BASE64.encode(serialized)))
Ok(Some(BASE64.encode_to_string(serialized)))
}
fn map_vault_error(key_id: &str, error: vaultrs::error::ClientError, operation: &str) -> KmsError {
@@ -524,7 +524,7 @@ impl VaultTransitKmsClient {
plaintext: &[u8],
encryption_context: &HashMap<String, String>,
) -> Result<String> {
let plaintext_b64 = BASE64.encode(plaintext);
let plaintext_b64 = BASE64.encode_to_string(plaintext);
let plaintext_b64 = plaintext_b64.as_str();
let aad = Self::canonicalize_context(encryption_context)?;
let aad = aad.as_deref();
@@ -568,7 +568,7 @@ impl VaultTransitKmsClient {
.await?;
BASE64
.decode(response.plaintext)
.decode_to_vec(response.plaintext)
.map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string()))
}
@@ -3031,7 +3031,7 @@ mod tests {
ScriptedResponse::ok(kv2_write_ack()),
// decrypt of the pre-rotation envelope; Vault owns the transit
// crypto, so the recovered material is the responder's to hand back.
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })),
])
.await;
@@ -3243,7 +3243,7 @@ mod tests {
// rewrap, context-bound route: latest-version read, then decrypt,
// then re-encrypt under the newest version.
ScriptedResponse::ok(transit_key_read_data_up_to("wired-key", 2)),
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })),
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v2:rewrapped" })),
])
.await;
+2 -2
View File
@@ -945,7 +945,7 @@ async fn tree_digest(root: &Path) -> Result<ContentDigest> {
lines.push(format!(
"{relative}\u{1f}{}\u{1f}{modified}\u{1f}{}",
metadata.len(),
hex::encode(Sha256::digest(&content))
hex_simd::encode_to_string(Sha256::digest(&content), hex_simd::AsciiCase::Lower)
));
}
lines.sort();
@@ -1199,7 +1199,7 @@ mod tests {
let text = String::from_utf8(encoded.clone()).expect("evidence is utf-8");
assert!(!text.contains(DRILL_MASTER_KEY), "the evidence must not carry the master key");
assert!(
!text.contains(&hex::encode([0x37u8; 32])),
!text.contains(&hex_simd::encode_to_string([0x37u8; 32], hex_simd::AsciiCase::Lower)),
"the evidence must not carry backup KEK material"
);
+4 -1
View File
@@ -568,7 +568,10 @@ pub(crate) fn compute_master_key_verifier(master_key: &str, salt: Option<&[u8]>,
let mut hasher = Sha256::new();
hasher.update(&framing);
hasher.update(derived.as_slice());
Ok(format!("{prefix}{}", hex::encode(hasher.finalize())))
Ok(format!(
"{prefix}{}",
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
))
}
/// The bundle-level protection label is the weakest state observed across
+2 -2
View File
@@ -72,7 +72,7 @@ use aes_gcm::{
Aes256Gcm, Nonce,
aead::{Aead, KeyInit},
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use tokio::fs;
@@ -640,7 +640,7 @@ fn decode_key_record(
return Err(BackupError::corrupted(format!("bundled key record '{stem}' carries no key material")).into());
}
let material =
Zeroizing::new(BASE64.decode(&probe.encrypted_key_material).map_err(|error| {
Zeroizing::new(BASE64.decode_to_vec(&probe.encrypted_key_material).map_err(|error| {
BackupError::corrupted(format!("bundled key record '{stem}' material is not valid base64: {error}"))
})?);
if !allowed_modes.contains(&protection_mode(probe.at_rest_protection)) {
+1 -1
View File
@@ -69,7 +69,7 @@ impl ContentDigest {
pub fn sha256_of(bytes: &[u8]) -> Self {
Self {
algorithm: DigestAlgorithm::Sha256,
hex: hex::encode(Sha256::digest(bytes)),
hex: hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower),
}
}
+5 -10
View File
@@ -367,9 +367,8 @@ impl StaticConfig {
/// Decode the base64-encoded secret key into raw bytes.
/// Returns an error if the key is not valid base64 or is not exactly 32 bytes.
pub fn decode_key(&self) -> Result<[u8; 32]> {
use base64::Engine as _;
let bytes = base64::engine::general_purpose::STANDARD
.decode(&self.secret_key)
let bytes = base64_simd::STANDARD
.decode_to_vec(&self.secret_key)
.map_err(|e| KmsError::configuration_error(format!("Static KMS secret key is not valid base64: {e}")))?;
if bytes.len() != 32 {
return Err(KmsError::configuration_error(format!(
@@ -1963,9 +1962,7 @@ mod tests {
#[test]
fn static_kms_config_serialization_does_not_expose_key_material() {
use base64::Engine as _;
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
let config = KmsConfig::static_kms("static-key".to_string(), encoded_key.clone());
let serialized = serde_json::to_string(&config).expect("static KMS config should serialize");
@@ -2569,14 +2566,12 @@ mod tests {
#[test]
fn test_from_env_reads_static_secret_file_and_sets_default_key() {
use base64::Engine as _;
let temp_dir = TempDir::new().expect("create temp dir for static KMS secret");
let secret_path = temp_dir.path().join("static-kms-secret");
// Named `*_key_b64` (not `*_secret`) so the logging-guardrails check does not
// flag these fixture interpolations as secrets leaking into log strings.
let file_key_b64 = base64::engine::general_purpose::STANDARD.encode([7u8; 32]);
let env_key_b64 = base64::engine::general_purpose::STANDARD.encode([9u8; 32]);
let file_key_b64 = base64_simd::STANDARD.encode_to_string([7u8; 32]);
let env_key_b64 = base64_simd::STANDARD.encode_to_string([9u8; 32]);
std::fs::write(&secret_path, format!("file-key:{file_key_b64}\n")).expect("write static KMS secret file");
with_vars(
+5 -4
View File
@@ -46,8 +46,7 @@ use crate::error::{KmsError, Result};
use aes_gcm::aead::{Aead, Payload};
use aes_gcm::{Aes256Gcm, Key, KeyInit, Nonce};
use argon2::{Algorithm, Argon2, Params, Version};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64_simd::STANDARD as BASE64_STANDARD;
use rand::RngExt;
use serde_json::Value;
use sha2::{Digest, Sha256};
@@ -284,14 +283,16 @@ fn seal_value(label: &str, plaintext: &str, secret: &str) -> Result<String> {
payload.extend_from_slice(&salt);
payload.extend_from_slice(&nonce);
payload.extend_from_slice(&ciphertext);
Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode(payload)))
Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode_to_string(payload)))
}
fn open_value(label: &str, sealed: &str, secret: &str) -> Result<String> {
let encoded = sealed
.strip_prefix(SEALED_VALUE_PREFIX)
.expect("caller checks the sealed prefix");
let payload = BASE64_STANDARD.decode(encoded).map_err(|_| sealed_value_unreadable(label))?;
let payload = BASE64_STANDARD
.decode_to_vec(encoded)
.map_err(|_| sealed_value_unreadable(label))?;
if payload.len() <= LOCAL_KMS_MASTER_KEY_SALT_LEN + NONCE_LEN {
return Err(sealed_value_unreadable(label));
}
+7 -3
View File
@@ -675,7 +675,6 @@ mod tests {
use crate::error::KmsError;
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage};
use async_trait::async_trait;
use base64::Engine as _;
use jiff::Zoned;
use std::collections::HashMap;
use std::sync::Mutex;
@@ -1110,8 +1109,13 @@ mod tests {
.await
.expect("enable should succeed");
let base64 = base64::engine::general_purpose::STANDARD;
let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)];
let base64 = base64_simd::STANDARD;
let encodings = |bytes: &[u8]| {
vec![
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower),
base64.encode_to_string(bytes),
]
};
let mut forbidden = vec![grant_token.to_string()];
forbidden.extend(encodings(&data_key.plaintext_key));
forbidden.extend(encodings(&decrypted.plaintext));
+1 -3
View File
@@ -551,7 +551,6 @@ mod tests {
ListKeysRequest, ListKeysResponse,
};
use async_trait::async_trait;
use base64::Engine as _;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use std::future::Future;
@@ -562,8 +561,7 @@ mod tests {
}
async fn static_backend() -> Arc<dyn KmsBackend> {
let config =
KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
let config = KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x42u8; 32]));
Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
}
+10 -14
View File
@@ -23,7 +23,6 @@ use crate::encryption::context_aad;
use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::types::*;
use base64::Engine;
use jiff::Zoned;
use md5::{Digest as Md5Digest, Md5};
use rand::random;
@@ -40,7 +39,7 @@ use zeroize::Zeroize;
fn md5_hex(input: impl AsRef<[u8]>) -> String {
let mut hasher = Md5::new();
hasher.update(input.as_ref());
hex::encode(hasher.finalize())
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)
}
/// Data key for object encryption
@@ -836,19 +835,16 @@ impl ObjectEncryptionService {
// Internal headers for decryption
headers.insert(
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(&metadata.iv),
base64_simd::STANDARD.encode_to_string(&metadata.iv),
);
if let Some(ref tag) = metadata.tag {
headers.insert(
INTERNAL_ENCRYPTION_TAG_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(tag),
);
headers.insert(INTERNAL_ENCRYPTION_TAG_HEADER.to_string(), base64_simd::STANDARD.encode_to_string(tag));
}
headers.insert(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key),
base64_simd::STANDARD.encode_to_string(&metadata.encrypted_data_key),
);
// Whatever the object was sealed under is what gets stored: for a
@@ -906,14 +902,14 @@ impl ObjectEncryptionService {
let iv = headers
.get(INTERNAL_ENCRYPTION_IV_HEADER)
.ok_or_else(|| KmsError::validation_error("Missing IV header"))?;
let iv = base64::engine::general_purpose::STANDARD
.decode(iv)
let iv = base64_simd::STANDARD
.decode_to_vec(iv)
.map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?;
let tag = if let Some(tag_str) = headers.get(INTERNAL_ENCRYPTION_TAG_HEADER) {
Some(
base64::engine::general_purpose::STANDARD
.decode(tag_str)
base64_simd::STANDARD
.decode_to_vec(tag_str)
.map_err(|e| KmsError::validation_error(format!("Invalid tag: {e}")))?,
)
} else {
@@ -921,8 +917,8 @@ impl ObjectEncryptionService {
};
let encrypted_data_key = if let Some(key_str) = headers.get(INTERNAL_ENCRYPTION_KEY_HEADER) {
base64::engine::general_purpose::STANDARD
.decode(key_str)
base64_simd::STANDARD
.decode_to_vec(key_str)
.map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))?
} else {
Vec::new() // Empty for SSE-C
+4 -5
View File
@@ -763,10 +763,10 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
#[cfg(test)]
mod tests {
use super::*;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
fn static_config(key_id: &str, fill: u8) -> KmsConfig {
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode_to_string([fill; 32]))
}
/// End-to-end wiring check for the AWS backend: an admin configure request
@@ -822,7 +822,7 @@ mod tests {
#[tokio::test]
async fn redacted_config_omits_static_key_material() {
let manager = KmsServiceManager::new();
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
manager
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
.await
@@ -1020,7 +1020,6 @@ mod tests {
#[tokio::test]
async fn configure_cannot_replace_existing_local_backend() {
use base64::Engine as _;
use tempfile::TempDir;
let key_dir = TempDir::new().expect("create local KMS directory");
@@ -1029,7 +1028,7 @@ mod tests {
let manager = KmsServiceManager::new();
manager.configure(local.clone()).await.expect("configure local KMS");
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]);
let error = manager
.configure(KmsConfig::static_kms("static-key".to_string(), encoded_key))
.await
+1 -2
View File
@@ -451,6 +451,5 @@ async fn harness_restart_brings_the_service_back_over_the_same_state() {
}
fn base64_of(bytes: &[u8]) -> String {
use base64::Engine as _;
base64::engine::general_purpose::STANDARD.encode(bytes)
base64_simd::STANDARD.encode_to_string(bytes)
}
+1 -1
View File
@@ -667,7 +667,7 @@ async fn sse_c_round_trips_and_rejects_the_wrong_key() {
async fn sse_c_validates_the_supplied_key_md5() {
let (_kms, service) = service_with_key("sse-c-md5-unused").await;
let customer_key = [0x33u8; 32];
let correct_md5 = hex::encode(md5_of(&customer_key));
let correct_md5 = hex_simd::encode_to_string(md5_of(&customer_key), hex_simd::AsciiCase::Lower);
service
.encrypt_object_with_customer_key(BUCKET, "md5-ok.bin", payload(64).as_slice(), &customer_key, Some(&correct_md5))
+2 -3
View File
@@ -34,8 +34,7 @@ use std::future::Future;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use base64_simd::STANDARD as BASE64;
use rustfs_kms::backends::BackendCapabilities;
use rustfs_kms::{
CreateKeyRequest, DeleteKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus,
@@ -51,7 +50,7 @@ pub const STATIC_KEY_ID: &str = "behavior-static-key";
/// Fixed rather than random so a failure is reproducible; it is test-only
/// material and never leaves this crate's test binaries.
pub fn static_secret_key() -> String {
BASE64.encode([0x5au8; 32])
BASE64.encode_to_string([0x5au8; 32])
}
/// Which backend a harness instance is running.
+5 -5
View File
@@ -99,14 +99,14 @@ swift = [
"dep:md-5",
"dep:hmac",
"dep:sha1",
"dep:hex",
"dep:hex-simd",
"dep:ipnetwork",
"dep:rustfs-trusted-proxies",
"dep:astral-tokio-tar",
"dep:base64",
"dep:base64-simd",
"dep:async-compression",
]
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64-simd", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"]
sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"]
[dependencies]
@@ -163,11 +163,11 @@ urlencoding = { workspace = true, optional = true }
md-5 = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
sha1 = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
hex-simd = { workspace = true, optional = true }
ipnetwork = { workspace = true, optional = true }
rustfs-trusted-proxies = { workspace = true, optional = true }
astral-tokio-tar = { workspace = true, optional = true }
base64 = { workspace = true, optional = true }
base64-simd = { workspace = true, optional = true }
async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] }
# WebDAV specific dependencies (optional)
+1 -1
View File
@@ -97,7 +97,7 @@ fn get_account_metadata_bucket_name(account: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(account.as_bytes());
let hash_bytes = hasher.finalize();
let hash = hex::encode(hash_bytes);
let hash = hex_simd::encode_to_string(hash_bytes, hex_simd::AsciiCase::Lower);
format!("swift-account-{}", &hash[0..16])
}
+4 -4
View File
@@ -182,7 +182,7 @@ pub fn generate_signature(
mac.update(message.as_bytes());
let result = mac.finalize();
let signature = hex::encode(result.into_bytes());
let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower);
Ok(signature)
}
@@ -213,10 +213,10 @@ pub fn validate_formpost(path: &str, request: &FormPostRequest, key: &str) -> Sw
// the sibling TempURL/SFTP checks. Decode the hex first so the comparison runs
// over the raw HMAC bytes and does not leak via string length; a non-hex
// provided signature can never match and is rejected the same way.
let expected_bytes =
hex::decode(&expected_sig).map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?;
let expected_bytes = hex_simd::decode_to_vec(&expected_sig)
.map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?;
let signatures_match = match hex::decode(request.signature.trim()) {
let signatures_match = match hex_simd::decode_to_vec(request.signature.trim()) {
Ok(provided_bytes) => super::tempurl::constant_time_compare(&provided_bytes, &expected_bytes),
Err(_) => false,
};
+5 -1
View File
@@ -84,7 +84,11 @@ impl SLOManifest {
let mut hasher = Md5::new();
hasher.update(etag_concat.as_bytes());
format!("\"{}-{}\"", hex::encode(hasher.finalize()), self.segments.len())
format!(
"\"{}-{}\"",
hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower),
self.segments.len()
)
}
/// Validate manifest against actual segments
+1 -1
View File
@@ -300,7 +300,7 @@ pub fn generate_sync_signature(path: &str, key: &str) -> SwiftResult<String> {
mac.update(path.as_bytes());
let result = mac.finalize();
Ok(hex::encode(result.into_bytes()))
Ok(hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower))
}
/// Verify sync signature
+1 -1
View File
@@ -125,7 +125,7 @@ impl TempURL {
// Hex-encode result
let result = mac.finalize();
let signature = hex::encode(result.into_bytes());
let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower);
Ok(signature)
}
+1 -2
View File
@@ -675,8 +675,7 @@ fn fixed_body(message: impl Into<Bytes>) -> WebDavBody {
/// Decode base64 string
fn base64_decode(encoded: &str) -> Result<Vec<u8>, ()> {
use base64::Engine;
base64::engine::general_purpose::STANDARD.decode(encoded).map_err(|_| ())
base64_simd::STANDARD.decode_to_vec(encoded).map_err(|_| ())
}
#[cfg(test)]
@@ -56,7 +56,7 @@ fn keystone_credentials(project_id: &str) -> Credentials {
fn account_metadata_bucket_name(account: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(account.as_bytes());
let hash = hex::encode(hasher.finalize());
let hash = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
format!("swift-account-{}", &hash[0..16])
}
+1 -1
View File
@@ -57,7 +57,7 @@ hotpath.workspace = true
aes-gcm = { workspace = true, features = ["rand_core"] }
bytes = { workspace = true, features = ["serde"] }
chacha20poly1305.workspace = true
hex.workspace = true
hex-simd.workspace = true
hmac.workspace = true
minlz.workspace = true
pin-project-lite.workspace = true
+4 -1
View File
@@ -731,12 +731,15 @@ fn random_stream_nonce() -> [u8; 12] {
#[cfg(test)]
mod tests {
use super::*;
use hex::encode as hex_encode;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE;
fn hex_encode(data: impl AsRef<[u8]>) -> String {
hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
}
#[tokio::test]
async fn decrypt_reader_can_start_from_non_zero_sequence_number() {
let plaintext = vec![0xAB; DARE_PAYLOAD_SIZE * 2 + 19];
+1 -1
View File
@@ -81,7 +81,7 @@ serde_json = { workspace = true, features = ["raw_value"] }
md-5 = { workspace = true }
tracing.workspace = true
thiserror.workspace = true
base64.workspace = true
base64-simd.workspace = true
sha1.workspace = true
sha2.workspace = true
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
+14 -11
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::errors::ChecksumMismatch;
use base64::{Engine as _, engine::general_purpose};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use http::HeaderMap;
use sha1::Sha1;
@@ -330,7 +330,7 @@ impl Checksum {
let mut hasher = checksum_type.hasher()?;
hasher.write_all(data).ok()?;
let raw = hasher.finalize();
let encoded = general_purpose::STANDARD.encode(&raw);
let encoded = BASE64_STANDARD.encode_to_string(&raw);
let checksum = Checksum {
checksum_type,
@@ -369,7 +369,7 @@ impl Checksum {
value_string = value.to_string();
}
// let raw = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(&value_string).ok()?;
let raw = general_purpose::STANDARD.decode(&value_string).ok()?;
let raw = BASE64_STANDARD.decode_to_vec(&value_string).ok()?;
let checksum = Checksum {
checksum_type,
@@ -413,14 +413,14 @@ impl Checksum {
if self.want_parts > 0 && self.want_parts != parts {
return Err(ChecksumMismatch {
want: format!("{}-{}", self.encoded, self.want_parts),
got: format!("{}-{}", general_purpose::STANDARD.encode(&sum), parts),
got: format!("{}-{}", base64_simd::STANDARD.encode_to_string(&sum), parts),
});
}
if sum != self.raw {
return Err(ChecksumMismatch {
want: self.encoded.clone(),
got: general_purpose::STANDARD.encode(&sum),
got: base64_simd::STANDARD.encode_to_string(&sum),
});
}
@@ -569,7 +569,7 @@ impl Checksum {
}
}
self.encoded = general_purpose::STANDARD.encode(&self.raw);
self.encoded = base64_simd::STANDARD.encode_to_string(&self.raw);
Ok(())
}
}
@@ -1160,7 +1160,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
let checksum_bytes = &buf[..length];
buf = &buf[length..];
let mut checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
let mut checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
if checksum_type.is(ChecksumType::MULTIPART) {
is_multipart = true;
@@ -1190,7 +1190,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
if part > 0 && (part as u64) <= parts_count {
let offset = ((part - 1) as usize) * length;
let part_checksum = &buf[offset..offset + length];
checksum_str = general_purpose::STANDARD.encode(part_checksum);
checksum_str = base64_simd::STANDARD.encode_to_string(part_checksum);
}
buf = &buf[want_len..];
}
@@ -1249,7 +1249,7 @@ pub fn read_part_checksums(mut buf: &[u8]) -> Vec<HashMap<String, String>> {
let checksum_bytes = &buf[..length];
buf = &buf[length..];
let checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
let checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
part_checksum.insert(checksum_type.to_string(), checksum_str);
}
@@ -1553,7 +1553,6 @@ mod tests {
// asserted alongside the raw hex so both the digest and its encoding are pinned.
#[test]
fn xxhash_sha512_regression_lock_non_empty() {
use base64::{Engine as _, engine::general_purpose::STANDARD};
let data = b"The quick brown fox jumps over the lazy dog";
// XXH3-64(fox) = 0xce7d19a5418fb365 is the official upstream vector.
@@ -1565,7 +1564,11 @@ mod tests {
let got_hex: String = c.raw.iter().map(|b| format!("{b:02x}")).collect();
assert_eq!(got_hex, want_hex, "{t:?} raw hex drifted");
// encoded field must be the standard-base64 of raw (S3 wire form)
assert_eq!(c.encoded, STANDARD.encode(&c.raw), "{t:?} encoded field != base64(raw)");
assert_eq!(
c.encoded,
base64_simd::STANDARD.encode_to_string(&c.raw),
"{t:?} encoded field != base64(raw)"
);
}
}
+4 -5
View File
@@ -91,8 +91,7 @@ use crate::Sha256Hasher;
use crate::compress_index::{Index, TryGetIndex};
use crate::get_content_checksum;
use crate::{DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, boxed_reader, wrap_reader};
use base64::Engine;
use base64::engine::general_purpose;
use http::HeaderMap;
use pin_project_lite::pin_project;
use s3s::TrailingHeaders;
@@ -582,8 +581,8 @@ impl AsyncRead for HashReader {
})
{
expected_content_hash.encoded = checksum_str;
expected_content_hash.raw = general_purpose::STANDARD
.decode(&expected_content_hash.encoded)
expected_content_hash.raw = base64_simd::STANDARD
.decode_to_vec(&expected_content_hash.encoded)
.map_err(|_| std::io::Error::other("Invalid base64 checksum"))?;
if expected_content_hash.raw.is_empty() {
@@ -598,7 +597,7 @@ impl AsyncRead for HashReader {
&& !expected_content_hash.checksum_type.trailing()
{
expected_content_hash.raw = content_hash;
expected_content_hash.encoded = general_purpose::STANDARD.encode(&expected_content_hash.raw);
expected_content_hash.encoded = base64_simd::STANDARD.encode_to_string(&expected_content_hash.raw);
} else if content_hash != expected_content_hash.raw {
let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower);
let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower);
-1
View File
@@ -310,7 +310,6 @@ astral-tokio-tar = { workspace = true }
atoi = { workspace = true }
atomic_enum = { workspace = true }
async_zip = { workspace = true, default-features = false, features = ["tokio", "deflate"] }
base64 = { workspace = true }
zeroize = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
+2 -3
View File
@@ -439,7 +439,6 @@ fn dispatch(entry: AuditEntry) {
#[cfg(test)]
mod tests {
use super::*;
use base64::Engine;
use rustfs_kms::backends::local::LocalKmsBackend;
use rustfs_kms::config::KmsConfig;
use rustfs_kms::types::{CreateKeyRequest, DeleteKeyRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec};
@@ -689,8 +688,8 @@ mod tests {
.expect("data key should be generated");
// What the endpoint hands back, and therefore what must not reappear.
let plaintext_b64 = base64::prelude::BASE64_STANDARD.encode(&response.plaintext_key);
let ciphertext_b64 = base64::prelude::BASE64_STANDARD.encode(&response.ciphertext_blob);
let plaintext_b64 = base64_simd::STANDARD.encode_to_string(&response.plaintext_key);
let ciphertext_b64 = base64_simd::STANDARD.encode_to_string(&response.ciphertext_blob);
assert!(!response.plaintext_key.is_empty(), "the test must drive real key material");
let redacted = rustfs_kms::redact_encryption_context(&std::collections::HashMap::from([
+9 -7
View File
@@ -46,7 +46,7 @@ use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::runtime_sources::{current_deployment_id, current_kms_runtime_service_manager};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, RemoteAddr};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use base64_simd::STANDARD as BASE64;
use hyper::{HeaderMap, Method, StatusCode};
use matchit::Params;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
@@ -429,7 +429,7 @@ impl BackupEnvironment {
let decoded = Zeroizing::new(
BASE64
.decode(raw_kek.trim())
.decode_to_vec(raw_kek.trim())
.map_err(|_| (StatusCode::PRECONDITION_FAILED, format!("{ENV_KMS_BACKUP_KEK} must be base64-encoded")))?,
);
if decoded.len() != 32 {
@@ -516,7 +516,9 @@ fn reuses_business_secret(kek_material: &[u8], raw_kek: &str, config: &KmsConfig
if raw_kek == secret.as_str() || secret.as_bytes() == kek_material {
return true;
}
BASE64.decode(secret.as_str()).is_ok_and(|decoded| decoded == kek_material)
BASE64
.decode_to_vec(secret.as_str())
.is_ok_and(|decoded| decoded == kek_material)
})
}
@@ -1185,7 +1187,7 @@ mod tests {
const DEPLOYMENT: &str = "deployment-under-test";
fn test_kek_bytes() -> Vec<u8> {
BASE64.decode(TEST_KEK_B64).expect("test KEK must decode")
BASE64.decode_to_vec(TEST_KEK_B64).expect("test KEK must decode")
}
fn local_config(key_dir: PathBuf) -> KmsConfig {
@@ -1310,7 +1312,7 @@ mod tests {
.expect_err("an empty KEK must be refused");
assert_eq!(error.0, StatusCode::PRECONDITION_FAILED);
let short = BASE64.encode([0x11; 16]);
let short = BASE64.encode_to_string([0x11; 16]);
let error = BackupEnvironment::build(PathBuf::from("/tmp/root"), &short, "kek".to_string(), 1, &config)
.expect_err("a KEK that is not 32 bytes must be refused");
assert_eq!(error.0, StatusCode::PRECONDITION_FAILED);
@@ -1353,7 +1355,7 @@ mod tests {
);
// An unrelated KEK is accepted.
let independent = BASE64.encode([0x5a; 32]);
let independent = BASE64.encode_to_string([0x5a; 32]);
assert!(BackupEnvironment::build(PathBuf::from("/tmp/root"), &independent, "kek".to_string(), 1, &config).is_ok());
}
@@ -1372,7 +1374,7 @@ mod tests {
let master_key = "local-master-key-super-secret";
let vault_token = "hvs.vault-token-super-secret";
let approle_secret = "approle-secret-id-super-secret";
let static_key = BASE64.encode([0x7c; 32]);
let static_key = BASE64.encode_to_string([0x7c; 32]);
let local = local_config_with_master_key(PathBuf::from("/var/lib/rustfs/kms"), master_key);
let kv2 = vault_kv2_config(VaultAuthMethod::Token {
+2 -6
View File
@@ -1422,12 +1422,8 @@ mod tests {
#[test]
fn static_kms_config_is_not_persisted_with_cluster_configuration() {
use base64::Engine as _;
let config = rustfs_kms::KmsConfig::static_kms(
"static-key".to_string(),
base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]),
);
let config =
rustfs_kms::KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x5au8; 32]));
assert!(ensure_kms_config_persistable(&config).is_err());
}
@@ -408,7 +408,6 @@ impl Operation for UntagKmsKeyHandler {
mod tests {
use super::*;
use crate::admin::handlers::kms_keys::stable_json_value;
use base64::Engine as _;
use rustfs_kms::KmsManager;
use rustfs_kms::backends::local::LocalKmsBackend;
use rustfs_kms::backends::static_kms::StaticKmsBackend;
@@ -431,8 +430,7 @@ mod tests {
/// is the backend that must answer every metadata update with a capability
/// gap rather than a failure of the request.
async fn static_service() -> ObjectEncryptionService {
let config =
KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
let config = KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x42u8; 32]));
let backend = Arc::new(
StaticKmsBackend::new(config.clone())
.await
+2 -3
View File
@@ -21,7 +21,6 @@ use crate::admin::runtime_sources::{current_kms_runtime_service_manager, current
use crate::auth::{check_key_valid, get_session_token};
use crate::kms_deletion_gate::current_key_impact;
use crate::server::{ADMIN_PREFIX, RemoteAddr};
use base64::Engine;
use hyper::{HeaderMap, Method, StatusCode};
use matchit::Params;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
@@ -1535,8 +1534,8 @@ impl Operation for GenerateDataKeyHandler {
Ok(response) => {
let api_response = GenerateDataKeyApiResponse {
key_id: response.key_id,
plaintext_key: base64::prelude::BASE64_STANDARD.encode(&response.plaintext_key),
ciphertext_blob: base64::prelude::BASE64_STANDARD.encode(&response.ciphertext_blob),
plaintext_key: base64_simd::STANDARD.encode_to_string(&response.plaintext_key),
ciphertext_blob: base64_simd::STANDARD.encode_to_string(&response.ciphertext_blob),
};
let data = serde_json::to_vec(&api_response)
+21 -22
View File
@@ -56,9 +56,8 @@ use crate::storage::storage_api::{
delete_config_no_lock, lock_bucket_targets_metadata, read_config_no_lock, save_config_no_lock, with_config_object_read_lock,
with_config_object_write_lock,
};
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use base64_simd::STANDARD as BASE64_STANDARD;
use base64_simd::URL_SAFE_NO_PAD;
use futures::StreamExt;
use hmac::{Hmac, Mac};
use http::header::{CONTENT_TYPE, HOST};
@@ -1645,7 +1644,7 @@ fn hash_client_secret(secret: Option<&str>) -> String {
let mut hasher = Sha256::new();
hasher.update(secret.as_bytes());
URL_SAFE_NO_PAD.encode(hasher.finalize())
URL_SAFE_NO_PAD.encode_to_string(hasher.finalize())
}
fn config_enabled(value: Option<String>) -> bool {
@@ -3775,7 +3774,7 @@ impl SiteReplicationRepairTask<'_> {
digest.update(self.path().as_bytes());
digest.update([0]);
digest.update(payload);
Ok(URL_SAFE_NO_PAD.encode(digest.finalize()))
Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize()))
}
async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result<Vec<u8>> {
@@ -3862,7 +3861,7 @@ fn site_replication_repair_plan_token(state: &SiteReplicationState, plan: &SiteR
for (_, task) in site_replication_repair_tasks(plan) {
digest.update(task.id()?.as_bytes());
}
Ok(URL_SAFE_NO_PAD.encode(digest.finalize()))
Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize()))
}
fn site_replication_repair_preflight_token(
@@ -3892,7 +3891,7 @@ fn site_replication_repair_preflight_token(
digest.update(event.path.as_bytes());
digest.update(&[0]);
}
Ok(URL_SAFE_NO_PAD.encode(digest.finalize().into_bytes()))
Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes()))
}
fn site_replication_repair_task_checkpoint_id(
@@ -3906,7 +3905,7 @@ fn site_replication_repair_task_checkpoint_id(
digest.update(peer_deployment_id.as_bytes());
digest.update(&[0]);
digest.update(task.id()?.as_bytes());
Ok(URL_SAFE_NO_PAD.encode(digest.finalize().into_bytes()))
Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes()))
}
fn site_replication_repair_sites(
@@ -4488,11 +4487,11 @@ fn raw_config_to_string(raw: &[u8]) -> Option<String> {
}
fn raw_config_to_base64(raw: &[u8]) -> Option<String> {
(!raw.is_empty()).then(|| BASE64_STANDARD.encode(raw))
(!raw.is_empty()).then(|| BASE64_STANDARD.encode_to_string(raw))
}
fn encode_bucket_meta_wire_value(value: Option<String>) -> Option<String> {
value.map(|raw| BASE64_STANDARD.encode(raw.as_bytes()))
value.map(|raw| BASE64_STANDARD.encode_to_string(raw.as_bytes()))
}
fn encode_bucket_meta_wire_item(mut item: SRBucketMeta) -> SRBucketMeta {
@@ -4508,7 +4507,7 @@ fn encode_bucket_meta_wire_item(mut item: SRBucketMeta) -> SRBucketMeta {
fn decode_bucket_meta_wire_value(raw: &str) -> Vec<u8> {
BASE64_STANDARD
.decode(raw.as_bytes())
.decode_to_vec(raw.as_bytes())
.ok()
.filter(|decoded| std::str::from_utf8(decoded).is_ok())
.unwrap_or_else(|| raw.as_bytes().to_vec())
@@ -7717,7 +7716,7 @@ fn site_resync_page(status: &SRResyncOpStatus, limit: usize, offset: usize) -> S
};
let encoded = serde_json::to_vec(&token)
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("encode resync cursor failed: {err}")))?;
URL_SAFE_NO_PAD.encode(encoded)
URL_SAFE_NO_PAD.encode_to_string(encoded)
} else {
String::new()
};
@@ -7736,7 +7735,7 @@ fn parse_site_resync_page(query: &HashMap<String, String>, status: &SRResyncOpSt
}
let offset = if let Some(value) = query.get("continuationToken") {
let decoded = URL_SAFE_NO_PAD
.decode(value)
.decode_to_vec(value)
.map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?;
let token: SiteResyncContinuationToken =
serde_json::from_slice(&decoded).map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?;
@@ -14098,7 +14097,7 @@ mod tests {
let bucket = SRBucketInfo {
bucket: "photos".to_string(),
created_at: Some(OffsetDateTime::UNIX_EPOCH),
object_lock_config: Some(BASE64_STANDARD.encode("<ObjectLockConfiguration/>")),
object_lock_config: Some(BASE64_STANDARD.encode_to_string("<ObjectLockConfiguration/>")),
..Default::default()
};
let bootstrap = bootstrap_bucket_make_op_path(&bucket);
@@ -14625,10 +14624,10 @@ mod tests {
SRBucketInfo {
bucket: "photos".to_string(),
policy: Some(serde_json::json!({"Statement": []})),
versioning: Some(BASE64_STANDARD.encode("<VersioningConfiguration/>")),
quota_config: Some(BASE64_STANDARD.encode(r#"{"quota":1024}"#)),
expiry_lc_config: Some(BASE64_STANDARD.encode("<LifecycleConfiguration/>")),
object_lock_config: Some(BASE64_STANDARD.encode("<ObjectLockConfiguration/>")),
versioning: Some(BASE64_STANDARD.encode_to_string("<VersioningConfiguration/>")),
quota_config: Some(BASE64_STANDARD.encode_to_string(r#"{"quota":1024}"#)),
expiry_lc_config: Some(BASE64_STANDARD.encode_to_string("<LifecycleConfiguration/>")),
object_lock_config: Some(BASE64_STANDARD.encode_to_string("<ObjectLockConfiguration/>")),
created_at: Some(OffsetDateTime::UNIX_EPOCH),
api_version: Some(SITE_REPL_API_VERSION.to_string()),
..Default::default()
@@ -14667,7 +14666,7 @@ mod tests {
"photos".to_string(),
SRBucketInfo {
bucket: "photos".to_string(),
expiry_lc_config: Some(BASE64_STANDARD.encode("<LifecycleConfiguration/>")),
expiry_lc_config: Some(BASE64_STANDARD.encode_to_string("<LifecycleConfiguration/>")),
api_version: Some(SITE_REPL_API_VERSION.to_string()),
..Default::default()
},
@@ -17637,7 +17636,7 @@ mod tests {
fn test_metainfo_bucket_config_values_are_base64_encoded() {
let raw = br#"<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>"#;
assert_eq!(raw_config_to_base64(raw), Some(BASE64_STANDARD.encode(raw)));
assert_eq!(raw_config_to_base64(raw), Some(BASE64_STANDARD.encode_to_string(raw)));
assert_ne!(raw_config_to_base64(raw), raw_config_to_string(raw));
assert_eq!(raw_config_to_base64(&[]), None);
}
@@ -18182,8 +18181,8 @@ mod tests {
};
let dep_a_xml = site_config_xml("dep-b");
let dep_b_xml = site_config_xml("dep-a");
let dep_a_b64 = BASE64_STANDARD.encode(dep_a_xml.as_bytes());
let dep_b_b64 = BASE64_STANDARD.encode(dep_b_xml.as_bytes());
let dep_a_b64 = BASE64_STANDARD.encode_to_string(dep_a_xml.as_bytes());
let dep_b_b64 = BASE64_STANDARD.encode_to_string(dep_b_xml.as_bytes());
// Both sites present the complete config in base64 wire form → NOT a mismatch.
assert_eq!(
+2 -3
View File
@@ -14,7 +14,6 @@
use std::time::Duration;
use base64::Engine as _;
use chrono::{DateTime, Utc};
use reqwest::{Client, StatusCode, Url, header};
use rustls::RootCertStore;
@@ -228,8 +227,8 @@ impl ConnectClient {
pending: &PendingRegistration,
identity: &super::identity::DeviceIdentity,
) -> Result<DeviceCredential, ClientError> {
let csr_der = base64::engine::general_purpose::STANDARD
.decode(&pending.certificate_request)
let csr_der = base64_simd::STANDARD
.decode_to_vec(&pending.certificate_request)
.map_err(|_| ClientError::PendingRegistration)?;
let transcript = RegistrationTranscript::build(
&token.registration_token_uid,
+7 -8
View File
@@ -20,8 +20,7 @@
//! module produces, so any divergence is a protocol break rather than a
//! local behaviour change.
use base64::Engine as _;
use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
use base64_simd::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
use p256::ecdsa::signature::{Signer as _, Verifier as _};
use p256::ecdsa::{Signature, SigningKey};
use p256::elliptic_curve::Generate as _;
@@ -112,7 +111,7 @@ impl RegistrationTranscript {
}
let expiry = expires_unix.to_string();
let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(certificate_request));
let csr_digest = BASE64_URL_NO_PAD.encode_to_string(Sha256::digest(certificate_request));
let fields: [(&'static str, &str); FIELD_COUNT] = [
("registrationTokenUid", registration_token_uid),
@@ -238,7 +237,7 @@ impl DeviceIdentity {
/// Standard padded base64 of the certificate request, as the body carries it.
pub fn certificate_request_base64(&self) -> Result<String, IdentityError> {
Ok(BASE64_STANDARD.encode(self.certificate_request_der()?))
Ok(BASE64_STANDARD.encode_to_string(self.certificate_request_der()?))
}
/// Sign a transcript, producing the low-S fixed-width proof.
@@ -252,20 +251,20 @@ impl DeviceIdentity {
RegistrationProof {
algorithm: PROOF_ALGORITHM.to_string(),
value: BASE64_URL_NO_PAD.encode(canonical.to_bytes()),
value: BASE64_URL_NO_PAD.encode_to_string(canonical.to_bytes()),
}
}
pub(crate) fn sign_pending_registration_state(&self, state: &[u8]) -> String {
let signature: Signature = self.signing_key.sign(state);
BASE64_URL_NO_PAD.encode(signature.normalize_s().to_bytes())
BASE64_URL_NO_PAD.encode_to_string(signature.normalize_s().to_bytes())
}
pub(crate) fn verifies_pending_registration_state(&self, state: &[u8], proof: &str) -> bool {
let Ok(octets) = BASE64_URL_NO_PAD.decode(proof) else {
let Ok(octets) = BASE64_URL_NO_PAD.decode_to_vec(proof) else {
return false;
};
if BASE64_URL_NO_PAD.encode(&octets) != proof {
if BASE64_URL_NO_PAD.encode_to_string(&octets) != proof {
return false;
}
let Ok(signature) = Signature::from_slice(&octets) else {
+3 -3
View File
@@ -25,7 +25,7 @@ use std::path::Path;
use std::path::PathBuf;
#[cfg(target_os = "linux")]
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use base64_simd::URL_SAFE_NO_PAD;
#[cfg(target_os = "linux")]
use p256::ecdsa::{Signature, SigningKey, signature::Signer as _};
#[cfg(target_os = "linux")]
@@ -163,7 +163,7 @@ fn write_offline_bundle_unix(
let produced_at = OffsetDateTime::from_unix_timestamp(context.produced_at_unix).map_err(|_| BundleError::InvalidMetadata)?;
let produced_at = produced_at.format(&Rfc3339).map_err(|_| BundleError::InvalidMetadata)?;
let nonce = URL_SAFE_NO_PAD.encode(context.nonce);
let nonce = URL_SAFE_NO_PAD.encode_to_string(context.nonce);
let device_key_id = hex_lower(&Sha256::digest(key.public_key_der()));
let manifest_entries = entries
.iter()
@@ -397,7 +397,7 @@ fn sign(key: &DeviceIdentity, manifest: &[u8]) -> Result<String, BundleError> {
input.push(0);
input.extend_from_slice(manifest);
let signature: Signature = signing_key.sign(&input);
Ok(URL_SAFE_NO_PAD.encode(signature.normalize_s().to_bytes()))
Ok(URL_SAFE_NO_PAD.encode_to_string(signature.normalize_s().to_bytes()))
}
#[cfg(target_os = "linux")]
+9 -10
View File
@@ -32,8 +32,7 @@
//! are frozen beside it. Reordering the checks changes which reason a given
//! artifact produces, which is itself part of the contract.
use base64::Engine as _;
use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
use base64_simd::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
use p256::ecdsa::signature::{Signer as _, Verifier as _};
use p256::ecdsa::{Signature, SigningKey, VerifyingKey};
use p256::pkcs8::DecodePrivateKey as _;
@@ -367,7 +366,7 @@ impl OfflineEnrollment {
// The octets that were transmitted. They are never re-serialised: every
// later step signs and parses this same buffer.
let bytes = BASE64_STANDARD
.decode(envelope.bytes.as_bytes())
.decode_to_vec(envelope.bytes.as_bytes())
.map_err(|_| EnrollmentError::MalformedDocument)?;
// Step 2: routing only.
@@ -440,8 +439,8 @@ impl OfflineEnrollment {
challenge_nonce: &challenge.nonce,
challenge_proof: &challenge.challenge_proof,
device_key_id: key_id(&point),
device_public_key: BASE64_URL_NO_PAD.encode(point),
device_nonce: BASE64_URL_NO_PAD.encode(device_nonce),
device_public_key: BASE64_URL_NO_PAD.encode_to_string(point),
device_nonce: BASE64_URL_NO_PAD.encode_to_string(device_nonce),
produced_at,
};
@@ -451,7 +450,7 @@ impl OfflineEnrollment {
let signature = sign(key, TAG_RESPONSE, &bytes)?;
let envelope = SignedDocument {
bytes: BASE64_STANDARD.encode(&bytes),
bytes: BASE64_STANDARD.encode_to_string(&bytes),
signature: DocumentSignature {
algorithm: SIGNATURE_ALGORITHM.to_owned(),
key_id: document.device_key_id,
@@ -537,7 +536,7 @@ fn verify_trust_chain(
/// checked against these, never against a re-encoding of the parsed link.
fn decode_trust_link(entry: &SignedDocument) -> Result<(TrustLink, Vec<u8>), EnrollmentError> {
let bytes = BASE64_STANDARD
.decode(entry.bytes.as_bytes())
.decode_to_vec(entry.bytes.as_bytes())
.map_err(|_| EnrollmentError::MalformedDocument)?;
let link = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::TrustChainInvalid)?;
Ok((link, bytes))
@@ -559,7 +558,7 @@ fn decode_signature(signature: &DocumentSignature) -> Result<Signature, Enrollme
}
let decoded = BASE64_URL_NO_PAD
.decode(value)
.decode_to_vec(value)
.map_err(|_| EnrollmentError::SignatureMalformed)?;
let octets: [u8; SIGNATURE_OCTETS] = decoded
.as_slice()
@@ -602,7 +601,7 @@ fn sign(key: &DeviceIdentity, tag: &[u8], bytes: &[u8]) -> Result<String, Enroll
let signature: Signature = signing_key.sign(&signature_input(tag, bytes));
let canonical = signature.normalize_s();
Ok(BASE64_URL_NO_PAD.encode(canonical.to_bytes()))
Ok(BASE64_URL_NO_PAD.encode_to_string(canonical.to_bytes()))
}
/// The device's public point, recovered from the DER encoding the identity
@@ -627,7 +626,7 @@ fn decode_public_key(value: &str) -> Option<(VerifyingKey, [u8; PUBLIC_KEY_OCTET
return None;
}
let point: [u8; PUBLIC_KEY_OCTETS] = BASE64_URL_NO_PAD.decode(value).ok()?.try_into().ok()?;
let point: [u8; PUBLIC_KEY_OCTETS] = BASE64_URL_NO_PAD.decode_to_vec(value).ok()?.try_into().ok()?;
if point[0] != UNCOMPRESSED_POINT {
return None;
}
+9 -10
View File
@@ -15,8 +15,7 @@
use std::io::Read;
use std::sync::Arc;
use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
use base64_simd::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
use p256::ecdsa::signature::Signer as _;
use p256::ecdsa::{Signature, SigningKey};
use p256::pkcs8::DecodePrivateKey as _;
@@ -89,10 +88,10 @@ impl RegistrationToken {
}
let document: RegistrationTokenDocument = serde_json::from_slice(&bytes).map_err(TokenError::Invalid)?;
let decoded = BASE64_URL_NO_PAD
.decode(&document.registration_token_secret)
.decode_to_vec(&document.registration_token_secret)
.map(Zeroizing::new)
.map_err(|_| TokenError::SecretShape)?;
if decoded.len() != 32 || BASE64_URL_NO_PAD.encode(&decoded) != document.registration_token_secret {
if decoded.len() != 32 || BASE64_URL_NO_PAD.encode_to_string(&decoded) != document.registration_token_secret {
return Err(TokenError::SecretShape);
}
if !is_uuid_v7(&document.registration_token_uid)
@@ -198,10 +197,10 @@ impl<'a> RotationRequest<'a> {
request_id: &'a str,
certificate_request: &'a str,
) -> Result<Self, CredentialValidationError> {
let csr_der = base64::engine::general_purpose::STANDARD
.decode(certificate_request)
let csr_der = base64_simd::STANDARD
.decode_to_vec(certificate_request)
.map_err(|_| CredentialValidationError::CertificateRequest)?;
let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(&csr_der));
let csr_digest = BASE64_URL_NO_PAD.encode_to_string(Sha256::digest(&csr_der));
let transcript = rotation_transcript(credential_fingerprint, device_name, request_id, &csr_digest)?;
let key = identity
.to_pkcs8_der()
@@ -216,7 +215,7 @@ impl<'a> RotationRequest<'a> {
certificate_request,
proof: ProofOwned {
algorithm: "ES256".to_string(),
value: BASE64_URL_NO_PAD.encode(canonical.to_bytes()),
value: BASE64_URL_NO_PAD.encode_to_string(canonical.to_bytes()),
},
})
}
@@ -475,8 +474,8 @@ pub(crate) fn public_key_fingerprint(identity: &DeviceIdentity) -> String {
}
pub(crate) fn certificate_request_matches(encoded: &str, identity: &DeviceIdentity) -> Result<bool, CredentialValidationError> {
let der = base64::engine::general_purpose::STANDARD
.decode(encoded)
let der = base64_simd::STANDARD
.decode_to_vec(encoded)
.map_err(|_| CredentialValidationError::CertificateRequest)?;
let (remaining, request) =
X509CertificationRequest::from_der(&der).map_err(|_| CredentialValidationError::CertificateRequest)?;
+2 -6
View File
@@ -446,7 +446,6 @@ mod tests {
use crate::server::{refresh_audit_module_enabled, refresh_notify_module_enabled};
use crate::storage::access::ReqInfo;
use crate::storage::request_context::RequestContext;
use base64::Engine as _;
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit};
use rustfs_audit::ObjectVersion;
@@ -765,10 +764,7 @@ mod tests {
std::collections::HashMap::from([
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
("x-rustfs-encryption-key-id".to_string(), "finance-key".to_string()),
(
"x-rustfs-encryption-key".to_string(),
base64::engine::general_purpose::STANDARD.encode([7u8; 48]),
),
("x-rustfs-encryption-key".to_string(), base64_simd::STANDARD.encode_to_string([7u8; 48])),
("x-rustfs-encryption-algorithm".to_string(), "aws:kms".to_string()),
])
}
@@ -840,7 +836,7 @@ mod tests {
let rendered = serde_json::to_string(&tags).expect("audit tags serialize");
assert!(
!rendered.contains(&base64::engine::general_purpose::STANDARD.encode([7u8; 48])),
!rendered.contains(&base64_simd::STANDARD.encode_to_string([7u8; 48])),
"the audit entry must not carry the wrapped data key: {rendered}"
);
},
+154 -114
View File
@@ -83,7 +83,7 @@ use aes_gcm::{
aead::{Aead, KeyInit},
};
use async_trait::async_trait;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
#[cfg(feature = "rio-v2")]
use chacha20poly1305::ChaCha20Poly1305;
#[cfg(feature = "rio-v2")]
@@ -153,7 +153,7 @@ fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] {
}
fn md5_base64(input: impl AsRef<[u8]>) -> String {
BASE64_STANDARD.encode(md5_bytes(input))
BASE64_STANDARD.encode_to_string(md5_bytes(input))
}
use super::Error;
@@ -562,7 +562,7 @@ pub(crate) fn extract_ssekms_context_from_headers(headers: &HeaderMap) -> Result
let value = v
.to_str()
.map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid UTF-8."))?;
let decoded = BASE64_STANDARD.decode(value).map_err(|_| {
let decoded = BASE64_STANDARD.decode_to_vec(value).map_err(|_| {
sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid base64-encoded JSON.")
})?;
@@ -1320,7 +1320,7 @@ fn stored_envelope_master_key_version(metadata: &HashMap<String, String>) -> Opt
// this lookup never reads, so the normalized result is identical without it.
let encoded = normalize_managed_metadata(metadata, None);
let encoded = encoded.get(INTERNAL_ENCRYPTION_KEY_HEADER)?;
let envelope = BASE64_STANDARD.decode(encoded).ok()?;
let envelope = BASE64_STANDARD.decode_to_vec(encoded).ok()?;
envelope_master_key_version(&envelope)
}
@@ -1516,7 +1516,7 @@ fn build_object_encryption_context(
fn encode_minio_kms_context(context: &HashMap<String, String>) -> Result<String, ApiError> {
let encoded = serde_json::to_vec(context)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to serialize KMS context: {e}"))))?;
Ok(BASE64_STANDARD.encode(encoded))
Ok(BASE64_STANDARD.encode_to_string(encoded))
}
fn decode_minio_kms_context(metadata: &HashMap<String, String>) -> Result<Option<HashMap<String, String>>, ApiError> {
@@ -1524,7 +1524,7 @@ fn decode_minio_kms_context(metadata: &HashMap<String, String>) -> Result<Option
return Ok(None);
};
let decoded = BASE64_STANDARD
.decode(encoded)
.decode_to_vec(encoded)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode MinIO KMS context: {e}"))))?;
serde_json::from_slice(&decoded)
.map(Some)
@@ -1680,7 +1680,7 @@ fn unseal_object_key(
#[cfg(feature = "rio-v2")]
fn try_decode_minio_sealed_key(bytes: &str) -> Result<Option<[u8; SEALED_KEY_SIZE]>, ApiError> {
let decoded = BASE64_STANDARD
.decode(bytes)
.decode_to_vec(bytes)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode sealed object key: {e}"))))?;
match decoded.as_slice().try_into() {
Ok(sealed_key) => Ok(Some(sealed_key)),
@@ -1691,7 +1691,7 @@ fn try_decode_minio_sealed_key(bytes: &str) -> Result<Option<[u8; SEALED_KEY_SIZ
#[cfg(feature = "rio-v2")]
fn try_decode_minio_sealing_iv(bytes: &str) -> Result<Option<[u8; SEALED_KEY_IV_SIZE]>, ApiError> {
let decoded = BASE64_STANDARD
.decode(bytes)
.decode_to_vec(bytes)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode sealing IV: {e}"))))?;
match decoded.as_slice().try_into() {
Ok(iv) => Ok(Some(iv)),
@@ -1758,23 +1758,29 @@ pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> Result<
// `rio-v2` the SSE-C path uses `EncryptionKeyKind::Object` and the sealed-key
// block below, so this branch is not taken.
if material.key_kind == EncryptionKeyKind::Direct {
metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(material.base_nonce));
metadata.insert(
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode_to_string(material.base_nonce),
);
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode(material.base_nonce),
BASE64_STANDARD.encode_to_string(material.base_nonce),
);
}
#[cfg(feature = "rio-v2")]
if let Some(sealed) = &material.managed_sealed_key {
metadata.insert(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealed.iv));
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode_to_string(sealed.iv),
);
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
);
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(sealed.sealed_key),
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
);
}
}
@@ -1816,14 +1822,23 @@ pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> Result<
}
if material.key_kind == EncryptionKeyKind::Direct {
metadata.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(encrypted_data_key));
metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(material.base_nonce));
metadata.insert(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
BASE64_STANDARD.encode_to_string(encrypted_data_key),
);
metadata.insert(
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode_to_string(material.base_nonce),
);
metadata.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), material.algorithm.as_str().to_string());
}
#[cfg(feature = "rio-v2")]
if let Some(sealed) = &material.managed_sealed_key {
metadata.insert(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealed.iv));
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode_to_string(sealed.iv),
);
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
@@ -1835,25 +1850,25 @@ pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> Result<
SSEType::SseS3 => {
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(sealed.sealed_key),
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
);
}
SSEType::SseKms => {
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(sealed.sealed_key),
BASE64_STANDARD.encode_to_string(sealed.sealed_key),
);
}
SSEType::SseC => {}
}
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(encrypted_data_key),
BASE64_STANDARD.encode_to_string(encrypted_data_key),
);
} else if cfg!(feature = "rio-v2") {
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode(material.base_nonce),
BASE64_STANDARD.encode_to_string(material.base_nonce),
);
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
@@ -1862,7 +1877,7 @@ pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> Result<
if let Some(kms_key_id) = &material.kms_key_id {
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), kms_key_id.to_string());
}
let encoded_key = BASE64_STANDARD.encode(encrypted_data_key);
let encoded_key = BASE64_STANDARD.encode_to_string(encrypted_data_key);
match material.sse_type {
SSEType::SseS3 => {
metadata.insert(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), encoded_key);
@@ -2352,7 +2367,7 @@ fn read_stored_ssec_nonce(metadata: &HashMap<String, String>, bucket: &str, key:
metadata
.get(INTERNAL_ENCRYPTION_IV_HEADER)
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER))
.and_then(|encoded| BASE64_STANDARD.decode(encoded).ok())
.and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok())
.and_then(|bytes| <[u8; 12]>::try_from(bytes.as_slice()).ok())
.unwrap_or_else(|| generate_ssec_nonce(bucket, key))
}
@@ -2651,7 +2666,7 @@ async fn apply_managed_decryption_material_inner(
.or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER))
.ok_or_else(|| ApiError::from(StorageError::other("Missing encrypted key in metadata")))?;
let encrypted_data_key = BASE64_STANDARD
.decode(encrypted_key_b64)
.decode_to_vec(encrypted_key_b64)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode encrypted key: {e}"))))?;
(
encrypted_data_key,
@@ -2678,14 +2693,14 @@ async fn apply_managed_decryption_material_inner(
.get(INTERNAL_ENCRYPTION_KEY_HEADER)
.ok_or_else(|| ApiError::from(StorageError::other("Missing encrypted key in metadata")))?;
let encrypted_data_key = BASE64_STANDARD
.decode(encrypted_key_b64)
.decode_to_vec(encrypted_key_b64)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode encrypted key: {e}"))))?;
let iv_b64 = normalized_metadata
.get(INTERNAL_ENCRYPTION_IV_HEADER)
.ok_or_else(|| ApiError::from(StorageError::other("Missing IV in metadata")))?;
let iv = BASE64_STANDARD
.decode(iv_b64)
.decode_to_vec(iv_b64)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode IV: {e}"))))?;
if iv.len() != 12 {
@@ -2876,7 +2891,7 @@ pub(crate) async fn rewrap_object_encryption_metadata(
return Ok(ObjectDekRewrapOutcome::NotApplicable);
};
let encrypted_data_key = BASE64_STANDARD
.decode(envelope_b64)
.decode_to_vec(envelope_b64)
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode encrypted key: {e}"))))?;
// Only RustFS envelopes are rewrappable here; MinIO's builtin-KMS
// ciphertext is opaque bytes owned by a different root of trust.
@@ -2918,7 +2933,7 @@ pub(crate) async fn rewrap_object_encryption_metadata(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
];
let old_envelope_b64 = envelope_b64.clone();
let new_envelope_b64 = BASE64_STANDARD.encode(&response.ciphertext);
let new_envelope_b64 = BASE64_STANDARD.encode_to_string(&response.ciphertext);
let mut overrides = HashMap::new();
for (stored_name, stored_value) in metadata {
let is_envelope_slot = REWRAP_ENVELOPE_HEADERS
@@ -3256,7 +3271,7 @@ fn decrypt_minio_kms_data_key(encrypted_dek: &[u8], master_key: &[u8; 32], aad:
{
let decode = |what: &str, value: &str| -> Result<Vec<u8>, ApiError> {
BASE64_STANDARD
.decode(value)
.decode_to_vec(value)
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid MinIO KMS {what}: {e}"))))
};
let mut body = decode("ciphertext", &json.bytes)?;
@@ -3346,7 +3361,7 @@ fn parse_simple_sse_cmk(cmk_value: &str) -> Result<[u8; 32], ApiError> {
)));
}
let decoded = BASE64_STANDARD
.decode(trimmed)
.decode_to_vec(trimmed)
.map_err(|e| ApiError::from(StorageError::other(format!("__RUSTFS_SSE_SIMPLE_CMK must be valid base64: {e}"))))?;
let master_key: [u8; 32] = decoded.try_into().map_err(|v: Vec<u8>| {
ApiError::from(StorageError::other(format!(
@@ -3401,7 +3416,7 @@ impl LocalSseDekProvider {
));
};
let decoded = BASE64_STANDARD.decode(raw_value.trim()).map_err(|err| {
let decoded = BASE64_STANDARD.decode_to_vec(raw_value.trim()).map_err(|err| {
sse_not_configured(format!(
"RUSTFS_SSE_S3_MASTER_KEY must be valid base64 for SSE-S3 when KMS is not configured: {err}"
))
@@ -3427,8 +3442,8 @@ impl LocalSseDekProvider {
.encrypt(&nonce, dek.as_slice())
.map_err(|_| ApiError::from(StorageError::other("Failed to encrypt DEK")))?;
let nonce = BASE64_STANDARD.encode(nonce);
let ciphertext = BASE64_STANDARD.encode(ciphertext);
let nonce = BASE64_STANDARD.encode_to_string(nonce);
let ciphertext = BASE64_STANDARD.encode_to_string(ciphertext);
serde_json::to_string(&LocalSseDekEnvelope {
version: LOCAL_SSE_DEK_FORMAT_VERSION,
nonce: &nonce,
@@ -3468,10 +3483,10 @@ impl LocalSseDekProvider {
}
};
let nonce_vec = BASE64_STANDARD
.decode(nonce)
.decode_to_vec(nonce)
.map_err(|_| ApiError::from(StorageError::other("Invalid nonce format")))?;
let ciphertext = BASE64_STANDARD
.decode(ciphertext)
.decode_to_vec(ciphertext)
.map_err(|_| ApiError::from(StorageError::other("Invalid ciphertext format")))?;
let key = Key::<Aes256Gcm>::from(cmk_value);
@@ -3792,7 +3807,7 @@ fn parse_minio_managed_sealed_key(
/// crate carries no JSON codec; any decode failure returns `None`, which skips
/// the context mapping exactly like the historical inline `if let Ok` chain.
fn recode_minio_kms_context(value: &str) -> Option<String> {
let decoded = BASE64_STANDARD.decode(value).ok()?;
let decoded = BASE64_STANDARD.decode_to_vec(value).ok()?;
let context = serde_json::from_slice::<HashMap<String, String>>(&decoded).ok()?;
serde_json::to_string(&context).ok()
}
@@ -3817,7 +3832,7 @@ pub fn validate_ssec_params(params: SsecParams) -> Result<ValidatedSsecParams, A
)));
}
let key_bytes = BASE64_STANDARD.decode(&params.key).map_err(|e| {
let key_bytes = BASE64_STANDARD.decode_to_vec(&params.key).map_err(|e| {
error!("Failed to decode SSE-C key: {}", e);
ssec_invalid_request("Invalid SSE-C key: not valid Base64.")
})?;
@@ -4035,10 +4050,10 @@ mod tests {
// Not valid base64.
assert!(super::parse_simple_sse_cmk("@@@not-base64@@@").is_err());
// Valid base64 but wrong length (16 bytes).
let short = BASE64_STANDARD.encode([1u8; 16]);
let short = BASE64_STANDARD.encode_to_string([1u8; 16]);
assert!(super::parse_simple_sse_cmk(&short).is_err());
// All-zero 32-byte key is rejected.
let zero = BASE64_STANDARD.encode([0u8; 32]);
let zero = BASE64_STANDARD.encode_to_string([0u8; 32]);
assert!(super::parse_simple_sse_cmk(&zero).is_err());
}
@@ -4055,14 +4070,14 @@ mod tests {
fn parse_simple_sse_cmk_accepts_valid_32_byte_key() {
let mut key = [0u8; 32];
key[0] = 7;
let encoded = BASE64_STANDARD.encode(key);
let encoded = BASE64_STANDARD.encode_to_string(key);
let got = super::parse_simple_sse_cmk(&encoded).expect("valid 32-byte key must parse");
assert_eq!(got, key);
}
use aes_gcm::aead::{Aead, KeyInit};
use aes_gcm::{Aes256Gcm, Key, Nonce};
use async_trait::async_trait;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use http::{HeaderMap, HeaderValue};
use rustfs_kms::types::ObjectEncryptionContext;
use rustfs_rio::{DecryptReader, EncryptReader};
@@ -4094,13 +4109,13 @@ mod tests {
#[tokio::test]
async fn object_encryption_resolver_returns_ssec_read_material() {
let key = [0x31; 32];
let key_b64 = BASE64_STANDARD.encode(key);
let key_b64 = BASE64_STANDARD.encode_to_string(key);
let key_md5 = md5_base64(key);
let nonce = [0x42; 12];
let metadata = HashMap::from([
("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()),
("X-Amz-Server-Side-Encryption-Customer-Key-Md5".to_string(), key_md5.clone()),
("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode(nonce)),
("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode_to_string(nonce)),
]);
let mut headers = HeaderMap::new();
headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
@@ -4131,7 +4146,7 @@ mod tests {
#[tokio::test]
async fn object_encryption_resolver_rejects_missing_or_invalid_ssec_algorithm() {
let key = [0x31; 32];
let key_b64 = BASE64_STANDARD.encode(key);
let key_b64 = BASE64_STANDARD.encode_to_string(key);
let key_md5 = md5_base64(key);
let metadata = HashMap::from([
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
@@ -4246,7 +4261,7 @@ mod tests {
}
fn local_sse_master_key_b64() -> String {
BASE64_STANDARD.encode([0x24u8; 32])
BASE64_STANDARD.encode_to_string([0x24u8; 32])
}
#[test]
@@ -4303,8 +4318,8 @@ mod tests {
#[test]
fn test_extract_ssekms_context_from_headers_decodes_base64_json() {
let mut headers = http::HeaderMap::new();
let encoded =
BASE64_STANDARD.encode(serde_json::to_vec(&HashMap::from([("tenant".to_string(), "alpha".to_string())])).unwrap());
let encoded = BASE64_STANDARD
.encode_to_string(serde_json::to_vec(&HashMap::from([("tenant".to_string(), "alpha".to_string())])).unwrap());
headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, HeaderValue::from_str(&encoded).unwrap());
let context = extract_ssekms_context_from_headers(&headers)
@@ -4401,7 +4416,7 @@ mod tests {
#[test]
fn test_validate_ssec_params_success() {
let key = BASE64_STANDARD.encode([42u8; 32]);
let key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let key_md5 = md5_base64([42u8; 32]);
let params = SsecParams {
@@ -4418,7 +4433,7 @@ mod tests {
#[test]
fn test_validate_ssec_params_wrong_algorithm() {
let key = BASE64_STANDARD.encode([42u8; 32]);
let key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let key_md5 = md5_base64([42u8; 32]);
let params = SsecParams {
@@ -4433,7 +4448,7 @@ mod tests {
#[test]
fn test_validate_ssec_params_wrong_key_length() {
let key = BASE64_STANDARD.encode([42u8; 16]); // Only 16 bytes
let key = BASE64_STANDARD.encode_to_string([42u8; 16]); // Only 16 bytes
let key_md5 = md5_base64([42u8; 16]);
let params = SsecParams {
@@ -4448,8 +4463,8 @@ mod tests {
#[test]
fn test_validate_ssec_params_wrong_md5() {
let key = BASE64_STANDARD.encode([42u8; 32]);
let key_md5 = BASE64_STANDARD.encode([99u8; 16]); // Wrong MD5
let key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let key_md5 = BASE64_STANDARD.encode_to_string([99u8; 16]); // Wrong MD5
let params = SsecParams {
algorithm: "AES256".to_string(),
@@ -4465,7 +4480,7 @@ mod tests {
async fn test_sse_encryption_rejects_partial_ssec_headers() {
let bucket = "test-bucket";
let key = "test-key";
let sse_key = BASE64_STANDARD.encode([42u8; 32]);
let sse_key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let sse_key_md5 = md5_base64([42u8; 32]);
let content_size = 1024;
@@ -4612,7 +4627,7 @@ mod tests {
let bucket = "bucket";
let key = "object";
let customer_key_bytes = [0x24u8; 32];
let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
let customer_key = BASE64_STANDARD.encode_to_string(customer_key_bytes);
let customer_key_md5 = md5_base64(customer_key_bytes);
let metadata_one = ssec_direct_put_metadata(bucket, key, &customer_key, &customer_key_md5).await;
@@ -4636,7 +4651,7 @@ mod tests {
.await
.expect("sse-c decryption material");
assert_eq!(
BASE64_STANDARD.encode(decrypted.base_nonce),
BASE64_STANDARD.encode_to_string(decrypted.base_nonce),
*iv,
"decrypt must read the persisted random nonce back"
);
@@ -4653,7 +4668,7 @@ mod tests {
let bucket = "bucket";
let key = "object";
let customer_key_bytes = [0x24u8; 32];
let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
let customer_key = BASE64_STANDARD.encode_to_string(customer_key_bytes);
let customer_key_md5 = md5_base64(customer_key_bytes);
let mut metadata = HashMap::new();
@@ -4685,7 +4700,7 @@ mod tests {
let bucket = "bucket";
let key = "object";
let customer_key_bytes = [0x51u8; 32];
let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
let customer_key = BASE64_STANDARD.encode_to_string(customer_key_bytes);
let customer_key_md5 = md5_base64(customer_key_bytes);
let plaintext = b"attack at dawn - sse-c round trip".to_vec();
@@ -4693,7 +4708,7 @@ mod tests {
// Encrypt with the key + nonce that were persisted at PUT time.
let enc_iv = BASE64_STANDARD
.decode(metadata.get(INTERNAL_ENCRYPTION_IV_HEADER).expect("persisted IV"))
.decode_to_vec(metadata.get(INTERNAL_ENCRYPTION_IV_HEADER).expect("persisted IV"))
.expect("valid base64 IV");
let cipher = Aes256Gcm::new_from_slice(&customer_key_bytes).expect("cipher");
let ciphertext = cipher
@@ -4724,7 +4739,7 @@ mod tests {
let bucket = "bucket";
let key = "object";
let customer_key_bytes = [0x33u8; 32];
let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
let customer_key = BASE64_STANDARD.encode_to_string(customer_key_bytes);
let customer_key_md5 = md5_base64(customer_key_bytes);
let material = sse_prepare_encryption(PrepareEncryptionRequest {
@@ -4750,7 +4765,7 @@ mod tests {
.clone();
// Random, not the deterministic bucket/key derivation.
assert_ne!(
BASE64_STANDARD.decode(&session_iv).expect("valid IV")[..],
BASE64_STANDARD.decode_to_vec(&session_iv).expect("valid IV")[..],
generate_ssec_nonce(bucket, key)[..]
);
@@ -4779,7 +4794,7 @@ mod tests {
let part_two_nonce = resolve_part_nonce(2).await;
assert_eq!(part_one_nonce, part_two_nonce, "all parts of one upload must share the persisted nonce");
assert_eq!(BASE64_STANDARD.encode(part_one_nonce), session_iv);
assert_eq!(BASE64_STANDARD.encode_to_string(part_one_nonce), session_iv);
}
#[cfg(feature = "rio-v2")]
@@ -4788,7 +4803,7 @@ mod tests {
let bucket = "test-bucket";
let key = "test-key";
let customer_key_bytes = [0x24u8; 32];
let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
let customer_key = BASE64_STANDARD.encode_to_string(customer_key_bytes);
let sse_key_md5 = md5_base64(customer_key_bytes);
let request = PrepareEncryptionRequest {
@@ -4966,7 +4981,7 @@ mod tests {
let bucket = "test-bucket";
let key = "test-key";
let content_size = 1024;
let sse_key = BASE64_STANDARD.encode([42u8; 32]);
let sse_key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let sse_key_md5 = md5_base64([42u8; 32]);
let request = EncryptionRequest {
@@ -5084,7 +5099,7 @@ mod tests {
reset_sse_dek_provider();
let envelope = probe_envelope_json();
let envelope_b64 = BASE64_STANDARD.encode(&envelope);
let envelope_b64 = BASE64_STANDARD.encode_to_string(&envelope);
let client_context = HashMap::from([("tenant".to_string(), "alpha".to_string())]);
let metadata = HashMap::from([
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
@@ -5095,7 +5110,7 @@ mod tests {
// A sealed object key: different bytes, must not be rewritten.
(
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(b"sealed-object-key-not-the-envelope"),
BASE64_STANDARD.encode_to_string(b"sealed-object-key-not-the-envelope"),
),
(
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
@@ -5118,7 +5133,7 @@ mod tests {
panic!("expected a rewrapped outcome, got {outcome:?}");
};
let new_b64 = BASE64_STANDARD.encode(&new_ciphertext);
let new_b64 = BASE64_STANDARD.encode_to_string(&new_ciphertext);
assert_eq!(
overrides,
HashMap::from([
@@ -5149,7 +5164,7 @@ mod tests {
let _guard = lock_sse_test_state().await;
reset_sse_dek_provider();
let envelope_b64 = BASE64_STANDARD.encode(probe_envelope_json());
let envelope_b64 = BASE64_STANDARD.encode_to_string(probe_envelope_json());
let metadata = HashMap::from([
("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), envelope_b64),
@@ -5221,7 +5236,7 @@ mod tests {
// SSE-C object: customer-key encryption never reaches KMS.
let ssec = HashMap::from([
("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([1u8; 12])),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([1u8; 12])),
]);
let outcome = rewrap_object_encryption_metadata("bucket", "object", &ssec)
.await
@@ -5233,7 +5248,7 @@ mod tests {
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
(
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(b"opaque-minio-sealed-bytes"),
BASE64_STANDARD.encode_to_string(b"opaque-minio-sealed-bytes"),
),
]);
let outcome = rewrap_object_encryption_metadata("bucket", "object", &minio)
@@ -5327,7 +5342,7 @@ mod tests {
.get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
.expect("minio kms context header should exist");
let decoded_context: HashMap<String, String> =
serde_json::from_slice(&BASE64_STANDARD.decode(encoded_context).expect("decode base64 context"))
serde_json::from_slice(&BASE64_STANDARD.decode_to_vec(encoded_context).expect("decode base64 context"))
.expect("decode json context");
assert_eq!(decoded_context, client_context);
@@ -5347,7 +5362,8 @@ mod tests {
let mut wrong_metadata = metadata.clone();
wrong_metadata.insert(
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
BASE64_STANDARD.encode(serde_json::to_vec(&HashMap::from([("tenant".to_string(), "beta".to_string())])).unwrap()),
BASE64_STANDARD
.encode_to_string(serde_json::to_vec(&HashMap::from([("tenant".to_string(), "beta".to_string())])).unwrap()),
);
let err = sse_decryption(DecryptionRequest {
bucket: "bucket",
@@ -5370,8 +5386,8 @@ mod tests {
#[cfg(feature = "rio-v2")]
#[test]
fn test_encryption_material_to_metadata_persists_minio_managed_headers() {
let encoded_nonce = BASE64_STANDARD.encode([9u8; 12]);
let encoded_key = BASE64_STANDARD.encode([1u8, 2, 3, 4]);
let encoded_nonce = BASE64_STANDARD.encode_to_string([9u8; 12]);
let encoded_key = BASE64_STANDARD.encode_to_string([1u8, 2, 3, 4]);
let metadata = encryption_material_to_metadata(&EncryptionMaterial {
sse_type: SSEType::SseKms,
server_side_encryption: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
@@ -5608,9 +5624,12 @@ mod tests {
let metadata = HashMap::from([
(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(b"encrypted-key"),
BASE64_STANDARD.encode_to_string(b"encrypted-key"),
),
(
MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode_to_string([0x11u8; 12]),
),
(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x11u8; 12])),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
@@ -5622,9 +5641,12 @@ mod tests {
assert_eq!(
normalized.get(INTERNAL_ENCRYPTION_KEY_HEADER),
Some(&BASE64_STANDARD.encode(b"encrypted-key"))
Some(&BASE64_STANDARD.encode_to_string(b"encrypted-key"))
);
assert_eq!(
normalized.get(INTERNAL_ENCRYPTION_IV_HEADER),
Some(&BASE64_STANDARD.encode_to_string([0x11u8; 12]))
);
assert_eq!(normalized.get(INTERNAL_ENCRYPTION_IV_HEADER), Some(&BASE64_STANDARD.encode([0x11u8; 12])));
assert_eq!(
normalized.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER),
Some(&MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string())
@@ -5677,8 +5699,11 @@ mod tests {
let sealed_key = metadata
.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
.expect("minio sealed key should be stored");
assert_eq!(BASE64_STANDARD.decode(sealing_iv).expect("decode iv").len(), SEALED_KEY_IV_SIZE);
assert_eq!(BASE64_STANDARD.decode(sealed_key).expect("decode sealed key").len(), SEALED_KEY_SIZE);
assert_eq!(BASE64_STANDARD.decode_to_vec(sealing_iv).expect("decode iv").len(), SEALED_KEY_IV_SIZE);
assert_eq!(
BASE64_STANDARD.decode_to_vec(sealed_key).expect("decode sealed key").len(),
SEALED_KEY_SIZE
);
let decrypted = sse_decryption(DecryptionRequest {
bucket: "bucket",
@@ -5726,7 +5751,7 @@ mod tests {
#[tokio::test]
async fn test_ssec_rio_v2_uses_sealed_object_key_metadata_roundtrip() {
let customer_key_bytes = [0x42u8; 32];
let customer_key = BASE64_STANDARD.encode(customer_key_bytes);
let customer_key = BASE64_STANDARD.encode_to_string(customer_key_bytes);
let customer_key_md5 = md5_base64(customer_key_bytes);
let material = sse_encryption(EncryptionRequest {
@@ -5831,7 +5856,7 @@ mod tests {
ssekms_key_id: None,
ssekms_context: None,
sse_customer_algorithm: Some("AES256".to_string()),
sse_customer_key: Some(BASE64_STANDARD.encode(key_bytes)),
sse_customer_key: Some(BASE64_STANDARD.encode_to_string(key_bytes)),
sse_customer_key_md5: Some(md5_base64(key_bytes)),
content_size: 1,
principal: None,
@@ -6111,7 +6136,11 @@ mod tests {
let ciphertext = cipher
.encrypt(&legacy_nonce, dek.as_slice())
.expect("legacy wrap should succeed");
let legacy_payload = format!("{}:{}", BASE64_STANDARD.encode(legacy_nonce), BASE64_STANDARD.encode(ciphertext));
let legacy_payload = format!(
"{}:{}",
BASE64_STANDARD.encode_to_string(legacy_nonce),
BASE64_STANDARD.encode_to_string(ciphertext)
);
let decrypted = TestSseDekProvider::decrypt_dek(&legacy_payload, cmk).expect("legacy payload should remain decryptable");
assert_eq!(decrypted, dek);
@@ -6121,8 +6150,8 @@ mod tests {
fn test_decrypt_dek_rejects_unknown_json_version() {
let envelope = serde_json::json!({
"version": super::LOCAL_SSE_DEK_FORMAT_VERSION + 1,
"nonce": BASE64_STANDARD.encode([0u8; 12]),
"ciphertext": BASE64_STANDARD.encode([0u8; 48]),
"nonce": BASE64_STANDARD.encode_to_string([0u8; 12]),
"ciphertext": BASE64_STANDARD.encode_to_string([0u8; 48]),
})
.to_string();
@@ -6293,13 +6322,19 @@ mod tests {
async_with_vars(
[
("__RUSTFS_SSE_SIMPLE_CMK", None::<String>),
("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode(local_master_key))),
("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode_to_string(local_master_key))),
],
async {
let metadata = HashMap::from([
("x-amz-server-side-encryption".to_string(), ServerSideEncryption::AWS_KMS.to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(encrypted_dek)),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(data_key.nonce)),
(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
BASE64_STANDARD.encode_to_string(encrypted_dek),
),
(
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
BASE64_STANDARD.encode_to_string(data_key.nonce),
),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "legacy-local-key".to_string()),
]);
@@ -6327,8 +6362,8 @@ mod tests {
}"#;
let metadata = HashMap::from([
("x-amz-server-side-encryption".to_string(), ServerSideEncryption::AES256.to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(kms_envelope)),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x14; 12])),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(kms_envelope)),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([0x14; 12])),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "test-key-id".to_string()),
]);
let error = match apply_managed_decryption_material("bucket", "object", &metadata, None).await {
@@ -6374,8 +6409,8 @@ mod tests {
}"#;
let metadata = HashMap::from([
("x-amz-server-side-encryption".to_string(), ServerSideEncryption::AES256.to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(kms_envelope)),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x14; 12])),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(kms_envelope)),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([0x14; 12])),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "envelope-key".to_string()),
]);
@@ -6431,9 +6466,9 @@ mod tests {
("x-amz-server-side-encryption".to_string(), ServerSideEncryption::AES256.to_string()),
(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(b"local-provider-format"),
BASE64_STANDARD.encode_to_string(b"local-provider-format"),
),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x14; 12])),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([0x14; 12])),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "test-key-id".to_string()),
]);
let error = match apply_managed_decryption_material("bucket", "object", &metadata, None).await {
@@ -6447,14 +6482,16 @@ mod tests {
#[tokio::test]
async fn test_kms_sse_dek_provider_uses_latest_reconfigured_service() {
use base64::Engine as _;
use rustfs_kms::config::KmsConfig;
let _guard = lock_sse_test_state().await;
let manager = Arc::new(rustfs_kms::KmsServiceManager::new());
manager
.reconfigure(KmsConfig::static_kms("first-key".to_string(), BASE64_STANDARD.encode([0x11; 32])))
.reconfigure(KmsConfig::static_kms(
"first-key".to_string(),
BASE64_STANDARD.encode_to_string([0x11; 32]),
))
.await
.expect("first KMS reconfigure should succeed");
@@ -6468,7 +6505,10 @@ mod tests {
.expect("provider should use the initial service");
manager
.reconfigure(KmsConfig::static_kms("second-key".to_string(), BASE64_STANDARD.encode([0x22; 32])))
.reconfigure(KmsConfig::static_kms(
"second-key".to_string(),
BASE64_STANDARD.encode_to_string([0x22; 32]),
))
.await
.expect("second KMS reconfigure should succeed");
@@ -6556,7 +6596,7 @@ mod tests {
// Key B is a different key; its MD5 won't match stored MD5.
let key_b = [99u8; 32];
let key_b_b64 = BASE64_STANDARD.encode(key_b);
let key_b_b64 = BASE64_STANDARD.encode_to_string(key_b);
let key_b_md5 = md5_base64(key_b);
let err = validate_ssec_for_read(&metadata, Some(&key_b_b64), Some(&key_b_md5)).unwrap_err();
@@ -6566,7 +6606,7 @@ mod tests {
#[test]
fn test_validate_ssec_for_read_correct_key() {
let key_bytes = [42u8; 32];
let key_b64 = BASE64_STANDARD.encode(key_bytes);
let key_b64 = BASE64_STANDARD.encode_to_string(key_bytes);
let key_md5 = md5_base64(key_bytes);
let mut metadata = HashMap::new();
@@ -6591,7 +6631,7 @@ mod tests {
// Attacker has a different key but tries to pass the stored MD5 as their header
let fake_key = [99u8; 32];
let fake_key_b64 = BASE64_STANDARD.encode(fake_key);
let fake_key_b64 = BASE64_STANDARD.encode_to_string(fake_key);
let err = validate_ssec_for_read(&metadata, Some(&fake_key_b64), Some(&stored_md5)).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
@@ -6709,7 +6749,7 @@ mod tests {
#[test]
fn test_validate_ssec_params_returns_invalid_request_on_bad_algorithm() {
let key = BASE64_STANDARD.encode([42u8; 32]);
let key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let key_md5 = md5_base64([42u8; 32]);
let params = SsecParams {
algorithm: "AES128".to_string(),
@@ -6722,11 +6762,11 @@ mod tests {
#[test]
fn test_validate_ssec_params_returns_invalid_request_on_bad_md5() {
let key = BASE64_STANDARD.encode([42u8; 32]);
let key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let params = SsecParams {
algorithm: "AES256".to_string(),
key,
key_md5: BASE64_STANDARD.encode([99u8; 16]),
key_md5: BASE64_STANDARD.encode_to_string([99u8; 16]),
};
let err = validate_ssec_params(params).unwrap_err();
assert_eq!(err.code, S3ErrorCode::InvalidRequest);
@@ -6772,8 +6812,8 @@ mod tests {
async fn test_sse_encryption_errors_on_invalid_ssec_params() {
let bucket = "test-bucket";
let key = "test-key";
let sse_key = BASE64_STANDARD.encode([42u8; 32]);
let wrong_md5 = BASE64_STANDARD.encode([99u8; 16]);
let sse_key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let wrong_md5 = BASE64_STANDARD.encode_to_string([99u8; 16]);
let request_wrong_md5 = EncryptionRequest {
bucket,
@@ -6898,8 +6938,8 @@ mod tests {
("x-amz-server-side-encryption".to_string(), ServerSideEncryption::AWS_KMS.to_string()),
("x-amz-server-side-encryption-aws-kms-key-id".to_string(), "finance-key".to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "finance-key".to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode([7u8; 48])),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([9u8; 12])),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string([7u8; 48])),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([9u8; 12])),
(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "aws:kms".to_string()),
])
}
@@ -6907,8 +6947,8 @@ mod tests {
fn sse_s3_object_metadata() -> HashMap<String, String> {
HashMap::from([
("x-amz-server-side-encryption".to_string(), ServerSideEncryption::AES256.to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode([7u8; 48])),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([9u8; 12])),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string([7u8; 48])),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode_to_string([9u8; 12])),
(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "AES256".to_string()),
])
}
@@ -7001,7 +7041,7 @@ mod tests {
#[tokio::test]
async fn ssec_requests_are_exempt_from_kms_key_authorization() {
let (principal, authorizer) = enforcing_principal(false);
let customer_key = BASE64_STANDARD.encode([42u8; 32]);
let customer_key = BASE64_STANDARD.encode_to_string([42u8; 32]);
let customer_key_md5 = md5_base64([42u8; 32]);
let outcome = sse_encryption(EncryptionRequest {
@@ -7166,8 +7206,8 @@ mod tests {
let rendered = format!("{write_tags:?}");
let encrypted_data_key = material.encrypted_data_key.clone().expect("managed sse wraps a data key");
for secret in [
BASE64_STANDARD.encode(&encrypted_data_key),
BASE64_STANDARD.encode(material.key_bytes),
BASE64_STANDARD.encode_to_string(&encrypted_data_key),
BASE64_STANDARD.encode_to_string(material.key_bytes),
format!("{:?}", material.key_bytes),
format!("{encrypted_data_key:?}"),
"acct-4711".to_string(),
@@ -7207,7 +7247,7 @@ mod tests {
ssekms_key_id: None,
ssekms_context: None,
sse_customer_algorithm: Some(SSECustomerAlgorithm::from("AES256".to_string())),
sse_customer_key: Some(SSECustomerKey::from(BASE64_STANDARD.encode(key))),
sse_customer_key: Some(SSECustomerKey::from(BASE64_STANDARD.encode_to_string(key))),
sse_customer_key_md5: Some(SSECustomerKeyMD5::from(md5_base64(key))),
content_size: 128,
principal: Some(&principal),
@@ -7353,7 +7393,7 @@ mod tests {
#[tokio::test]
async fn classification_matches_ssec_validation_and_headers() {
let key = [0x42u8; 32];
let key_b64 = BASE64_STANDARD.encode(key);
let key_b64 = BASE64_STANDARD.encode_to_string(key);
let key_md5 = md5_base64(key);
let metadata = HashMap::from([
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
@@ -7395,7 +7435,7 @@ mod tests {
bucket: "finance",
key: "ledger.csv",
metadata: &metadata,
sse_customer_key: Some(&SSECustomerKey::from(BASE64_STANDARD.encode(other_key))),
sse_customer_key: Some(&SSECustomerKey::from(BASE64_STANDARD.encode_to_string(other_key))),
sse_customer_key_md5: Some(&SSECustomerKeyMD5::from(md5_base64(other_key))),
principal: None,
})
@@ -7405,7 +7445,7 @@ mod tests {
bucket: "finance",
key: "ledger.csv",
metadata: &metadata,
sse_customer_key: Some(&SSECustomerKey::from(BASE64_STANDARD.encode(other_key))),
sse_customer_key: Some(&SSECustomerKey::from(BASE64_STANDARD.encode_to_string(other_key))),
sse_customer_key_md5: Some(&SSECustomerKeyMD5::from(md5_base64(other_key))),
principal: None,
})
@@ -7510,7 +7550,7 @@ mod tests {
#[test]
fn stored_envelope_master_key_version_reads_both_metadata_families() {
let envelope = BASE64_STANDARD.encode(audit_test_envelope(Some(2)));
let envelope = BASE64_STANDARD.encode_to_string(audit_test_envelope(Some(2)));
// RustFS-branded stored key.
let metadata = HashMap::from([(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), envelope.clone())]);
+11 -12
View File
@@ -20,8 +20,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
use base64_simd::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
use rustfs::connect::identity::{DeviceIdentity, IdentityError, RegistrationTranscript};
use rustfs::connect::identity_store::{IdentityStore, StoreError};
@@ -66,8 +65,8 @@ fn transcript_reproduces_every_accept_vector() {
let token = &vector["tokenRecord"];
let request = &vector["request"];
let csr = base64::engine::general_purpose::STANDARD
.decode(
let csr = base64_simd::STANDARD
.decode_to_vec(
request["certificateRequest"]
.as_str()
.expect("vector carries a certificate request"),
@@ -118,8 +117,8 @@ fn published_proofs_verify_over_locally_rebuilt_transcripts() {
let token = &vector["tokenRecord"];
let request = &vector["request"];
let csr = base64::engine::general_purpose::STANDARD
.decode(request["certificateRequest"].as_str().unwrap())
let csr = base64_simd::STANDARD
.decode_to_vec(request["certificateRequest"].as_str().unwrap())
.expect("certificate request is base64");
let transcript = RegistrationTranscript::build(
@@ -134,7 +133,7 @@ fn published_proofs_verify_over_locally_rebuilt_transcripts() {
.expect("transcript builds");
let raw = BASE64_URL_NO_PAD
.decode(request["proof"]["value"].as_str().expect("vector carries a proof"))
.decode_to_vec(request["proof"]["value"].as_str().expect("vector carries a proof"))
.expect("proof decodes");
let signature = p256::ecdsa::Signature::from_slice(&raw).expect("signature parses");
assert_eq!(
@@ -183,10 +182,10 @@ fn csr_octets_matching_golden_digest() -> Vec<u8> {
let Some(encoded) = vector["request"]["certificateRequest"].as_str() else {
continue;
};
let der = base64::engine::general_purpose::STANDARD
.decode(encoded)
let der = base64_simd::STANDARD
.decode_to_vec(encoded)
.expect("certificate request is base64");
let digest = BASE64_URL_NO_PAD.encode(<sha2::Sha256 as sha2::Digest>::digest(&der));
let digest = BASE64_URL_NO_PAD.encode_to_string(<sha2::Sha256 as sha2::Digest>::digest(&der));
if digest == want {
return der;
}
@@ -302,7 +301,7 @@ fn proof_is_a_canonical_low_s_signature_that_verifies() {
"the proof must use the base64url alphabet with no padding"
);
let raw = BASE64_URL_NO_PAD.decode(&proof.value).expect("proof decodes");
let raw = BASE64_URL_NO_PAD.decode_to_vec(&proof.value).expect("proof decodes");
assert_eq!(raw.len(), 64, "the signature is a fixed-width r || s");
let signature = p256::ecdsa::Signature::from_slice(&raw).expect("signature parses");
@@ -334,7 +333,7 @@ fn proof_does_not_verify_over_a_different_transcript() {
let other = transcript_from_fixture_inputs(b"a different certificate request").expect("transcript builds");
assert_ne!(transcript.as_bytes(), other.as_bytes());
let raw = BASE64_URL_NO_PAD.decode(&proof.value).expect("proof decodes");
let raw = BASE64_URL_NO_PAD.decode_to_vec(&proof.value).expect("proof decodes");
let signature = p256::ecdsa::Signature::from_slice(&raw).expect("signature parses");
let verifying = <p256::ecdsa::VerifyingKey as p256::pkcs8::DecodePublicKey>::from_public_key_der(&identity.public_key_der())
.expect("public key decodes");
+3 -3
View File
@@ -22,7 +22,7 @@ use std::process::Command;
#[cfg(target_os = "linux")]
use std::time::Duration;
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use base64_simd::URL_SAFE_NO_PAD;
use p256::ecdsa::{Signature, VerifyingKey, signature::Verifier as _};
use p256::pkcs8::DecodePublicKey as _;
use rustfs::connect::DeviceIdentity;
@@ -169,7 +169,7 @@ fn connect_offline_bundle_is_deterministic_bounded_and_signed_over_exact_manifes
assert_eq!(signature_document["signedFile"], "manifest.json");
assert_eq!(signature_document["domainSeparationTag"], "rustfs-support-bundle-v1");
let signature_bytes: [u8; 64] = URL_SAFE_NO_PAD
.decode(signature_document["value"].as_str().expect("signature value"))
.decode_to_vec(signature_document["value"].as_str().expect("signature value"))
.expect("signature base64url")
.try_into()
.expect("fixed-width signature");
@@ -197,7 +197,7 @@ fn connect_offline_bundle_is_deterministic_bounded_and_signed_over_exact_manifes
"organizations/0198f3a1-4c00-7a10-8b21-0c1d2e3f4a50/clusters/0198f3a1-5d00-7b20-9c31-1d2e3f4a5b61"
);
assert_eq!(manifest["deviceName"], DEVICE_NAME);
assert_eq!(manifest["nonce"], URL_SAFE_NO_PAD.encode([0x2a; 32]));
assert_eq!(manifest["nonce"], URL_SAFE_NO_PAD.encode_to_string([0x2a; 32]));
assert_eq!(manifest["producedAt"], "2026-05-04T02:00:00Z");
assert_eq!(manifest["redactionVersion"], REDACTION_VERSION);
assert_eq!(manifest["rulesetHash"], RULESET_HASH);
+23 -18
View File
@@ -29,9 +29,8 @@
use std::fs;
use std::path::PathBuf;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
use base64_simd::STANDARD as BASE64_STANDARD;
use base64_simd::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD;
use rustfs::connect::identity::DeviceIdentity;
use rustfs::connect::offline::{EnrollmentError, OfflineEnrollment, VerifiedChallenge};
use serde_json::Value;
@@ -133,7 +132,7 @@ fn envelope(document: &Value) -> Vec<u8> {
/// The raw octets the signature covers, exactly as transmitted.
fn signed_octets(document: &Value) -> Vec<u8> {
BASE64_STANDARD
.decode(field(document, "bytes"))
.decode_to_vec(field(document, "bytes"))
.expect("document bytes are padded base64")
}
@@ -158,7 +157,9 @@ fn hex_to_bytes(hex: &str) -> Vec<u8> {
/// Turn a fixture's unpadded-base64url SEC1 point into a usable verifying key.
fn verifying_key(sec1_base64url: &str) -> p256::ecdsa::VerifyingKey {
let point = BASE64_URL_NO_PAD.decode(sec1_base64url).expect("public key is base64url");
let point = BASE64_URL_NO_PAD
.decode_to_vec(sec1_base64url)
.expect("public key is base64url");
assert_eq!(point.len(), 65, "the protocol freezes a 65 octet uncompressed SEC1 point");
let mut der = hex_to_bytes(SPKI_PREFIX_HEX);
@@ -215,7 +216,7 @@ fn answered_challenge(response_vector: &Value) -> (Value, VerifiedChallenge) {
fn device_nonce_of(document: &Value) -> [u8; 32] {
let raw = BASE64_URL_NO_PAD
.decode(field(&signed_document(document), "deviceNonce"))
.decode_to_vec(field(&signed_document(document), "deviceNonce"))
.expect("deviceNonce is base64url");
raw.try_into().expect("replay.nonceLengthBytes freezes a 32 octet nonce")
}
@@ -330,7 +331,7 @@ fn e2e_public_chain_matches_the_challenge_and_every_signature_verifies() {
for link in chain.as_array().expect("E2E chain is a list") {
assert_eq!(field(&link["signature"], "keyId"), issuer_id.as_str());
let signature = BASE64_URL_NO_PAD
.decode(field(&link["signature"], "value"))
.decode_to_vec(field(&link["signature"], "value"))
.expect("trust-link signature is base64url");
issuer
.verify(
@@ -345,7 +346,7 @@ fn e2e_public_chain_matches_the_challenge_and_every_signature_verifies() {
assert_eq!(field(&challenge, "connectKeyId"), issuer_id.as_str());
let signature = BASE64_URL_NO_PAD
.decode(field(&challenge_envelope["signature"], "value"))
.decode_to_vec(field(&challenge_envelope["signature"], "value"))
.expect("challenge signature is base64url");
issuer
.verify(
@@ -571,7 +572,7 @@ fn response_reject_vectors_are_artifacts_build_response_cannot_emit() {
let presented = verifying_key(field(&refused, "devicePublicKey"));
let raw = BASE64_URL_NO_PAD
.decode(field(&vector["document"]["signature"], "value"))
.decode_to_vec(field(&vector["document"]["signature"], "value"))
.expect("signature is base64url");
let signature = p256::ecdsa::Signature::from_slice(&raw).expect("signature parses");
assert!(
@@ -685,8 +686,12 @@ fn malleated_high_s_signature_is_refused_although_it_verifies_mathematically() {
let malleated_value = field(&malleated, "value").to_string();
assert_ne!(genuine_value, malleated_value, "the malleation must be a different encoding");
let genuine = BASE64_URL_NO_PAD.decode(&genuine_value).expect("signature is base64url");
let raw = BASE64_URL_NO_PAD.decode(&malleated_value).expect("signature is base64url");
let genuine = BASE64_URL_NO_PAD
.decode_to_vec(&genuine_value)
.expect("signature is base64url");
let raw = BASE64_URL_NO_PAD
.decode_to_vec(&malleated_value)
.expect("signature is base64url");
assert_eq!(raw.len(), 64, "the malleation is well formed at 64 octets");
assert_eq!(raw[..32], genuine[..32], "the malleation shares r with the genuine signature");
assert_ne!(raw[32..], genuine[32..], "the malleation replaces s with n - s");
@@ -797,7 +802,7 @@ fn assert_response_proves_possession(built_envelope: &Value, label: &str) {
"{label}: the signature must use the base64url alphabet with no padding"
);
let bytes = BASE64_URL_NO_PAD.decode(value).expect("signature is base64url");
let bytes = BASE64_URL_NO_PAD.decode_to_vec(value).expect("signature is base64url");
assert_eq!(bytes.len(), 64, "{label}: the signature is a fixed-width r || s");
let signature = p256::ecdsa::Signature::from_slice(&bytes).expect("signature parses");
assert_eq!(
@@ -815,7 +820,7 @@ fn assert_response_proves_possession(built_envelope: &Value, label: &str) {
// SubjectPublicKeyInfo, not of the bare point and not of the transfer
// encoding.
let mut spki = hex_to_bytes(SPKI_PREFIX_HEX);
spki.extend_from_slice(&BASE64_URL_NO_PAD.decode(presented).expect("public key is base64url"));
spki.extend_from_slice(&BASE64_URL_NO_PAD.decode_to_vec(presented).expect("public key is base64url"));
let fingerprint = sha256_hex(&spki);
assert_eq!(
field(&built, "deviceKeyId"),
@@ -859,12 +864,12 @@ fn built_response_binds_the_challenge_proof_and_proves_possession_of_the_device_
assert_eq!(
field(&built, "devicePublicKey"),
BASE64_URL_NO_PAD.encode(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]),
BASE64_URL_NO_PAD.encode_to_string(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]),
"the presented key must be the key that was passed in"
);
assert_eq!(
field(&built, "deviceNonce"),
BASE64_URL_NO_PAD.encode([0x11; 32]),
BASE64_URL_NO_PAD.encode_to_string([0x11; 32]),
"the device nonce must be the one that was passed in"
);
assert!(field(&built, "producedAt").ends_with('Z'), "producedAt is a UTC RFC 3339 instant");
@@ -901,8 +906,8 @@ fn built_response_carries_no_private_key_material() {
for (description, needle) in [
("the PKCS#8 encoding", pkcs8.to_vec()),
("the raw private scalar", scalar.to_vec()),
("the scalar in base64url", BASE64_URL_NO_PAD.encode(scalar).into_bytes()),
("the scalar in standard base64", BASE64_STANDARD.encode(scalar).into_bytes()),
("the scalar in base64url", BASE64_URL_NO_PAD.encode_to_string(scalar).into_bytes()),
("the scalar in standard base64", BASE64_STANDARD.encode_to_string(scalar).into_bytes()),
("the scalar in hex", scalar_hex.into_bytes()),
] {
assert!(
@@ -914,7 +919,7 @@ fn built_response_carries_no_private_key_material() {
// The public half must be there, so the absence above is a statement about
// what was excluded rather than about a haystack that would not have found
// the private half either.
let point = BASE64_URL_NO_PAD.encode(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]);
let point = BASE64_URL_NO_PAD.encode_to_string(&key.public_key_der()[hex_to_bytes(SPKI_PREFIX_HEX).len()..]);
assert!(
haystack.windows(point.len()).any(|window| window == point.as_bytes()),
"the response must still present the public key"
+4 -5
View File
@@ -18,8 +18,7 @@ use std::io::Write as _;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use base64::Engine as _;
use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
use base64_simd::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD};
use bytes::Bytes;
use http_body_util::{BodyExt as _, Full};
use hyper::service::service_fn;
@@ -359,9 +358,9 @@ fn verify_rotation_request(request: &Value, current_public_key: &[u8], fingerpri
assert_eq!(request["protocolVersion"], "v1");
assert_eq!(request["proof"]["algorithm"], "ES256");
let csr = BASE64_STANDARD
.decode(request["certificateRequest"].as_str().expect("certificateRequest"))
.decode_to_vec(request["certificateRequest"].as_str().expect("certificateRequest"))
.expect("CSR base64");
let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(&csr));
let csr_digest = BASE64_URL_NO_PAD.encode_to_string(Sha256::digest(&csr));
let request_id = request["requestId"].as_str().expect("requestId");
let transcript = rebuilt_rotation_transcript(
b"RUSTFS-CONNECT-CREDENTIAL-ROTATION-V1",
@@ -369,7 +368,7 @@ fn verify_rotation_request(request: &Value, current_public_key: &[u8], fingerpri
);
let encoded = request["proof"]["value"].as_str().expect("proof value");
assert_eq!(encoded.len(), 86);
let raw = BASE64_URL_NO_PAD.decode(encoded).expect("proof base64url");
let raw = BASE64_URL_NO_PAD.decode_to_vec(encoded).expect("proof base64url");
let signature = Signature::from_slice(&raw).expect("fixed-width signature");
assert_eq!(signature.normalize_s(), signature, "rotation proof must be low-S");
let verifying = VerifyingKey::from_public_key_der(current_public_key).expect("current public key");