From ba7785d61dbf4ed1cb678cfb379e8c49a98795bc Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 27 Aug 2026 03:17:24 +0800 Subject: [PATCH] chore(deps): migrate direct encoding deps to simd (#6690) --- Cargo.lock | 20 +- Cargo.toml | 2 - crates/e2e_test/Cargo.toml | 4 +- crates/e2e_test/src/checksum_upload_test.rs | 5 +- crates/e2e_test/src/compression_test.rs | 3 +- .../e2e_test/src/copy_object_checksum_test.rs | 9 +- crates/e2e_test/src/fake_s3_target/mod.rs | 6 +- .../src/inline_fast_path_cluster_test.rs | 7 +- crates/e2e_test/src/kms/common.rs | 14 +- .../src/kms/kms_comprehensive_test.rs | 2 +- .../e2e_test/src/kms/kms_edge_cases_test.rs | 15 +- crates/e2e_test/src/kms/kms_local_test.rs | 6 +- crates/e2e_test/src/kms/kms_vault_test.rs | 4 +- .../src/kms/multipart_encryption_test.rs | 2 +- crates/e2e_test/src/multipart_auth_test.rs | 23 +- crates/e2e_test/src/protocols/webdav_core.rs | 3 +- .../src/replication_extension_test.rs | 20 +- crates/e2e_test/src/ssec_copy_test.rs | 5 +- crates/ecstore/Cargo.toml | 1 - .../replication_target_boundary.rs | 4 +- crates/ecstore/src/cluster/rpc/http_auth.rs | 31 +- crates/ecstore/src/object_api/readers.rs | 38 +-- crates/ecstore/src/store/list_objects.rs | 6 +- crates/heal/Cargo.toml | 2 +- crates/heal/src/heal/resume/checkpoint.rs | 3 +- crates/heal/src/heal/storage.rs | 16 +- crates/kms/Cargo.toml | 4 +- crates/kms/examples/kms_dr_drill.rs | 8 +- crates/kms/examples/local_kms_key_decrypt.rs | 4 +- crates/kms/src/audit.rs | 2 +- crates/kms/src/backends/aws.rs | 13 +- crates/kms/src/backends/contract_tests.rs | 5 +- crates/kms/src/backends/local.rs | 18 +- crates/kms/src/backends/mod.rs | 5 +- crates/kms/src/backends/static_kms.rs | 5 +- crates/kms/src/backends/vault.rs | 26 +- crates/kms/src/backends/vault_transit.rs | 12 +- crates/kms/src/backup/drill.rs | 4 +- crates/kms/src/backup/local_export.rs | 5 +- crates/kms/src/backup/local_restore.rs | 4 +- crates/kms/src/backup/manifest.rs | 2 +- crates/kms/src/config.rs | 15 +- crates/kms/src/config_secret.rs | 9 +- crates/kms/src/manager.rs | 10 +- crates/kms/src/probe.rs | 4 +- crates/kms/src/service.rs | 24 +- crates/kms/src/service_manager.rs | 9 +- crates/kms/tests/behavior_lifecycle.rs | 3 +- crates/kms/tests/behavior_objects.rs | 2 +- crates/kms/tests/common/mod.rs | 5 +- crates/protocols/Cargo.toml | 10 +- crates/protocols/src/swift/account.rs | 2 +- crates/protocols/src/swift/formpost.rs | 8 +- crates/protocols/src/swift/slo.rs | 6 +- crates/protocols/src/swift/sync.rs | 2 +- crates/protocols/src/swift/tempurl.rs | 2 +- crates/protocols/src/webdav/server.rs | 3 +- .../tests/swift_metadata_persistence.rs | 2 +- crates/rio-v2/Cargo.toml | 2 +- crates/rio-v2/src/encrypt_reader.rs | 5 +- crates/rio/Cargo.toml | 2 +- crates/rio/src/checksum.rs | 25 +- crates/rio/src/hash_reader.rs | 9 +- rustfs/Cargo.toml | 1 - rustfs/src/admin/handlers/kms_audit.rs | 5 +- rustfs/src/admin/handlers/kms_backup.rs | 16 +- rustfs/src/admin/handlers/kms_dynamic.rs | 8 +- rustfs/src/admin/handlers/kms_key_metadata.rs | 4 +- rustfs/src/admin/handlers/kms_keys.rs | 5 +- rustfs/src/admin/handlers/site_replication.rs | 43 ++- rustfs/src/connect/client.rs | 5 +- rustfs/src/connect/identity.rs | 15 +- rustfs/src/connect/offline/bundle_writer.rs | 6 +- rustfs/src/connect/offline/enrollment.rs | 19 +- rustfs/src/connect/registration.rs | 19 +- rustfs/src/storage/helper.rs | 8 +- rustfs/src/storage/sse.rs | 268 ++++++++++-------- rustfs/tests/connect_identity.rs | 23 +- rustfs/tests/connect_offline_bundle.rs | 6 +- rustfs/tests/connect_offline_enrollment.rs | 41 +-- rustfs/tests/connect_registration.rs | 9 +- 81 files changed, 523 insertions(+), 505 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5bb08ede7..07023e21e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3868,7 +3868,7 @@ dependencies = [ "aws-sdk-s3", "aws-sdk-sts", "aws-smithy-http-client", - "base64 0.23.1", + "base64-simd", "bytes", "chrono", "clap", @@ -3876,7 +3876,7 @@ dependencies = [ "flatbuffers", "flate2", "futures", - "hex", + "hex-simd", "hmac 0.13.0", "hotpath", "http 1.5.0", @@ -9313,7 +9313,6 @@ dependencies = [ "aws-config", "aws-sdk-s3", "axum", - "base64 0.23.1", "base64-simd", "bytes", "chacha20poly1305", @@ -9573,7 +9572,6 @@ dependencies = [ "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", - "base64 0.23.1", "base64-simd", "byteorder", "bytes", @@ -9739,7 +9737,7 @@ name = "rustfs-heal" version = "1.0.0-rc.3" dependencies = [ "async-trait", - "base64 0.23.1", + "base64-simd", "bytes", "crc-fast", "futures", @@ -9944,9 +9942,9 @@ dependencies = [ "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", - "base64 0.23.1", + "base64-simd", "chacha20poly1305", - "hex", + "hex-simd", "hotpath", "http 1.5.0", "insta", @@ -10251,12 +10249,12 @@ dependencies = [ "async-compression", "async-trait", "axum", - "base64 0.23.1", + "base64-simd", "bytes", "dav-server", "futures", "futures-util", - "hex", + "hex-simd", "hmac 0.13.0", "hotpath", "http 1.5.0", @@ -10355,7 +10353,7 @@ dependencies = [ "aes-gcm", "arc-swap", "axum", - "base64 0.23.1", + "base64-simd", "bytes", "crc-fast", "faster-hex", @@ -10395,7 +10393,7 @@ dependencies = [ "aes-gcm", "bytes", "chacha20poly1305", - "hex", + "hex-simd", "hmac 0.13.0", "hotpath", "minlz", diff --git a/Cargo.toml b/Cargo.toml index 7f0b31277..0f7a7f001 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -243,7 +243,6 @@ aws-sdk-sts = { default-features = false, version = "1.113.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" } aws-smithy-runtime-api = { version = "1.15.0" } aws-smithy-types = { version = "1.6.2" } -base64 = "0.23.1" base64-simd = "0.8.0" brotli = "8.0.4" clap = { version = "4.6.6" } @@ -266,7 +265,6 @@ hashbrown = { version = "0.17.1" } # Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet # every authenticator app expects). Already in the graph transitively. data-encoding = "2.11.1" -hex = "0.4.3" hex-simd = "0.8.0" highway = { version = "1.3.0" } hostname = "0.4.2" diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index 81184ad52..75a539561 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -121,10 +121,10 @@ tracing-subscriber = { workspace = true, features = ["env-filter", "time"] } uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] } urlencoding.workspace = true walkdir.workspace = true -base64 = { workspace = true } +base64-simd = { workspace = true } rand = { workspace = true, features = ["serde"] } chrono = { workspace = true, features = ["serde"] } -hex = { workspace = true } +hex-simd = { workspace = true } md-5 = { workspace = true } opentelemetry-proto = { workspace = true } prost.workspace = true diff --git a/crates/e2e_test/src/checksum_upload_test.rs b/crates/e2e_test/src/checksum_upload_test.rs index 78b5d9fdd..3a242c3b2 100644 --- a/crates/e2e_test/src/checksum_upload_test.rs +++ b/crates/e2e_test/src/checksum_upload_test.rs @@ -24,7 +24,6 @@ mod tests { use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart}; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; - use base64::Engine; use md5::{Digest as Md5Digest, Md5}; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; use sha2::Sha256; @@ -74,12 +73,12 @@ mod tests { let mut hasher = Md5::new(); hasher.update(body); let digest = hasher.finalize(); - base64::engine::general_purpose::STANDARD.encode(digest.as_slice()) + base64_simd::STANDARD.encode_to_string(digest.as_slice()) } fn checksum_sha256_base64(body: &[u8]) -> String { let digest = Sha256::digest(body); - base64::engine::general_purpose::STANDARD.encode(digest.as_slice()) + base64_simd::STANDARD.encode_to_string(digest.as_slice()) } fn checksum_crc64nvme_base64(body: &[u8]) -> String { diff --git a/crates/e2e_test/src/compression_test.rs b/crates/e2e_test/src/compression_test.rs index 1ed0269ec..f9df6167d 100644 --- a/crates/e2e_test/src/compression_test.rs +++ b/crates/e2e_test/src/compression_test.rs @@ -652,11 +652,10 @@ const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket"; async fn start_rustfs_with_compression_and_sse( env: &mut RustFSTestEnvironment, ) -> Result<(), Box> { - use base64::Engine; env.cleanup_existing_processes().await?; let binary_path = rustfs_binary_path(); - let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + let master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); // Server output goes to a file inside the per-test temp dir so a failing // run can be diagnosed from the child's logs. let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?; diff --git a/crates/e2e_test/src/copy_object_checksum_test.rs b/crates/e2e_test/src/copy_object_checksum_test.rs index b4ac11bbb..a39b04141 100644 --- a/crates/e2e_test/src/copy_object_checksum_test.rs +++ b/crates/e2e_test/src/copy_object_checksum_test.rs @@ -27,8 +27,7 @@ mod tests { VersioningConfiguration, }; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; - use base64::Engine as _; - use base64::engine::general_purpose::STANDARD as BASE64; + use base64_simd::STANDARD as BASE64; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; use sha2::{Digest, Sha256}; use tracing::info; @@ -465,7 +464,7 @@ mod tests { create_versioned_bucket(&client, dst_bucket).await; let content = b"deterministic synthetic payload for copy-object checksum #4996"; - let expected_sha256 = BASE64.encode(Sha256::digest(content)); + let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content)); client .put_object() @@ -534,7 +533,7 @@ mod tests { create_versioned_bucket(&client, dst_bucket).await; let content = b"another deterministic payload whose source checksum must survive the copy"; - let expected_sha256 = BASE64.encode(Sha256::digest(content)); + let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content)); // Store the source WITH a SHA-256 checksum so it has one to preserve. let put_src = client @@ -614,7 +613,7 @@ mod tests { create_versioned_bucket(&client, dst_bucket).await; let content = b"payload whose copy must be re-checksummed with a different algorithm"; - let expected_sha256 = BASE64.encode(Sha256::digest(content)); + let expected_sha256 = BASE64.encode_to_string(Sha256::digest(content)); // Source is stored WITH a SHA-256 checksum. client diff --git a/crates/e2e_test/src/fake_s3_target/mod.rs b/crates/e2e_test/src/fake_s3_target/mod.rs index aad643043..c5d7edffb 100644 --- a/crates/e2e_test/src/fake_s3_target/mod.rs +++ b/crates/e2e_test/src/fake_s3_target/mod.rs @@ -1113,7 +1113,7 @@ fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] { fn md5_hex(input: impl AsRef<[u8]>) -> String { let mut hasher = Md5::new(); hasher.update(input.as_ref()); - hex::encode(hasher.finalize()) + hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower) } fn ensure_store_budget(state: &StoreState, removed_bytes: usize, added_bytes: usize, adds_version: bool) -> S3Result { @@ -1375,7 +1375,7 @@ impl S3 for FakeBackend { Some(value) => value, None => { let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?; - hex::encode(digest) + hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower) } }; let version = ObjectVersion { @@ -1660,7 +1660,7 @@ impl S3 for FakeBackend { } let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?; let (digest, _body_permit) = md5_digest(body.clone(), _body_permit).await?; - let e_tag = hex::encode(digest); + let e_tag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower); let mut state = lock(&self.store); let existing_bytes = state .uploads diff --git a/crates/e2e_test/src/inline_fast_path_cluster_test.rs b/crates/e2e_test/src/inline_fast_path_cluster_test.rs index 8de63303f..5583a1b84 100644 --- a/crates/e2e_test/src/inline_fast_path_cluster_test.rs +++ b/crates/e2e_test/src/inline_fast_path_cluster_test.rs @@ -28,7 +28,6 @@ use aws_sdk_s3::types::{ BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus, LifecycleRule, LifecycleRuleFilter, ServerSideEncryption, Transition, TransitionStorageClass, VersioningConfiguration, }; -use base64::Engine; use bytes::Bytes; use flate2::read::GzDecoder; use http::header::{CONTENT_ENCODING, HOST}; @@ -1808,7 +1807,7 @@ async fn four_node_inline_fallback_controls() -> TestResult { let collector = OtlpMetricCollector::start().await?; let mut cluster = RustFSTestClusterEnvironment::new(4).await?; configure_reader_metric_cluster(&mut cluster, &collector); - let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", &sse_master_key); cluster.start().await?; @@ -2017,7 +2016,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te let collector = OtlpMetricCollector::start().await?; let mut cluster = RustFSTestClusterEnvironment::new(4).await?; - let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key); cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true"); cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"); @@ -2489,7 +2488,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_ hot.set_env("RUSTFS_SCANNER_CYCLE", "1"); hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1"); - let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); hot.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key); hot.set_env("RUSTFS_COMPRESSION_ENABLED", "true"); hot.start().await?; diff --git a/crates/e2e_test/src/kms/common.rs b/crates/e2e_test/src/kms/common.rs index b551a9849..af9b410db 100644 --- a/crates/e2e_test/src/kms/common.rs +++ b/crates/e2e_test/src/kms/common.rs @@ -27,7 +27,7 @@ use aws_sdk_s3::Client; use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::ServerSideEncryption; -use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; +use base64_simd::STANDARD as BASE64; use http::header::{CONTENT_TYPE, HOST}; use md5::{Digest as Md5Digest, Md5}; use rustfs_signer::constants::UNSIGNED_PAYLOAD; @@ -64,7 +64,7 @@ pub fn init_logging() { pub fn sse_customer_key_md5_base64(key: &str) -> String { let mut hasher = Md5::new(); hasher.update(key.as_bytes()); - BASE64.encode(hasher.finalize()) + BASE64.encode_to_string(hasher.finalize()) } pub fn assert_s3_error(result: Result>, status: u16, code: &str, message: &str, context: &str) @@ -365,7 +365,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result< "created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()), "rotated_at": serde_json::Value::Null, "created_by": "e2e-test", - "encrypted_key_material": BASE64.encode(key_data), + "encrypted_key_material": BASE64.encode_to_string(key_data), "nonce": Vec::::new() }); @@ -383,7 +383,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<( info!("Testing SSE-C encryption"); let test_key = "01234567890123456789012345678901"; // 32-byte key - let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key); + let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key); let test_key_md5 = sse_customer_key_md5_base64(test_key); let test_data = b"Hello, KMS SSE-C World!"; let object_key = "test-sse-c-object"; @@ -551,8 +551,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<() // Test SSE-C with wrong key for download let test_key = "01234567890123456789012345678901"; let wrong_key = "98765432109876543210987654321098"; - let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key); - let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key); + let test_key_b64 = base64_simd::STANDARD.encode_to_string(test_key); + let wrong_key_b64 = base64_simd::STANDARD.encode_to_string(wrong_key); let test_key_md5 = sse_customer_key_md5_base64(test_key); let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key); let test_data = b"Test data for error scenarios"; @@ -807,7 +807,7 @@ pub async fn test_multipart_upload_with_config( // Prepare encryption parameters let (sse_c_key_b64, sse_c_key_md5) = match &config.encryption_type { EncryptionType::SSEC { key, key_md5 } => { - let key_b64 = base64::engine::general_purpose::STANDARD.encode(key); + let key_b64 = base64_simd::STANDARD.encode_to_string(key); (Some(key_b64), Some(key_md5.clone())) } _ => (None, None), diff --git a/crates/e2e_test/src/kms/kms_comprehensive_test.rs b/crates/e2e_test/src/kms/kms_comprehensive_test.rs index d4bbf604a..8569118bf 100644 --- a/crates/e2e_test/src/kms/kms_comprehensive_test.rs +++ b/crates/e2e_test/src/kms/kms_comprehensive_test.rs @@ -177,7 +177,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box { // SSE-C let key = format!("testkey{i:026}"); // 32-byte key - let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key); + let key_b64 = base64_simd::STANDARD.encode_to_string(&key); let key_md5 = sse_customer_key_md5_base64(&key); client @@ -535,8 +534,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box Result<(), Box) -> String { "conditions": conditions, }); - base64::engine::general_purpose::STANDARD.encode(policy.to_string()) + base64_simd::STANDARD.encode_to_string(policy.to_string()) } fn sse_customer_key_md5_base64(key: &str) -> String { let mut hasher = Md5::new(); hasher.update(key.as_bytes()); - base64::engine::general_purpose::STANDARD.encode(hasher.finalize()) + base64_simd::STANDARD.encode_to_string(hasher.finalize()) } fn md5_hex(input: impl AsRef<[u8]>) -> String { let mut hasher = Md5::new(); hasher.update(input.as_ref()); - hex::encode(hasher.finalize()) + hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower) } async fn create_restricted_user( @@ -97,7 +96,7 @@ fn restricted_user_client(env: &RustFSTestEnvironment, username: &str, secret_ke const LOCAL_SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY"; fn local_sse_master_key_value() -> String { - base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]) + base64_simd::STANDARD.encode_to_string([0x42u8; 32]) } async fn make_tar(files: &[(&str, &[u8])], dirs: &[&str]) -> Vec { @@ -1887,7 +1886,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition let object_key = "sse-c-object.txt"; let expected_body = b"anonymous-post-sse-c".to_vec(); let customer_key = "01234567890123456789012345678901"; - let customer_key_b64 = base64::engine::general_purpose::STANDARD.encode(customer_key); + let customer_key_b64 = base64_simd::STANDARD.encode_to_string(customer_key); let customer_key_md5 = sse_customer_key_md5_base64(customer_key); let admin_client = env.create_s3_client(); @@ -1941,7 +1940,7 @@ async fn test_anonymous_post_object_allows_sse_c_fields_outside_policy_condition .bucket(bucket) .key(object_key) .sse_customer_algorithm("AES256") - .sse_customer_key(base64::engine::general_purpose::STANDARD.encode(customer_key)) + .sse_customer_key(base64_simd::STANDARD.encode_to_string(customer_key)) .sse_customer_key_md5(customer_key_md5) .send() .await?; @@ -1963,8 +1962,8 @@ async fn test_anonymous_post_object_rejects_sse_c_exact_policy_mismatch() -> Res let object_key = "sse-c-mismatch-object.txt"; let policy_key = "01234567890123456789012345678901"; let request_key = "abcdefghijklmnopqrstuvwxyzABCDEF"; - let policy_key_b64 = base64::engine::general_purpose::STANDARD.encode(policy_key); - let request_key_b64 = base64::engine::general_purpose::STANDARD.encode(request_key); + let policy_key_b64 = base64_simd::STANDARD.encode_to_string(policy_key); + let request_key_b64 = base64_simd::STANDARD.encode_to_string(request_key); let admin_client = env.create_s3_client(); admin_client.create_bucket().bucket(bucket).send().await?; @@ -3526,7 +3525,7 @@ async fn test_signed_put_object_extract_preserves_sse_s3_and_redirect() -> Resul init_logging(); let mut env = RustFSTestEnvironment::new().await?; - let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())]) .await?; @@ -3799,7 +3798,7 @@ async fn test_signed_put_object_extract_uses_bucket_default_sse_s3() -> Result<( init_logging(); let mut env = RustFSTestEnvironment::new().await?; - let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]); + let sse_master_key = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key.as_str())]) .await?; @@ -3925,7 +3924,7 @@ async fn test_signed_put_object_extract_preserves_sse_c() -> Result<(), Box String { fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String { let credentials = format!("{}:{}", access_key, secret_key); - let encoded = base64::engine::general_purpose::STANDARD.encode(credentials); + let encoded = base64_simd::STANDARD.encode_to_string(credentials); format!("Basic {}", encoded) } diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 479fbeeed..98b9a5d8c 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -34,7 +34,7 @@ use aws_sdk_s3::types::{ VersioningConfiguration, }; use aws_sdk_s3::{Client, Config}; -use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use base64_simd::STANDARD as BASE64_STANDARD; use bytes::Bytes; use flate2::read::GzDecoder; use futures::{Stream, StreamExt}; @@ -1244,7 +1244,7 @@ async fn wait_for_source_replication_pending_or_failed( } async fn wait_for_source_replication_status(client: &Client, bucket: &str, key: &str, expected: &str, ssec: bool) -> TestResult { - let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY); let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); let wait = async { loop { @@ -1339,7 +1339,7 @@ async fn assert_failed_replication_stays_absent_for( ssec: bool, duration: Duration, ) -> TestResult { - let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY); let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); let wait = async { let deadline = tokio::time::Instant::now() + duration; @@ -4332,7 +4332,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult { let target_client = target_env.create_s3_client(); let key = "ssec-contract.txt"; let body = b"repl-17 SSE-C payload"; - let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY); let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); source_client @@ -4387,7 +4387,7 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult { ); // A wrong customer key must fail too. - let wrong_key = BASE64_STANDARD.encode("99999999999999999999999999999999"); + let wrong_key = BASE64_STANDARD.encode_to_string("99999999999999999999999999999999"); let wrong_key_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999"); let wrong_read = target_client .get_object() @@ -4423,7 +4423,7 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult { let source_client = source_env.create_s3_client(); let target_client = target_env.create_s3_client(); let key = "ssec-mp-contract.bin"; - let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY); let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); let created = source_client @@ -4568,7 +4568,7 @@ async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_header .await?; put_bucket_replication(&source_env, source_bucket, &target_arn).await?; - let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY); let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); let put_ssec = |key: &'static str| { source_client @@ -4741,7 +4741,7 @@ async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult let source_client = source_env.create_s3_client(); let key = "ssec-heal-contract.txt"; let body = b"repl-22 ssec heal payload".to_vec(); - let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY); let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); // Target outage: the SSE-C write cannot replicate. @@ -4856,7 +4856,7 @@ async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult { // The SSE-C object exists before any replication wiring. let key = "ssec-existing-contract.txt"; let body = b"repl-22 ssec existing-object payload".to_vec(); - let customer_key = BASE64_STANDARD.encode(REPL17_SSEC_KEY); + let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY); let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY); source_client .put_object() @@ -9152,7 +9152,7 @@ async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> // the real SSE-C decryption; the plaintext fake simply ignores them). target.take_requests(); let ssec_key = "01234567890123456789012345678901"; - let ssec_key_b64 = BASE64_STANDARD.encode(ssec_key); + let ssec_key_b64 = BASE64_STANDARD.encode_to_string(ssec_key); let ssec_key_md5 = sse_customer_key_md5_base64(ssec_key); let _ = source_client .get_object() diff --git a/crates/e2e_test/src/ssec_copy_test.rs b/crates/e2e_test/src/ssec_copy_test.rs index e84ed9b11..0ca783ff3 100644 --- a/crates/e2e_test/src/ssec_copy_test.rs +++ b/crates/e2e_test/src/ssec_copy_test.rs @@ -21,7 +21,6 @@ use aws_sdk_s3::error::BoxError; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration}; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; -use base64::Engine; use md5::{Digest as Md5Digest, Md5}; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -107,8 +106,8 @@ fn customer_key(byte: u8) -> CustomerKey { hasher.update(raw); CustomerKey { raw: String::from_utf8_lossy(&raw).into_owned(), - encoded: base64::engine::general_purpose::STANDARD.encode(raw), - md5: base64::engine::general_purpose::STANDARD.encode(hasher.finalize()), + encoded: base64_simd::STANDARD.encode_to_string(raw), + md5: base64_simd::STANDARD.encode_to_string(hasher.finalize()), } } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index cbfafd4a9..c2d605510 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -176,7 +176,6 @@ rmp.workspace = true rmp-serde.workspace = true tokio-util = { workspace = true, features = ["io", "compat"] } tokio-stream = { workspace = true, features = ["sync"] } -base64 = { workspace = true } hmac = { workspace = true } sha2 = { workspace = true } hex-simd = { workspace = true } diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index 50aa14cd7..dbe3a29d5 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -200,7 +200,7 @@ impl ReplicationTargetStore { } pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) -> Result<(PutObjectOptions, bool)> { - use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; + use base64_simd::STANDARD as BASE64_STANDARD; use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT}; let mut meta = HashMap::new(); @@ -252,7 +252,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) && !checksum_data.is_empty() { if is_ssec { - let encoded = BASE64_STANDARD.encode(checksum_data); + let encoded = BASE64_STANDARD.encode_to_string(checksum_data); insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded); } else if object_info.is_encrypted() { // Encrypted checksums cannot be exposed as plaintext headers, and diff --git a/crates/ecstore/src/cluster/rpc/http_auth.rs b/crates/ecstore/src/cluster/rpc/http_auth.rs index 00b0dd2bf..157da2c57 100644 --- a/crates/ecstore/src/cluster/rpc/http_auth.rs +++ b/crates/ecstore/src/cluster/rpc/http_auth.rs @@ -31,8 +31,7 @@ use crate::storage_api_contracts::internode::{ NS_SCANNER_PROTOCOL_VERSION, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION, }; -use base64::Engine as _; -use base64::engine::general_purpose; + use hmac::{Hmac, KeyInit, Mac}; use http::uri::Authority; use http::{HeaderMap, HeaderValue, Method, Uri}; @@ -523,11 +522,11 @@ fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) let mut mac = ::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size"); mac.update(data.as_bytes()); let result = mac.finalize(); - general_purpose::STANDARD.encode(result.into_bytes()) + base64_simd::STANDARD.encode_to_string(result.into_bytes()) } fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool { - let Ok(signature) = general_purpose::STANDARD.decode(signature) else { + let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else { return false; }; @@ -745,11 +744,11 @@ fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io:: let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; update_signature_v2(&mut mac, scope); - Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes())) + Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes())) } fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool { - let Ok(signature) = general_purpose::STANDARD.decode(signature) else { + let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else { return false; }; let Ok(mut mac) = ::new_from_slice(secret.as_bytes()) else { @@ -792,11 +791,11 @@ fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std: let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; update_replay_scope(&mut mac, scope); - Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes())) + Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes())) } fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool { - let Ok(signature) = general_purpose::STANDARD.decode(signature) else { + let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else { return false; }; let Ok(mut mac) = ::new_from_slice(secret.as_bytes()) else { @@ -821,15 +820,15 @@ fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch); - Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes())) + Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes())) } fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> { if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() { return Err(std::io::Error::other("Invalid RPC boot epoch proof scope")); } - let proof = general_purpose::STANDARD - .decode(proof) + let proof = base64_simd::STANDARD + .decode_to_vec(proof) .map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?; let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; @@ -862,7 +861,7 @@ fn generate_replay_cache_capability_proof( let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch); - Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes())) + Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes())) } fn verify_replay_cache_capability_proof( @@ -872,8 +871,8 @@ fn verify_replay_cache_capability_proof( boot_epoch: Uuid, proof: &str, ) -> std::io::Result<()> { - let proof = general_purpose::STANDARD - .decode(proof) + let proof = base64_simd::STANDARD + .decode_to_vec(proof) .map_err(|_| std::io::Error::other("Invalid RPC replay cache capability proof"))?; let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; @@ -1988,9 +1987,9 @@ mod tests { let method = Method::GET; let timestamp = 1640995200; let signature = generate_signature(secret, url, &method, timestamp); - let mut tampered = general_purpose::STANDARD.decode(&signature).unwrap(); + let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap(); tampered[0] ^= 1; - let tampered_signature = general_purpose::STANDARD.encode(tampered); + let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered); assert!(verify_signature(secret, url, &method, timestamp, &signature)); assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature)); diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 9fc69118d..8bcc23974 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -1407,8 +1407,7 @@ fn multipart_part_numbers(parts: &[ObjectPartInfo]) -> Vec { #[cfg(test)] mod tests { use super::*; - use base64::Engine; - use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; + use base64_simd::STANDARD as BASE64_STANDARD; use md5::{Digest, Md5}; use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER}; use std::collections::HashMap; @@ -1468,7 +1467,7 @@ mod tests { request: ReadEncryptionRequest<'_>, ) -> std::result::Result, EncryptionResolutionError> { if let Some(encoded) = request.metadata.get(TEST_OBJECT_KEY_HEADER) { - let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| { + let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| { EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidMetadata, "invalid test object key") })?; let key_bytes = decoded.try_into().map_err(|_| { @@ -1493,7 +1492,7 @@ mod tests { .map_err(|_| { EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key") })?; - let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| { + let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| { EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key") })?; let key_bytes = decoded.try_into().map_err(|_| { @@ -1505,7 +1504,7 @@ mod tests { let base_nonce = request .metadata .get(TEST_NONCE_HEADER) - .and_then(|encoded| BASE64_STANDARD.decode(encoded).ok()) + .and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok()) .and_then(|bytes| bytes.try_into().ok()) .unwrap_or_else(|| fixture_nonce(request.bucket, request.object)); Ok(Some(ReadEncryptionMaterial { @@ -1533,7 +1532,7 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert( TEST_DIRECT_KEY_HEADER, - HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("test key header is valid"), + HeaderValue::from_str(&BASE64_STANDARD.encode_to_string(key_bytes)).expect("test key header is valid"), ); headers } @@ -2439,7 +2438,10 @@ mod tests { user_defined: Arc::new(HashMap::from([ ("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()), ("X-Amz-Server-Side-Encryption-Iv".to_string(), "AAAAAAAAAAAAAAAA".to_string()), - ("X-Amz-Server-Side-Encryption-Key".to_string(), BASE64_STANDARD.encode([7_u8; 32])), + ( + "X-Amz-Server-Side-Encryption-Key".to_string(), + BASE64_STANDARD.encode_to_string([7_u8; 32]), + ), ("x-rustfs-encryption-original-size".to_string(), "64".to_string()), ])), ..Default::default() @@ -2748,7 +2750,7 @@ mod tests { ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), - BASE64_STANDARD.encode(md5_bytes(key_bytes)), + BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)), ), ( "x-amz-server-side-encryption-customer-original-size".to_string(), @@ -2796,11 +2798,11 @@ mod tests { name: object.to_string(), size: encrypted.len() as i64, user_defined: Arc::new(HashMap::from([ - (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), - BASE64_STANDARD.encode(md5_bytes(customer_key)), + BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)), ), ( "x-amz-server-side-encryption-customer-original-size".to_string(), @@ -2856,7 +2858,7 @@ mod tests { ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), - BASE64_STANDARD.encode(md5_bytes(key_bytes)), + BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)), ), ( "x-amz-server-side-encryption-customer-original-size".to_string(), @@ -3008,13 +3010,13 @@ mod tests { ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), - BASE64_STANDARD.encode(md5_bytes(key_bytes)), + BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)), ), ( "x-amz-server-side-encryption-customer-original-size".to_string(), total_plaintext.to_string(), ), - (TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE)), + (TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode_to_string(LEGACY_FIXTURE_BASE_NONCE)), ]) } @@ -3751,11 +3753,11 @@ mod tests { name: object.to_string(), size: encrypted.len() as i64, user_defined: Arc::new(HashMap::from([ - (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), - BASE64_STANDARD.encode(md5_bytes(customer_key)), + BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)), ), ( "x-amz-server-side-encryption-customer-original-size".to_string(), @@ -3821,7 +3823,7 @@ mod tests { ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), - BASE64_STANDARD.encode(md5_bytes(key_bytes)), + BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)), ), ( "x-amz-server-side-encryption-customer-original-size".to_string(), @@ -3964,11 +3966,11 @@ mod tests { ..Default::default() }]), user_defined: Arc::new(HashMap::from([ - (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), - BASE64_STANDARD.encode(md5_bytes(customer_key)), + BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)), ), ( "x-amz-server-side-encryption-customer-original-size".to_string(), diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 683e9eeb3..e3ed34d24 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -42,7 +42,7 @@ use crate::storage_api_contracts::{ }; use crate::store::ECStore; use crate::store::utils::is_reserved_or_invalid_bucket; -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use base64_simd::STANDARD as BASE64_STANDARD; use bytes::Bytes; use futures::future::join_all; use rand::seq::SliceRandom; @@ -1398,11 +1398,11 @@ async fn persist_observed_list_objects_mutation(store: Option<&ECStore>, bucket: } fn encode_persistent_list_metadata_string(value: &str) -> String { - BASE64_STANDARD.encode(value.as_bytes()) + BASE64_STANDARD.encode_to_string(value.as_bytes()) } fn decode_persistent_list_metadata_string(value: &str) -> Option { - let bytes = BASE64_STANDARD.decode(value).ok()?; + let bytes = BASE64_STANDARD.decode_to_vec(value).ok()?; String::from_utf8(bytes).ok() } diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index ddedc83d4..511595429 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -90,7 +90,7 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnos async-trait = { workspace = true } futures = { workspace = true } metrics = { workspace = true } -base64 = { workspace = true } +base64-simd = { workspace = true } bytes = { workspace = true } crc-fast = { workspace = true } sha2 = { workspace = true } diff --git a/crates/heal/src/heal/resume/checkpoint.rs b/crates/heal/src/heal/resume/checkpoint.rs index b3f097c9f..505f10a4d 100644 --- a/crates/heal/src/heal/resume/checkpoint.rs +++ b/crates/heal/src/heal/resume/checkpoint.rs @@ -13,7 +13,6 @@ // limitations under the License. use crate::{Error, Result}; -use base64::Engine as _; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::HashSet; @@ -827,7 +826,7 @@ impl CheckpointManager { } fn checkpoint_digest(checkpoint_data: &[u8]) -> String { - base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data)) + base64_simd::STANDARD.encode_to_string(Sha256::digest(checkpoint_data)) } fn digest_path(task_id: &str) -> std::path::PathBuf { diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index f84a75a60..1f857aca1 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -14,8 +14,7 @@ use crate::{Error, Result}; use async_trait::async_trait; -use base64::Engine as _; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64_simd::URL_SAFE_NO_PAD; use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode}; use rustfs_madmin::heal_commands::HealResultItem; use serde::{Deserialize, Serialize}; @@ -147,7 +146,7 @@ pub(crate) fn encode_heal_token(marker: Option<&str>, version_marker: Option<&st // serde_json of a simple two-Option struct cannot fail; fall back to an // empty object rather than panicking if it somehow does. let json = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{}".to_vec()); - format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(json)) + format!("{HEAL_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(json)) } /// Decode an opaque heal continuation token back into `(marker, version_marker)`. @@ -174,7 +173,7 @@ pub(crate) fn decode_heal_token(token: &str) -> (Option, Option) return (None, None); }; - let bytes = match URL_SAFE_NO_PAD.decode(encoded) { + let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) { Ok(bytes) => bytes, Err(e) => { warn!( @@ -234,7 +233,7 @@ const DISK_WALK_TOKEN_PREFIX: &str = "dw1:"; /// enumerators can never misread each other's cursor: a `dw1:` token decodes to /// `(None, None)` under the B5 decoder, and a `v1:` token decodes to `None` here. pub(crate) fn encode_disk_walk_token(next_forward: &str) -> String { - format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode(next_forward.as_bytes())) + format!("{DISK_WALK_TOKEN_PREFIX}{}", URL_SAFE_NO_PAD.encode_to_string(next_forward.as_bytes())) } /// Decode a disk-walk continuation token back into the `next_forward` object key. @@ -261,7 +260,7 @@ pub(crate) fn decode_disk_walk_token(token: &str) -> Option { return None; }; - let bytes = match URL_SAFE_NO_PAD.decode(encoded) { + let bytes = match URL_SAFE_NO_PAD.decode_to_vec(encoded) { Ok(bytes) => bytes, Err(e) => { warn!( @@ -1513,7 +1512,6 @@ mod tests { decode_disk_walk_token, decode_heal_token, encode_disk_walk_token, encode_heal_token, is_transient_object_exists_error, is_transient_object_exists_message, next_heal_listing_token, }; - use base64::Engine as _; #[test] fn next_heal_listing_token_returns_none_for_complete_page() { @@ -1564,7 +1562,7 @@ mod tests { assert_eq!(decode_heal_token("no-prefix-here"), (None, None)); assert_eq!(decode_heal_token("v1:!!!not-base64!!!"), (None, None)); // valid base64 of non-JSON bytes. - let bad_json = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json")); + let bad_json = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(b"not json")); assert_eq!(decode_heal_token(&bad_json), (None, None)); // a raw v2-style list_objects_v2 token (no "v1:" prefix) resets cleanly. assert_eq!(decode_heal_token("some-opaque-legacy-token"), (None, None)); @@ -1576,7 +1574,7 @@ mod tests { // list_object_versions returns NotImplemented for that pairing. // Craft a token whose JSON encodes (None, Some) directly and confirm coercion. let json = br#"{"m":null,"v":"orphan-version"}"#; - let token = format!("v1:{}", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json)); + let token = format!("v1:{}", base64_simd::URL_SAFE_NO_PAD.encode_to_string(json)); assert_eq!(decode_heal_token(&token), (None, None), "version-only marker must coerce to (None, None)"); } diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index f8491d5a2..77f853097 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -50,8 +50,8 @@ aes-gcm = { workspace = true, features = ["rand_core"] } argon2 = { workspace = true } chacha20poly1305 = { workspace = true } rand = { workspace = true, features = ["serde"] } -base64 = { workspace = true } -hex = { workspace = true } +base64-simd = { workspace = true } +hex-simd = { workspace = true } sha2 = { workspace = true } subtle = { workspace = true } zeroize = { workspace = true, features = ["derive"] } diff --git a/crates/kms/examples/kms_dr_drill.rs b/crates/kms/examples/kms_dr_drill.rs index 9074bb202..7c673b4e3 100644 --- a/crates/kms/examples/kms_dr_drill.rs +++ b/crates/kms/examples/kms_dr_drill.rs @@ -23,7 +23,7 @@ //! Exit status is the verdict: 0 when every check held, 1 otherwise, so a //! scheduled drill fails its job instead of quietly filing a bad report. -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use base64_simd::STANDARD as BASE64; use rustfs_kms::backup::{BackupKek, DrillDataset, DrillDisaster, DrillRequest, DrillVerdict, run_local_drill}; use std::path::PathBuf; use std::process::ExitCode; @@ -100,7 +100,11 @@ fn disaster_from_env() -> Result { fn kek_from_env() -> Result { let raw = Zeroizing::new(required(ENV_KEK)?); - let decoded = Zeroizing::new(BASE64.decode(raw.trim()).map_err(|_| format!("{ENV_KEK} must be base64"))?); + let decoded = Zeroizing::new( + BASE64 + .decode_to_vec(raw.trim()) + .map_err(|_| format!("{ENV_KEK} must be base64"))?, + ); if decoded.len() != 32 { return Err(format!("{ENV_KEK} must decode to exactly 32 bytes")); } diff --git a/crates/kms/examples/local_kms_key_decrypt.rs b/crates/kms/examples/local_kms_key_decrypt.rs index 37f174ccc..75365d90a 100644 --- a/crates/kms/examples/local_kms_key_decrypt.rs +++ b/crates/kms/examples/local_kms_key_decrypt.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use base64_simd::STANDARD as BASE64_STANDARD; use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; @@ -69,7 +69,7 @@ async fn run() -> Result<(), String> { .decrypt_key_material_for_export(&key_id) .await .map_err(|error| error.to_string())?; - let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref())); + let encoded = Zeroizing::new(BASE64_STANDARD.encode_to_string(key_material.as_ref())); let mut stdout = io::stdout().lock(); writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}")) diff --git a/crates/kms/src/audit.rs b/crates/kms/src/audit.rs index df809926e..87a09e080 100644 --- a/crates/kms/src/audit.rs +++ b/crates/kms/src/audit.rs @@ -305,7 +305,7 @@ pub fn redact_encryption_context(encryption_context: &HashMap) - } fn digest_value(value: &str) -> String { - let digest = hex::encode(Sha256::digest(value.as_bytes())); + let digest = hex_simd::encode_to_string(Sha256::digest(value.as_bytes()), hex_simd::AsciiCase::Lower); format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN]) } diff --git a/crates/kms/src/backends/aws.rs b/crates/kms/src/backends/aws.rs index 93b85c956..87ee53ae7 100644 --- a/crates/kms/src/backends/aws.rs +++ b/crates/kms/src/backends/aws.rs @@ -848,8 +848,7 @@ mod tests { use aws_sdk_kms::config::{BehaviorVersion, Credentials, Region}; use aws_smithy_http_client::test_util::{NeverClient, ReplayEvent, StaticReplayClient}; use aws_smithy_types::body::SdkBody; - use base64::Engine as _; - use base64::engine::general_purpose::STANDARD as BASE64; + use base64_simd::STANDARD as BASE64; use std::sync::atomic::{AtomicU64, Ordering}; /// AWS KMS speaks awsJson1_1; every request goes to `/` on the regional @@ -977,8 +976,8 @@ mod tests { let ciphertext = b"encrypted-data-key".to_vec(); let (http_client, backend) = scripted_backend(vec![ok_event(serde_json::json!({ "KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key", - "Plaintext": BASE64.encode(&plaintext), - "CiphertextBlob": BASE64.encode(&ciphertext), + "Plaintext": BASE64.encode_to_string(&plaintext), + "CiphertextBlob": BASE64.encode_to_string(&ciphertext), }))]); let response = backend @@ -997,7 +996,7 @@ mod tests { let plaintext = b"recovered-data-key".to_vec(); let (_http, backend) = scripted_backend(vec![ok_event(serde_json::json!({ "KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key", - "Plaintext": BASE64.encode(&plaintext), + "Plaintext": BASE64.encode_to_string(&plaintext), "EncryptionAlgorithm": "SYMMETRIC_DEFAULT", }))]); @@ -1061,8 +1060,8 @@ mod tests { error_event(400, "ThrottlingException", "rate exceeded"), ok_event(serde_json::json!({ "KeyId": "test-key", - "Plaintext": BASE64.encode([1u8; 32]), - "CiphertextBlob": BASE64.encode(b"blob"), + "Plaintext": BASE64.encode_to_string([1u8; 32]), + "CiphertextBlob": BASE64.encode_to_string(b"blob"), })), ]); diff --git a/crates/kms/src/backends/contract_tests.rs b/crates/kms/src/backends/contract_tests.rs index ed1342f1f..8b25643b0 100644 --- a/crates/kms/src/backends/contract_tests.rs +++ b/crates/kms/src/backends/contract_tests.rs @@ -41,8 +41,7 @@ use crate::types::{ DescribeKeyRequest, EncryptRequest, GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext, RewrapDataKeyRequest, }; -use base64::Engine as _; -use base64::engine::general_purpose::STANDARD as BASE64; +use base64_simd::STANDARD as BASE64; use rand::RngExt as _; use std::collections::HashMap; use std::sync::Arc; @@ -285,7 +284,7 @@ async fn static_backend_stateless_contract() { let key_id = "static-contract-key"; let mut raw_key = [0u8; 32]; rand::rng().fill(&mut raw_key[..]); - let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key)); + let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode_to_string(raw_key)); let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build"); let backend: &dyn KmsBackend = &static_backend; diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index b694f366a..ac10274fc 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -33,7 +33,7 @@ use aes_gcm::{ }; use argon2::{Algorithm, Argon2, Params, Version}; use async_trait::async_trait; -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use base64_simd::STANDARD as BASE64; use jiff::Zoned; use rand::RngExt; use serde::de::{self, IgnoredAny, MapAccess, Visitor}; @@ -1268,7 +1268,7 @@ impl LocalKmsClient { } let encrypted_bytes = BASE64 - .decode(&stored_key.encrypted_key_material) + .decode_to_vec(&stored_key.encrypted_key_material) .map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?; let effective_protection = if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified { @@ -1406,13 +1406,17 @@ impl LocalKmsClient { .encrypt(&nonce, key_material) .map_err(|e| KmsError::cryptographic_error("encrypt", e.to_string()))?; // Encode encrypted bytes to base64 string - (BASE64.encode(&encrypted), nonce.to_vec(), StoredKeyProtection::EncryptedMasterKey) + ( + BASE64.encode_to_string(&encrypted), + nonce.to_vec(), + StoredKeyProtection::EncryptedMasterKey, + ) } else { warn!( key_id = %master_key.key_id, "Local KMS is storing key material as plaintext-dev-only because no master key is configured" ); - (BASE64.encode(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly) + (BASE64.encode_to_string(key_material), Vec::new(), StoredKeyProtection::PlaintextDevOnly) }; let stored_key = StoredMasterKey { @@ -2648,10 +2652,10 @@ mod tests { let tampered_material = { let mut material = BASE64 - .decode(pristine["encrypted_key_material"].as_str().expect("material is a string")) + .decode_to_vec(pristine["encrypted_key_material"].as_str().expect("material is a string")) .expect("decode pristine material"); *material.last_mut().expect("material is not empty") ^= 0x01; - BASE64.encode(&material) + BASE64.encode_to_string(&material) }; type PoisonCase = (&'static str, Vec, fn(&KmsError) -> bool); @@ -2976,7 +2980,7 @@ mod tests { "created_at": "2024-01-01T00:00:00+00:00", "rotated_at": serde_json::Value::Null, "created_by": "legacy-test", - "encrypted_key_material": BASE64.encode([7u8; 32]), + "encrypted_key_material": BASE64.encode_to_string([7u8; 32]), "nonce": Vec::::new() }); diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index a6213724e..6fcb26a9b 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -722,8 +722,7 @@ impl Default for BackendCapabilities { mod tests { use super::*; use crate::config::KmsConfig; - use base64::Engine as _; - use base64::engine::general_purpose::STANDARD as BASE64; + use base64_simd::STANDARD as BASE64; /// Backend that implements only the trait-mandated operations and relies /// on the default `capabilities` implementation. @@ -958,7 +957,7 @@ mod tests { #[tokio::test] async fn static_backend_capabilities_golden() { - let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode([0u8; 32])); + let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode_to_string([0u8; 32])); let backend = static_kms::StaticKmsBackend::new(config) .await .expect("static backend should build"); diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index 2f3560497..2ad2d3e1a 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -434,8 +434,7 @@ mod tests { use crate::backends::KmsBackend as KmsBackendTrait; use crate::config::{BackendConfig, KmsBackend, StaticConfig}; use crate::encryption::is_data_key_envelope; - use base64::Engine as _; - use base64::engine::general_purpose::STANDARD as BASE64; + use base64_simd::STANDARD as BASE64; /// Generate a random 32-byte key and return (key_id, raw_key). fn random_static_key(key_id: &str) -> (String, [u8; 32]) { @@ -447,7 +446,7 @@ mod tests { fn static_config(key_id: &str, raw_key: &[u8; 32]) -> StaticConfig { StaticConfig { key_id: key_id.to_string(), - secret_key: BASE64.encode(raw_key), + secret_key: BASE64.encode_to_string(raw_key), } } diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 0a787631a..2bfb97c6c 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -33,7 +33,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar use crate::policy::{self, AttemptError, OpClass, RetryPolicy}; use crate::types::*; use async_trait::async_trait; -use base64::{Engine as _, engine::general_purpose}; +use base64_simd::STANDARD as BASE64; use jiff::Zoned; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -527,8 +527,8 @@ fn decode_stored_key_material(key_id: &str, encrypted_material: &str) -> Result< // Mirrors `decrypt_key_material`: stored material is currently base64 without an // additional encryption layer. - let key_material = general_purpose::STANDARD - .decode(encrypted_material) + let key_material = BASE64 + .decode_to_vec(encrypted_material) .map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?; // Key material must be exactly 32 bytes for AES-256. @@ -693,7 +693,7 @@ impl VaultKmsClient { /// confidentiality. Any identity with KV read access to the key path can recover the /// plaintext master key. async fn encrypt_key_material(&self, key_material: &[u8]) -> Result { - Ok(general_purpose::STANDARD.encode(key_material)) + Ok(base64_simd::STANDARD.encode_to_string(key_material)) } /// Read the immutable material record of one key version. @@ -2405,7 +2405,7 @@ mod tests { tags: HashMap::new(), deletion_date: None, rotated_at: None, - encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]), + encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]), baseline_version: None, wrap_budget_reserved: 0, } @@ -2869,21 +2869,21 @@ mod tests { )); // Truncated material: valid base64 of fewer than 32 bytes. - let truncated = general_purpose::STANDARD.encode([0x42u8; 16]); + let truncated = base64_simd::STANDARD.encode_to_string([0x42u8; 16]); assert!(matches!( decode_stored_key_material("poisoned", &truncated), Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned" )); // Oversized material: valid base64 of more than 32 bytes. - let oversized = general_purpose::STANDARD.encode([0x42u8; 33]); + let oversized = base64_simd::STANDARD.encode_to_string([0x42u8; 33]); assert!(matches!( decode_stored_key_material("poisoned", &oversized), Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned" )); // Well-formed material still decodes. - let valid = general_purpose::STANDARD.encode([0x42u8; 32]); + let valid = base64_simd::STANDARD.encode_to_string([0x42u8; 32]); assert_eq!( decode_stored_key_material("healthy", &valid).expect("valid material must decode"), vec![0x42u8; 32] @@ -3046,7 +3046,7 @@ mod tests { description: None, metadata: HashMap::new(), tags: HashMap::new(), - encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]), + encrypted_key_material: base64_simd::STANDARD.encode_to_string([0x42u8; 32]), baseline_version: Some(1), deletion_date: None, rotated_at: None, @@ -3760,7 +3760,7 @@ mod tests { /// Base64 material distinct from `healthy_key_data`'s, standing in for the /// material a concurrent rotation committed. fn rotated_material() -> String { - general_purpose::STANDARD.encode([0x43u8; 32]) + base64_simd::STANDARD.encode_to_string([0x43u8; 32]) } /// The issue's lost-update scenario: node A disables a key while node B's @@ -4358,7 +4358,7 @@ mod tests { let material_v2 = [0x43u8; 32]; let record_v2 = VaultKeyVersionRecord { version: 2, - encrypted_key_material: general_purpose::STANDARD.encode(material_v2), + encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2), created_at: Zoned::now(), }; // A well-formed envelope wrapped under version 2 — under a reverted @@ -4900,8 +4900,8 @@ mod tests { #[tokio::test] async fn wired_decrypt_of_pre_versioning_envelope_adds_no_request() { let key_data = healthy_key_data(); - let key_material = general_purpose::STANDARD - .decode(&key_data.encrypted_key_material) + let key_material = BASE64 + .decode_to_vec(&key_data.encrypted_key_material) .expect("decode fixture material"); let (encrypted_key, nonce) = AesDekCrypto::new() .encrypt(&key_material, b"dek-plaintext", &[]) diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 5b49ed87b..b4177e950 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -31,7 +31,7 @@ use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummar use crate::policy::{self, AttemptError, OpClass, RetryPolicy}; use crate::types::*; use async_trait::async_trait; -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use base64_simd::STANDARD as BASE64; use jiff::Zoned; use moka::future::Cache; use serde::{Deserialize, Serialize}; @@ -479,7 +479,7 @@ impl VaultTransitKmsClient { .map(|(key, value)| (key.clone(), value.clone())) .collect(); let serialized = serde_json::to_vec(&ordered)?; - Ok(Some(BASE64.encode(serialized))) + Ok(Some(BASE64.encode_to_string(serialized))) } fn map_vault_error(key_id: &str, error: vaultrs::error::ClientError, operation: &str) -> KmsError { @@ -524,7 +524,7 @@ impl VaultTransitKmsClient { plaintext: &[u8], encryption_context: &HashMap, ) -> Result { - let plaintext_b64 = BASE64.encode(plaintext); + let plaintext_b64 = BASE64.encode_to_string(plaintext); let plaintext_b64 = plaintext_b64.as_str(); let aad = Self::canonicalize_context(encryption_context)?; let aad = aad.as_deref(); @@ -568,7 +568,7 @@ impl VaultTransitKmsClient { .await?; BASE64 - .decode(response.plaintext) + .decode_to_vec(response.plaintext) .map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string())) } @@ -3031,7 +3031,7 @@ mod tests { ScriptedResponse::ok(kv2_write_ack()), // decrypt of the pre-rotation envelope; Vault owns the transit // crypto, so the recovered material is the responder's to hand back. - ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })), + ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })), ]) .await; @@ -3243,7 +3243,7 @@ mod tests { // rewrap, context-bound route: latest-version read, then decrypt, // then re-encrypt under the newest version. ScriptedResponse::ok(transit_key_read_data_up_to("wired-key", 2)), - ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })), + ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode_to_string(RECOVERED_DEK) })), ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v2:rewrapped" })), ]) .await; diff --git a/crates/kms/src/backup/drill.rs b/crates/kms/src/backup/drill.rs index 1ffc77d5b..2ba81fedf 100644 --- a/crates/kms/src/backup/drill.rs +++ b/crates/kms/src/backup/drill.rs @@ -945,7 +945,7 @@ async fn tree_digest(root: &Path) -> Result { lines.push(format!( "{relative}\u{1f}{}\u{1f}{modified}\u{1f}{}", metadata.len(), - hex::encode(Sha256::digest(&content)) + hex_simd::encode_to_string(Sha256::digest(&content), hex_simd::AsciiCase::Lower) )); } lines.sort(); @@ -1199,7 +1199,7 @@ mod tests { let text = String::from_utf8(encoded.clone()).expect("evidence is utf-8"); assert!(!text.contains(DRILL_MASTER_KEY), "the evidence must not carry the master key"); assert!( - !text.contains(&hex::encode([0x37u8; 32])), + !text.contains(&hex_simd::encode_to_string([0x37u8; 32], hex_simd::AsciiCase::Lower)), "the evidence must not carry backup KEK material" ); diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs index e26f7cd35..ca0ab58da 100644 --- a/crates/kms/src/backup/local_export.rs +++ b/crates/kms/src/backup/local_export.rs @@ -568,7 +568,10 @@ pub(crate) fn compute_master_key_verifier(master_key: &str, salt: Option<&[u8]>, let mut hasher = Sha256::new(); hasher.update(&framing); hasher.update(derived.as_slice()); - Ok(format!("{prefix}{}", hex::encode(hasher.finalize()))) + Ok(format!( + "{prefix}{}", + hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower) + )) } /// The bundle-level protection label is the weakest state observed across diff --git a/crates/kms/src/backup/local_restore.rs b/crates/kms/src/backup/local_restore.rs index 3fe2612f2..9fadb89d1 100644 --- a/crates/kms/src/backup/local_restore.rs +++ b/crates/kms/src/backup/local_restore.rs @@ -72,7 +72,7 @@ use aes_gcm::{ Aes256Gcm, Nonce, aead::{Aead, KeyInit}, }; -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use base64_simd::STANDARD as BASE64; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tokio::fs; @@ -640,7 +640,7 @@ fn decode_key_record( return Err(BackupError::corrupted(format!("bundled key record '{stem}' carries no key material")).into()); } let material = - Zeroizing::new(BASE64.decode(&probe.encrypted_key_material).map_err(|error| { + Zeroizing::new(BASE64.decode_to_vec(&probe.encrypted_key_material).map_err(|error| { BackupError::corrupted(format!("bundled key record '{stem}' material is not valid base64: {error}")) })?); if !allowed_modes.contains(&protection_mode(probe.at_rest_protection)) { diff --git a/crates/kms/src/backup/manifest.rs b/crates/kms/src/backup/manifest.rs index 28ea05f76..0b75fc924 100644 --- a/crates/kms/src/backup/manifest.rs +++ b/crates/kms/src/backup/manifest.rs @@ -69,7 +69,7 @@ impl ContentDigest { pub fn sha256_of(bytes: &[u8]) -> Self { Self { algorithm: DigestAlgorithm::Sha256, - hex: hex::encode(Sha256::digest(bytes)), + hex: hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower), } } diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index bb670609b..506e09a18 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -367,9 +367,8 @@ impl StaticConfig { /// Decode the base64-encoded secret key into raw bytes. /// Returns an error if the key is not valid base64 or is not exactly 32 bytes. pub fn decode_key(&self) -> Result<[u8; 32]> { - use base64::Engine as _; - let bytes = base64::engine::general_purpose::STANDARD - .decode(&self.secret_key) + let bytes = base64_simd::STANDARD + .decode_to_vec(&self.secret_key) .map_err(|e| KmsError::configuration_error(format!("Static KMS secret key is not valid base64: {e}")))?; if bytes.len() != 32 { return Err(KmsError::configuration_error(format!( @@ -1963,9 +1962,7 @@ mod tests { #[test] fn static_kms_config_serialization_does_not_expose_key_material() { - use base64::Engine as _; - - let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]); + let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]); let config = KmsConfig::static_kms("static-key".to_string(), encoded_key.clone()); let serialized = serde_json::to_string(&config).expect("static KMS config should serialize"); @@ -2569,14 +2566,12 @@ mod tests { #[test] fn test_from_env_reads_static_secret_file_and_sets_default_key() { - use base64::Engine as _; - let temp_dir = TempDir::new().expect("create temp dir for static KMS secret"); let secret_path = temp_dir.path().join("static-kms-secret"); // Named `*_key_b64` (not `*_secret`) so the logging-guardrails check does not // flag these fixture interpolations as secrets leaking into log strings. - let file_key_b64 = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); - let env_key_b64 = base64::engine::general_purpose::STANDARD.encode([9u8; 32]); + let file_key_b64 = base64_simd::STANDARD.encode_to_string([7u8; 32]); + let env_key_b64 = base64_simd::STANDARD.encode_to_string([9u8; 32]); std::fs::write(&secret_path, format!("file-key:{file_key_b64}\n")).expect("write static KMS secret file"); with_vars( diff --git a/crates/kms/src/config_secret.rs b/crates/kms/src/config_secret.rs index 4903b384d..f08a7a988 100644 --- a/crates/kms/src/config_secret.rs +++ b/crates/kms/src/config_secret.rs @@ -46,8 +46,7 @@ use crate::error::{KmsError, Result}; use aes_gcm::aead::{Aead, Payload}; use aes_gcm::{Aes256Gcm, Key, KeyInit, Nonce}; use argon2::{Algorithm, Argon2, Params, Version}; -use base64::Engine as _; -use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64_simd::STANDARD as BASE64_STANDARD; use rand::RngExt; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -284,14 +283,16 @@ fn seal_value(label: &str, plaintext: &str, secret: &str) -> Result { payload.extend_from_slice(&salt); payload.extend_from_slice(&nonce); payload.extend_from_slice(&ciphertext); - Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode(payload))) + Ok(format!("{SEALED_VALUE_PREFIX}{}", BASE64_STANDARD.encode_to_string(payload))) } fn open_value(label: &str, sealed: &str, secret: &str) -> Result { let encoded = sealed .strip_prefix(SEALED_VALUE_PREFIX) .expect("caller checks the sealed prefix"); - let payload = BASE64_STANDARD.decode(encoded).map_err(|_| sealed_value_unreadable(label))?; + let payload = BASE64_STANDARD + .decode_to_vec(encoded) + .map_err(|_| sealed_value_unreadable(label))?; if payload.len() <= LOCAL_KMS_MASTER_KEY_SALT_LEN + NONCE_LEN { return Err(sealed_value_unreadable(label)); } diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index f2cc936c5..07e4daaaa 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -675,7 +675,6 @@ mod tests { use crate::error::KmsError; use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage}; use async_trait::async_trait; - use base64::Engine as _; use jiff::Zoned; use std::collections::HashMap; use std::sync::Mutex; @@ -1110,8 +1109,13 @@ mod tests { .await .expect("enable should succeed"); - let base64 = base64::engine::general_purpose::STANDARD; - let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)]; + let base64 = base64_simd::STANDARD; + let encodings = |bytes: &[u8]| { + vec![ + hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower), + base64.encode_to_string(bytes), + ] + }; let mut forbidden = vec![grant_token.to_string()]; forbidden.extend(encodings(&data_key.plaintext_key)); forbidden.extend(encodings(&decrypted.plaintext)); diff --git a/crates/kms/src/probe.rs b/crates/kms/src/probe.rs index 3ec8f8b03..f65a71a44 100644 --- a/crates/kms/src/probe.rs +++ b/crates/kms/src/probe.rs @@ -551,7 +551,6 @@ mod tests { ListKeysRequest, ListKeysResponse, }; use async_trait::async_trait; - use base64::Engine as _; use metrics_util::MetricKind; use metrics_util::debugging::{DebugValue, DebuggingRecorder}; use std::future::Future; @@ -562,8 +561,7 @@ mod tests { } async fn static_backend() -> Arc { - let config = - KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32])); + let config = KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x42u8; 32])); Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build")) } diff --git a/crates/kms/src/service.rs b/crates/kms/src/service.rs index 7efe65b94..c4f0a6054 100644 --- a/crates/kms/src/service.rs +++ b/crates/kms/src/service.rs @@ -23,7 +23,6 @@ use crate::encryption::context_aad; use crate::error::{KmsError, Result}; use crate::manager::KmsManager; use crate::types::*; -use base64::Engine; use jiff::Zoned; use md5::{Digest as Md5Digest, Md5}; use rand::random; @@ -40,7 +39,7 @@ use zeroize::Zeroize; fn md5_hex(input: impl AsRef<[u8]>) -> String { let mut hasher = Md5::new(); hasher.update(input.as_ref()); - hex::encode(hasher.finalize()) + hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower) } /// Data key for object encryption @@ -836,19 +835,16 @@ impl ObjectEncryptionService { // Internal headers for decryption headers.insert( INTERNAL_ENCRYPTION_IV_HEADER.to_string(), - base64::engine::general_purpose::STANDARD.encode(&metadata.iv), + base64_simd::STANDARD.encode_to_string(&metadata.iv), ); if let Some(ref tag) = metadata.tag { - headers.insert( - INTERNAL_ENCRYPTION_TAG_HEADER.to_string(), - base64::engine::general_purpose::STANDARD.encode(tag), - ); + headers.insert(INTERNAL_ENCRYPTION_TAG_HEADER.to_string(), base64_simd::STANDARD.encode_to_string(tag)); } headers.insert( INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), - base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key), + base64_simd::STANDARD.encode_to_string(&metadata.encrypted_data_key), ); // Whatever the object was sealed under is what gets stored: for a @@ -906,14 +902,14 @@ impl ObjectEncryptionService { let iv = headers .get(INTERNAL_ENCRYPTION_IV_HEADER) .ok_or_else(|| KmsError::validation_error("Missing IV header"))?; - let iv = base64::engine::general_purpose::STANDARD - .decode(iv) + let iv = base64_simd::STANDARD + .decode_to_vec(iv) .map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?; let tag = if let Some(tag_str) = headers.get(INTERNAL_ENCRYPTION_TAG_HEADER) { Some( - base64::engine::general_purpose::STANDARD - .decode(tag_str) + base64_simd::STANDARD + .decode_to_vec(tag_str) .map_err(|e| KmsError::validation_error(format!("Invalid tag: {e}")))?, ) } else { @@ -921,8 +917,8 @@ impl ObjectEncryptionService { }; let encrypted_data_key = if let Some(key_str) = headers.get(INTERNAL_ENCRYPTION_KEY_HEADER) { - base64::engine::general_purpose::STANDARD - .decode(key_str) + base64_simd::STANDARD + .decode_to_vec(key_str) .map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))? } else { Vec::new() // Empty for SSE-C diff --git a/crates/kms/src/service_manager.rs b/crates/kms/src/service_manager.rs index ec67af821..e52186a4b 100644 --- a/crates/kms/src/service_manager.rs +++ b/crates/kms/src/service_manager.rs @@ -763,10 +763,10 @@ pub async fn get_global_encryption_service() -> Option KmsConfig { - KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32])) + KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode_to_string([fill; 32])) } /// End-to-end wiring check for the AWS backend: an admin configure request @@ -822,7 +822,7 @@ mod tests { #[tokio::test] async fn redacted_config_omits_static_key_material() { let manager = KmsServiceManager::new(); - let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]); + let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]); manager .configure(KmsConfig::static_kms("static-key".to_string(), encoded_key)) .await @@ -1020,7 +1020,6 @@ mod tests { #[tokio::test] async fn configure_cannot_replace_existing_local_backend() { - use base64::Engine as _; use tempfile::TempDir; let key_dir = TempDir::new().expect("create local KMS directory"); @@ -1029,7 +1028,7 @@ mod tests { let manager = KmsServiceManager::new(); manager.configure(local.clone()).await.expect("configure local KMS"); - let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]); + let encoded_key = base64_simd::STANDARD.encode_to_string([0x5au8; 32]); let error = manager .configure(KmsConfig::static_kms("static-key".to_string(), encoded_key)) .await diff --git a/crates/kms/tests/behavior_lifecycle.rs b/crates/kms/tests/behavior_lifecycle.rs index c0867b209..9a85ec940 100644 --- a/crates/kms/tests/behavior_lifecycle.rs +++ b/crates/kms/tests/behavior_lifecycle.rs @@ -451,6 +451,5 @@ async fn harness_restart_brings_the_service_back_over_the_same_state() { } fn base64_of(bytes: &[u8]) -> String { - use base64::Engine as _; - base64::engine::general_purpose::STANDARD.encode(bytes) + base64_simd::STANDARD.encode_to_string(bytes) } diff --git a/crates/kms/tests/behavior_objects.rs b/crates/kms/tests/behavior_objects.rs index 979cf0bb8..dbab6b311 100644 --- a/crates/kms/tests/behavior_objects.rs +++ b/crates/kms/tests/behavior_objects.rs @@ -667,7 +667,7 @@ async fn sse_c_round_trips_and_rejects_the_wrong_key() { async fn sse_c_validates_the_supplied_key_md5() { let (_kms, service) = service_with_key("sse-c-md5-unused").await; let customer_key = [0x33u8; 32]; - let correct_md5 = hex::encode(md5_of(&customer_key)); + let correct_md5 = hex_simd::encode_to_string(md5_of(&customer_key), hex_simd::AsciiCase::Lower); service .encrypt_object_with_customer_key(BUCKET, "md5-ok.bin", payload(64).as_slice(), &customer_key, Some(&correct_md5)) diff --git a/crates/kms/tests/common/mod.rs b/crates/kms/tests/common/mod.rs index e795e5f28..2c0cb9a92 100644 --- a/crates/kms/tests/common/mod.rs +++ b/crates/kms/tests/common/mod.rs @@ -34,8 +34,7 @@ use std::future::Future; use std::path::PathBuf; use std::sync::{Arc, Mutex}; -use base64::Engine as _; -use base64::engine::general_purpose::STANDARD as BASE64; +use base64_simd::STANDARD as BASE64; use rustfs_kms::backends::BackendCapabilities; use rustfs_kms::{ CreateKeyRequest, DeleteKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus, @@ -51,7 +50,7 @@ pub const STATIC_KEY_ID: &str = "behavior-static-key"; /// Fixed rather than random so a failure is reproducible; it is test-only /// material and never leaves this crate's test binaries. pub fn static_secret_key() -> String { - BASE64.encode([0x5au8; 32]) + BASE64.encode_to_string([0x5au8; 32]) } /// Which backend a harness instance is running. diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index a121ebef9..7f8170d9c 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -99,14 +99,14 @@ swift = [ "dep:md-5", "dep:hmac", "dep:sha1", - "dep:hex", + "dep:hex-simd", "dep:ipnetwork", "dep:rustfs-trusted-proxies", "dep:astral-tokio-tar", - "dep:base64", + "dep:base64-simd", "dep:async-compression", ] -webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"] +webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64-simd", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"] sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"] [dependencies] @@ -163,11 +163,11 @@ urlencoding = { workspace = true, optional = true } md-5 = { workspace = true, optional = true } hmac = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } -hex = { workspace = true, optional = true } +hex-simd = { workspace = true, optional = true } ipnetwork = { workspace = true, optional = true } rustfs-trusted-proxies = { workspace = true, optional = true } astral-tokio-tar = { workspace = true, optional = true } -base64 = { workspace = true, optional = true } +base64-simd = { workspace = true, optional = true } async-compression = { workspace = true, optional = true, features = ["tokio", "gzip", "bzip2"] } # WebDAV specific dependencies (optional) diff --git a/crates/protocols/src/swift/account.rs b/crates/protocols/src/swift/account.rs index 3cdab2114..aeae24b6a 100644 --- a/crates/protocols/src/swift/account.rs +++ b/crates/protocols/src/swift/account.rs @@ -97,7 +97,7 @@ fn get_account_metadata_bucket_name(account: &str) -> String { let mut hasher = Sha256::new(); hasher.update(account.as_bytes()); let hash_bytes = hasher.finalize(); - let hash = hex::encode(hash_bytes); + let hash = hex_simd::encode_to_string(hash_bytes, hex_simd::AsciiCase::Lower); format!("swift-account-{}", &hash[0..16]) } diff --git a/crates/protocols/src/swift/formpost.rs b/crates/protocols/src/swift/formpost.rs index 052dad787..a63b5f67f 100644 --- a/crates/protocols/src/swift/formpost.rs +++ b/crates/protocols/src/swift/formpost.rs @@ -182,7 +182,7 @@ pub fn generate_signature( mac.update(message.as_bytes()); let result = mac.finalize(); - let signature = hex::encode(result.into_bytes()); + let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower); Ok(signature) } @@ -213,10 +213,10 @@ pub fn validate_formpost(path: &str, request: &FormPostRequest, key: &str) -> Sw // the sibling TempURL/SFTP checks. Decode the hex first so the comparison runs // over the raw HMAC bytes and does not leak via string length; a non-hex // provided signature can never match and is rejected the same way. - let expected_bytes = - hex::decode(&expected_sig).map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?; + let expected_bytes = hex_simd::decode_to_vec(&expected_sig) + .map_err(|e| SwiftError::InternalServerError(format!("Signature encoding error: {}", e)))?; - let signatures_match = match hex::decode(request.signature.trim()) { + let signatures_match = match hex_simd::decode_to_vec(request.signature.trim()) { Ok(provided_bytes) => super::tempurl::constant_time_compare(&provided_bytes, &expected_bytes), Err(_) => false, }; diff --git a/crates/protocols/src/swift/slo.rs b/crates/protocols/src/swift/slo.rs index c53cd1f4d..8e29e6ecd 100644 --- a/crates/protocols/src/swift/slo.rs +++ b/crates/protocols/src/swift/slo.rs @@ -84,7 +84,11 @@ impl SLOManifest { let mut hasher = Md5::new(); hasher.update(etag_concat.as_bytes()); - format!("\"{}-{}\"", hex::encode(hasher.finalize()), self.segments.len()) + format!( + "\"{}-{}\"", + hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower), + self.segments.len() + ) } /// Validate manifest against actual segments diff --git a/crates/protocols/src/swift/sync.rs b/crates/protocols/src/swift/sync.rs index 60a8749ec..2d9b9e1d6 100644 --- a/crates/protocols/src/swift/sync.rs +++ b/crates/protocols/src/swift/sync.rs @@ -300,7 +300,7 @@ pub fn generate_sync_signature(path: &str, key: &str) -> SwiftResult { mac.update(path.as_bytes()); let result = mac.finalize(); - Ok(hex::encode(result.into_bytes())) + Ok(hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower)) } /// Verify sync signature diff --git a/crates/protocols/src/swift/tempurl.rs b/crates/protocols/src/swift/tempurl.rs index 2c0e7fbad..87b6fcd39 100644 --- a/crates/protocols/src/swift/tempurl.rs +++ b/crates/protocols/src/swift/tempurl.rs @@ -125,7 +125,7 @@ impl TempURL { // Hex-encode result let result = mac.finalize(); - let signature = hex::encode(result.into_bytes()); + let signature = hex_simd::encode_to_string(result.into_bytes(), hex_simd::AsciiCase::Lower); Ok(signature) } diff --git a/crates/protocols/src/webdav/server.rs b/crates/protocols/src/webdav/server.rs index fd1f56c09..8162e8601 100644 --- a/crates/protocols/src/webdav/server.rs +++ b/crates/protocols/src/webdav/server.rs @@ -675,8 +675,7 @@ fn fixed_body(message: impl Into) -> WebDavBody { /// Decode base64 string fn base64_decode(encoded: &str) -> Result, ()> { - use base64::Engine; - base64::engine::general_purpose::STANDARD.decode(encoded).map_err(|_| ()) + base64_simd::STANDARD.decode_to_vec(encoded).map_err(|_| ()) } #[cfg(test)] diff --git a/crates/protocols/tests/swift_metadata_persistence.rs b/crates/protocols/tests/swift_metadata_persistence.rs index 6b0b2df8d..ad636cb09 100644 --- a/crates/protocols/tests/swift_metadata_persistence.rs +++ b/crates/protocols/tests/swift_metadata_persistence.rs @@ -56,7 +56,7 @@ fn keystone_credentials(project_id: &str) -> Credentials { fn account_metadata_bucket_name(account: &str) -> String { let mut hasher = Sha256::new(); hasher.update(account.as_bytes()); - let hash = hex::encode(hasher.finalize()); + let hash = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower); format!("swift-account-{}", &hash[0..16]) } diff --git a/crates/rio-v2/Cargo.toml b/crates/rio-v2/Cargo.toml index b941ed7fc..10e36d0ff 100644 --- a/crates/rio-v2/Cargo.toml +++ b/crates/rio-v2/Cargo.toml @@ -57,7 +57,7 @@ hotpath.workspace = true aes-gcm = { workspace = true, features = ["rand_core"] } bytes = { workspace = true, features = ["serde"] } chacha20poly1305.workspace = true -hex.workspace = true +hex-simd.workspace = true hmac.workspace = true minlz.workspace = true pin-project-lite.workspace = true diff --git a/crates/rio-v2/src/encrypt_reader.rs b/crates/rio-v2/src/encrypt_reader.rs index 56f119724..9dab95c2a 100644 --- a/crates/rio-v2/src/encrypt_reader.rs +++ b/crates/rio-v2/src/encrypt_reader.rs @@ -731,12 +731,15 @@ fn random_stream_nonce() -> [u8; 12] { #[cfg(test)] mod tests { use super::*; - use hex::encode as hex_encode; use std::io::Cursor; use tokio::io::AsyncReadExt; const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE; + fn hex_encode(data: impl AsRef<[u8]>) -> String { + hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower) + } + #[tokio::test] async fn decrypt_reader_can_start_from_non_zero_sequence_number() { let plaintext = vec![0xAB; DARE_PAYLOAD_SIZE * 2 + 19]; diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index 87ecc79cb..46723ff2b 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -81,7 +81,7 @@ serde_json = { workspace = true, features = ["raw_value"] } md-5 = { workspace = true } tracing.workspace = true thiserror.workspace = true -base64.workspace = true +base64-simd.workspace = true sha1.workspace = true sha2.workspace = true xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] } diff --git a/crates/rio/src/checksum.rs b/crates/rio/src/checksum.rs index cd50db57b..1f211476d 100644 --- a/crates/rio/src/checksum.rs +++ b/crates/rio/src/checksum.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::errors::ChecksumMismatch; -use base64::{Engine as _, engine::general_purpose}; +use base64_simd::STANDARD as BASE64_STANDARD; use bytes::Bytes; use http::HeaderMap; use sha1::Sha1; @@ -330,7 +330,7 @@ impl Checksum { let mut hasher = checksum_type.hasher()?; hasher.write_all(data).ok()?; let raw = hasher.finalize(); - let encoded = general_purpose::STANDARD.encode(&raw); + let encoded = BASE64_STANDARD.encode_to_string(&raw); let checksum = Checksum { checksum_type, @@ -369,7 +369,7 @@ impl Checksum { value_string = value.to_string(); } // let raw = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(&value_string).ok()?; - let raw = general_purpose::STANDARD.decode(&value_string).ok()?; + let raw = BASE64_STANDARD.decode_to_vec(&value_string).ok()?; let checksum = Checksum { checksum_type, @@ -413,14 +413,14 @@ impl Checksum { if self.want_parts > 0 && self.want_parts != parts { return Err(ChecksumMismatch { want: format!("{}-{}", self.encoded, self.want_parts), - got: format!("{}-{}", general_purpose::STANDARD.encode(&sum), parts), + got: format!("{}-{}", base64_simd::STANDARD.encode_to_string(&sum), parts), }); } if sum != self.raw { return Err(ChecksumMismatch { want: self.encoded.clone(), - got: general_purpose::STANDARD.encode(&sum), + got: base64_simd::STANDARD.encode_to_string(&sum), }); } @@ -569,7 +569,7 @@ impl Checksum { } } - self.encoded = general_purpose::STANDARD.encode(&self.raw); + self.encoded = base64_simd::STANDARD.encode_to_string(&self.raw); Ok(()) } } @@ -1160,7 +1160,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap, bo let checksum_bytes = &buf[..length]; buf = &buf[length..]; - let mut checksum_str = general_purpose::STANDARD.encode(checksum_bytes); + let mut checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes); if checksum_type.is(ChecksumType::MULTIPART) { is_multipart = true; @@ -1190,7 +1190,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap, bo if part > 0 && (part as u64) <= parts_count { let offset = ((part - 1) as usize) * length; let part_checksum = &buf[offset..offset + length]; - checksum_str = general_purpose::STANDARD.encode(part_checksum); + checksum_str = base64_simd::STANDARD.encode_to_string(part_checksum); } buf = &buf[want_len..]; } @@ -1249,7 +1249,7 @@ pub fn read_part_checksums(mut buf: &[u8]) -> Vec> { let checksum_bytes = &buf[..length]; buf = &buf[length..]; - let checksum_str = general_purpose::STANDARD.encode(checksum_bytes); + let checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes); part_checksum.insert(checksum_type.to_string(), checksum_str); } @@ -1553,7 +1553,6 @@ mod tests { // asserted alongside the raw hex so both the digest and its encoding are pinned. #[test] fn xxhash_sha512_regression_lock_non_empty() { - use base64::{Engine as _, engine::general_purpose::STANDARD}; let data = b"The quick brown fox jumps over the lazy dog"; // XXH3-64(fox) = 0xce7d19a5418fb365 is the official upstream vector. @@ -1565,7 +1564,11 @@ mod tests { let got_hex: String = c.raw.iter().map(|b| format!("{b:02x}")).collect(); assert_eq!(got_hex, want_hex, "{t:?} raw hex drifted"); // encoded field must be the standard-base64 of raw (S3 wire form) - assert_eq!(c.encoded, STANDARD.encode(&c.raw), "{t:?} encoded field != base64(raw)"); + assert_eq!( + c.encoded, + base64_simd::STANDARD.encode_to_string(&c.raw), + "{t:?} encoded field != base64(raw)" + ); } } diff --git a/crates/rio/src/hash_reader.rs b/crates/rio/src/hash_reader.rs index f33cb7661..49efc1d6b 100644 --- a/crates/rio/src/hash_reader.rs +++ b/crates/rio/src/hash_reader.rs @@ -91,8 +91,7 @@ use crate::Sha256Hasher; use crate::compress_index::{Index, TryGetIndex}; use crate::get_content_checksum; use crate::{DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, boxed_reader, wrap_reader}; -use base64::Engine; -use base64::engine::general_purpose; + use http::HeaderMap; use pin_project_lite::pin_project; use s3s::TrailingHeaders; @@ -582,8 +581,8 @@ impl AsyncRead for HashReader { }) { expected_content_hash.encoded = checksum_str; - expected_content_hash.raw = general_purpose::STANDARD - .decode(&expected_content_hash.encoded) + expected_content_hash.raw = base64_simd::STANDARD + .decode_to_vec(&expected_content_hash.encoded) .map_err(|_| std::io::Error::other("Invalid base64 checksum"))?; if expected_content_hash.raw.is_empty() { @@ -598,7 +597,7 @@ impl AsyncRead for HashReader { && !expected_content_hash.checksum_type.trailing() { expected_content_hash.raw = content_hash; - expected_content_hash.encoded = general_purpose::STANDARD.encode(&expected_content_hash.raw); + expected_content_hash.encoded = base64_simd::STANDARD.encode_to_string(&expected_content_hash.raw); } else if content_hash != expected_content_hash.raw { let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower); let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower); diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 6fbedae51..394831cd9 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -310,7 +310,6 @@ astral-tokio-tar = { workspace = true } atoi = { workspace = true } atomic_enum = { workspace = true } async_zip = { workspace = true, default-features = false, features = ["tokio", "deflate"] } -base64 = { workspace = true } zeroize = { workspace = true } hmac = { workspace = true } sha2 = { workspace = true } diff --git a/rustfs/src/admin/handlers/kms_audit.rs b/rustfs/src/admin/handlers/kms_audit.rs index f819a4a90..883131fdf 100644 --- a/rustfs/src/admin/handlers/kms_audit.rs +++ b/rustfs/src/admin/handlers/kms_audit.rs @@ -439,7 +439,6 @@ fn dispatch(entry: AuditEntry) { #[cfg(test)] mod tests { use super::*; - use base64::Engine; use rustfs_kms::backends::local::LocalKmsBackend; use rustfs_kms::config::KmsConfig; use rustfs_kms::types::{CreateKeyRequest, DeleteKeyRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec}; @@ -689,8 +688,8 @@ mod tests { .expect("data key should be generated"); // What the endpoint hands back, and therefore what must not reappear. - let plaintext_b64 = base64::prelude::BASE64_STANDARD.encode(&response.plaintext_key); - let ciphertext_b64 = base64::prelude::BASE64_STANDARD.encode(&response.ciphertext_blob); + let plaintext_b64 = base64_simd::STANDARD.encode_to_string(&response.plaintext_key); + let ciphertext_b64 = base64_simd::STANDARD.encode_to_string(&response.ciphertext_blob); assert!(!response.plaintext_key.is_empty(), "the test must drive real key material"); let redacted = rustfs_kms::redact_encryption_context(&std::collections::HashMap::from([ diff --git a/rustfs/src/admin/handlers/kms_backup.rs b/rustfs/src/admin/handlers/kms_backup.rs index 02682e7da..f94651874 100644 --- a/rustfs/src/admin/handlers/kms_backup.rs +++ b/rustfs/src/admin/handlers/kms_backup.rs @@ -46,7 +46,7 @@ use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{current_deployment_id, current_kms_runtime_service_manager}; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use base64_simd::STANDARD as BASE64; use hyper::{HeaderMap, Method, StatusCode}; use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; @@ -429,7 +429,7 @@ impl BackupEnvironment { let decoded = Zeroizing::new( BASE64 - .decode(raw_kek.trim()) + .decode_to_vec(raw_kek.trim()) .map_err(|_| (StatusCode::PRECONDITION_FAILED, format!("{ENV_KMS_BACKUP_KEK} must be base64-encoded")))?, ); if decoded.len() != 32 { @@ -516,7 +516,9 @@ fn reuses_business_secret(kek_material: &[u8], raw_kek: &str, config: &KmsConfig if raw_kek == secret.as_str() || secret.as_bytes() == kek_material { return true; } - BASE64.decode(secret.as_str()).is_ok_and(|decoded| decoded == kek_material) + BASE64 + .decode_to_vec(secret.as_str()) + .is_ok_and(|decoded| decoded == kek_material) }) } @@ -1185,7 +1187,7 @@ mod tests { const DEPLOYMENT: &str = "deployment-under-test"; fn test_kek_bytes() -> Vec { - BASE64.decode(TEST_KEK_B64).expect("test KEK must decode") + BASE64.decode_to_vec(TEST_KEK_B64).expect("test KEK must decode") } fn local_config(key_dir: PathBuf) -> KmsConfig { @@ -1310,7 +1312,7 @@ mod tests { .expect_err("an empty KEK must be refused"); assert_eq!(error.0, StatusCode::PRECONDITION_FAILED); - let short = BASE64.encode([0x11; 16]); + let short = BASE64.encode_to_string([0x11; 16]); let error = BackupEnvironment::build(PathBuf::from("/tmp/root"), &short, "kek".to_string(), 1, &config) .expect_err("a KEK that is not 32 bytes must be refused"); assert_eq!(error.0, StatusCode::PRECONDITION_FAILED); @@ -1353,7 +1355,7 @@ mod tests { ); // An unrelated KEK is accepted. - let independent = BASE64.encode([0x5a; 32]); + let independent = BASE64.encode_to_string([0x5a; 32]); assert!(BackupEnvironment::build(PathBuf::from("/tmp/root"), &independent, "kek".to_string(), 1, &config).is_ok()); } @@ -1372,7 +1374,7 @@ mod tests { let master_key = "local-master-key-super-secret"; let vault_token = "hvs.vault-token-super-secret"; let approle_secret = "approle-secret-id-super-secret"; - let static_key = BASE64.encode([0x7c; 32]); + let static_key = BASE64.encode_to_string([0x7c; 32]); let local = local_config_with_master_key(PathBuf::from("/var/lib/rustfs/kms"), master_key); let kv2 = vault_kv2_config(VaultAuthMethod::Token { diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index 65431c977..5bee555f4 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -1422,12 +1422,8 @@ mod tests { #[test] fn static_kms_config_is_not_persisted_with_cluster_configuration() { - use base64::Engine as _; - - let config = rustfs_kms::KmsConfig::static_kms( - "static-key".to_string(), - base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]), - ); + let config = + rustfs_kms::KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x5au8; 32])); assert!(ensure_kms_config_persistable(&config).is_err()); } diff --git a/rustfs/src/admin/handlers/kms_key_metadata.rs b/rustfs/src/admin/handlers/kms_key_metadata.rs index 679e501fd..8e4bd4a3a 100644 --- a/rustfs/src/admin/handlers/kms_key_metadata.rs +++ b/rustfs/src/admin/handlers/kms_key_metadata.rs @@ -408,7 +408,6 @@ impl Operation for UntagKmsKeyHandler { mod tests { use super::*; use crate::admin::handlers::kms_keys::stable_json_value; - use base64::Engine as _; use rustfs_kms::KmsManager; use rustfs_kms::backends::local::LocalKmsBackend; use rustfs_kms::backends::static_kms::StaticKmsBackend; @@ -431,8 +430,7 @@ mod tests { /// is the backend that must answer every metadata update with a capability /// gap rather than a failure of the request. async fn static_service() -> ObjectEncryptionService { - let config = - KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32])); + let config = KmsConfig::static_kms("static-key".to_string(), base64_simd::STANDARD.encode_to_string([0x42u8; 32])); let backend = Arc::new( StaticKmsBackend::new(config.clone()) .await diff --git a/rustfs/src/admin/handlers/kms_keys.rs b/rustfs/src/admin/handlers/kms_keys.rs index de99ae07c..6e0bbdf08 100644 --- a/rustfs/src/admin/handlers/kms_keys.rs +++ b/rustfs/src/admin/handlers/kms_keys.rs @@ -21,7 +21,6 @@ use crate::admin::runtime_sources::{current_kms_runtime_service_manager, current use crate::auth::{check_key_valid, get_session_token}; use crate::kms_deletion_gate::current_key_impact; use crate::server::{ADMIN_PREFIX, RemoteAddr}; -use base64::Engine; use hyper::{HeaderMap, Method, StatusCode}; use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; @@ -1535,8 +1534,8 @@ impl Operation for GenerateDataKeyHandler { Ok(response) => { let api_response = GenerateDataKeyApiResponse { key_id: response.key_id, - plaintext_key: base64::prelude::BASE64_STANDARD.encode(&response.plaintext_key), - ciphertext_blob: base64::prelude::BASE64_STANDARD.encode(&response.ciphertext_blob), + plaintext_key: base64_simd::STANDARD.encode_to_string(&response.plaintext_key), + ciphertext_blob: base64_simd::STANDARD.encode_to_string(&response.ciphertext_blob), }; let data = serde_json::to_vec(&api_response) diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 3b7637dd5..2dfa83a18 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -56,9 +56,8 @@ use crate::storage::storage_api::{ delete_config_no_lock, lock_bucket_targets_metadata, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock, }; -use base64::Engine; -use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64_simd::STANDARD as BASE64_STANDARD; +use base64_simd::URL_SAFE_NO_PAD; use futures::StreamExt; use hmac::{Hmac, Mac}; use http::header::{CONTENT_TYPE, HOST}; @@ -1645,7 +1644,7 @@ fn hash_client_secret(secret: Option<&str>) -> String { let mut hasher = Sha256::new(); hasher.update(secret.as_bytes()); - URL_SAFE_NO_PAD.encode(hasher.finalize()) + URL_SAFE_NO_PAD.encode_to_string(hasher.finalize()) } fn config_enabled(value: Option) -> bool { @@ -3775,7 +3774,7 @@ impl SiteReplicationRepairTask<'_> { digest.update(self.path().as_bytes()); digest.update([0]); digest.update(payload); - Ok(URL_SAFE_NO_PAD.encode(digest.finalize())) + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize())) } async fn send(&self, transport: &PeerTransport, access_key: &str, secret_key: &str) -> S3Result> { @@ -3862,7 +3861,7 @@ fn site_replication_repair_plan_token(state: &SiteReplicationState, plan: &SiteR for (_, task) in site_replication_repair_tasks(plan) { digest.update(task.id()?.as_bytes()); } - Ok(URL_SAFE_NO_PAD.encode(digest.finalize())) + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize())) } fn site_replication_repair_preflight_token( @@ -3892,7 +3891,7 @@ fn site_replication_repair_preflight_token( digest.update(event.path.as_bytes()); digest.update(&[0]); } - Ok(URL_SAFE_NO_PAD.encode(digest.finalize().into_bytes())) + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes())) } fn site_replication_repair_task_checkpoint_id( @@ -3906,7 +3905,7 @@ fn site_replication_repair_task_checkpoint_id( digest.update(peer_deployment_id.as_bytes()); digest.update(&[0]); digest.update(task.id()?.as_bytes()); - Ok(URL_SAFE_NO_PAD.encode(digest.finalize().into_bytes())) + Ok(URL_SAFE_NO_PAD.encode_to_string(digest.finalize().into_bytes())) } fn site_replication_repair_sites( @@ -4488,11 +4487,11 @@ fn raw_config_to_string(raw: &[u8]) -> Option { } fn raw_config_to_base64(raw: &[u8]) -> Option { - (!raw.is_empty()).then(|| BASE64_STANDARD.encode(raw)) + (!raw.is_empty()).then(|| BASE64_STANDARD.encode_to_string(raw)) } fn encode_bucket_meta_wire_value(value: Option) -> Option { - value.map(|raw| BASE64_STANDARD.encode(raw.as_bytes())) + value.map(|raw| BASE64_STANDARD.encode_to_string(raw.as_bytes())) } fn encode_bucket_meta_wire_item(mut item: SRBucketMeta) -> SRBucketMeta { @@ -4508,7 +4507,7 @@ fn encode_bucket_meta_wire_item(mut item: SRBucketMeta) -> SRBucketMeta { fn decode_bucket_meta_wire_value(raw: &str) -> Vec { BASE64_STANDARD - .decode(raw.as_bytes()) + .decode_to_vec(raw.as_bytes()) .ok() .filter(|decoded| std::str::from_utf8(decoded).is_ok()) .unwrap_or_else(|| raw.as_bytes().to_vec()) @@ -7717,7 +7716,7 @@ fn site_resync_page(status: &SRResyncOpStatus, limit: usize, offset: usize) -> S }; let encoded = serde_json::to_vec(&token) .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("encode resync cursor failed: {err}")))?; - URL_SAFE_NO_PAD.encode(encoded) + URL_SAFE_NO_PAD.encode_to_string(encoded) } else { String::new() }; @@ -7736,7 +7735,7 @@ fn parse_site_resync_page(query: &HashMap, status: &SRResyncOpSt } let offset = if let Some(value) = query.get("continuationToken") { let decoded = URL_SAFE_NO_PAD - .decode(value) + .decode_to_vec(value) .map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?; let token: SiteResyncContinuationToken = serde_json::from_slice(&decoded).map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?; @@ -14098,7 +14097,7 @@ mod tests { let bucket = SRBucketInfo { bucket: "photos".to_string(), created_at: Some(OffsetDateTime::UNIX_EPOCH), - object_lock_config: Some(BASE64_STANDARD.encode("")), + object_lock_config: Some(BASE64_STANDARD.encode_to_string("")), ..Default::default() }; let bootstrap = bootstrap_bucket_make_op_path(&bucket); @@ -14625,10 +14624,10 @@ mod tests { SRBucketInfo { bucket: "photos".to_string(), policy: Some(serde_json::json!({"Statement": []})), - versioning: Some(BASE64_STANDARD.encode("")), - quota_config: Some(BASE64_STANDARD.encode(r#"{"quota":1024}"#)), - expiry_lc_config: Some(BASE64_STANDARD.encode("")), - object_lock_config: Some(BASE64_STANDARD.encode("")), + versioning: Some(BASE64_STANDARD.encode_to_string("")), + quota_config: Some(BASE64_STANDARD.encode_to_string(r#"{"quota":1024}"#)), + expiry_lc_config: Some(BASE64_STANDARD.encode_to_string("")), + object_lock_config: Some(BASE64_STANDARD.encode_to_string("")), created_at: Some(OffsetDateTime::UNIX_EPOCH), api_version: Some(SITE_REPL_API_VERSION.to_string()), ..Default::default() @@ -14667,7 +14666,7 @@ mod tests { "photos".to_string(), SRBucketInfo { bucket: "photos".to_string(), - expiry_lc_config: Some(BASE64_STANDARD.encode("")), + expiry_lc_config: Some(BASE64_STANDARD.encode_to_string("")), api_version: Some(SITE_REPL_API_VERSION.to_string()), ..Default::default() }, @@ -17637,7 +17636,7 @@ mod tests { fn test_metainfo_bucket_config_values_are_base64_encoded() { let raw = br#""#; - assert_eq!(raw_config_to_base64(raw), Some(BASE64_STANDARD.encode(raw))); + assert_eq!(raw_config_to_base64(raw), Some(BASE64_STANDARD.encode_to_string(raw))); assert_ne!(raw_config_to_base64(raw), raw_config_to_string(raw)); assert_eq!(raw_config_to_base64(&[]), None); } @@ -18182,8 +18181,8 @@ mod tests { }; let dep_a_xml = site_config_xml("dep-b"); let dep_b_xml = site_config_xml("dep-a"); - let dep_a_b64 = BASE64_STANDARD.encode(dep_a_xml.as_bytes()); - let dep_b_b64 = BASE64_STANDARD.encode(dep_b_xml.as_bytes()); + let dep_a_b64 = BASE64_STANDARD.encode_to_string(dep_a_xml.as_bytes()); + let dep_b_b64 = BASE64_STANDARD.encode_to_string(dep_b_xml.as_bytes()); // Both sites present the complete config in base64 wire form → NOT a mismatch. assert_eq!( diff --git a/rustfs/src/connect/client.rs b/rustfs/src/connect/client.rs index 4855adb46..9d283460f 100644 --- a/rustfs/src/connect/client.rs +++ b/rustfs/src/connect/client.rs @@ -14,7 +14,6 @@ use std::time::Duration; -use base64::Engine as _; use chrono::{DateTime, Utc}; use reqwest::{Client, StatusCode, Url, header}; use rustls::RootCertStore; @@ -228,8 +227,8 @@ impl ConnectClient { pending: &PendingRegistration, identity: &super::identity::DeviceIdentity, ) -> Result { - let csr_der = base64::engine::general_purpose::STANDARD - .decode(&pending.certificate_request) + let csr_der = base64_simd::STANDARD + .decode_to_vec(&pending.certificate_request) .map_err(|_| ClientError::PendingRegistration)?; let transcript = RegistrationTranscript::build( &token.registration_token_uid, diff --git a/rustfs/src/connect/identity.rs b/rustfs/src/connect/identity.rs index e3028c770..8ce7bb890 100644 --- a/rustfs/src/connect/identity.rs +++ b/rustfs/src/connect/identity.rs @@ -20,8 +20,7 @@ //! module produces, so any divergence is a protocol break rather than a //! local behaviour change. -use base64::Engine as _; -use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD}; +use base64_simd::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD}; use p256::ecdsa::signature::{Signer as _, Verifier as _}; use p256::ecdsa::{Signature, SigningKey}; use p256::elliptic_curve::Generate as _; @@ -112,7 +111,7 @@ impl RegistrationTranscript { } let expiry = expires_unix.to_string(); - let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(certificate_request)); + let csr_digest = BASE64_URL_NO_PAD.encode_to_string(Sha256::digest(certificate_request)); let fields: [(&'static str, &str); FIELD_COUNT] = [ ("registrationTokenUid", registration_token_uid), @@ -238,7 +237,7 @@ impl DeviceIdentity { /// Standard padded base64 of the certificate request, as the body carries it. pub fn certificate_request_base64(&self) -> Result { - Ok(BASE64_STANDARD.encode(self.certificate_request_der()?)) + Ok(BASE64_STANDARD.encode_to_string(self.certificate_request_der()?)) } /// Sign a transcript, producing the low-S fixed-width proof. @@ -252,20 +251,20 @@ impl DeviceIdentity { RegistrationProof { algorithm: PROOF_ALGORITHM.to_string(), - value: BASE64_URL_NO_PAD.encode(canonical.to_bytes()), + value: BASE64_URL_NO_PAD.encode_to_string(canonical.to_bytes()), } } pub(crate) fn sign_pending_registration_state(&self, state: &[u8]) -> String { let signature: Signature = self.signing_key.sign(state); - BASE64_URL_NO_PAD.encode(signature.normalize_s().to_bytes()) + BASE64_URL_NO_PAD.encode_to_string(signature.normalize_s().to_bytes()) } pub(crate) fn verifies_pending_registration_state(&self, state: &[u8], proof: &str) -> bool { - let Ok(octets) = BASE64_URL_NO_PAD.decode(proof) else { + let Ok(octets) = BASE64_URL_NO_PAD.decode_to_vec(proof) else { return false; }; - if BASE64_URL_NO_PAD.encode(&octets) != proof { + if BASE64_URL_NO_PAD.encode_to_string(&octets) != proof { return false; } let Ok(signature) = Signature::from_slice(&octets) else { diff --git a/rustfs/src/connect/offline/bundle_writer.rs b/rustfs/src/connect/offline/bundle_writer.rs index e46eb1a43..aea3194c8 100644 --- a/rustfs/src/connect/offline/bundle_writer.rs +++ b/rustfs/src/connect/offline/bundle_writer.rs @@ -25,7 +25,7 @@ use std::path::Path; use std::path::PathBuf; #[cfg(target_os = "linux")] -use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use base64_simd::URL_SAFE_NO_PAD; #[cfg(target_os = "linux")] use p256::ecdsa::{Signature, SigningKey, signature::Signer as _}; #[cfg(target_os = "linux")] @@ -163,7 +163,7 @@ fn write_offline_bundle_unix( let produced_at = OffsetDateTime::from_unix_timestamp(context.produced_at_unix).map_err(|_| BundleError::InvalidMetadata)?; let produced_at = produced_at.format(&Rfc3339).map_err(|_| BundleError::InvalidMetadata)?; - let nonce = URL_SAFE_NO_PAD.encode(context.nonce); + let nonce = URL_SAFE_NO_PAD.encode_to_string(context.nonce); let device_key_id = hex_lower(&Sha256::digest(key.public_key_der())); let manifest_entries = entries .iter() @@ -397,7 +397,7 @@ fn sign(key: &DeviceIdentity, manifest: &[u8]) -> Result { input.push(0); input.extend_from_slice(manifest); let signature: Signature = signing_key.sign(&input); - Ok(URL_SAFE_NO_PAD.encode(signature.normalize_s().to_bytes())) + Ok(URL_SAFE_NO_PAD.encode_to_string(signature.normalize_s().to_bytes())) } #[cfg(target_os = "linux")] diff --git a/rustfs/src/connect/offline/enrollment.rs b/rustfs/src/connect/offline/enrollment.rs index a6c7c69d1..ef36ce4cd 100644 --- a/rustfs/src/connect/offline/enrollment.rs +++ b/rustfs/src/connect/offline/enrollment.rs @@ -32,8 +32,7 @@ //! are frozen beside it. Reordering the checks changes which reason a given //! artifact produces, which is itself part of the contract. -use base64::Engine as _; -use base64::engine::general_purpose::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD}; +use base64_simd::{STANDARD as BASE64_STANDARD, URL_SAFE_NO_PAD as BASE64_URL_NO_PAD}; use p256::ecdsa::signature::{Signer as _, Verifier as _}; use p256::ecdsa::{Signature, SigningKey, VerifyingKey}; use p256::pkcs8::DecodePrivateKey as _; @@ -367,7 +366,7 @@ impl OfflineEnrollment { // The octets that were transmitted. They are never re-serialised: every // later step signs and parses this same buffer. let bytes = BASE64_STANDARD - .decode(envelope.bytes.as_bytes()) + .decode_to_vec(envelope.bytes.as_bytes()) .map_err(|_| EnrollmentError::MalformedDocument)?; // Step 2: routing only. @@ -440,8 +439,8 @@ impl OfflineEnrollment { challenge_nonce: &challenge.nonce, challenge_proof: &challenge.challenge_proof, device_key_id: key_id(&point), - device_public_key: BASE64_URL_NO_PAD.encode(point), - device_nonce: BASE64_URL_NO_PAD.encode(device_nonce), + device_public_key: BASE64_URL_NO_PAD.encode_to_string(point), + device_nonce: BASE64_URL_NO_PAD.encode_to_string(device_nonce), produced_at, }; @@ -451,7 +450,7 @@ impl OfflineEnrollment { let signature = sign(key, TAG_RESPONSE, &bytes)?; let envelope = SignedDocument { - bytes: BASE64_STANDARD.encode(&bytes), + bytes: BASE64_STANDARD.encode_to_string(&bytes), signature: DocumentSignature { algorithm: SIGNATURE_ALGORITHM.to_owned(), key_id: document.device_key_id, @@ -537,7 +536,7 @@ fn verify_trust_chain( /// checked against these, never against a re-encoding of the parsed link. fn decode_trust_link(entry: &SignedDocument) -> Result<(TrustLink, Vec), EnrollmentError> { let bytes = BASE64_STANDARD - .decode(entry.bytes.as_bytes()) + .decode_to_vec(entry.bytes.as_bytes()) .map_err(|_| EnrollmentError::MalformedDocument)?; let link = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::TrustChainInvalid)?; Ok((link, bytes)) @@ -559,7 +558,7 @@ fn decode_signature(signature: &DocumentSignature) -> Result Result Option<(VerifyingKey, [u8; PUBLIC_KEY_OCTET return None; } - let point: [u8; PUBLIC_KEY_OCTETS] = BASE64_URL_NO_PAD.decode(value).ok()?.try_into().ok()?; + let point: [u8; PUBLIC_KEY_OCTETS] = BASE64_URL_NO_PAD.decode_to_vec(value).ok()?.try_into().ok()?; if point[0] != UNCOMPRESSED_POINT { return None; } diff --git a/rustfs/src/connect/registration.rs b/rustfs/src/connect/registration.rs index 82d1dcae4..829002c9b 100644 --- a/rustfs/src/connect/registration.rs +++ b/rustfs/src/connect/registration.rs @@ -15,8 +15,7 @@ use std::io::Read; use std::sync::Arc; -use base64::Engine as _; -use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD; +use base64_simd::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD; use p256::ecdsa::signature::Signer as _; use p256::ecdsa::{Signature, SigningKey}; use p256::pkcs8::DecodePrivateKey as _; @@ -89,10 +88,10 @@ impl RegistrationToken { } let document: RegistrationTokenDocument = serde_json::from_slice(&bytes).map_err(TokenError::Invalid)?; let decoded = BASE64_URL_NO_PAD - .decode(&document.registration_token_secret) + .decode_to_vec(&document.registration_token_secret) .map(Zeroizing::new) .map_err(|_| TokenError::SecretShape)?; - if decoded.len() != 32 || BASE64_URL_NO_PAD.encode(&decoded) != document.registration_token_secret { + if decoded.len() != 32 || BASE64_URL_NO_PAD.encode_to_string(&decoded) != document.registration_token_secret { return Err(TokenError::SecretShape); } if !is_uuid_v7(&document.registration_token_uid) @@ -198,10 +197,10 @@ impl<'a> RotationRequest<'a> { request_id: &'a str, certificate_request: &'a str, ) -> Result { - let csr_der = base64::engine::general_purpose::STANDARD - .decode(certificate_request) + let csr_der = base64_simd::STANDARD + .decode_to_vec(certificate_request) .map_err(|_| CredentialValidationError::CertificateRequest)?; - let csr_digest = BASE64_URL_NO_PAD.encode(Sha256::digest(&csr_der)); + let csr_digest = BASE64_URL_NO_PAD.encode_to_string(Sha256::digest(&csr_der)); let transcript = rotation_transcript(credential_fingerprint, device_name, request_id, &csr_digest)?; let key = identity .to_pkcs8_der() @@ -216,7 +215,7 @@ impl<'a> RotationRequest<'a> { certificate_request, proof: ProofOwned { algorithm: "ES256".to_string(), - value: BASE64_URL_NO_PAD.encode(canonical.to_bytes()), + value: BASE64_URL_NO_PAD.encode_to_string(canonical.to_bytes()), }, }) } @@ -475,8 +474,8 @@ pub(crate) fn public_key_fingerprint(identity: &DeviceIdentity) -> String { } pub(crate) fn certificate_request_matches(encoded: &str, identity: &DeviceIdentity) -> Result { - let der = base64::engine::general_purpose::STANDARD - .decode(encoded) + let der = base64_simd::STANDARD + .decode_to_vec(encoded) .map_err(|_| CredentialValidationError::CertificateRequest)?; let (remaining, request) = X509CertificationRequest::from_der(&der).map_err(|_| CredentialValidationError::CertificateRequest)?; diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index 20104dac5..5b6a2f88e 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -446,7 +446,6 @@ mod tests { use crate::server::{refresh_audit_module_enabled, refresh_notify_module_enabled}; use crate::storage::access::ReqInfo; use crate::storage::request_context::RequestContext; - use base64::Engine as _; use http::{Extensions, HeaderMap, HeaderValue, Method, Uri}; use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit}; use rustfs_audit::ObjectVersion; @@ -765,10 +764,7 @@ mod tests { std::collections::HashMap::from([ ("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()), ("x-rustfs-encryption-key-id".to_string(), "finance-key".to_string()), - ( - "x-rustfs-encryption-key".to_string(), - base64::engine::general_purpose::STANDARD.encode([7u8; 48]), - ), + ("x-rustfs-encryption-key".to_string(), base64_simd::STANDARD.encode_to_string([7u8; 48])), ("x-rustfs-encryption-algorithm".to_string(), "aws:kms".to_string()), ]) } @@ -840,7 +836,7 @@ mod tests { let rendered = serde_json::to_string(&tags).expect("audit tags serialize"); assert!( - !rendered.contains(&base64::engine::general_purpose::STANDARD.encode([7u8; 48])), + !rendered.contains(&base64_simd::STANDARD.encode_to_string([7u8; 48])), "the audit entry must not carry the wrapped data key: {rendered}" ); }, diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 6da76d0b2..97a116d93 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -83,7 +83,7 @@ use aes_gcm::{ aead::{Aead, KeyInit}, }; use async_trait::async_trait; -use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use base64_simd::STANDARD as BASE64_STANDARD; #[cfg(feature = "rio-v2")] use chacha20poly1305::ChaCha20Poly1305; #[cfg(feature = "rio-v2")] @@ -153,7 +153,7 @@ fn md5_bytes(input: impl AsRef<[u8]>) -> [u8; 16] { } fn md5_base64(input: impl AsRef<[u8]>) -> String { - BASE64_STANDARD.encode(md5_bytes(input)) + BASE64_STANDARD.encode_to_string(md5_bytes(input)) } use super::Error; @@ -562,7 +562,7 @@ pub(crate) fn extract_ssekms_context_from_headers(headers: &HeaderMap) -> Result let value = v .to_str() .map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid UTF-8."))?; - let decoded = BASE64_STANDARD.decode(value).map_err(|_| { + let decoded = BASE64_STANDARD.decode_to_vec(value).map_err(|_| { sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid base64-encoded JSON.") })?; @@ -1320,7 +1320,7 @@ fn stored_envelope_master_key_version(metadata: &HashMap) -> Opt // this lookup never reads, so the normalized result is identical without it. let encoded = normalize_managed_metadata(metadata, None); let encoded = encoded.get(INTERNAL_ENCRYPTION_KEY_HEADER)?; - let envelope = BASE64_STANDARD.decode(encoded).ok()?; + let envelope = BASE64_STANDARD.decode_to_vec(encoded).ok()?; envelope_master_key_version(&envelope) } @@ -1516,7 +1516,7 @@ fn build_object_encryption_context( fn encode_minio_kms_context(context: &HashMap) -> Result { let encoded = serde_json::to_vec(context) .map_err(|e| ApiError::from(StorageError::other(format!("Failed to serialize KMS context: {e}"))))?; - Ok(BASE64_STANDARD.encode(encoded)) + Ok(BASE64_STANDARD.encode_to_string(encoded)) } fn decode_minio_kms_context(metadata: &HashMap) -> Result>, ApiError> { @@ -1524,7 +1524,7 @@ fn decode_minio_kms_context(metadata: &HashMap) -> Result