gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 08:27:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(deps): migrate direct encoding deps to simd (#6690)

Browse Source
This commit is contained in:
houseme
2026-08-27 03:17:24 +08:00
committed by GitHub
parent 31031f2a46
commit ba7785d61d
81 changed files with 523 additions and 505 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/Cargo.toml
-1
View File
@@ -176,7 +176,6 @@ rmp.workspace = true
rmp-serde.workspace = true
tokio-util = { workspace = true, features = ["io", "compat"] }
tokio-stream = { workspace = true, features = ["sync"] }
base64 = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
hex-simd = { workspace = true }
crates/ecstore/src/bucket/replication/replication_target_boundary.rs
+2 -2
View File
@@ -200,7 +200,7 @@ impl ReplicationTargetStore {
}
pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) -> Result<(PutObjectOptions, bool)> {
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT};
let mut meta = HashMap::new();
@@ -252,7 +252,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
&& !checksum_data.is_empty()
{
if is_ssec {
let encoded = BASE64_STANDARD.encode(checksum_data);
let encoded = BASE64_STANDARD.encode_to_string(checksum_data);
insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded);
} else if object_info.is_encrypted() {
// Encrypted checksums cannot be exposed as plaintext headers, and
crates/ecstore/src/cluster/rpc/http_auth.rs
+15 -16
View File
@@ -31,8 +31,7 @@ use crate::storage_api_contracts::internode::{
NS_SCANNER_PROTOCOL_VERSION, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use base64::Engine as _;
use base64::engine::general_purpose;
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
@@ -523,11 +522,11 @@ fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64)
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
mac.update(data.as_bytes());
let result = mac.finalize();
general_purpose::STANDARD.encode(result.into_bytes())
base64_simd::STANDARD.encode_to_string(result.into_bytes())
}
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
@@ -745,11 +744,11 @@ fn generate_signature_v2(secret: &str, scope: SignatureV2Scope<'_>) -> std::io::
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_signature_v2(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
@@ -792,11 +791,11 @@ fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std:
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_scope(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
@@ -821,15 +820,15 @@ fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let proof = general_purpose::STANDARD
.decode(proof)
let proof = base64_simd::STANDARD
.decode_to_vec(proof)
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
@@ -862,7 +861,7 @@ fn generate_replay_cache_capability_proof(
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
}
fn verify_replay_cache_capability_proof(
@@ -872,8 +871,8 @@ fn verify_replay_cache_capability_proof(
boot_epoch: Uuid,
proof: &str,
) -> std::io::Result<()> {
let proof = general_purpose::STANDARD
.decode(proof)
let proof = base64_simd::STANDARD
.decode_to_vec(proof)
.map_err(|_| std::io::Error::other("Invalid RPC replay cache capability proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
@@ -1988,9 +1987,9 @@ mod tests {
let method = Method::GET;
let timestamp = 1640995200;
let signature = generate_signature(secret, url, &method, timestamp);
let mut tampered = general_purpose::STANDARD.decode(&signature).unwrap();
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
tampered[0] ^= 1;
let tampered_signature = general_purpose::STANDARD.encode(tampered);
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
assert!(verify_signature(secret, url, &method, timestamp, &signature));
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
crates/ecstore/src/object_api/readers.rs
+20 -18
View File
@@ -1407,8 +1407,7 @@ fn multipart_part_numbers(parts: &[ObjectPartInfo]) -> Vec<usize> {
#[cfg(test)]
mod tests {
use super::*;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use base64_simd::STANDARD as BASE64_STANDARD;
use md5::{Digest, Md5};
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER};
use std::collections::HashMap;
@@ -1468,7 +1467,7 @@ mod tests {
request: ReadEncryptionRequest<'_>,
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
if let Some(encoded) = request.metadata.get(TEST_OBJECT_KEY_HEADER) {
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidMetadata, "invalid test object key")
})?;
let key_bytes = decoded.try_into().map_err(|_| {
@@ -1493,7 +1492,7 @@ mod tests {
.map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
})?;
let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| {
let decoded = BASE64_STANDARD.decode_to_vec(encoded).map_err(|_| {
EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key")
})?;
let key_bytes = decoded.try_into().map_err(|_| {
@@ -1505,7 +1504,7 @@ mod tests {
let base_nonce = request
.metadata
.get(TEST_NONCE_HEADER)
.and_then(|encoded| BASE64_STANDARD.decode(encoded).ok())
.and_then(|encoded| BASE64_STANDARD.decode_to_vec(encoded).ok())
.and_then(|bytes| bytes.try_into().ok())
.unwrap_or_else(|| fixture_nonce(request.bucket, request.object));
Ok(Some(ReadEncryptionMaterial {
@@ -1533,7 +1532,7 @@ mod tests {
let mut headers = HeaderMap::new();
headers.insert(
TEST_DIRECT_KEY_HEADER,
HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("test key header is valid"),
HeaderValue::from_str(&BASE64_STANDARD.encode_to_string(key_bytes)).expect("test key header is valid"),
);
headers
}
@@ -2439,7 +2438,10 @@ mod tests {
user_defined: Arc::new(HashMap::from([
("X-Amz-Server-Side-Encryption".to_string(), "aws:kms".to_string()),
("X-Amz-Server-Side-Encryption-Iv".to_string(), "AAAAAAAAAAAAAAAA".to_string()),
("X-Amz-Server-Side-Encryption-Key".to_string(), BASE64_STANDARD.encode([7_u8; 32])),
(
"X-Amz-Server-Side-Encryption-Key".to_string(),
BASE64_STANDARD.encode_to_string([7_u8; 32]),
),
("x-rustfs-encryption-original-size".to_string(), "64".to_string()),
])),
..Default::default()
@@ -2748,7 +2750,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -2796,11 +2798,11 @@ mod tests {
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -2856,7 +2858,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3008,13 +3010,13 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
total_plaintext.to_string(),
),
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE)),
(TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode_to_string(LEGACY_FIXTURE_BASE_NONCE)),
])
}
@@ -3751,11 +3753,11 @@ mod tests {
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3821,7 +3823,7 @@ mod tests {
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(key_bytes)),
BASE64_STANDARD.encode_to_string(md5_bytes(key_bytes)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
@@ -3964,11 +3966,11 @@ mod tests {
..Default::default()
}]),
user_defined: Arc::new(HashMap::from([
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)),
(TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode_to_string(object_key)),
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
(
"x-amz-server-side-encryption-customer-key-md5".to_string(),
BASE64_STANDARD.encode(md5_bytes(customer_key)),
BASE64_STANDARD.encode_to_string(md5_bytes(customer_key)),
),
(
"x-amz-server-side-encryption-customer-original-size".to_string(),
crates/ecstore/src/store/list_objects.rs
+3 -3
View File
@@ -42,7 +42,7 @@ use crate::storage_api_contracts::{
};
use crate::store::ECStore;
use crate::store::utils::is_reserved_or_invalid_bucket;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use futures::future::join_all;
use rand::seq::SliceRandom;
@@ -1398,11 +1398,11 @@ async fn persist_observed_list_objects_mutation(store: Option<&ECStore>, bucket:
}
fn encode_persistent_list_metadata_string(value: &str) -> String {
BASE64_STANDARD.encode(value.as_bytes())
BASE64_STANDARD.encode_to_string(value.as_bytes())
}
fn decode_persistent_list_metadata_string(value: &str) -> Option<String> {
let bytes = BASE64_STANDARD.decode(value).ok()?;
let bytes = BASE64_STANDARD.decode_to_vec(value).ok()?;
String::from_utf8(bytes).ok()
}