gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 08:27:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(deps): migrate direct encoding deps to simd (#6690)

Browse Source
This commit is contained in:
houseme
2026-08-27 03:17:24 +08:00
committed by GitHub
parent 31031f2a46
commit ba7785d61d
81 changed files with 523 additions and 505 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/rio/Cargo.toml
+1 -1
View File
@@ -81,7 +81,7 @@ serde_json = { workspace = true, features = ["raw_value"] }
md-5 = { workspace = true }
tracing.workspace = true
thiserror.workspace = true
base64.workspace = true
base64-simd.workspace = true
sha1.workspace = true
sha2.workspace = true
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
crates/rio/src/checksum.rs
+14 -11
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::errors::ChecksumMismatch;
use base64::{Engine as _, engine::general_purpose};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use http::HeaderMap;
use sha1::Sha1;
@@ -330,7 +330,7 @@ impl Checksum {
let mut hasher = checksum_type.hasher()?;
hasher.write_all(data).ok()?;
let raw = hasher.finalize();
let encoded = general_purpose::STANDARD.encode(&raw);
let encoded = BASE64_STANDARD.encode_to_string(&raw);
let checksum = Checksum {
checksum_type,
@@ -369,7 +369,7 @@ impl Checksum {
value_string = value.to_string();
}
// let raw = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(&value_string).ok()?;
let raw = general_purpose::STANDARD.decode(&value_string).ok()?;
let raw = BASE64_STANDARD.decode_to_vec(&value_string).ok()?;
let checksum = Checksum {
checksum_type,
@@ -413,14 +413,14 @@ impl Checksum {
if self.want_parts > 0 && self.want_parts != parts {
return Err(ChecksumMismatch {
want: format!("{}-{}", self.encoded, self.want_parts),
got: format!("{}-{}", general_purpose::STANDARD.encode(&sum), parts),
got: format!("{}-{}", base64_simd::STANDARD.encode_to_string(&sum), parts),
});
}
if sum != self.raw {
return Err(ChecksumMismatch {
want: self.encoded.clone(),
got: general_purpose::STANDARD.encode(&sum),
got: base64_simd::STANDARD.encode_to_string(&sum),
});
}
@@ -569,7 +569,7 @@ impl Checksum {
}
}
self.encoded = general_purpose::STANDARD.encode(&self.raw);
self.encoded = base64_simd::STANDARD.encode_to_string(&self.raw);
Ok(())
}
}
@@ -1160,7 +1160,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
let checksum_bytes = &buf[..length];
buf = &buf[length..];
let mut checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
let mut checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
if checksum_type.is(ChecksumType::MULTIPART) {
is_multipart = true;
@@ -1190,7 +1190,7 @@ pub fn read_checksums(mut buf: &[u8], part: i32) -> (HashMap<String, String>, bo
if part > 0 && (part as u64) <= parts_count {
let offset = ((part - 1) as usize) * length;
let part_checksum = &buf[offset..offset + length];
checksum_str = general_purpose::STANDARD.encode(part_checksum);
checksum_str = base64_simd::STANDARD.encode_to_string(part_checksum);
}
buf = &buf[want_len..];
}
@@ -1249,7 +1249,7 @@ pub fn read_part_checksums(mut buf: &[u8]) -> Vec<HashMap<String, String>> {
let checksum_bytes = &buf[..length];
buf = &buf[length..];
let checksum_str = general_purpose::STANDARD.encode(checksum_bytes);
let checksum_str = base64_simd::STANDARD.encode_to_string(checksum_bytes);
part_checksum.insert(checksum_type.to_string(), checksum_str);
}
@@ -1553,7 +1553,6 @@ mod tests {
// asserted alongside the raw hex so both the digest and its encoding are pinned.
#[test]
fn xxhash_sha512_regression_lock_non_empty() {
use base64::{Engine as _, engine::general_purpose::STANDARD};
let data = b"The quick brown fox jumps over the lazy dog";
// XXH3-64(fox) = 0xce7d19a5418fb365 is the official upstream vector.
@@ -1565,7 +1564,11 @@ mod tests {
let got_hex: String = c.raw.iter().map(|b| format!("{b:02x}")).collect();
assert_eq!(got_hex, want_hex, "{t:?} raw hex drifted");
// encoded field must be the standard-base64 of raw (S3 wire form)
assert_eq!(c.encoded, STANDARD.encode(&c.raw), "{t:?} encoded field != base64(raw)");
assert_eq!(
c.encoded,
base64_simd::STANDARD.encode_to_string(&c.raw),
"{t:?} encoded field != base64(raw)"
);
}
}
crates/rio/src/hash_reader.rs
+4 -5
View File
@@ -91,8 +91,7 @@ use crate::Sha256Hasher;
use crate::compress_index::{Index, TryGetIndex};
use crate::get_content_checksum;
use crate::{DynReader, EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, WarpReader, boxed_reader, wrap_reader};
use base64::Engine;
use base64::engine::general_purpose;
use http::HeaderMap;
use pin_project_lite::pin_project;
use s3s::TrailingHeaders;
@@ -582,8 +581,8 @@ impl AsyncRead for HashReader {
})
{
expected_content_hash.encoded = checksum_str;
expected_content_hash.raw = general_purpose::STANDARD
.decode(&expected_content_hash.encoded)
expected_content_hash.raw = base64_simd::STANDARD
.decode_to_vec(&expected_content_hash.encoded)
.map_err(|_| std::io::Error::other("Invalid base64 checksum"))?;
if expected_content_hash.raw.is_empty() {
@@ -598,7 +597,7 @@ impl AsyncRead for HashReader {
&& !expected_content_hash.checksum_type.trailing()
{
expected_content_hash.raw = content_hash;
expected_content_hash.encoded = general_purpose::STANDARD.encode(&expected_content_hash.raw);
expected_content_hash.encoded = base64_simd::STANDARD.encode_to_string(&expected_content_hash.raw);
} else if content_hash != expected_content_hash.raw {
let expected_hex = hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower);
let actual_hex = hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower);