diff --git a/crates/ecstore/src/checksum.rs b/crates/ecstore/src/checksum.rs index e9920eca0..e338b6b1e 100644 --- a/crates/ecstore/src/checksum.rs +++ b/crates/ecstore/src/checksum.rs @@ -20,12 +20,12 @@ #![allow(clippy::all)] use lazy_static::lazy_static; +use rustfs_utils::HashAlgorithm; use std::collections::HashMap; use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart}; use crate::{disk::DiskAPI, store_api::GetObjectReader}; use rustfs_utils::crypto::{base64_decode, base64_encode}; -use rustfs_utils::hasher::{Hasher, Sha256}; use s3s::header::{ X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256, }; @@ -133,7 +133,7 @@ impl ChecksumMode { } } - pub fn hasher(&self) -> Result, std::io::Error> { + pub fn hasher(&self) -> Result { match /*C_ChecksumMask & **/self { /*ChecksumMode::ChecksumCRC32 => { return Ok(Box::new(crc32fast::Hasher::new())); @@ -145,7 +145,7 @@ impl ChecksumMode { return Ok(Box::new(sha1::new())); }*/ ChecksumMode::ChecksumSHA256 => { - return Ok(Box::new(Sha256::new())); + return Ok(HashAlgorithm::SHA256); } /*ChecksumMode::ChecksumCRC64NVME => { return Ok(Box::new(crc64nvme.New()); @@ -170,8 +170,8 @@ impl ChecksumMode { return Ok("".to_string()); } let mut h = self.hasher()?; - h.write(b); - Ok(base64_encode(h.sum().as_bytes())) + let hash = h.hash_encode(b); + Ok(base64_encode(hash.as_ref())) } pub fn to_string(&self) -> String { @@ -201,15 +201,15 @@ impl ChecksumMode { } } - pub fn check_sum_reader(&self, r: GetObjectReader) -> Result { - let mut h = self.hasher()?; - Ok(Checksum::new(self.clone(), h.sum().as_bytes())) - } + // pub fn check_sum_reader(&self, r: GetObjectReader) -> Result { + // let mut h = self.hasher()?; + // Ok(Checksum::new(self.clone(), h.sum().as_bytes())) + // } - pub fn check_sum_bytes(&self, b: &[u8]) -> Result { - let mut h = self.hasher()?; - Ok(Checksum::new(self.clone(), h.sum().as_bytes())) - } + // pub fn check_sum_bytes(&self, b: &[u8]) -> Result { + // let mut h = self.hasher()?; + // Ok(Checksum::new(self.clone(), h.sum().as_bytes())) + // } pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result { if !self.can_composite() { @@ -227,10 +227,10 @@ impl ChecksumMode { let c = self.base(); let crc_bytes = Vec::::with_capacity(p.len() * self.raw_byte_len() as usize); let mut h = self.hasher()?; - h.write(&crc_bytes); + let hash = h.hash_encode(crc_bytes.as_ref()); Ok(Checksum { checksum_type: self.clone(), - r: h.sum().as_bytes().to_vec(), + r: hash.as_ref().to_vec(), computed: false, }) } diff --git a/crates/ecstore/src/client/api_put_object.rs b/crates/ecstore/src/client/api_put_object.rs index 18e9bae60..0cfa03789 100644 --- a/crates/ecstore/src/client/api_put_object.rs +++ b/crates/ecstore/src/client/api_put_object.rs @@ -25,7 +25,6 @@ use std::{collections::HashMap, sync::Arc}; use time::{Duration, OffsetDateTime, macros::format_description}; use tracing::{error, info, warn}; -use rustfs_utils::hasher::Hasher; use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus}; use s3s::header::{ X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS, @@ -364,18 +363,14 @@ impl TransitionClient { if opts.send_content_md5 { let mut md5_hasher = self.md5_hasher.lock().unwrap(); let hash = md5_hasher.as_mut().expect("err"); - hash.write(&buf[..length]); - md5_base64 = base64_encode(hash.sum().as_bytes()); + let hash = hash.hash_encode(&buf[..length]); + md5_base64 = base64_encode(hash.as_ref()); } else { - let csum; - { - let mut crc = opts.auto_checksum.hasher()?; - crc.reset(); - crc.write(&buf[..length]); - csum = crc.sum(); - } + let mut crc = opts.auto_checksum.hasher()?; + let csum = crc.hash_encode(&buf[..length]); + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { - custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap()); + custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err")); } else { warn!("Invalid header name: {}", opts.auto_checksum.key()); } diff --git a/crates/ecstore/src/client/api_put_object_multipart.rs b/crates/ecstore/src/client/api_put_object_multipart.rs index d3a84c77a..4652314e6 100644 --- a/crates/ecstore/src/client/api_put_object_multipart.rs +++ b/crates/ecstore/src/client/api_put_object_multipart.rs @@ -31,7 +31,6 @@ use tracing::{error, info}; use url::form_urlencoded::Serializer; use uuid::Uuid; -use rustfs_utils::hasher::Hasher; use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; use s3s::{Body, dto::StreamingBlob}; //use crate::disk::{Reader, BufferReader}; @@ -117,8 +116,8 @@ impl TransitionClient { let length = buf.len(); for (k, v) in hash_algos.iter_mut() { - v.write(&buf[..length]); - hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap()); + let hash = v.hash_encode(&buf[..length]); + hash_sums.insert(k.to_string(), hash.as_ref().to_vec()); } //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress); @@ -134,15 +133,11 @@ impl TransitionClient { sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower); //} if hash_sums.len() == 0 { - let csum; - { - let mut crc = opts.auto_checksum.hasher()?; - crc.reset(); - crc.write(&buf[..length]); - csum = crc.sum(); - } + let mut crc = opts.auto_checksum.hasher()?; + let csum = crc.hash_encode(&buf[..length]); + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { - custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err")); + custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err")); } else { warn!("Invalid header name: {}", opts.auto_checksum.key()); } diff --git a/crates/ecstore/src/client/api_put_object_streaming.rs b/crates/ecstore/src/client/api_put_object_streaming.rs index 82ad9fea9..fa6d1c897 100644 --- a/crates/ecstore/src/client/api_put_object_streaming.rs +++ b/crates/ecstore/src/client/api_put_object_streaming.rs @@ -40,7 +40,7 @@ use crate::client::{ constants::ISO8601_DATEFORMAT, transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo}, }; -use rustfs_utils::hasher::Hasher; + use rustfs_utils::{crypto::base64_encode, path::trim_etag}; use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; @@ -153,21 +153,16 @@ impl TransitionClient { if opts.send_content_md5 { let mut md5_hasher = self.md5_hasher.lock().unwrap(); let md5_hash = md5_hasher.as_mut().expect("err"); - md5_hash.reset(); - md5_hash.write(&buf[..length]); - md5_base64 = base64_encode(md5_hash.sum().as_bytes()); + let hash = md5_hash.hash_encode(&buf[..length]); + md5_base64 = base64_encode(hash.as_ref()); } else { - let csum; - { - let mut crc = opts.auto_checksum.hasher()?; - crc.reset(); - crc.write(&buf[..length]); - csum = crc.sum(); - } - if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) { - custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err")); + let mut crc = opts.auto_checksum.hasher()?; + let csum = crc.hash_encode(&buf[..length]); + + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { + custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err")); } else { - warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized()); + warn!("Invalid header name: {}", opts.auto_checksum.key()); } } @@ -308,17 +303,11 @@ impl TransitionClient { let mut custom_header = HeaderMap::new(); if !opts.send_content_md5 { - let csum; - { - let mut crc = opts.auto_checksum.hasher()?; - crc.reset(); - crc.write(&buf[..length]); - csum = crc.sum(); - } + let mut crc = opts.auto_checksum.hasher()?; + let csum = crc.hash_encode(&buf[..length]); + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { - if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) { - custom_header.insert(header_name, header_value); - } + custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err")); } else { warn!("Invalid header name: {}", opts.auto_checksum.key()); } @@ -334,8 +323,8 @@ impl TransitionClient { if opts.send_content_md5 { let mut md5_hasher = clone_self.md5_hasher.lock().unwrap(); let md5_hash = md5_hasher.as_mut().expect("err"); - md5_hash.write(&buf[..length]); - md5_base64 = base64_encode(md5_hash.sum().as_bytes()); + let hash = md5_hash.hash_encode(&buf[..length]); + md5_base64 = base64_encode(hash.as_ref()); } //defer wg.Done() diff --git a/crates/ecstore/src/client/api_remove.rs b/crates/ecstore/src/client/api_remove.rs index b3f75e727..853ee6413 100644 --- a/crates/ecstore/src/client/api_remove.rs +++ b/crates/ecstore/src/client/api_remove.rs @@ -21,6 +21,7 @@ use bytes::Bytes; use http::{HeaderMap, HeaderValue, Method, StatusCode}; +use rustfs_utils::{HashAlgorithm, crypto::base64_encode}; use s3s::S3ErrorCode; use s3s::dto::ReplicationStatus; use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION; @@ -38,7 +39,6 @@ use crate::{ store_api::{GetObjectReader, ObjectInfo, StorageAPI}, }; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; -use rustfs_utils::hasher::{sum_md5_base64, sum_sha256_hex}; pub struct RemoveBucketOptions { _forced_elete: bool, @@ -330,8 +330,8 @@ impl TransitionClient { query_values: url_values.clone(), content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), content_length: remove_bytes.len() as i64, - content_md5_base64: sum_md5_base64(&remove_bytes), - content_sha256_hex: sum_sha256_hex(&remove_bytes), + content_md5_base64: base64_encode(&HashAlgorithm::Md5.hash_encode(&remove_bytes).as_ref()), + content_sha256_hex: base64_encode(&HashAlgorithm::SHA256.hash_encode(&remove_bytes).as_ref()), custom_header: headers, object_name: "".to_string(), stream_sha256: false, diff --git a/crates/ecstore/src/client/bucket_cache.rs b/crates/ecstore/src/client/bucket_cache.rs index 12e38d785..1ac29958d 100644 --- a/crates/ecstore/src/client/bucket_cache.rs +++ b/crates/ecstore/src/client/bucket_cache.rs @@ -31,7 +31,6 @@ use crate::client::{ transition_api::{Document, TransitionClient}, }; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; -use rustfs_utils::hasher::{Hasher, Sha256}; use s3s::Body; use s3s::S3ErrorCode; diff --git a/crates/ecstore/src/client/transition_api.rs b/crates/ecstore/src/client/transition_api.rs index 656641166..ea3fde4b9 100644 --- a/crates/ecstore/src/client/transition_api.rs +++ b/crates/ecstore/src/client/transition_api.rs @@ -28,8 +28,12 @@ use http::{ }; use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; use hyper_util::{client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor}; +use md5::Digest; +use md5::Md5; use rand::Rng; +use rustfs_utils::HashAlgorithm; use serde::{Deserialize, Serialize}; +use sha2::Sha256; use std::io::Cursor; use std::pin::Pin; use std::sync::atomic::{AtomicI32, Ordering}; @@ -60,7 +64,6 @@ use crate::client::{ }; use crate::{checksum::ChecksumMode, store_api::GetObjectReader}; use rustfs_rio::HashReader; -use rustfs_utils::hasher::{MD5, Sha256}; use rustfs_utils::{ net::get_endpoint_url, retry::{MAX_RETRY, new_retry_timer}, @@ -104,8 +107,8 @@ pub struct TransitionClient { pub random: u64, pub lookup: BucketLookupType, //pub lookupFn: func(u url.URL, bucketName string) BucketLookupType, - pub md5_hasher: Arc>>, - pub sha256_hasher: Option, + pub md5_hasher: Arc>>, + pub sha256_hasher: Option, pub health_status: AtomicI32, pub trailing_header_support: bool, pub max_retries: i64, @@ -122,8 +125,8 @@ pub struct Options { //pub custom_region_via_url: func(u url.URL) string, //pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType, pub trailing_headers: bool, - pub custom_md5: Option, - pub custom_sha256: Option, + pub custom_md5: Option, + pub custom_sha256: Option, pub max_retries: i64, } @@ -190,11 +193,11 @@ impl TransitionClient { { let mut md5_hasher = clnt.md5_hasher.lock().unwrap(); if md5_hasher.is_none() { - *md5_hasher = Some(MD5::new()); + *md5_hasher = Some(HashAlgorithm::Md5); } } if clnt.sha256_hasher.is_none() { - clnt.sha256_hasher = Some(Sha256::new()); + clnt.sha256_hasher = Some(HashAlgorithm::SHA256); } clnt.trailing_header_support = opts.trailing_headers && clnt.override_signer_type == SignatureType::SignatureV4; @@ -241,8 +244,8 @@ impl TransitionClient { &self, is_md5_requested: bool, is_sha256_requested: bool, - ) -> (HashMap, HashMap>) { - todo!(); + ) -> (HashMap, HashMap>) { + todo!() } fn is_online(&self) -> bool { diff --git a/crates/utils/src/hasher.rs b/crates/utils/src/hasher.rs deleted file mode 100644 index 26a02f80d..000000000 --- a/crates/utils/src/hasher.rs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use md5::{Digest as Md5Digest, Md5}; -use sha2::{ - Sha256 as sha_sha256, - digest::{Reset, Update}, -}; -pub trait Hasher { - fn write(&mut self, bytes: &[u8]); - fn reset(&mut self); - fn sum(&mut self) -> String; - fn size(&self) -> usize; - fn block_size(&self) -> usize; -} - -#[derive(Default)] -pub enum HashType { - #[default] - Undefined, - Uuid(Uuid), - Md5(MD5), - Sha256(Sha256), -} - -impl Hasher for HashType { - fn write(&mut self, bytes: &[u8]) { - match self { - HashType::Md5(md5) => md5.write(bytes), - HashType::Sha256(sha256) => sha256.write(bytes), - HashType::Uuid(uuid) => uuid.write(bytes), - HashType::Undefined => (), - } - } - - fn reset(&mut self) { - match self { - HashType::Md5(md5) => md5.reset(), - HashType::Sha256(sha256) => sha256.reset(), - HashType::Uuid(uuid) => uuid.reset(), - HashType::Undefined => (), - } - } - - fn sum(&mut self) -> String { - match self { - HashType::Md5(md5) => md5.sum(), - HashType::Sha256(sha256) => sha256.sum(), - HashType::Uuid(uuid) => uuid.sum(), - HashType::Undefined => "".to_owned(), - } - } - - fn size(&self) -> usize { - match self { - HashType::Md5(md5) => md5.size(), - HashType::Sha256(sha256) => sha256.size(), - HashType::Uuid(uuid) => uuid.size(), - HashType::Undefined => 0, - } - } - - fn block_size(&self) -> usize { - match self { - HashType::Md5(md5) => md5.block_size(), - HashType::Sha256(sha256) => sha256.block_size(), - HashType::Uuid(uuid) => uuid.block_size(), - HashType::Undefined => 64, - } - } -} - -#[derive(Debug)] -pub struct Sha256 { - hasher: sha_sha256, -} - -impl Sha256 { - pub fn new() -> Self { - Self { - hasher: sha_sha256::new(), - } - } -} -impl Default for Sha256 { - fn default() -> Self { - Self::new() - } -} - -impl Hasher for Sha256 { - fn write(&mut self, bytes: &[u8]) { - Update::update(&mut self.hasher, bytes); - } - - fn reset(&mut self) { - Reset::reset(&mut self.hasher); - } - - fn sum(&mut self) -> String { - hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower) - } - - fn size(&self) -> usize { - 32 - } - - fn block_size(&self) -> usize { - 64 - } -} - -#[derive(Debug)] -pub struct MD5 { - hasher: Md5, -} - -impl MD5 { - pub fn new() -> Self { - Self { hasher: Md5::new() } - } -} -impl Default for MD5 { - fn default() -> Self { - Self::new() - } -} - -impl Hasher for MD5 { - fn write(&mut self, bytes: &[u8]) { - Md5Digest::update(&mut self.hasher, bytes); - } - - fn reset(&mut self) {} - - fn sum(&mut self) -> String { - hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower) - } - - fn size(&self) -> usize { - 32 - } - - fn block_size(&self) -> usize { - 64 - } -} - -pub struct Uuid { - id: String, -} - -impl Uuid { - pub fn new(id: String) -> Self { - Self { id } - } -} - -impl Hasher for Uuid { - fn write(&mut self, _bytes: &[u8]) {} - - fn reset(&mut self) {} - - fn sum(&mut self) -> String { - self.id.clone() - } - - fn size(&self) -> usize { - self.id.len() - } - - fn block_size(&self) -> usize { - 64 - } -} - -pub fn sum_sha256_hex(data: &[u8]) -> String { - let mut hash = Sha256::new(); - hash.write(data); - base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum()) -} - -pub fn sum_md5_base64(data: &[u8]) -> String { - let mut hash = MD5::new(); - hash.write(data); - base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum()) -} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 0e1e8a9cc..b7ed63039 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -30,8 +30,6 @@ pub mod io; #[cfg(feature = "hash")] pub mod hash; -pub mod hasher; - #[cfg(feature = "os")] pub mod os;