ilm feature add

crates/utils/src/crypto.rs

@@ -1,3 +1,8 @@
+use std::mem::MaybeUninit;
+
+use hex_simd::{AsOut, AsciiCase};
+use hyper::body::Bytes;
+
 pub fn base64_encode(input: &[u8]) -> String {
     base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
 }
@@ -24,6 +29,80 @@ pub fn hex(data: impl AsRef<[u8]>) -> String {
 //     h.finish().unwrap()
 // }
 
+/// verify sha256 checksum string
+pub fn is_sha256_checksum(s: &str) -> bool {
+    // TODO: optimize
+    let is_lowercase_hex = |c: u8| matches!(c, b'0'..=b'9' | b'a'..=b'f');
+    s.len() == 64 && s.as_bytes().iter().copied().all(is_lowercase_hex)
+}
+
+/// `hmac_sha1(key, data)`
+pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
+    use hmac::{Hmac, Mac};
+    use sha1::Sha1;
+
+    let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).unwrap();
+    m.update(data.as_ref());
+    m.finalize().into_bytes().into()
+}
+
+/// `hmac_sha256(key, data)`
+pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] {
+    use hmac::{Hmac, Mac};
+    use sha2::Sha256;
+
+    let mut m = Hmac::<Sha256>::new_from_slice(key.as_ref()).unwrap();
+    m.update(data.as_ref());
+    m.finalize().into_bytes().into()
+}
+
+/// `f(hex(src))`
+fn hex_bytes32<R>(src: impl AsRef<[u8]>, f: impl FnOnce(&str) -> R) -> R {
+    let buf: &mut [_] = &mut [MaybeUninit::uninit(); 64];
+    let ans = hex_simd::encode_as_str(src.as_ref(), buf.as_out(), AsciiCase::Lower);
+    f(ans)
+}
+
+#[cfg(not(all(feature = "openssl", not(windows))))]
+fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> + use<> {
+    use sha2::{Digest, Sha256};
+    <Sha256 as Digest>::digest(data)
+}
+
+#[cfg(all(feature = "openssl", not(windows)))]
+fn sha256(data: &[u8]) -> impl AsRef<[u8]> {
+    use openssl::hash::{Hasher, MessageDigest};
+    let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
+    h.update(data).unwrap();
+    h.finish().unwrap()
+}
+
+#[cfg(not(all(feature = "openssl", not(windows))))]
+fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8; 32]> + use<> {
+    use sha2::{Digest, Sha256};
+    let mut h = <Sha256 as Digest>::new();
+    chunk.iter().for_each(|data| h.update(data));
+    h.finalize()
+}
+
+#[cfg(all(feature = "openssl", not(windows)))]
+fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8]> {
+    use openssl::hash::{Hasher, MessageDigest};
+    let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
+    chunk.iter().for_each(|data| h.update(data).unwrap());
+    h.finish().unwrap()
+}
+
+/// `f(hex(sha256(data)))`
+pub fn hex_sha256<R>(data: &[u8], f: impl FnOnce(&str) -> R) -> R {
+    hex_bytes32(sha256(data).as_ref(), f)
+}
+
+/// `f(hex(sha256(chunk)))`
+pub fn hex_sha256_chunk<R>(chunk: &[Bytes], f: impl FnOnce(&str) -> R) -> R {
+    hex_bytes32(sha256_chunk(chunk).as_ref(), f)
+}
+
 #[test]
 fn test_base64_encoding_decoding() {
     let original_uuid_timestamp = "c0194290-d911-45cb-8e12-79ec563f46a8x1735460504394878000";

crates/utils/src/hash.rs

@@ -61,6 +61,8 @@ impl HashAlgorithm {
 use crc32fast::Hasher;
 use siphasher::sip::SipHasher;
 
+pub const EMPTY_STRING_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
 pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
     //  你的密钥，必须是 16 字节
 

crates/utils/src/lib.rs

@@ -7,6 +7,8 @@ pub mod net;
 #[cfg(feature = "net")]
 pub use net::*;
 
+pub mod retry;
+
 #[cfg(feature = "io")]
 pub mod io;
 

crates/utils/src/net.rs

@@ -1,11 +1,17 @@
+use hyper::client::conn::http2::Builder;
+use hyper_util::rt::TokioExecutor;
 use lazy_static::lazy_static;
 use std::{
-    collections::HashSet,
+    collections::{HashMap, HashSet},
     fmt::Display,
     net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
 };
+use url::{Host, Url};
+//use hyper::{client::conn::http2::Builder, rt::Executor};
+//use tonic::{SharedExec, UserAgent};
+//use hyper_util::rt::TokioTimer;
 
-use url::Host;
+use s3s::header::X_AMZ_STORAGE_CLASS;
 
 lazy_static! {
     static ref LOCAL_IPS: Vec<IpAddr> = must_get_local_ips().unwrap();
@@ -105,6 +111,107 @@ pub fn must_get_local_ips() -> std::io::Result<Vec<IpAddr>> {
     }
 }
 
+pub fn get_default_location(u: Url, region_override: &str) -> String {
+    todo!();
+}
+
+pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, std::io::Error> {
+    let mut scheme = "https";
+    if !secure {
+        scheme = "http";
+    }
+
+    let endpoint_url_str = format!("{scheme}://{endpoint}");
+    let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else { return Err(std::io::Error::other("url parse error.")); };
+
+    //is_valid_endpoint_url(endpoint_url)?;
+    Ok(endpoint_url)
+}
+
+pub const DEFAULT_DIAL_TIMEOUT: i64 = 5;
+
+pub fn new_remotetarget_http_transport(insecure: bool) -> Builder<TokioExecutor> {
+    todo!();
+}
+
+lazy_static! {
+    static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("attributes".to_string(),                   true);
+        m.insert("partNumber".to_string(),                   true);
+        m.insert("versionId".to_string(),                    true);
+        m.insert("response-cache-control".to_string(),       true);
+        m.insert("response-content-disposition".to_string(), true);
+        m.insert("response-content-encoding".to_string(),    true);
+        m.insert("response-content-language".to_string(),    true);
+        m.insert("response-content-type".to_string(),        true);
+        m.insert("response-expires".to_string(),             true);
+        m
+    };
+
+    static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("content-type".to_string(),                        true);
+        m.insert("cache-control".to_string(),                       true);
+        m.insert("content-encoding".to_string(),                    true);
+        m.insert("content-disposition".to_string(),                 true);
+        m.insert("content-language".to_string(),                    true);
+        m.insert("x-amz-website-redirect-location".to_string(),     true);
+        m.insert("x-amz-object-lock-mode".to_string(),              true);
+        m.insert("x-amz-metadata-directive".to_string(),            true);
+        m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
+        m.insert("expires".to_string(),                             true);
+        m.insert("x-amz-replication-status".to_string(),            true);
+        m
+    };
+
+    static ref SSE_HEADERS: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("x-amz-server-side-encryption".to_string(),                    true);
+        m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(),     true);
+        m.insert("x-amz-server-side-encryption-context".to_string(),            true);
+        m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
+        m.insert("x-amz-server-side-encryption-customer-key".to_string(),       true);
+        m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(),   true);
+        m
+    };
+}
+
+pub fn is_standard_query_value(qs_key: &str) -> bool {
+    SUPPORTED_QUERY_VALUES[qs_key]
+}
+
+const ALLOWED_CUSTOM_QUERY_PREFIX: &str = "x-";
+
+pub fn is_custom_query_value(qs_key: &str) -> bool {
+    qs_key.starts_with(ALLOWED_CUSTOM_QUERY_PREFIX)
+}
+
+pub fn is_storageclass_header(header_key: &str) -> bool {
+    header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
+}
+
+pub fn is_standard_header(header_key: &str) -> bool {
+    *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
+}
+
+pub fn is_sse_header(header_key: &str) -> bool {
+    *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
+}
+
+pub fn is_amz_header(header_key: &str) -> bool {
+    let key = header_key.to_lowercase();
+    key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" || is_sse_header(header_key) || key.starts_with("x-amz-checksum-")
+}
+
+pub fn is_rustfs_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-rustfs-")
+}
+
+pub fn is_rustfs_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-rustfs-")
+}
+
 #[derive(Debug, Clone)]
 pub struct XHost {
     pub name: String,

crates/utils/src/path.rs

@@ -253,6 +253,11 @@ pub fn dir(path: &str) -> String {
     let (a, _) = split(path);
     clean(a)
 }
+
+pub fn trim_etag(etag: &str) -> String {
+    etag.trim_matches('"').to_string()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/utils/src/retry.rs

@@ -0,0 +1,179 @@
+//use tokio_stream::Stream;
+use std::future::Future;
+use std::time::{Duration, Instant};
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+// MaxRetry is the maximum number of retries before stopping.
+pub const MAX_RETRY: i64 = 10;
+
+// MaxJitter will randomize over the full exponential backoff time
+pub const MAX_JITTER: f64 = 1.0;
+
+// NoJitter disables the use of jitter for randomizing the exponential backoff time
+pub const NO_JITTER: f64 = 0.0;
+
+// DefaultRetryUnit - default unit multiplicative per retry.
+// defaults to 200 * time.Millisecond
+//const DefaultRetryUnit = 200 * time.Millisecond;
+
+// DefaultRetryCap - Each retry attempt never waits no longer than
+// this maximum time duration.
+//const DefaultRetryCap = time.Second;
+
+/*
+struct Delay {
+    when: Instant,
+}
+
+impl Future for Delay {
+    type Output = &'static str;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>)
+        -> Poll<&'static str>
+    {
+        if Instant::now() >= self.when {
+            println!("Hello world");
+            Poll::Ready("done")
+        } else {
+            // Ignore this line for now.
+            cx.waker().wake_by_ref();
+            Poll::Pending
+        }
+    }
+}
+
+struct RetryTimer {
+    rem: usize,
+    delay: Delay,
+}
+
+impl RetryTimer {
+    fn new() -> Self {
+        Self {
+            rem: 3,
+            delay: Delay { when: Instant::now() }
+        }
+    }
+}
+
+impl Stream for RetryTimer {
+    type Item = ();
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>)
+        -> Poll<Option<()>>
+    {
+        if self.rem == 0 {
+            // No more delays
+            return Poll::Ready(None);
+        }
+
+        match Pin::new(&mut self.delay).poll(cx) {
+            Poll::Ready(_) => {
+                let when = self.delay.when + Duration::from_millis(10);
+                self.delay = Delay { when };
+                self.rem -= 1;
+                Poll::Ready(Some(()))
+            }
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}*/
+
+pub fn new_retry_timer(max_retry: i32, base_sleep: Duration, max_sleep: Duration, jitter: f64) -> Vec<i32> {
+    /*attemptCh := make(chan int)
+
+    exponentialBackoffWait := func(attempt int) time.Duration {
+      // normalize jitter to the range [0, 1.0]
+      if jitter < NoJitter {
+        jitter = NoJitter
+      }
+      if jitter > MaxJitter {
+        jitter = MaxJitter
+      }
+
+      // sleep = random_between(0, min(maxSleep, base * 2 ** attempt))
+      sleep := baseSleep * time.Duration(1<<uint(attempt))
+      if sleep > maxSleep {
+        sleep = maxSleep
+      }
+      if jitter != NoJitter {
+        sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
+      }
+      return sleep
+    }
+
+    go func() {
+      defer close(attemptCh)
+      for i := 0; i < maxRetry; i++ {
+        select {
+        case attemptCh <- i + 1:
+        case <-ctx.Done():
+          return
+        }
+
+        select {
+        case <-time.After(exponentialBackoffWait(i)):
+        case <-ctx.Done():
+          return
+        }
+      }
+    }()
+    return attemptCh*/
+    todo!();
+}
+
+/*var retryableS3Codes = map[string]struct{}{
+  "RequestError":          {},
+  "RequestTimeout":        {},
+  "Throttling":            {},
+  "ThrottlingException":   {},
+  "RequestLimitExceeded":  {},
+  "RequestThrottled":      {},
+  "InternalError":         {},
+  "ExpiredToken":          {},
+  "ExpiredTokenException": {},
+  "SlowDown":              {},
+}
+
+fn isS3CodeRetryable(s3Code string) (ok bool) {
+  _, ok = retryableS3Codes[s3Code]
+  return ok
+}
+
+var retryableHTTPStatusCodes = map[int]struct{}{
+  http.StatusRequestTimeout:      {},
+  429:                            {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
+  499:                            {}, // client closed request, retry. A non-standard status code introduced by nginx.
+  http.StatusInternalServerError: {},
+  http.StatusBadGateway:          {},
+  http.StatusServiceUnavailable:  {},
+  http.StatusGatewayTimeout:      {},
+  520:                            {}, // It is used by Cloudflare as a catch-all response for when the origin server sends something unexpected.
+  // Add more HTTP status codes here.
+}
+
+fn isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
+  _, ok = retryableHTTPStatusCodes[httpStatusCode]
+  return ok
+}
+
+fn isRequestErrorRetryable(ctx context.Context, err error) bool {
+  if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+    // Retry if internal timeout in the HTTP call.
+    return ctx.Err() == nil
+  }
+  if ue, ok := err.(*url.Error); ok {
+    e := ue.Unwrap()
+    switch e.(type) {
+    // x509: certificate signed by unknown authority
+    case x509.UnknownAuthorityError:
+      return false
+    }
+    switch e.Error() {
+    case "http: server gave HTTP response to HTTPS client":
+      return false
+    }
+  }
+  return true
+}*/