Merge pull request #500 from rustfs/feature/ilm

Feature/ilm

.vscode/launch.json

@@ -20,33 +20,19 @@
                 }
             },
             "env": {
-                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug",
-                "RUSTFS_VOLUMES": "./target/volume/test{0...3}",
-                "RUSTFS_ADDRESS": "[::]:9000",
-                "RUSTFS_CONSOLE_ENABLE": "true",
-                "RUSTFS_CONSOLE_ADDRESS": "[::]:9002",
-                "RUSTFS_SERVER_DOMAINS": "localhost:9000",
-                "RUSTFS_TLS_PATH": "./deploy/certs",
-                "RUSTFS_OBS_CONFIG": "./deploy/config/obs.example.toml",
-                "RUSTFS__OBSERVABILITY__ENDPOINT": "http://localhost:4317",
-                "RUSTFS__OBSERVABILITY__USE_STDOUT": "true",
-                "RUSTFS__OBSERVABILITY__SAMPLE_RATIO": "2.0",
-                "RUSTFS__OBSERVABILITY__METER_INTERVAL": "30",
-                "RUSTFS__OBSERVABILITY__SERVICE_NAME": "rustfs",
-                "RUSTFS__OBSERVABILITY__SERVICE_VERSION": "0.1.0",
-                "RUSTFS__OBSERVABILITY__ENVIRONMENT": "develop",
-                "RUSTFS__OBSERVABILITY__LOGGER_LEVEL": "info",
-                "RUSTFS__SINKS__FILE__ENABLED": "true",
-                "RUSTFS__SINKS__FILE__PATH": "./deploy/logs/rustfs.log",
-                "RUSTFS__SINKS__WEBHOOK__ENABLED": "false",
-                "RUSTFS__SINKS__WEBHOOK__ENDPOINT": "", 
-                "RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN": "",
-                "RUSTFS__SINKS__KAFKA__ENABLED": "false",
-                "RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS": "",
-                "RUSTFS__SINKS__KAFKA__TOPIC": "",
-                "RUSTFS__LOGGER__QUEUE_CAPACITY": "10"
-
+                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug"
             },
+            "args": [
+                "--access-key",
+                "AKEXAMPLERUSTFS",
+                "--secret-key",
+                "SKEXAMPLERUSTFS",
+                "--address",
+                "0.0.0.0:9010",
+                "--domain-name",
+                "127.0.0.1:9010",
+                "./target/volume/test{0...4}"
+            ],
             "cwd": "${workspaceFolder}"
         },
         {
@@ -86,6 +72,19 @@
             },
             "args": [],
             "cwd": "${workspaceFolder}"
+        },
+        {
+            "name": "Debug executable target/debug/rustfs",
+            "type":  "lldb",
+            "request": "launch",
+            "program": "${workspaceFolder}/target/debug/rustfs",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            //"stopAtEntry": false,
+            //"preLaunchTask": "cargo build",
+            "sourceLanguages": [
+                "rust"
+            ],
         }
     ]
 }

Cargo.toml

@@ -21,7 +21,7 @@ members = [
     "rustfs", # Core file system implementation
     "s3select/api", # S3 Select API interface
     "s3select/query", # S3 Select query engine
-
+    "reader",
 
 ]
 resolver = "2"
@@ -60,6 +60,7 @@ rustfs-utils = { path = "crates/utils", version = "0.0.1" }
 rustfs-rio = { path = "crates/rio", version = "0.0.1" }
 rustfs-filemeta = { path = "crates/filemeta", version = "0.0.1" }
 workers = { path = "./common/workers", version = "0.0.1" }
+reader = { path = "./reader", version = "0.0.1" }
 aes-gcm = { version = "0.10.3", features = ["std"] }
 arc-swap = "1.7.1"
 argon2 = { version = "0.5.3", features = ["std"] }
@@ -90,7 +91,7 @@ derive_builder = "0.20.2"
 dioxus = { version = "0.6.3", features = ["router"] }
 dirs = "6.0.0"
 flatbuffers = "25.2.10"
-flexi_logger = { version = "0.30.2", features = ["trc", "dont_minimize_extra_stacks"] }
+flexi_logger = { version = "0.30.2", features = ["trc","dont_minimize_extra_stacks"] }  
 form_urlencoded = "1.2.1"
 futures = "0.3.31"
 futures-core = "0.3.31"
@@ -106,6 +107,7 @@ hyper-util = { version = "0.1.14", features = [
     "server-auto",
     "server-graceful",
 ] }
+hyper-rustls = "0.27.5"
 http = "1.3.1"
 http-body = "1.0.1"
 humantime = "2.2.0"
@@ -196,8 +198,12 @@ scopeguard = "1.2.0"
 shadow-rs = { version = "1.1.1", default-features = false }
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.140"
+serde-xml-rs = "0.8.1"
 serde_urlencoded = "0.7.1"
+sha1 = "0.10.6"
 sha2 = "0.10.9"
+hmac = "0.12.1"
+std-next = "0.1.8"
 siphasher = "1.0.1"
 smallvec = { version = "1.15.1", features = ["serde"] }
 snafu = "0.8.6"
@@ -223,6 +229,7 @@ tokio-tar = "0.3.1"
 tokio-util = { version = "0.7.15", features = ["io", "compat"] }
 tonic = { version = "0.13.1", features = ["gzip"] }
 tonic-build = { version = "0.13.1" }
+async-channel = "2.3.1"
 tower = { version = "0.5.2", features = ["timeout"] }
 tower-http = { version = "0.6.6", features = ["cors"] }
 tracing = "0.1.41"

common/lock/src/local_locker.rs

@@ -7,7 +7,7 @@ use std::{
 
 use crate::{Locker, lock_args::LockArgs};
 
-const MAX_DELETE_LIST: usize = 1000;
+pub const MAX_DELETE_LIST: usize = 1000;
 
 #[derive(Clone, Debug)]
 struct LockRequesterInfo {

crates/filemeta/Cargo.toml

@@ -21,6 +21,8 @@ byteorder = "1.5.0"
 tracing.workspace = true
 thiserror.workspace = true
 
+s3s.workspace = true
+
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
 

crates/filemeta/src/fileinfo.rs

@@ -17,6 +17,10 @@ pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
 pub const NULL_VERSION_ID: &str = "null";
 // pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded";
 
+pub const TIER_FV_ID: &str = "tier-free-versionID";
+pub const TIER_FV_MARKER: &str = "tier-free-marker";
+pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
+
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
 pub struct ObjectPartInfo {
     pub etag: String,
@@ -152,11 +156,10 @@ pub struct FileInfo {
     pub version_id: Option<Uuid>,
     pub is_latest: bool,
     pub deleted: bool,
-    // Transition related fields
-    pub transition_status: Option<String>,
-    pub transitioned_obj_name: Option<String>,
-    pub transition_tier: Option<String>,
-    pub transition_version_id: Option<String>,
+    pub transition_status: String,
+    pub transitioned_objname: String,
+    pub transition_tier: String,
+    pub transition_version_id: Option<Uuid>,
     pub expire_restored: bool,
     pub data_dir: Option<Uuid>,
     pub mod_time: Option<OffsetDateTime>,
@@ -225,7 +228,11 @@ impl FileInfo {
     }
 
     pub fn get_etag(&self) -> Option<String> {
-        self.metadata.get("etag").cloned()
+        if let Some(meta) = self.metadata.get("etag") {
+            Some(meta.clone())
+        } else {
+            None
+        }
     }
 
     pub fn write_quorum(&self, quorum: usize) -> usize {
@@ -307,6 +314,35 @@ impl FileInfo {
         self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
     }
 
+    pub fn set_tier_free_version_id(&mut self, version_id: &str) {
+        self.metadata
+            .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string());
+    }
+
+    pub fn tier_free_version_id(&self) -> String {
+        self.metadata[&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID)].clone()
+    }
+
+    pub fn set_tier_free_version(&mut self) {
+        self.metadata
+            .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string());
+    }
+
+    pub fn set_skip_tier_free_version(&mut self) {
+        self.metadata
+            .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string());
+    }
+
+    pub fn skip_tier_free_version(&self) -> bool {
+        self.metadata
+            .contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID))
+    }
+
+    pub fn tier_free_version(&self) -> bool {
+        self.metadata
+            .contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER))
+    }
+
     pub fn set_inline_data(&mut self) {
         self.metadata
             .insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
@@ -331,7 +367,7 @@ impl FileInfo {
 
     /// Check if the object is remote (transitioned to another tier)
     pub fn is_remote(&self) -> bool {
-        !self.transition_tier.as_ref().is_none_or(|s| s.is_empty())
+        !self.transition_tier.is_empty()
     }
 
     /// Get the data directory for this object
@@ -387,7 +423,7 @@ impl FileInfo {
     pub fn transition_info_equals(&self, other: &FileInfo) -> bool {
         self.transition_status == other.transition_status
             && self.transition_tier == other.transition_tier
-            && self.transitioned_obj_name == other.transitioned_obj_name
+            && self.transitioned_objname == other.transitioned_objname
             && self.transition_version_id == other.transition_version_id
     }
 

crates/filemeta/src/filemeta.rs

@@ -8,6 +8,7 @@ use crate::headers::{
 use byteorder::ByteOrder;
 use bytes::Bytes;
 use rmp::Marker;
+use s3s::header::X_AMZ_RESTORE;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::convert::TryFrom;
@@ -38,6 +39,19 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2;
 const META_DATA_READ_DEFAULT: usize = 4 << 10;
 const MSGP_UINT32_SIZE: usize = 5;
 
+pub const TRANSITION_COMPLETE: &str = "complete";
+pub const TRANSITION_PENDING: &str = "pending";
+
+pub const FREE_VERSION: &str = "free-version";
+
+pub const TRANSITION_STATUS: &str = "transition-status";
+pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
+pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
+pub const TRANSITION_TIER: &str = "transition-tier";
+
+const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days";
+const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date";
+
 // type ScanHeaderVersionFn = Box<dyn Fn(usize, &[u8], &[u8]) -> Result<()>>;
 
 #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -468,6 +482,42 @@ impl FileMeta {
         Err(Error::other("add_version failed"))
     }
 
+    pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> {
+        let mod_time = ver.get_mod_time().unwrap().nanosecond();
+        if !ver.valid() {
+            return Err(Error::other("attempted to add invalid version"));
+        }
+        let encoded = ver.marshal_msg()?;
+
+        if self.versions.len() + 1 > 100 {
+            return Err(Error::other(
+                "You've exceeded the limit on the number of versions you can create on this object",
+            ));
+        }
+
+        self.versions.push(FileMetaShallowVersion {
+            header: FileMetaVersionHeader {
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?),
+                ..Default::default()
+            },
+            ..Default::default()
+        });
+
+        let len = self.versions.len();
+        for (i, existing) in self.versions.iter().enumerate() {
+            if existing.header.mod_time.unwrap().nanosecond() <= mod_time {
+                let vers = self.versions[i..len - 1].to_vec();
+                self.versions[i + 1..].clone_from_slice(vers.as_slice());
+                self.versions[i] = FileMetaShallowVersion {
+                    header: ver.header(),
+                    meta: encoded,
+                };
+                return Ok(());
+            }
+        }
+        Err(Error::other("addVersion: Internal error, unable to add version"))
+    }
+
     // delete_version deletes version, returns data_dir
     pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
         let mut ventry = FileMetaVersion::default();
@@ -503,6 +553,42 @@ impl FileMeta {
             }
         }
 
+        for (i, version) in self.versions.iter().enumerate() {
+            if version.header.version_type != VersionType::Object || version.header.version_id != fi.version_id {
+                continue;
+            }
+
+            let mut ver = self.get_idx(i)?;
+
+            if fi.expire_restored {
+                ver.object.as_mut().unwrap().remove_restore_hdrs();
+                let _ = self.set_idx(i, ver.clone());
+            } else if fi.transition_status == TRANSITION_COMPLETE {
+                ver.object.as_mut().unwrap().set_transition(fi);
+                ver.object.as_mut().unwrap().reset_inline_data();
+                self.set_idx(i, ver.clone())?;
+            } else {
+                let vers = self.versions[i + 1..].to_vec();
+                self.versions.extend(vers.iter().cloned());
+                let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi);
+                if to_free {
+                    self.add_version_filemata(free_version)?;
+                }
+            }
+
+            if fi.deleted {
+                self.add_version_filemata(ventry)?;
+            }
+            if self.shared_data_dir_count(ver.object.as_ref().unwrap().version_id, ver.object.as_ref().unwrap().data_dir) > 0 {
+                return Ok(None);
+            }
+            return Ok(ver.object.as_ref().unwrap().data_dir);
+        }
+
+        if fi.deleted {
+            self.add_version_filemata(ventry)?;
+        }
+
         Err(Error::FileVersionNotFound)
     }
 
@@ -1850,10 +1936,29 @@ impl MetaObject {
         }
     }
 
-    /// Set transition metadata
-    pub fn set_transition(&mut self, _fi: &FileInfo) {
-        // Implementation for object lifecycle transitions
-        // This would handle storage class transitions
+    pub fn set_transition(&mut self, fi: &FileInfo) {
+        self.meta_sys.insert(
+            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS),
+            fi.transition_status.as_bytes().to_vec(),
+        );
+        self.meta_sys.insert(
+            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME),
+            fi.transitioned_objname.as_bytes().to_vec(),
+        );
+        self.meta_sys.insert(
+            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID),
+            fi.transition_version_id.unwrap().as_bytes().to_vec(),
+        );
+        self.meta_sys.insert(
+            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER),
+            fi.transition_tier.as_bytes().to_vec(),
+        );
+    }
+
+    pub fn remove_restore_hdrs(&mut self) {
+        self.meta_user.remove(X_AMZ_RESTORE.as_str());
+        self.meta_user.remove(X_AMZ_RESTORE_EXPIRY_DAYS);
+        self.meta_user.remove(X_AMZ_RESTORE_REQUEST_DATE);
     }
 
     pub fn uses_data_dir(&self) -> bool {
@@ -1889,6 +1994,66 @@ impl MetaObject {
         let bytes = hash.to_le_bytes();
         [bytes[0], bytes[1], bytes[2], bytes[3]]
     }
+
+    pub fn init_free_version(&self, fi: &FileInfo) -> (FileMetaVersion, bool) {
+        if fi.skip_tier_free_version() {
+            return (FileMetaVersion::default(), false);
+        }
+        if let Some(status) = self
+            .meta_sys
+            .get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS))
+        {
+            if *status == TRANSITION_COMPLETE.as_bytes().to_vec() {
+                let vid = Uuid::parse_str(&fi.tier_free_version_id());
+                if let Err(err) = vid {
+                    panic!(
+                        "Invalid Tier Object delete marker versionId {} {}",
+                        fi.tier_free_version_id(),
+                        err.to_string()
+                    );
+                }
+                let vid = vid.unwrap();
+                let mut free_entry = FileMetaVersion {
+                    version_type: VersionType::Delete,
+                    write_version: 0,
+                    ..Default::default()
+                };
+                free_entry.delete_marker = Some(MetaDeleteMarker {
+                    version_id: Some(vid),
+                    mod_time: self.mod_time,
+                    meta_sys: Some(HashMap::<String, Vec<u8>>::new()),
+                });
+
+                free_entry
+                    .delete_marker
+                    .as_mut()
+                    .unwrap()
+                    .meta_sys
+                    .as_mut()
+                    .unwrap()
+                    .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]);
+                let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER);
+                let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME);
+                let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID);
+
+                let aa = [tier_key, tier_obj_key, tier_obj_vid_key];
+                for (k, v) in &self.meta_sys {
+                    if aa.contains(&k) {
+                        free_entry
+                            .delete_marker
+                            .as_mut()
+                            .unwrap()
+                            .meta_sys
+                            .as_mut()
+                            .unwrap()
+                            .insert(k.clone(), v.clone());
+                    }
+                }
+                return (free_entry, true);
+            }
+        }
+        (FileMetaVersion::default(), false)
+    }
 }
 
 impl From<FileInfo> for MetaObject {

crates/filemeta/src/lib.rs

@@ -1,5 +1,5 @@
 mod error;
-mod fileinfo;
+pub mod fileinfo;
 mod filemeta;
 mod filemeta_inline;
 pub mod headers;

crates/utils/Cargo.toml

@@ -7,28 +7,27 @@ rust-version.workspace = true
 version.workspace = true
 
 [dependencies]
-base64-simd = { workspace = true, optional = true }
+base64-simd= { workspace = true , optional = true}
 blake3 = { workspace = true, optional = true }
 crc32fast.workspace = true
-hex-simd = { workspace = true, optional = true }
+hex-simd= { workspace = true , optional = true}
 highway = { workspace = true, optional = true }
-lazy_static = { workspace = true, optional = true }
+lazy_static= { workspace = true , optional = true}
 local-ip-address = { workspace = true, optional = true }
 md-5 = { workspace = true, optional = true }
-netif = { workspace = true, optional = true }
+netif= { workspace = true , optional = true}
 nix = { workspace = true, optional = true }
-regex = { workspace = true, optional = true }
+regex= { workspace = true, optional = true }
 rustfs-config = { workspace = true, features = ["constants"] }
 rustls = { workspace = true, optional = true }
 rustls-pemfile = { workspace = true, optional = true }
 rustls-pki-types = { workspace = true, optional = true }
 serde = { workspace = true, optional = true }
-sha2 = { workspace = true, optional = true }
 siphasher = { workspace = true, optional = true }
 tempfile = { workspace = true, optional = true }
 tokio = { workspace = true, optional = true, features = ["io-util", "macros"] }
 tracing = { workspace = true }
-url = { workspace = true, optional = true }
+url = { workspace = true , optional = true}
 flate2 = { workspace = true, optional = true }
 brotli = { workspace = true, optional = true }
 zstd = { workspace = true, optional = true }
@@ -39,6 +38,13 @@ futures = { workspace = true, optional = true }
 transform-stream = { workspace = true, optional = true }
 bytes = { workspace = true, optional = true }
 sysinfo = { workspace = true, optional = true }
+hyper.workspace = true
+hyper-util.workspace = true
+common.workspace = true
+sha1 = { workspace = true }
+sha2 = { workspace = true, optional = true }
+hmac.workspace = true
+s3s.workspace = true
 
 [dev-dependencies]
 tempfile = { workspace = true }
@@ -59,7 +65,7 @@ io = ["dep:tokio"]
 path = []
 compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
 string = ["dep:regex", "dep:lazy_static", "dep:rand"]
-crypto = ["dep:base64-simd", "dep:hex-simd"]
+crypto = ["dep:base64-simd","dep:hex-simd"]
 hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher"]
 os = ["dep:nix", "dep:tempfile", "winapi"]  # operating system utilities
 integration = []  # integration test features

crates/utils/src/crypto.rs

@@ -1,3 +1,8 @@
+use std::mem::MaybeUninit;
+
+use hex_simd::{AsOut, AsciiCase};
+use hyper::body::Bytes;
+
 pub fn base64_encode(input: &[u8]) -> String {
     base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
 }
@@ -24,6 +29,80 @@ pub fn hex(data: impl AsRef<[u8]>) -> String {
 //     h.finish().unwrap()
 // }
 
+/// verify sha256 checksum string
+pub fn is_sha256_checksum(s: &str) -> bool {
+    // TODO: optimize
+    let is_lowercase_hex = |c: u8| matches!(c, b'0'..=b'9' | b'a'..=b'f');
+    s.len() == 64 && s.as_bytes().iter().copied().all(is_lowercase_hex)
+}
+
+/// `hmac_sha1(key, data)`
+pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
+    use hmac::{Hmac, Mac};
+    use sha1::Sha1;
+
+    let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).unwrap();
+    m.update(data.as_ref());
+    m.finalize().into_bytes().into()
+}
+
+/// `hmac_sha256(key, data)`
+pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] {
+    use hmac::{Hmac, Mac};
+    use sha2::Sha256;
+
+    let mut m = Hmac::<Sha256>::new_from_slice(key.as_ref()).unwrap();
+    m.update(data.as_ref());
+    m.finalize().into_bytes().into()
+}
+
+/// `f(hex(src))`
+fn hex_bytes32<R>(src: impl AsRef<[u8]>, f: impl FnOnce(&str) -> R) -> R {
+    let buf: &mut [_] = &mut [MaybeUninit::uninit(); 64];
+    let ans = hex_simd::encode_as_str(src.as_ref(), buf.as_out(), AsciiCase::Lower);
+    f(ans)
+}
+
+#[cfg(not(all(feature = "openssl", not(windows))))]
+fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> + use<> {
+    use sha2::{Digest, Sha256};
+    <Sha256 as Digest>::digest(data)
+}
+
+#[cfg(all(feature = "openssl", not(windows)))]
+fn sha256(data: &[u8]) -> impl AsRef<[u8]> {
+    use openssl::hash::{Hasher, MessageDigest};
+    let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
+    h.update(data).unwrap();
+    h.finish().unwrap()
+}
+
+#[cfg(not(all(feature = "openssl", not(windows))))]
+fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8; 32]> + use<> {
+    use sha2::{Digest, Sha256};
+    let mut h = <Sha256 as Digest>::new();
+    chunk.iter().for_each(|data| h.update(data));
+    h.finalize()
+}
+
+#[cfg(all(feature = "openssl", not(windows)))]
+fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8]> {
+    use openssl::hash::{Hasher, MessageDigest};
+    let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
+    chunk.iter().for_each(|data| h.update(data).unwrap());
+    h.finish().unwrap()
+}
+
+/// `f(hex(sha256(data)))`
+pub fn hex_sha256<R>(data: &[u8], f: impl FnOnce(&str) -> R) -> R {
+    hex_bytes32(sha256(data).as_ref(), f)
+}
+
+/// `f(hex(sha256(chunk)))`
+pub fn hex_sha256_chunk<R>(chunk: &[Bytes], f: impl FnOnce(&str) -> R) -> R {
+    hex_bytes32(sha256_chunk(chunk).as_ref(), f)
+}
+
 #[test]
 fn test_base64_encoding_decoding() {
     let original_uuid_timestamp = "c0194290-d911-45cb-8e12-79ec563f46a8x1735460504394878000";

crates/utils/src/hash.rs

@@ -93,6 +93,8 @@ impl HashAlgorithm {
 use crc32fast::Hasher;
 use siphasher::sip::SipHasher;
 
+pub const EMPTY_STRING_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
 pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
     //  你的密钥，必须是 16 字节
 

crates/utils/src/lib.rs

@@ -7,6 +7,8 @@ pub mod net;
 #[cfg(feature = "net")]
 pub use net::*;
 
+pub mod retry;
+
 #[cfg(feature = "io")]
 pub mod io;
 

crates/utils/src/net.rs

@@ -1,14 +1,21 @@
 use bytes::Bytes;
 use futures::pin_mut;
 use futures::{Stream, StreamExt};
+use hyper::client::conn::http2::Builder;
+use hyper_util::rt::TokioExecutor;
 use lazy_static::lazy_static;
 use std::{
-    collections::HashSet,
+    collections::{HashMap, HashSet},
     fmt::Display,
     net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
 };
 use transform_stream::AsyncTryStream;
-use url::Host;
+use url::{Host, Url};
+//use hyper::{client::conn::http2::Builder, rt::Executor};
+//use tonic::{SharedExec, UserAgent};
+//use hyper_util::rt::TokioTimer;
+
+use s3s::header::X_AMZ_STORAGE_CLASS;
 
 lazy_static! {
     static ref LOCAL_IPS: Vec<IpAddr> = must_get_local_ips().unwrap();
@@ -108,6 +115,111 @@ pub fn must_get_local_ips() -> std::io::Result<Vec<IpAddr>> {
     }
 }
 
+pub fn get_default_location(u: Url, region_override: &str) -> String {
+    todo!();
+}
+
+pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result<Url, std::io::Error> {
+    let mut scheme = "https";
+    if !secure {
+        scheme = "http";
+    }
+
+    let endpoint_url_str = format!("{scheme}://{endpoint}");
+    let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else {
+        return Err(std::io::Error::other("url parse error."));
+    };
+
+    //is_valid_endpoint_url(endpoint_url)?;
+    Ok(endpoint_url)
+}
+
+pub const DEFAULT_DIAL_TIMEOUT: i64 = 5;
+
+pub fn new_remotetarget_http_transport(insecure: bool) -> Builder<TokioExecutor> {
+    todo!();
+}
+
+lazy_static! {
+    static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("attributes".to_string(), true);
+        m.insert("partNumber".to_string(), true);
+        m.insert("versionId".to_string(), true);
+        m.insert("response-cache-control".to_string(), true);
+        m.insert("response-content-disposition".to_string(), true);
+        m.insert("response-content-encoding".to_string(), true);
+        m.insert("response-content-language".to_string(), true);
+        m.insert("response-content-type".to_string(), true);
+        m.insert("response-expires".to_string(), true);
+        m
+    };
+    static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("content-type".to_string(), true);
+        m.insert("cache-control".to_string(), true);
+        m.insert("content-encoding".to_string(), true);
+        m.insert("content-disposition".to_string(), true);
+        m.insert("content-language".to_string(), true);
+        m.insert("x-amz-website-redirect-location".to_string(), true);
+        m.insert("x-amz-object-lock-mode".to_string(), true);
+        m.insert("x-amz-metadata-directive".to_string(), true);
+        m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
+        m.insert("expires".to_string(), true);
+        m.insert("x-amz-replication-status".to_string(), true);
+        m
+    };
+    static ref SSE_HEADERS: HashMap<String, bool> = {
+        let mut m = HashMap::new();
+        m.insert("x-amz-server-side-encryption".to_string(), true);
+        m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
+        m.insert("x-amz-server-side-encryption-context".to_string(), true);
+        m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
+        m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
+        m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
+        m
+    };
+}
+
+pub fn is_standard_query_value(qs_key: &str) -> bool {
+    SUPPORTED_QUERY_VALUES[qs_key]
+}
+
+const ALLOWED_CUSTOM_QUERY_PREFIX: &str = "x-";
+
+pub fn is_custom_query_value(qs_key: &str) -> bool {
+    qs_key.starts_with(ALLOWED_CUSTOM_QUERY_PREFIX)
+}
+
+pub fn is_storageclass_header(header_key: &str) -> bool {
+    header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
+}
+
+pub fn is_standard_header(header_key: &str) -> bool {
+    *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
+}
+
+pub fn is_sse_header(header_key: &str) -> bool {
+    *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
+}
+
+pub fn is_amz_header(header_key: &str) -> bool {
+    let key = header_key.to_lowercase();
+    key.starts_with("x-amz-meta-")
+        || key.starts_with("x-amz-grant-")
+        || key == "x-amz-acl"
+        || is_sse_header(header_key)
+        || key.starts_with("x-amz-checksum-")
+}
+
+pub fn is_rustfs_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-rustfs-")
+}
+
+pub fn is_rustfs_header(header_key: &str) -> bool {
+    header_key.to_lowercase().starts_with("x-rustfs-")
+}
+
 #[derive(Debug, Clone)]
 pub struct XHost {
     pub name: String,

crates/utils/src/path.rs

@@ -253,6 +253,11 @@ pub fn dir(path: &str) -> String {
     let (a, _) = split(path);
     clean(a)
 }
+
+pub fn trim_etag(etag: &str) -> String {
+    etag.trim_matches('"').to_string()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/utils/src/retry.rs

@@ -0,0 +1,175 @@
+use std::time::Duration;
+
+// MaxRetry is the maximum number of retries before stopping.
+pub const MAX_RETRY: i64 = 10;
+
+// MaxJitter will randomize over the full exponential backoff time
+pub const MAX_JITTER: f64 = 1.0;
+
+// NoJitter disables the use of jitter for randomizing the exponential backoff time
+pub const NO_JITTER: f64 = 0.0;
+
+// DefaultRetryUnit - default unit multiplicative per retry.
+// defaults to 200 * time.Millisecond
+//const DefaultRetryUnit = 200 * time.Millisecond;
+
+// DefaultRetryCap - Each retry attempt never waits no longer than
+// this maximum time duration.
+//const DefaultRetryCap = time.Second;
+
+/*
+struct Delay {
+    when: Instant,
+}
+
+impl Future for Delay {
+    type Output = &'static str;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>)
+        -> Poll<&'static str>
+    {
+        if Instant::now() >= self.when {
+            println!("Hello world");
+            Poll::Ready("done")
+        } else {
+            // Ignore this line for now.
+            cx.waker().wake_by_ref();
+            Poll::Pending
+        }
+    }
+}
+
+struct RetryTimer {
+    rem: usize,
+    delay: Delay,
+}
+
+impl RetryTimer {
+    fn new() -> Self {
+        Self {
+            rem: 3,
+            delay: Delay { when: Instant::now() }
+        }
+    }
+}
+
+impl Stream for RetryTimer {
+    type Item = ();
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>)
+        -> Poll<Option<()>>
+    {
+        if self.rem == 0 {
+            // No more delays
+            return Poll::Ready(None);
+        }
+
+        match Pin::new(&mut self.delay).poll(cx) {
+            Poll::Ready(_) => {
+                let when = self.delay.when + Duration::from_millis(10);
+                self.delay = Delay { when };
+                self.rem -= 1;
+                Poll::Ready(Some(()))
+            }
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}*/
+
+pub fn new_retry_timer(max_retry: i32, base_sleep: Duration, max_sleep: Duration, jitter: f64) -> Vec<i32> {
+    /*attemptCh := make(chan int)
+
+    exponentialBackoffWait := func(attempt int) time.Duration {
+      // normalize jitter to the range [0, 1.0]
+      if jitter < NoJitter {
+        jitter = NoJitter
+      }
+      if jitter > MaxJitter {
+        jitter = MaxJitter
+      }
+
+      // sleep = random_between(0, min(maxSleep, base * 2 ** attempt))
+      sleep := baseSleep * time.Duration(1<<uint(attempt))
+      if sleep > maxSleep {
+        sleep = maxSleep
+      }
+      if jitter != NoJitter {
+        sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
+      }
+      return sleep
+    }
+
+    go func() {
+      defer close(attemptCh)
+      for i := 0; i < maxRetry; i++ {
+        select {
+        case attemptCh <- i + 1:
+        case <-ctx.Done():
+          return
+        }
+
+        select {
+        case <-time.After(exponentialBackoffWait(i)):
+        case <-ctx.Done():
+          return
+        }
+      }
+    }()
+    return attemptCh*/
+    todo!();
+}
+
+/*var retryableS3Codes = map[string]struct{}{
+  "RequestError":          {},
+  "RequestTimeout":        {},
+  "Throttling":            {},
+  "ThrottlingException":   {},
+  "RequestLimitExceeded":  {},
+  "RequestThrottled":      {},
+  "InternalError":         {},
+  "ExpiredToken":          {},
+  "ExpiredTokenException": {},
+  "SlowDown":              {},
+}
+
+fn isS3CodeRetryable(s3Code string) (ok bool) {
+  _, ok = retryableS3Codes[s3Code]
+  return ok
+}
+
+var retryableHTTPStatusCodes = map[int]struct{}{
+  http.StatusRequestTimeout:      {},
+  429:                            {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet
+  499:                            {}, // client closed request, retry. A non-standard status code introduced by nginx.
+  http.StatusInternalServerError: {},
+  http.StatusBadGateway:          {},
+  http.StatusServiceUnavailable:  {},
+  http.StatusGatewayTimeout:      {},
+  520:                            {}, // It is used by Cloudflare as a catch-all response for when the origin server sends something unexpected.
+  // Add more HTTP status codes here.
+}
+
+fn isHTTPStatusRetryable(httpStatusCode int) (ok bool) {
+  _, ok = retryableHTTPStatusCodes[httpStatusCode]
+  return ok
+}
+
+fn isRequestErrorRetryable(ctx context.Context, err error) bool {
+  if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+    // Retry if internal timeout in the HTTP call.
+    return ctx.Err() == nil
+  }
+  if ue, ok := err.(*url.Error); ok {
+    e := ue.Unwrap()
+    switch e.(type) {
+    // x509: certificate signed by unknown authority
+    case x509.UnknownAuthorityError:
+      return false
+    }
+    switch e.Error() {
+    case "http: server gave HTTP response to HTTPS client":
+      return false
+    }
+  }
+  return true
+}*/

ecstore/Cargo.toml

@@ -32,9 +32,12 @@ serde.workspace = true
 time.workspace = true
 bytesize.workspace = true
 serde_json.workspace = true
+serde-xml-rs.workspace = true
+serde_urlencoded.workspace = true
 tracing-error.workspace = true
 s3s.workspace = true
 http.workspace = true
+http-body-util = "0.1.1"
 highway = { workspace = true }
 url.workspace = true
 uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
@@ -53,16 +56,29 @@ tokio-util = { workspace = true, features = ["io", "compat"] }
 crc32fast = { workspace = true }
 siphasher = { workspace = true }
 base64-simd = { workspace = true }
+
 base64 = { workspace = true }
 hmac = { workspace = true }
 sha2 = { workspace = true }
+sha1 = { workspace = true }
+
 hex-simd = { workspace = true }
 path-clean = { workspace = true }
 tempfile.workspace = true
+hyper.workspace = true
+hyper-util.workspace = true
+hyper-rustls.workspace = true
+rustls.workspace = true
 tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
 tokio-stream = { workspace = true }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
+tower.workspace = true
+async-channel.workspace = true
+num = "0.4.3"
+enumset = "1.1.5"
+hmac.workspace = true
+std-next.workspace = true
 num_cpus = { workspace = true }
 rand.workspace = true
 pin-project-lite.workspace = true
@@ -77,10 +93,11 @@ urlencoding = { workspace = true }
 smallvec = { workspace = true }
 shadow-rs.workspace = true
 rustfs-filemeta.workspace = true
-rustfs-utils = { workspace = true, features = ["full"] }
+rustfs-utils ={workspace = true, features=["full"]}
 rustfs-rio.workspace = true
 futures-util.workspace = true
 serde_urlencoded.workspace = true
+reader = { workspace = true }
 
 [target.'cfg(not(windows))'.dependencies]
 nix = { workspace = true }

ecstore/src/admin_server_info.rs

@@ -23,7 +23,7 @@ use protos::{
 };
 use std::{
     collections::{HashMap, HashSet},
-    time::SystemTime,
+    time::{SystemTime, UNIX_EPOCH},
 };
 use time::OffsetDateTime;
 use tonic::Request;

ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs

@@ -0,0 +1,29 @@
+use super::lifecycle;
+
+#[derive(Debug, Clone, Default)]
+pub enum LcEventSrc {
+    #[default]
+    None,
+    Heal,
+    Scanner,
+    Decom,
+    Rebal,
+    S3HeadObject,
+    S3GetObject,
+    S3ListObjects,
+    S3PutObject,
+    S3CopyObject,
+    S3CompleteMultipartUpload,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct LcAuditEvent {
+    pub event: lifecycle::Event,
+    pub source: LcEventSrc,
+}
+
+impl LcAuditEvent {
+    pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self {
+        Self { event, source }
+    }
+}

ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -0,0 +1,843 @@
+use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
+use futures::Future;
+use http::HeaderMap;
+use lazy_static::lazy_static;
+use s3s::Body;
+use sha2::{Digest, Sha256};
+use std::any::{Any, TypeId};
+use std::collections::HashMap;
+use std::env;
+use std::io::{Cursor, Write};
+use std::pin::Pin;
+use std::sync::atomic::{AtomicI32, AtomicI64, Ordering};
+use std::sync::{Arc, Mutex};
+use tokio::select;
+use tokio::sync::mpsc::{Receiver, Sender};
+use tokio::sync::{RwLock, mpsc};
+use tracing::{error, info, warn};
+use uuid::Uuid;
+use xxhash_rust::xxh64;
+
+use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
+use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions};
+use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
+use super::tier_sweeper::{Jentry, delete_object_from_remote_tier};
+use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
+use crate::client::object_api_utils::new_getobjectreader;
+use crate::error::Error;
+use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
+use crate::event::name::EventName;
+use crate::event_notification::{EventArgs, send_event};
+use crate::global::GLOBAL_LocalNodeName;
+use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
+use crate::heal::{
+    data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object},
+    data_scanner_metric::ScannerMetrics,
+    data_usage_cache::TierStats,
+};
+use crate::store::ECStore;
+use crate::store_api::StorageAPI;
+use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete};
+use crate::tier::warm_backend::WarmBackendGetOpts;
+use s3s::dto::BucketLifecycleConfiguration;
+
+pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type TraceFn =
+    Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
+
+static XXHASH_SEED: u64 = 0;
+
+const DISABLED: &str = "Disabled";
+
+//pub const ERR_INVALID_STORAGECLASS: &str = "invalid storage class.";
+pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
+
+lazy_static! {
+    pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
+    pub static ref GLOBAL_TransitionState: Arc<TransitionState> = TransitionState::new();
+}
+
+pub struct LifecycleSys;
+
+impl LifecycleSys {
+    pub fn new() -> Arc<Self> {
+        Arc::new(Self)
+    }
+
+    pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
+        let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0;
+        Some(lc)
+    }
+
+    pub fn trace(oi: &ObjectInfo) -> TraceFn {
+        todo!();
+    }
+}
+
+struct ExpiryTask {
+    obj_info: ObjectInfo,
+    event: lifecycle::Event,
+    src: LcEventSrc,
+}
+
+impl ExpiryOp for ExpiryTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
+        let _ = hasher.write(format!("{}", self.obj_info.name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+struct ExpiryStats {
+    missed_expiry_tasks: AtomicI64,
+    missed_freevers_tasks: AtomicI64,
+    missed_tier_journal_tasks: AtomicI64,
+    workers: AtomicI64,
+}
+
+impl ExpiryStats {
+    pub fn missed_tasks(&self) -> i64 {
+        self.missed_expiry_tasks.load(Ordering::SeqCst)
+    }
+
+    fn missed_free_vers_tasks(&self) -> i64 {
+        self.missed_freevers_tasks.load(Ordering::SeqCst)
+    }
+
+    fn missed_tier_journal_tasks(&self) -> i64 {
+        self.missed_tier_journal_tasks.load(Ordering::SeqCst)
+    }
+
+    fn num_workers(&self) -> i64 {
+        self.workers.load(Ordering::SeqCst)
+    }
+}
+
+pub trait ExpiryOp: 'static {
+    fn op_hash(&self) -> u64;
+    fn as_any(&self) -> &dyn Any;
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct TransitionedObject {
+    pub name: String,
+    pub version_id: String,
+    pub tier: String,
+    pub free_version: bool,
+    pub status: String,
+}
+
+struct FreeVersionTask(ObjectInfo);
+
+impl ExpiryOp for FreeVersionTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.0.transitioned_object.tier).as_bytes());
+        let _ = hasher.write(format!("{}", self.0.transitioned_object.name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+struct NewerNoncurrentTask {
+    bucket: String,
+    versions: Vec<ObjectToDelete>,
+    event: lifecycle::Event,
+}
+
+impl ExpiryOp for NewerNoncurrentTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.bucket).as_bytes());
+        let _ = hasher.write(format!("{}", self.versions[0].object_name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+pub struct ExpiryState {
+    tasks_tx: Vec<Sender<Option<ExpiryOpType>>>,
+    tasks_rx: Vec<Arc<tokio::sync::Mutex<Receiver<Option<ExpiryOpType>>>>>,
+    stats: Option<ExpiryStats>,
+}
+
+impl ExpiryState {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new() -> Arc<RwLock<Self>> {
+        Arc::new(RwLock::new(Self {
+            tasks_tx: vec![],
+            tasks_rx: vec![],
+            stats: Some(ExpiryStats {
+                missed_expiry_tasks: AtomicI64::new(0),
+                missed_freevers_tasks: AtomicI64::new(0),
+                missed_tier_journal_tasks: AtomicI64::new(0),
+                workers: AtomicI64::new(0),
+            }),
+        }))
+    }
+
+    pub async fn pending_tasks(&self) -> usize {
+        let rxs = &self.tasks_rx;
+        if rxs.len() == 0 {
+            return 0;
+        }
+        let mut tasks = 0;
+        for rx in rxs.iter() {
+            tasks += rx.lock().await.len();
+        }
+        tasks
+    }
+
+    pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> {
+        let wrkr = self.get_worker_ch(je.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
+        }
+        let wrkr = wrkr.expect("err");
+        select! {
+            //_ -> GlobalContext.Done() => ()
+            _ = wrkr.send(Some(Box::new(je.clone()))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
+            }
+        }
+        return Ok(());
+    }
+
+    pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) {
+        let task = FreeVersionTask(oi);
+        let wrkr = self.get_worker_ch(task.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
+            return;
+        }
+        let wrkr = wrkr.expect("err!");
+        select! {
+            //_ -> GlobalContext.Done() => {}
+            _ = wrkr.send(Some(Box::new(task))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
+            }
+        }
+    }
+
+    pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
+        let task = ExpiryTask {
+            obj_info: oi.clone(),
+            event: event.clone(),
+            src: src.clone(),
+        };
+        let wrkr = self.get_worker_ch(task.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            return;
+        }
+        let wrkr = wrkr.expect("err!");
+        select! {
+            //_ -> GlobalContext.Done() => {}
+            _ = wrkr.send(Some(Box::new(task))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            }
+        }
+    }
+
+    pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
+        if versions.len() == 0 {
+            return;
+        }
+
+        let task = NewerNoncurrentTask {
+            bucket: String::from(bucket),
+            versions: versions,
+            event: lc_event,
+        };
+        let wrkr = self.get_worker_ch(task.op_hash());
+        if wrkr.is_none() {
+            *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            return;
+        }
+        let wrkr = wrkr.expect("err!");
+        select! {
+            //_ -> GlobalContext.Done() => {}
+            _ = wrkr.send(Some(Box::new(task))) => (),
+            else => {
+                *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            }
+        }
+    }
+
+    pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
+        if self.tasks_tx.len() == 0 {
+            return None;
+        }
+        Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
+    }
+
+    pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
+        if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 {
+            return;
+        }
+
+        let mut state = GLOBAL_ExpiryState.write().await;
+
+        while state.tasks_tx.len() < n {
+            let (tx, mut rx) = mpsc::channel(10000);
+            let api = api.clone();
+            let rx = Arc::new(tokio::sync::Mutex::new(rx));
+            state.tasks_tx.push(tx);
+            state.tasks_rx.push(rx.clone());
+            *state.stats.as_mut().expect("err").workers.get_mut() += 1;
+            tokio::spawn(async move {
+                let mut rx = rx.lock().await;
+                //let mut expiry_state = GLOBAL_ExpiryState.read().await;
+                ExpiryState::worker(&mut *rx, api).await;
+            });
+        }
+
+        let mut l = state.tasks_tx.len();
+        while l > n {
+            let worker = state.tasks_tx[l - 1].clone();
+            worker.send(None).await.unwrap_or(());
+            state.tasks_tx.remove(l - 1);
+            state.tasks_rx.remove(l - 1);
+            *state.stats.as_mut().expect("err").workers.get_mut() -= 1;
+            l -= 1;
+        }
+    }
+
+    pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
+        loop {
+            select! {
+                _ = tokio::signal::ctrl_c() => {
+                    info!("got ctrl+c, exits");
+                    break;
+                }
+                v = rx.recv() => {
+                    if v.is_none() {
+                        break;
+                    }
+                    let v = v.expect("err!");
+                    if v.is_none() {
+                        //rx.close();
+                        //drop(rx);
+                        let _ = rx;
+                        return;
+                    }
+                    let v = v.expect("err!");
+                    if v.as_any().is::<ExpiryTask>() {
+                        let v = v.as_any().downcast_ref::<ExpiryTask>().expect("err!");
+                        if v.obj_info.transitioned_object.status != "" {
+                            apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
+                        } else {
+                            apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await;
+                        }
+                    }
+                    else if v.as_any().is::<NewerNoncurrentTask>() {
+                        let v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("err!");
+                        //delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
+                    }
+                    else if v.as_any().is::<Jentry>() {
+                        //transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+                    }
+                    else if v.as_any().is::<FreeVersionTask>() {
+                        let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
+                        let oi = v.0.clone();
+
+                    }
+                    else {
+                        //info!("Invalid work type - {:?}", v);
+                        todo!();
+                    }
+                }
+            }
+        }
+    }
+}
+
+struct TransitionTask {
+    obj_info: ObjectInfo,
+    src: LcEventSrc,
+    event: lifecycle::Event,
+}
+
+impl ExpiryOp for TransitionTask {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes());
+        //let _ = hasher.write(format!("{}", self.obj_info.versions[0].object_name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+pub struct TransitionState {
+    transition_tx: A_Sender<Option<TransitionTask>>,
+    transition_rx: A_Receiver<Option<TransitionTask>>,
+    pub num_workers: AtomicI64,
+    kill_tx: A_Sender<()>,
+    kill_rx: A_Receiver<()>,
+    active_tasks: AtomicI64,
+    missed_immediate_tasks: AtomicI64,
+    last_day_stats: Arc<Mutex<HashMap<String, LastDayTierStats>>>,
+}
+
+type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
+//type RetKill = impl Future<Output = Option<()>> + Send + 'static;
+//type RetTransitionTask = impl Future<Output = Option<Option<TransitionTask>>> + Send + 'static;
+
+impl TransitionState {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new() -> Arc<Self> {
+        let (tx1, rx1) = bounded(100000);
+        let (tx2, rx2) = bounded(1);
+        Arc::new(Self {
+            transition_tx: tx1,
+            transition_rx: rx1,
+            num_workers: AtomicI64::new(0),
+            kill_tx: tx2,
+            kill_rx: rx2,
+            active_tasks: AtomicI64::new(0),
+            missed_immediate_tasks: AtomicI64::new(0),
+            last_day_stats: Arc::new(Mutex::new(HashMap::new())),
+        })
+    }
+
+    pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
+        let task = TransitionTask {
+            obj_info: oi.clone(),
+            src: src.clone(),
+            event: event.clone(),
+        };
+        select! {
+            //_ -> t.ctx.Done() => (),
+            _ = self.transition_tx.send(Some(task)) => (),
+            else => {
+                match src {
+                    LcEventSrc::S3PutObject | LcEventSrc::S3CopyObject | LcEventSrc::S3CompleteMultipartUpload => {
+                        self.missed_immediate_tasks.fetch_add(1, Ordering::SeqCst);
+                    }
+                    _ => ()
+                }
+            },
+        }
+    }
+
+    pub async fn init(api: Arc<ECStore>) {
+        let mut n = 10; //globalAPIConfig.getTransitionWorkers();
+        let tw = 10; //globalILMConfig.getTransitionWorkers(); 
+        if tw > 0 {
+            n = tw;
+        }
+
+        //let mut transition_state = GLOBAL_TransitionState.write().await;
+        //self.objAPI = objAPI
+        Self::update_workers(api, n).await;
+    }
+
+    pub fn pending_tasks(&self) -> usize {
+        //let transition_rx = GLOBAL_TransitionState.transition_rx.lock().unwrap();
+        let transition_rx = &GLOBAL_TransitionState.transition_rx;
+        transition_rx.len()
+    }
+
+    pub fn active_tasks(&self) -> i64 {
+        self.active_tasks.load(Ordering::SeqCst)
+    }
+
+    pub fn missed_immediate_tasks(&self) -> i64 {
+        self.missed_immediate_tasks.load(Ordering::SeqCst)
+    }
+
+    pub async fn worker(api: Arc<ECStore>) {
+        loop {
+            select! {
+                _ = GLOBAL_TransitionState.kill_rx.recv() => {
+                    return;
+                }
+                task = GLOBAL_TransitionState.transition_rx.recv() => {
+                    if task.is_err() {
+                        break;
+                    }
+                    let task = task.expect("err!");
+                    if task.is_none() {
+                        //self.transition_rx.close();
+                        //drop(self.transition_rx);
+                        return;
+                    }
+                    let task = task.expect("err!");
+                    if task.as_any().is::<TransitionTask>() {
+                        let task = task.as_any().downcast_ref::<TransitionTask>().expect("err!");
+
+                        GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst);
+                        if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
+                            if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) {
+                                if !err.to_string().contains("use of closed network connection") {
+                                    error!("Transition to {} failed for {}/{} version:{} with {}",
+                                        task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.expect("err"), err.to_string());
+                                }
+                            }
+                        } else {
+                            let mut ts = TierStats {
+                                total_size: task.obj_info.size as u64,
+                                num_versions: 1,
+                                ..Default::default()
+                            };
+                            if task.obj_info.is_latest {
+                                ts.num_objects = 1;
+                            }
+                            GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts);
+                        }
+                        GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst);
+                    }
+                }
+                else => ()
+            }
+        }
+    }
+
+    pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
+        let mut tier_stats = self.last_day_stats.lock().unwrap();
+        tier_stats
+            .entry(tier.to_string())
+            .and_modify(|e| e.add_stats(ts))
+            .or_insert(LastDayTierStats::default());
+    }
+
+    pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
+        let tier_stats = self.last_day_stats.lock().unwrap();
+        let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
+        for (tier, st) in tier_stats.iter() {
+            res.insert(tier.clone(), st.clone());
+        }
+        res
+    }
+
+    pub async fn update_workers(api: Arc<ECStore>, n: i64) {
+        Self::update_workers_inner(api, n).await;
+    }
+
+    pub async fn update_workers_inner(api: Arc<ECStore>, n: i64) {
+        let mut n = n;
+        if n == 0 {
+            n = 100;
+        }
+
+        let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
+        while num_workers < n {
+            let clone_api = api.clone();
+            tokio::spawn(async move {
+                TransitionState::worker(clone_api).await;
+            });
+            num_workers = num_workers + 1;
+            GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
+        }
+
+        let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
+        while num_workers > n {
+            let worker = GLOBAL_TransitionState.kill_tx.clone();
+            worker.send(()).await;
+            num_workers = num_workers - 1;
+            GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
+        }
+    }
+}
+
+struct AuditTierOp {
+    tier: String,
+    time_to_responsens: i64,
+    output_bytes: i64,
+    error: String,
+}
+
+impl AuditTierOp {
+    #[allow(clippy::new_ret_no_self)]
+    pub async fn new() -> Result<Self, std::io::Error> {
+        Ok(Self {
+            tier: String::from("tier"),
+            time_to_responsens: 0,
+            output_bytes: 0,
+            error: String::from(""),
+        })
+    }
+
+    pub fn string(&self) -> String {
+        format!(
+            "tier:{},respNS:{},tx:{},err:{}",
+            self.tier, self.time_to_responsens, self.output_bytes, self.error
+        )
+    }
+}
+
+pub async fn init_background_expiry(api: Arc<ECStore>) {
+    let mut workers = num_cpus::get() / 2;
+    //globalILMConfig.getExpirationWorkers()
+    if let Ok(env_expiration_workers) = env::var("_RUSTFS_EXPIRATION_WORKERS") {
+        if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
+            workers = num_expirations;
+        }
+    }
+
+    if workers == 0 {
+        workers = 100;
+    }
+
+    //let expiry_state = GLOBAL_ExpiryStSate.write().await;
+    ExpiryState::resize_workers(workers, api).await;
+}
+
+pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
+    for rule in &lc.rules {
+        if let Some(transitions) = &rule.transitions {
+            for transition in transitions {
+                if let Some(storage_class) = &transition.storage_class {
+                    if storage_class.as_str() != "" {
+                        let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
+                        if !valid {
+                            return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
+                        }
+                    }
+                }
+            }
+        }
+        if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions {
+            for noncurrent_version_transition in noncurrent_version_transitions {
+                if let Some(storage_class) = &noncurrent_version_transition.storage_class {
+                    if storage_class.as_str() != "" {
+                        let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
+                        if !valid {
+                            return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
+                        }
+                    }
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
+    let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await;
+    if !lc.is_none() {
+        let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await;
+        match event.action {
+            lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
+                if oi.delete_marker || oi.is_dir {
+                    return;
+                }
+                GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await;
+            }
+            _ => (),
+        }
+    }
+}
+
+pub async fn expire_transitioned_object(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    src: &LcEventSrc,
+) -> Result<ObjectInfo, std::io::Error> {
+    //let traceFn = GLOBAL_LifecycleSys.trace(oi);
+    let mut opts = ObjectOptions {
+        versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
+        expiration: ExpirationOptions { expire: true },
+        ..Default::default()
+    };
+    if lc_event.action == IlmAction::DeleteVersionAction {
+        opts.version_id = oi.version_id.map(|id| id.to_string());
+    }
+    //let tags = LcAuditEvent::new(src, lcEvent).Tags();
+    if lc_event.action == IlmAction::DeleteRestoredAction {
+        opts.transition.expire_restored = true;
+        match api.delete_object(&oi.bucket, &oi.name, opts).await {
+            Ok(dobj) => {
+                //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
+                return Ok(dobj);
+            }
+            Err(err) => return Err(std::io::Error::other(err)),
+        }
+    }
+
+    let ret = delete_object_from_remote_tier(
+        &oi.transitioned_object.name,
+        &oi.transitioned_object.version_id,
+        &oi.transitioned_object.tier,
+    )
+    .await;
+    if ret.is_ok() {
+        opts.skip_decommissioned = true;
+    } else {
+        //transitionLogIf(ctx, err);
+    }
+
+    let dobj = api.delete_object(&oi.bucket, &oi.name, opts).await?;
+
+    //defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+
+    let mut event_name = EventName::ObjectRemovedDelete;
+    if oi.delete_marker {
+        event_name = EventName::ObjectRemovedDeleteMarkerCreated;
+    }
+    let obj_info = ObjectInfo {
+        name: oi.name.clone(),
+        version_id: oi.version_id,
+        delete_marker: oi.delete_marker,
+        ..Default::default()
+    };
+    send_event(EventArgs {
+        event_name: event_name.as_ref().to_string(),
+        bucket_name: obj_info.bucket.clone(),
+        object: obj_info,
+        user_agent: "Internal: [ILM-Expiry]".to_string(),
+        host: GLOBAL_LocalNodeName.to_string(),
+        ..Default::default()
+    });
+
+    Ok(dobj)
+}
+
+pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
+    let us = Uuid::new_v4().to_string();
+    let mut hasher = Sha256::new();
+    let _ = hasher.write(format!("{}/{}", get_global_deployment_id().unwrap_or_default(), bucket).as_bytes());
+    hasher.flush();
+    let hash = rustfs_utils::crypto::hex(hasher.clone().finalize().as_slice());
+    let obj = format!("{}/{}/{}/{}", &hash[0..16], &us[0..2], &us[2..4], &us);
+    Ok(obj)
+}
+
+pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
+    let time_ilm = ScannerMetrics::time_ilm(lae.event.action);
+
+    let opts = ObjectOptions {
+        transition: TransitionOptions {
+            status: lifecycle::TRANSITION_PENDING.to_string(),
+            tier: lae.event.storage_class,
+            etag: oi.etag.clone().expect("err").to_string(),
+            ..Default::default()
+        },
+        //lifecycle_audit_event: lae,
+        version_id: Some(oi.version_id.expect("err").to_string()),
+        versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
+        version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
+        mod_time: oi.mod_time,
+        ..Default::default()
+    };
+    time_ilm(1);
+    api.transition_object(&oi.bucket, &oi.name, &opts).await
+}
+
+pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn {
+    todo!();
+}
+
+pub async fn get_transitioned_object_reader(
+    bucket: &str,
+    object: &str,
+    rs: HTTPRangeSpec,
+    h: HeaderMap,
+    oi: ObjectInfo,
+    opts: &ObjectOptions,
+) -> Result<GetObjectReader, std::io::Error> {
+    let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
+    let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await {
+        Ok(d) => d,
+        Err(err) => return Err(std::io::Error::other(err)),
+    };
+
+    let ret = new_getobjectreader(rs, &oi, opts, &h);
+    if let Err(err) = ret {
+        return Err(error_resp_to_object_err(err, vec![bucket, object]));
+    }
+    let (get_fn, off, length) = ret.expect("err");
+    let mut gopts = WarmBackendGetOpts::default();
+
+    if off >= 0 && length >= 0 {
+        gopts.start_offset = off;
+        gopts.length = length;
+    }
+
+    //return Ok(HttpFileReader::new(rs, &oi, opts, &h));
+    //timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
+    let reader = tgt_client
+        .get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
+        .await?;
+    Ok(get_fn(reader, h))
+}
+
+pub fn post_restore_opts(r: http::Request<Body>, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
+    todo!();
+}
+
+pub fn put_restore_opts(bucket: &str, object: &str, rreq: &RestoreObjectRequest, oi: &ObjectInfo) -> ObjectOptions {
+    todo!();
+}
+
+pub trait LifecycleOps {
+    fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts;
+}
+
+impl LifecycleOps for ObjectInfo {
+    fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts {
+        lifecycle::ObjectOpts {
+            name: self.name.clone(),
+            user_tags: self.user_tags.clone(),
+            version_id: self.version_id.expect("err").to_string(),
+            mod_time: self.mod_time,
+            size: self.size,
+            is_latest: self.is_latest,
+            num_versions: self.num_versions,
+            delete_marker: self.delete_marker,
+            successor_mod_time: self.successor_mod_time,
+            //restore_ongoing:    self.restore_ongoing,
+            //restore_expires:    self.restore_expires,
+            transition_status: self.transitioned_object.status.clone(),
+            ..Default::default()
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct S3Location {
+    pub bucketname: String,
+    //pub encryption:    Encryption,
+    pub prefix: String,
+    pub storage_class: String,
+    //pub tagging:       Tags,
+    pub user_metadata: HashMap<String, String>,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct OutputLocation(pub S3Location);
+
+#[derive(Debug, Default, Clone)]
+pub struct RestoreObjectRequest {
+    pub days: i64,
+    pub ror_type: String,
+    pub tier: String,
+    pub description: String,
+    //pub select_parameters: SelectParameters,
+    pub output_location: OutputLocation,
+}
+
+const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;

ecstore/src/bucket/lifecycle/lifecycle.rs

@@ -0,0 +1,730 @@
+use s3s::dto::{
+    BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
+    ObjectLockConfiguration, ObjectLockEnabled, Transition,
+};
+use std::cmp::Ordering;
+use std::env;
+use std::fmt::Display;
+use time::macros::{datetime, offset};
+use time::{self, Duration, OffsetDateTime};
+
+use crate::bucket::lifecycle::rule::TransitionOps;
+
+use super::bucket_lifecycle_ops::RestoreObjectRequest;
+
+pub const TRANSITION_COMPLETE: &str = "complete";
+pub const TRANSITION_PENDING: &str = "pending";
+
+const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules";
+const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
+const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
+const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema";
+const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
+    "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum IlmAction {
+    NoneAction = 0,
+    DeleteAction,
+    DeleteVersionAction,
+    TransitionAction,
+    TransitionVersionAction,
+    DeleteRestoredAction,
+    DeleteRestoredVersionAction,
+    DeleteAllVersionsAction,
+    DelMarkerDeleteAllVersionsAction,
+    ActionCount,
+}
+
+impl IlmAction {
+    pub fn delete_restored(&self) -> bool {
+        *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction
+    }
+
+    pub fn delete_versioned(&self) -> bool {
+        *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction
+    }
+
+    pub fn delete_all(&self) -> bool {
+        *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction
+    }
+
+    pub fn delete(&self) -> bool {
+        if self.delete_restored() {
+            return true;
+        }
+        *self == Self::DeleteVersionAction
+            || *self == Self::DeleteAction
+            || *self == Self::DeleteAllVersionsAction
+            || *self == Self::DelMarkerDeleteAllVersionsAction
+    }
+}
+
+impl Display for IlmAction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+#[async_trait::async_trait]
+pub trait RuleValidate {
+    fn validate(&self) -> Result<(), std::io::Error>;
+}
+
+#[async_trait::async_trait]
+impl RuleValidate for LifecycleRule {
+    /*fn validate_id(&self) -> Result<()> {
+        if self.id.len() > 255 {
+            return errInvalidRuleID;
+        }
+        Ok(())
+    }
+
+    fn validate_status(&self) -> Result<()> {
+        if self.Status.len() == 0 {
+            return errEmptyRuleStatus;
+        }
+
+        if self.Status != Enabled && self.Status != Disabled {
+            return errInvalidRuleStatus;
+        }
+        Ok(())
+    }
+
+    fn validate_expiration(&self) -> Result<()> {
+        self.Expiration.Validate();
+    }
+
+    fn validate_noncurrent_expiration(&self) -> Result<()> {
+        self.NoncurrentVersionExpiration.Validate()
+    }
+
+    fn validate_prefix_and_filter(&self) -> Result<()> {
+        if !self.Prefix.set && self.Filter.IsEmpty() || self.Prefix.set && !self.Filter.IsEmpty() {
+            return errXMLNotWellFormed;
+        }
+        if !self.Prefix.set {
+            return self.Filter.Validate();
+        }
+        Ok(())
+    }
+
+    fn validate_transition(&self) -> Result<()> {
+        self.Transition.Validate()
+    }
+
+    fn validate_noncurrent_transition(&self) -> Result<()> {
+        self.NoncurrentVersionTransition.Validate()
+    }
+
+    fn get_prefix(&self) -> String {
+        if p := self.Prefix.String(); p != "" {
+          return p
+        }
+        if p := self.Filter.Prefix.String(); p != "" {
+          return p
+        }
+        if p := self.Filter.And.Prefix.String(); p != "" {
+          return p
+        }
+        "".to_string()
+    }*/
+
+    fn validate(&self) -> Result<(), std::io::Error> {
+        /*self.validate_id()?;
+        self.validate_status()?;
+        self.validate_expiration()?;
+        self.validate_noncurrent_expiration()?;
+        self.validate_prefix_and_filter()?;
+        self.validate_transition()?;
+        self.validate_noncurrent_transition()?;
+        if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() {
+          return errInvalidRuleDelMarkerExpiration
+        }
+        if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() {
+          return errXMLNotWellFormed
+        }*/
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+pub trait Lifecycle {
+    async fn has_transition(&self) -> bool;
+    fn has_expiry(&self) -> bool;
+    async fn has_active_rules(&self, prefix: &str) -> bool;
+    async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>;
+    async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>>;
+    async fn eval(&self, obj: &ObjectOpts) -> Event;
+    async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event;
+    //fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts);
+    async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event;
+}
+
+#[async_trait::async_trait]
+impl Lifecycle for BucketLifecycleConfiguration {
+    async fn has_transition(&self) -> bool {
+        for rule in self.rules.iter() {
+            if !rule.transitions.is_none() {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn has_expiry(&self) -> bool {
+        for rule in self.rules.iter() {
+            if !rule.expiration.is_none() || !rule.noncurrent_version_expiration.is_none() {
+                return true;
+            }
+        }
+        false
+    }
+
+    async fn has_active_rules(&self, prefix: &str) -> bool {
+        if self.rules.len() == 0 {
+            return false;
+        }
+        for rule in self.rules.iter() {
+            if rule.status.as_str() == ExpirationStatus::DISABLED {
+                continue;
+            }
+
+            let rule_prefix = rule.prefix.as_ref().expect("err!");
+            if prefix.len() > 0 && rule_prefix.len() > 0 {
+                if !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix) {
+                    continue;
+                }
+            }
+
+            let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!");
+            if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 {
+                return true;
+            }
+            if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 {
+                return true;
+            }
+            if !rule.noncurrent_version_transitions.is_none() {
+                return true;
+            }
+            let rule_expiration = rule.expiration.as_ref().expect("err!");
+            if !rule_expiration.date.is_none()
+                && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
+                    < OffsetDateTime::now_utc().unix_timestamp()
+            {
+                return true;
+            }
+            if !rule_expiration.date.is_none() {
+                return true;
+            }
+            if rule_expiration.expired_object_delete_marker.expect("err!") {
+                return true;
+            }
+            let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
+            let rule_transitions_0 = rule_transitions[0].clone();
+            if !rule_transitions_0.date.is_none()
+                && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
+                    < OffsetDateTime::now_utc().unix_timestamp()
+            {
+                return true;
+            }
+            if !rule.transitions.is_none() {
+                return true;
+            }
+        }
+        false
+    }
+
+    async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error> {
+        if self.rules.len() > 1000 {
+            return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES));
+        }
+        if self.rules.len() == 0 {
+            return Err(std::io::Error::other(ERR_LIFECYCLE_NO_RULE));
+        }
+
+        for r in &self.rules {
+            r.validate()?;
+            if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() {
+                if let Some(expiration) = r.expiration.as_ref() {
+                    if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
+                        if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED && (!expired_object_delete_marker) {
+                            return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
+                        }
+                    } /*else {
+                    if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED {
+                    return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED));
+                    }
+                    }*/
+                }
+            }
+        }
+        for (i, _) in self.rules.iter().enumerate() {
+            if i == self.rules.len() - 1 {
+                break;
+            }
+            let other_rules = &self.rules[i + 1..];
+            for other_rule in other_rules {
+                if self.rules[i].id == other_rule.id {
+                    return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    async fn filter_rules(&self, obj: &ObjectOpts) -> Option<Vec<LifecycleRule>> {
+        if obj.name == "" {
+            return None;
+        }
+        let mut rules = Vec::<LifecycleRule>::new();
+        for rule in self.rules.iter() {
+            if rule.status.as_str() == ExpirationStatus::DISABLED {
+                continue;
+            }
+            if let Some(prefix) = rule.prefix.clone() {
+                if !obj.name.starts_with(prefix.as_str()) {
+                    continue;
+                }
+            }
+            /*if !rule.filter.test_tags(obj.user_tags) {
+                continue;
+            }*/
+            //if !obj.delete_marker && !rule.filter.BySize(obj.size) {
+            if !obj.delete_marker && false {
+                continue;
+            }
+            rules.push(rule.clone());
+        }
+        Some(rules)
+    }
+
+    async fn eval(&self, obj: &ObjectOpts) -> Event {
+        self.eval_inner(obj, OffsetDateTime::now_utc()).await
+    }
+
+    async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event {
+        let mut events = Vec::<Event>::new();
+        if obj.mod_time.expect("err").unix_timestamp() == 0 {
+            return Event::default();
+        }
+
+        if let Some(restore_expires) = obj.restore_expires {
+            if !restore_expires.unix_timestamp() == 0 && now.unix_timestamp() > restore_expires.unix_timestamp() {
+                let mut action = IlmAction::DeleteRestoredAction;
+                if !obj.is_latest {
+                    action = IlmAction::DeleteRestoredVersionAction;
+                }
+
+                events.push(Event {
+                    action: action,
+                    due: Some(now),
+                    rule_id: "".into(),
+                    noncurrent_days: 0,
+                    newer_noncurrent_versions: 0,
+                    storage_class: "".into(),
+                });
+            }
+        }
+
+        if let Some(ref lc_rules) = self.filter_rules(obj).await {
+            for rule in lc_rules.iter() {
+                if obj.expired_object_deletemarker() {
+                    if let Some(expiration) = rule.expiration.as_ref() {
+                        if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
+                            events.push(Event {
+                                action: IlmAction::DeleteVersionAction,
+                                rule_id: rule.id.clone().expect("err!"),
+                                due: Some(now),
+                                noncurrent_days: 0,
+                                newer_noncurrent_versions: 0,
+                                storage_class: "".into(),
+                            });
+                            break;
+                        }
+                    }
+
+                    if let Some(expiration) = rule.expiration.as_ref() {
+                        if let Some(days) = expiration.days {
+                            let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/);
+                            if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
+                                events.push(Event {
+                                    action: IlmAction::DeleteVersionAction,
+                                    rule_id: rule.id.clone().expect("err!"),
+                                    due: Some(expected_expiry),
+                                    noncurrent_days: 0,
+                                    newer_noncurrent_versions: 0,
+                                    storage_class: "".into(),
+                                });
+                                break;
+                            }
+                        }
+                    }
+                }
+
+                if obj.is_latest {
+                    if let Some(ref expiration) = rule.expiration {
+                        if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
+                            if obj.delete_marker && expired_object_delete_marker {
+                                let due = expiration.next_due(obj);
+                                if let Some(due) = due {
+                                    if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() {
+                                        events.push(Event {
+                                            action: IlmAction::DelMarkerDeleteAllVersionsAction,
+                                            rule_id: rule.id.clone().expect("err!"),
+                                            due: Some(due),
+                                            noncurrent_days: 0,
+                                            newer_noncurrent_versions: 0,
+                                            storage_class: "".into(),
+                                        });
+                                    }
+                                }
+                                continue;
+                            }
+                        }
+                    }
+                }
+
+                if !obj.is_latest {
+                    if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
+                        if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
+                            if newer_noncurrent_versions > 0 {
+                                continue;
+                            }
+                        }
+                    }
+                }
+
+                if !obj.is_latest {
+                    if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
+                        if let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days {
+                            if noncurrent_days != 0 {
+                                if let Some(successor_mod_time) = obj.successor_mod_time {
+                                    let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
+                                    if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
+                                        events.push(Event {
+                                            action: IlmAction::DeleteVersionAction,
+                                            rule_id: rule.id.clone().expect("err!"),
+                                            due: Some(expected_expiry),
+                                            noncurrent_days: 0,
+                                            newer_noncurrent_versions: 0,
+                                            storage_class: "".into(),
+                                        });
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                if !obj.is_latest {
+                    if let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions {
+                        if let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class {
+                            if storage_class.as_str() != "" {
+                                if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE {
+                                    let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj);
+                                    if due.is_some()
+                                        && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp())
+                                    {
+                                        events.push(Event {
+                                            action: IlmAction::TransitionVersionAction,
+                                            rule_id: rule.id.clone().expect("err!"),
+                                            due,
+                                            storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
+                                                .storage_class
+                                                .clone()
+                                                .unwrap()
+                                                .as_str()
+                                                .to_string(),
+                                            ..Default::default()
+                                        });
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                if obj.is_latest && !obj.delete_marker {
+                    if let Some(ref expiration) = rule.expiration {
+                        if let Some(ref date) = expiration.date {
+                            let date0 = OffsetDateTime::from(date.clone());
+                            if date0.unix_timestamp() != 0 {
+                                if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() {
+                                    events.push(Event {
+                                        action: IlmAction::DeleteAction,
+                                        rule_id: rule.id.clone().expect("err!"),
+                                        due: Some(date0),
+                                        noncurrent_days: 0,
+                                        newer_noncurrent_versions: 0,
+                                        storage_class: "".into(),
+                                    });
+                                }
+                            }
+                        } else if let Some(days) = expiration.days {
+                            if days != 0 {
+                                let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days);
+                                if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() {
+                                    let mut event = Event {
+                                        action: IlmAction::DeleteAction,
+                                        rule_id: rule.id.clone().expect("err!"),
+                                        due: Some(expected_expiry),
+                                        noncurrent_days: 0,
+                                        newer_noncurrent_versions: 0,
+                                        storage_class: "".into(),
+                                    };
+                                    /*if rule.expiration.expect("err!").delete_all.val {
+                                        event.action = IlmAction::DeleteAllVersionsAction
+                                    }*/
+                                    events.push(event);
+                                }
+                            }
+                        }
+                    }
+
+                    if obj.transition_status != TRANSITION_COMPLETE {
+                        if let Some(ref transitions) = rule.transitions {
+                            let due = transitions[0].next_due(obj);
+                            if let Some(due) = due {
+                                if due.unix_timestamp() > 0
+                                    && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp())
+                                {
+                                    events.push(Event {
+                                        action: IlmAction::TransitionAction,
+                                        rule_id: rule.id.clone().expect("err!"),
+                                        due: Some(due),
+                                        storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
+                                        noncurrent_days: 0,
+                                        newer_noncurrent_versions: 0,
+                                    });
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        if events.len() > 0 {
+            events.sort_by(|a, b| {
+                if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
+                    && now.unix_timestamp() > b.due.expect("err").unix_timestamp()
+                    || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
+                {
+                    match a.action {
+                        IlmAction::DeleteAllVersionsAction
+                        | IlmAction::DelMarkerDeleteAllVersionsAction
+                        | IlmAction::DeleteAction
+                        | IlmAction::DeleteVersionAction => {
+                            return Ordering::Less;
+                        }
+                        _ => (),
+                    }
+                    match b.action {
+                        IlmAction::DeleteAllVersionsAction
+                        | IlmAction::DelMarkerDeleteAllVersionsAction
+                        | IlmAction::DeleteAction
+                        | IlmAction::DeleteVersionAction => {
+                            return Ordering::Greater;
+                        }
+                        _ => (),
+                    }
+                    return Ordering::Less;
+                }
+
+                if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() {
+                    return Ordering::Less;
+                }
+                return Ordering::Greater;
+            });
+            return events[0].clone();
+        }
+
+        Event::default()
+    }
+
+    async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event {
+        if let Some(filter_rules) = self.filter_rules(obj).await {
+            for rule in filter_rules.iter() {
+                if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration {
+                    if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions {
+                        if newer_noncurrent_versions == 0 {
+                            continue;
+                        }
+                        return Event {
+                            action: IlmAction::DeleteVersionAction,
+                            rule_id: rule.id.clone().expect("err"),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            newer_noncurrent_versions: newer_noncurrent_versions as usize,
+                            due: Some(OffsetDateTime::UNIX_EPOCH),
+                            storage_class: "".into(),
+                        };
+                    } else {
+                        return Event {
+                            action: IlmAction::DeleteVersionAction,
+                            rule_id: rule.id.clone().expect("err"),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            newer_noncurrent_versions: 0,
+                            due: Some(OffsetDateTime::UNIX_EPOCH),
+                            storage_class: "".into(),
+                        };
+                    }
+                }
+            }
+        }
+        Event::default()
+    }
+}
+
+#[async_trait::async_trait]
+pub trait LifecycleCalculate {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime>;
+}
+
+#[async_trait::async_trait]
+impl LifecycleCalculate for LifecycleExpiration {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
+        if !obj.is_latest || !obj.delete_marker {
+            return None;
+        }
+
+        Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
+    }
+}
+
+#[async_trait::async_trait]
+impl LifecycleCalculate for NoncurrentVersionTransition {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
+        if obj.is_latest || self.storage_class.is_none() {
+            return None;
+        }
+        if self.noncurrent_days.is_none() {
+            return obj.successor_mod_time;
+        }
+        Some(expected_expiry_time(obj.successor_mod_time.unwrap(), self.noncurrent_days.unwrap()))
+    }
+}
+
+#[async_trait::async_trait]
+impl LifecycleCalculate for Transition {
+    fn next_due(&self, obj: &ObjectOpts) -> Option<OffsetDateTime> {
+        if !obj.is_latest || self.days.is_none() {
+            return None;
+        }
+
+        if let Some(date) = self.date.clone() {
+            return Some(date.into());
+        }
+
+        if self.days.is_none() {
+            return obj.mod_time;
+        }
+        Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap()))
+    }
+}
+
+pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime {
+    if days == 0 {
+        return mod_time;
+    }
+    let t = mod_time
+        .to_offset(offset!(-0:00:00))
+        .saturating_add(Duration::days(0 /*days as i64*/)); //debug
+    let mut hour = 3600;
+    if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") {
+        if let Ok(num_hour) = env_ilm_hour.parse::<usize>() {
+            hour = num_hour;
+        }
+    }
+    //t.Truncate(24 * hour)
+    t
+}
+
+#[derive(Default)]
+pub struct ObjectOpts {
+    pub name: String,
+    pub user_tags: String,
+    pub mod_time: Option<OffsetDateTime>,
+    pub size: usize,
+    pub version_id: String,
+    pub is_latest: bool,
+    pub delete_marker: bool,
+    pub num_versions: usize,
+    pub successor_mod_time: Option<OffsetDateTime>,
+    pub transition_status: String,
+    pub restore_ongoing: bool,
+    pub restore_expires: Option<OffsetDateTime>,
+    pub versioned: bool,
+    pub version_suspended: bool,
+}
+
+impl ObjectOpts {
+    pub fn expired_object_deletemarker(&self) -> bool {
+        self.delete_marker && self.num_versions == 1
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Event {
+    pub action: IlmAction,
+    pub rule_id: String,
+    pub due: Option<OffsetDateTime>,
+    pub noncurrent_days: u32,
+    pub newer_noncurrent_versions: usize,
+    pub storage_class: String,
+}
+
+impl Default for Event {
+    fn default() -> Self {
+        Self {
+            action: IlmAction::NoneAction,
+            rule_id: "".into(),
+            due: Some(OffsetDateTime::UNIX_EPOCH),
+            noncurrent_days: 0,
+            newer_noncurrent_versions: 0,
+            storage_class: "".into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct ExpirationOptions {
+    pub expire: bool,
+}
+
+impl ExpirationOptions {
+    fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+        todo!();
+    }
+
+    fn unmarshal_msg(&self, bts: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+        todo!();
+    }
+
+    fn msg_size(&self) -> i64 {
+        1 + 7 + 10
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TransitionOptions {
+    pub status: String,
+    pub tier: String,
+    pub etag: String,
+    pub restore_request: RestoreObjectRequest,
+    pub restore_expiry: OffsetDateTime,
+    pub expire_restored: bool,
+}
+
+impl Default for TransitionOptions {
+    fn default() -> Self {
+        Self {
+            status: Default::default(),
+            tier: Default::default(),
+            etag: Default::default(),
+            restore_request: Default::default(),
+            restore_expiry: OffsetDateTime::now_utc(),
+            expire_restored: Default::default(),
+        }
+    }
+}

ecstore/src/bucket/lifecycle/mod.rs

@@ -0,0 +1,6 @@
+pub mod bucket_lifecycle_audit;
+pub mod bucket_lifecycle_ops;
+pub mod lifecycle;
+pub mod rule;
+pub mod tier_last_day_stats;
+pub mod tier_sweeper;

ecstore/src/bucket/lifecycle/rule.rs

@@ -0,0 +1,49 @@
+use s3s::dto::{LifecycleRuleFilter, Transition};
+
+const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition";
+const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format";
+const ERR_TRANSITION_INVALID: &str =
+    "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.";
+const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT";
+
+pub trait Filter {
+    fn test_tags(&self, user_tags: &str) -> bool;
+    fn by_size(&self, sz: i64) -> bool;
+}
+
+impl Filter for LifecycleRuleFilter {
+    fn test_tags(&self, user_tags: &str) -> bool {
+        true
+    }
+
+    fn by_size(&self, sz: i64) -> bool {
+        true
+    }
+}
+
+pub trait TransitionOps {
+    fn validate(&self) -> Result<(), std::io::Error>;
+}
+
+impl TransitionOps for Transition {
+    fn validate(&self) -> Result<(), std::io::Error> {
+        if !self.date.is_none() && self.days.expect("err!") > 0 {
+            return Err(std::io::Error::other(ERR_TRANSITION_INVALID));
+        }
+
+        if self.storage_class.is_none() {
+            return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED"));
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_rule() {
+        //assert!(skip_access_checks(p.to_str().unwrap()));
+    }
+}

ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -0,0 +1,83 @@
+use sha2::Sha256;
+
+use std::collections::HashMap;
+use std::ops::Sub;
+use time::OffsetDateTime;
+use tracing::{error, warn};
+
+use crate::heal::data_usage_cache::TierStats;
+
+pub type DailyAllTierStats = HashMap<String, LastDayTierStats>;
+
+#[derive(Clone)]
+pub struct LastDayTierStats {
+    bins: [TierStats; 24],
+    updated_at: OffsetDateTime,
+}
+
+impl Default for LastDayTierStats {
+    fn default() -> Self {
+        Self {
+            bins: Default::default(),
+            updated_at: OffsetDateTime::now_utc(),
+        }
+    }
+}
+
+impl LastDayTierStats {
+    pub fn add_stats(&mut self, ts: TierStats) {
+        let mut now = OffsetDateTime::now_utc();
+        self.forward_to(&mut now);
+
+        let now_idx = now.hour() as usize;
+        self.bins[now_idx] = self.bins[now_idx].add(&ts);
+    }
+
+    fn forward_to(&mut self, t: &mut OffsetDateTime) {
+        if t.unix_timestamp() == 0 {
+            *t = OffsetDateTime::now_utc();
+        }
+
+        let since = t.sub(self.updated_at).whole_hours();
+        if since < 1 {
+            return;
+        }
+
+        let (idx, mut last_idx) = (t.hour(), self.updated_at.hour());
+
+        self.updated_at = *t;
+
+        if since >= 24 {
+            self.bins = [TierStats::default(); 24];
+            return;
+        }
+
+        while last_idx != idx {
+            last_idx = (last_idx + 1) % 24;
+            self.bins[last_idx as usize] = TierStats::default();
+        }
+    }
+
+    fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
+        let mut cl = self.clone();
+        let mut cm = m.clone();
+        let mut merged = LastDayTierStats::default();
+
+        if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() {
+            cm.forward_to(&mut cl.updated_at);
+            merged.updated_at = cl.updated_at;
+        } else {
+            cl.forward_to(&mut cm.updated_at);
+            merged.updated_at = cm.updated_at;
+        }
+
+        for (i, _) in cl.bins.iter().enumerate() {
+            merged.bins[i] = cl.bins[i].add(&cm.bins[i]);
+        }
+
+        merged
+    }
+}
+
+#[cfg(test)]
+mod test {}

ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -0,0 +1,129 @@
+use sha2::{Digest, Sha256};
+use std::any::Any;
+use std::io::{Cursor, Write};
+use xxhash_rust::xxh64;
+
+use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
+use super::lifecycle::{self, ObjectOpts};
+use crate::global::GLOBAL_TierConfigMgr;
+
+static XXHASH_SEED: u64 = 0;
+
+#[derive(Default)]
+struct ObjSweeper {
+    object: String,
+    bucket: String,
+    version_id: String,
+    versioned: bool,
+    suspended: bool,
+    transition_status: String,
+    transition_tier: String,
+    transition_version_id: String,
+    remote_object: String,
+}
+
+impl ObjSweeper {
+    #[allow(clippy::new_ret_no_self)]
+    pub async fn new(bucket: &str, object: &str) -> Result<Self, std::io::Error> {
+        Ok(Self {
+            object: object.into(),
+            bucket: bucket.into(),
+            ..Default::default()
+        })
+    }
+
+    pub fn with_version(&mut self, vid: String) -> &Self {
+        self.version_id = vid;
+        self
+    }
+
+    pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self {
+        self.versioned = versioned;
+        self.suspended = suspended;
+        self
+    }
+
+    pub fn get_opts(&self) -> lifecycle::ObjectOpts {
+        let mut opts = ObjectOpts {
+            version_id: self.version_id.clone(),
+            versioned: self.versioned,
+            version_suspended: self.suspended,
+            ..Default::default()
+        };
+        if self.suspended && self.version_id == "" {
+            opts.version_id = String::from("");
+        }
+        opts
+    }
+
+    pub fn set_transition_state(&mut self, info: TransitionedObject) {
+        self.transition_tier = info.tier;
+        self.transition_status = info.status;
+        self.remote_object = info.name;
+        self.transition_version_id = info.version_id;
+    }
+
+    pub fn should_remove_remote_object(&self) -> Option<Jentry> {
+        if self.transition_status != lifecycle::TRANSITION_COMPLETE {
+            return None;
+        }
+
+        let mut del_tier = false;
+        if !self.versioned || self.suspended {
+            // 1, 2.a, 2.b
+            del_tier = true;
+        } else if self.versioned && self.version_id != "" {
+            // 3.a
+            del_tier = true;
+        }
+        if del_tier {
+            return Some(Jentry {
+                obj_name: self.remote_object.clone(),
+                version_id: self.transition_version_id.clone(),
+                tier_name: self.transition_tier.clone(),
+            });
+        }
+        None
+    }
+
+    pub async fn sweep(&self) {
+        let je = self.should_remove_remote_object();
+        if !je.is_none() {
+            let mut expiry_state = GLOBAL_ExpiryState.write().await;
+            expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Jentry {
+    obj_name: String,
+    version_id: String,
+    tier_name: String,
+}
+
+impl ExpiryOp for Jentry {
+    fn op_hash(&self) -> u64 {
+        let mut hasher = Sha256::new();
+        let _ = hasher.write(format!("{}", self.tier_name).as_bytes());
+        let _ = hasher.write(format!("{}", self.obj_name).as_bytes());
+        hasher.flush();
+        xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
+    let mut config_mgr = GLOBAL_TierConfigMgr.write().await;
+    let w = match config_mgr.get_driver(tier_name).await {
+        Ok(w) => w,
+        Err(e) => return Err(std::io::Error::other(e)),
+    };
+    w.remove(obj_name, rv_id).await
+}
+
+#[cfg(test)]
+mod test {}

ecstore/src/bucket/mod.rs

@@ -1,4 +1,5 @@
 pub mod error;
+pub mod lifecycle;
 pub mod metadata;
 pub mod metadata_sys;
 pub mod object_lock;

ecstore/src/bucket/object_lock/mod.rs

@@ -1,3 +1,6 @@
+pub mod objectlock;
+pub mod objectlock_sys;
+
 use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
 
 pub trait ObjectLockApi {

ecstore/src/bucket/object_lock/objectlock.rs

@@ -0,0 +1,100 @@
+use std::any::{Any, TypeId};
+use std::collections::HashMap;
+use time::{OffsetDateTime, format_description};
+use tracing::{error, warn};
+
+use s3s::dto::{Date, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode};
+use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
+
+//const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention";
+//const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str     = "X-Amz-Object-Lock-Retain-Until-Date";
+//const AMZ_OBJECTLOCK_MODE: &str                  = "X-Amz-Object-Lock-Mode";
+//const AMZ_OBJECTLOCK_LEGALHOLD: &str             = "X-Amz-Object-Lock-Legal-Hold";
+
+const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config";
+const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format";
+const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future";
+const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive";
+const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str =
+    "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters";
+const ERR_OBJECTLOCK_INVALID_HEADERS: &str =
+    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied";
+const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema";
+
+pub fn utc_now_ntp() -> OffsetDateTime {
+    return OffsetDateTime::now_utc();
+}
+
+pub fn get_object_retention_meta(meta: HashMap<String, String>) -> ObjectLockRetention {
+    let mode: ObjectLockRetentionMode;
+    let mut retain_until_date: Date = Date::from(OffsetDateTime::UNIX_EPOCH);
+
+    let mut mode_str = meta.get(X_AMZ_OBJECT_LOCK_MODE.as_str().to_lowercase().as_str());
+    if mode_str.is_none() {
+        mode_str = Some(&meta[X_AMZ_OBJECT_LOCK_MODE.as_str()]);
+    }
+    if let Some(mode_str) = mode_str {
+        mode = parse_ret_mode(mode_str.as_str());
+    } else {
+        return ObjectLockRetention {
+            mode: None,
+            retain_until_date: None,
+        };
+    }
+
+    let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str());
+    if till_str.is_none() {
+        till_str = Some(&meta[X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()]);
+    }
+    if let Some(till_str) = till_str {
+        let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT);
+        if t.is_err() {
+            retain_until_date = Date::from(t.expect("err")); //TODO: utc
+        }
+    }
+    ObjectLockRetention {
+        mode: Some(mode),
+        retain_until_date: Some(retain_until_date),
+    }
+}
+
+pub fn get_object_legalhold_meta(meta: HashMap<String, String>) -> ObjectLockLegalHold {
+    let mut hold_str = meta.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_lowercase().as_str());
+    if hold_str.is_none() {
+        hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]);
+    }
+    if let Some(hold_str) = hold_str {
+        return ObjectLockLegalHold {
+            status: Some(parse_legalhold_status(hold_str)),
+        };
+    }
+    ObjectLockLegalHold { status: None }
+}
+
+pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode {
+    let mut mode;
+    match mode_str.to_uppercase().as_str() {
+        "GOVERNANCE" => {
+            mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE);
+        }
+        "COMPLIANCE" => {
+            mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE);
+        }
+        _ => unreachable!(),
+    }
+    mode
+}
+
+pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus {
+    let mut st;
+    match hold_str {
+        "ON" => {
+            st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON);
+        }
+        "OFF" => {
+            st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF);
+        }
+        _ => unreachable!(),
+    }
+    st
+}

ecstore/src/bucket/object_lock/objectlock_sys.rs

@@ -0,0 +1,58 @@
+use std::any::{Any, TypeId};
+use std::sync::Arc;
+use time::OffsetDateTime;
+use tracing::{error, warn};
+
+use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
+
+use crate::bucket::metadata_sys::get_object_lock_config;
+use crate::store_api::ObjectInfo;
+
+use super::objectlock;
+
+pub struct BucketObjectLockSys {}
+
+impl BucketObjectLockSys {
+    #[allow(clippy::new_ret_no_self)]
+    pub async fn new() -> Arc<Self> {
+        Arc::new(Self {})
+    }
+
+    pub async fn get(bucket: &str) -> Option<DefaultRetention> {
+        if let Some(object_lock_rule) = get_object_lock_config(bucket)
+            .await
+            .expect("get_object_lock_config err!")
+            .0
+            .rule
+        {
+            return object_lock_rule.default_retention;
+        }
+        None
+    }
+}
+
+pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool {
+    if obj_info.delete_marker {
+        return false;
+    }
+
+    let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err"));
+    match lhold.status {
+        Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => {
+            return true;
+        }
+        _ => (),
+    }
+
+    let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err"));
+    match ret.mode {
+        Some(r) if (r.as_str() == ObjectLockRetentionMode::COMPLIANCE || r.as_str() == ObjectLockRetentionMode::GOVERNANCE) => {
+            let t = objectlock::utc_now_ntp();
+            if OffsetDateTime::from(ret.retain_until_date.expect("err!")).unix_timestamp() > t.unix_timestamp() {
+                return true;
+            }
+        }
+        _ => (),
+    }
+    false
+}

ecstore/src/checksum.rs

@@ -0,0 +1,298 @@
+#![allow(clippy::map_entry)]
+use lazy_static::lazy_static;
+use std::ops::{BitAnd, BitOr};
+use std::{collections::HashMap, sync::Arc};
+
+use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
+use crate::{disk::DiskAPI, store_api::GetObjectReader};
+use reader::hasher::{Hasher, Sha256};
+use rustfs_utils::crypto::{base64_decode, base64_encode};
+use s3s::header::{
+    X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
+};
+
+use enumset::{EnumSet, EnumSetType, enum_set};
+
+#[derive(Debug, EnumSetType, Default)]
+#[enumset(repr = "u8")]
+pub enum ChecksumMode {
+    #[default]
+    ChecksumNone,
+    ChecksumSHA256,
+    ChecksumSHA1,
+    ChecksumCRC32,
+    ChecksumCRC32C,
+    ChecksumCRC64NVME,
+    ChecksumFullObject,
+}
+
+lazy_static! {
+    static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
+        let mut s = EnumSet::all();
+        s.remove(ChecksumMode::ChecksumFullObject);
+        s
+    };
+    static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
+    static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
+}
+const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
+
+impl ChecksumMode {
+    //pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
+
+    pub fn base(&self) -> ChecksumMode {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            1_u8 => ChecksumMode::ChecksumNone,
+            2_u8 => ChecksumMode::ChecksumSHA256,
+            4_u8 => ChecksumMode::ChecksumSHA1,
+            8_u8 => ChecksumMode::ChecksumCRC32,
+            16_u8 => ChecksumMode::ChecksumCRC32C,
+            32_u8 => ChecksumMode::ChecksumCRC64NVME,
+            _ => panic!("enum err."),
+        }
+    }
+
+    pub fn is(&self, t: ChecksumMode) -> bool {
+        *self & t == t
+    }
+
+    pub fn key(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return X_AMZ_CHECKSUM_CRC32.to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return X_AMZ_CHECKSUM_CRC32C.to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return X_AMZ_CHECKSUM_SHA1.to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return X_AMZ_CHECKSUM_SHA256.to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return AMZ_CHECKSUM_CRC64NVME.to_string();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn can_composite(&self) -> bool {
+        todo!();
+    }
+
+    pub fn can_merge_crc(&self) -> bool {
+        todo!();
+    }
+
+    pub fn full_object_requested(&self) -> bool {
+        todo!();
+    }
+
+    pub fn key_capitalized(&self) -> String {
+        self.key()
+    }
+
+    pub fn raw_byte_len(&self) -> usize {
+        let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
+        if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
+            4
+        } else if u == ChecksumMode::ChecksumSHA1 as u8 {
+            4 //sha1.size
+        } else if u == ChecksumMode::ChecksumSHA256 as u8 {
+            4 //sha256.size
+        } else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
+            4 //crc64.size
+        } else {
+            0
+        }
+    }
+
+    pub fn hasher(&self) -> Result<Box<dyn Hasher>, std::io::Error> {
+        match /*C_ChecksumMask & **/self {
+            /*ChecksumMode::ChecksumCRC32 => {
+                return Ok(Box::new(crc32fast::Hasher::new()));
+            }*/
+            /*ChecksumMode::ChecksumCRC32C => {
+                return Ok(Box::new(crc32::new(crc32.MakeTable(crc32.Castagnoli))));
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return Ok(Box::new(sha1::new()));
+            }*/
+            ChecksumMode::ChecksumSHA256 => {
+                return Ok(Box::new(Sha256::new()));
+            }
+            /*ChecksumMode::ChecksumCRC64NVME => {
+                return Ok(Box::new(crc64nvme.New());
+            }*/
+            _ => return Err(std::io::Error::other("unsupported checksum type")),
+        }
+    }
+
+    pub fn is_set(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        s.len() == 1
+    }
+
+    pub fn set_default(&mut self, t: ChecksumMode) {
+        if !self.is_set() {
+            *self = t;
+        }
+    }
+
+    pub fn encode_to_string(&self, b: &[u8]) -> Result<String, std::io::Error> {
+        if !self.is_set() {
+            return Ok("".to_string());
+        }
+        let mut h = self.hasher()?;
+        h.write(b);
+        Ok(base64_encode(h.sum().as_bytes()))
+    }
+
+    pub fn to_string(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return "CRC32".to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return "CRC32C".to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return "SHA1".to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return "SHA256".to_string();
+            }
+            ChecksumMode::ChecksumNone => {
+                return "".to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return "CRC64NVME".to_string();
+            }
+            _ => {
+                return "<invalid>".to_string();
+            }
+        }
+    }
+
+    pub fn check_sum_reader(&self, r: GetObjectReader) -> Result<Checksum, std::io::Error> {
+        let mut h = self.hasher()?;
+        Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    }
+
+    pub fn check_sum_bytes(&self, b: &[u8]) -> Result<Checksum, std::io::Error> {
+        let mut h = self.hasher()?;
+        Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    }
+
+    pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        if !self.can_composite() {
+            return Err(std::io::Error::other("cannot do composite checksum"));
+        }
+        p.sort_by(|i, j| {
+            if i.part_num < j.part_num {
+                std::cmp::Ordering::Less
+            } else if i.part_num > j.part_num {
+                std::cmp::Ordering::Greater
+            } else {
+                std::cmp::Ordering::Equal
+            }
+        });
+        let c = self.base();
+        let mut crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
+        let mut h = self.hasher()?;
+        h.write(&crc_bytes);
+        Ok(Checksum {
+            checksum_type: self.clone(),
+            r: h.sum().as_bytes().to_vec(),
+        })
+    }
+
+    pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        todo!();
+    }
+}
+
+#[derive(Default)]
+struct Checksum {
+    checksum_type: ChecksumMode,
+    r: Vec<u8>,
+}
+
+impl Checksum {
+    fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Checksum {
+                checksum_type: t,
+                r: b.to_vec(),
+            };
+        }
+        Checksum::default()
+    }
+
+    fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
+        let b = match base64_decode(s.as_bytes()) {
+            Ok(b) => b,
+            Err(err) => return Err(std::io::Error::other(err.to_string())),
+        };
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Ok(Checksum { checksum_type: t, r: b });
+        }
+        Ok(Checksum::default())
+    }
+
+    fn is_set(&self) -> bool {
+        self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len()
+    }
+
+    fn encoded(&self) -> String {
+        if !self.is_set() {
+            return "".to_string();
+        }
+        base64_encode(&self.r)
+    }
+
+    fn raw(&self) -> Option<Vec<u8>> {
+        if !self.is_set() {
+            return None;
+        }
+        Some(self.r.clone())
+    }
+}
+
+pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
+    opts.user_metadata
+        .insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
+    if opts.auto_checksum.full_object_requested() {
+        opts.user_metadata
+            .insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+    }
+}
+
+pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
+    if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
+        let crc = opts.auto_checksum.composite_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key(), crc.encoded());
+            hm
+        }
+    } else if opts.auto_checksum.can_merge_crc() {
+        let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
+            hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+            hm
+        }
+    }
+
+    Ok(())
+}

ecstore/src/client/admin_handler_utils.rs

@@ -0,0 +1,33 @@
+use http::status::StatusCode;
+use std::fmt::{self, Display, Formatter};
+
+#[derive(Default, thiserror::Error, Debug, PartialEq)]
+pub struct AdminError {
+    pub code: &'static str,
+    pub message: &'static str,
+    pub status_code: StatusCode,
+}
+
+impl Display for AdminError {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl AdminError {
+    pub fn new(code: &'static str, message: &'static str, status_code: StatusCode) -> Self {
+        Self {
+            code,
+            message,
+            status_code,
+        }
+    }
+
+    pub fn msg(message: &'static str) -> Self {
+        Self {
+            code: "InternalError",
+            message,
+            status_code: StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}

ecstore/src/client/api_bucket_policy.rs

@@ -0,0 +1,123 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use http::{HeaderMap, StatusCode};
+use std::collections::HashMap;
+
+use crate::client::{
+    api_error_response::http_resp_to_error_response,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
+        if policy == "" {
+            return self.remove_bucket_policy(bucket_name).await;
+        }
+
+        self.put_bucket_policy(bucket_name, policy).await
+    }
+
+    pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            query_values: url_values,
+            content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
+            content_length: policy.len() as i64,
+            object_name: "".to_string(),
+            custom_header: HeaderMap::new(),
+            content_md5_base64: "".to_string(),
+            content_sha256_hex: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+        //defer closeResponse(resp)
+        //if resp != nil {
+        if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+        //}
+        Ok(())
+    }
+
+    pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    object_name: "".to_string(),
+                    custom_header: HeaderMap::new(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+        //defer closeResponse(resp)
+
+        if resp.status() != StatusCode::NO_CONTENT {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+
+        Ok(())
+    }
+
+    pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
+        Ok(bucket_policy)
+    }
+
+    pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("policy".to_string(), "".to_string());
+
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    object_name: "".to_string(),
+                    custom_header: HeaderMap::new(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
+        Ok(policy)
+    }
+}

ecstore/src/client/api_error_response.rs

@@ -0,0 +1,266 @@
+#![allow(clippy::map_entry)]
+use http::StatusCode;
+use serde::{Deserialize, Serialize};
+use serde::{de::Deserializer, ser::Serializer};
+use std::fmt::Display;
+
+use s3s::Body;
+use s3s::S3ErrorCode;
+
+const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
+
+#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ErrorResponse {
+    #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
+    pub code: S3ErrorCode,
+    pub message: String,
+    pub bucket_name: String,
+    pub key: String,
+    pub resource: String,
+    pub request_id: String,
+    pub host_id: String,
+    pub region: String,
+    pub server: String,
+    #[serde(skip)]
+    pub status_code: StatusCode,
+}
+
+fn serialize_code<S>(data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    s.serialize_str("")
+}
+
+fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
+}
+
+impl Default for ErrorResponse {
+    fn default() -> Self {
+        ErrorResponse {
+            code: S3ErrorCode::Custom("".into()),
+            message: Default::default(),
+            bucket_name: Default::default(),
+            key: Default::default(),
+            resource: Default::default(),
+            request_id: Default::default(),
+            host_id: Default::default(),
+            region: Default::default(),
+            server: Default::default(),
+            status_code: Default::default(),
+        }
+    }
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
+    if let Some(err) = err.get_ref() {
+        if err.is::<ErrorResponse>() {
+            err.downcast_ref::<ErrorResponse>().expect("err!").clone()
+        } else {
+            ErrorResponse::default()
+        }
+    } else {
+        ErrorResponse::default()
+    }
+}
+
+pub fn http_resp_to_error_response(
+    resp: http::Response<Body>,
+    b: Vec<u8>,
+    bucket_name: &str,
+    object_name: &str,
+) -> ErrorResponse {
+    let err_body = String::from_utf8(b).unwrap();
+    //let err_body = xml_decode_and_body(resp.body, &err_resp);
+    let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
+    let mut err_resp = ErrorResponse::default();
+    if err_resp_.is_err() {
+        match resp.status() {
+            StatusCode::NOT_FOUND => {
+                if object_name == "" {
+                    err_resp = ErrorResponse {
+                        status_code: resp.status(),
+                        code: S3ErrorCode::NoSuchBucket,
+                        message: "The specified bucket does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        ..Default::default()
+                    };
+                } else {
+                    err_resp = ErrorResponse {
+                        status_code: resp.status(),
+                        code: S3ErrorCode::NoSuchKey,
+                        message: "The specified key does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        key: object_name.to_string(),
+                        ..Default::default()
+                    };
+                }
+            }
+            StatusCode::FORBIDDEN => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::AccessDenied,
+                    message: "Access Denied.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key: object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::CONFLICT => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::BucketNotEmpty,
+                    message: "Bucket not empty.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            StatusCode::PRECONDITION_FAILED => {
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::PreconditionFailed,
+                    message: "Pre condition failed.".to_string(),
+                    bucket_name: bucket_name.to_string(),
+                    key: object_name.to_string(),
+                    ..Default::default()
+                };
+            }
+            _ => {
+                let mut msg = resp.status().to_string();
+                if err_body.len() > 0 {
+                    msg = err_body;
+                }
+                err_resp = ErrorResponse {
+                    status_code: resp.status(),
+                    code: S3ErrorCode::Custom(resp.status().to_string().into()),
+                    message: msg,
+                    bucket_name: bucket_name.to_string(),
+                    ..Default::default()
+                };
+            }
+        }
+    } else {
+        err_resp = err_resp_.unwrap();
+    }
+    err_resp.status_code = resp.status();
+    if let Some(server_name) = resp.headers().get("Server") {
+        err_resp.server = server_name.to_str().expect("err").to_string();
+    }
+
+    let code = resp.headers().get("x-minio-error-code");
+    if code.is_some() {
+        err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
+    }
+    let desc = resp.headers().get("x-minio-error-desc");
+    if desc.is_some() {
+        err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
+    }
+
+    if err_resp.request_id == "" {
+        if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") {
+            err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.host_id == "" {
+        if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") {
+            err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.region == "" {
+        if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") {
+            err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
+        }
+    }
+    if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
+        err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region);
+    }
+
+    err_resp
+}
+
+pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::InvalidArgument,
+        message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
+            .to_string(),
+        bucket_name: bucket_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
+        total_size, max_object_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::EntityTooLarge,
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.",
+        total_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::EntityTooSmall,
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
+    let msg = format!(
+        "Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.",
+        total_read, total_size
+    );
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::Custom("UnexpectedEOF".into()),
+        message: msg,
+        bucket_name: bucket_name.to_string(),
+        key: object_name.to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_invalid_argument(message: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::BAD_REQUEST,
+        code: S3ErrorCode::InvalidArgument,
+        message: message.to_string(),
+        request_id: "rustfs".to_string(),
+        ..Default::default()
+    }
+}
+
+pub fn err_api_not_supported(message: &str) -> ErrorResponse {
+    ErrorResponse {
+        status_code: StatusCode::NOT_IMPLEMENTED,
+        code: S3ErrorCode::Custom("APINotSupported".into()),
+        message: message.to_string(),
+        request_id: "rustfs".to_string(),
+        ..Default::default()
+    }
+}

ecstore/src/client/api_get_object.rs

@@ -0,0 +1,206 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use http::HeaderMap;
+use std::io::Cursor;
+use tokio::io::BufReader;
+
+use crate::client::{
+    api_error_response::err_invalid_argument,
+    api_get_options::GetObjectOptions,
+    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
+};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn get_object_inner(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &GetObjectOptions,
+    ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    query_values: opts.to_query_values(),
+                    custom_header: opts.header(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        let resp = &resp;
+        let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;
+
+        let b = resp.body().bytes().expect("err").to_vec();
+        Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b))))
+    }
+}
+
+#[derive(Default)]
+struct GetRequest {
+    pub buffer: Vec<u8>,
+    pub offset: i64,
+    pub did_offset_change: bool,
+    pub been_read: bool,
+    pub is_read_at: bool,
+    pub is_read_op: bool,
+    pub is_first_req: bool,
+    pub setting_object_info: bool,
+}
+
+struct GetResponse {
+    pub size: i64,
+    //pub error:       error,
+    pub did_read: bool,
+    pub object_info: ObjectInfo,
+}
+
+#[derive(Default)]
+struct Object {
+    //pub reqch:      chan<- getRequest,
+    //pub resch:      <-chan getResponse,
+    //pub cancel:     context.CancelFunc,
+    pub curr_offset: i64,
+    pub object_info: ObjectInfo,
+    pub seek_data: bool,
+    pub is_closed: bool,
+    pub is_started: bool,
+    //pub prev_err: error,
+    pub been_read: bool,
+    pub object_info_set: bool,
+}
+
+impl Object {
+    pub fn new() -> Object {
+        Self { ..Default::default() }
+    }
+
+    fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
+        todo!()
+    }
+
+    fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
+        self.curr_offset += bytes_read;
+
+        Ok(())
+    }
+
+    fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
+        let mut read_req = GetRequest {
+            is_read_op: true,
+            been_read: self.been_read,
+            buffer: b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_req.is_first_req = true;
+        }
+
+        read_req.did_offset_change = self.seek_data;
+        read_req.offset = self.curr_offset;
+
+        let response = self.do_get_request(&read_req)?;
+
+        let bytes_read = response.size;
+
+        let oerr = self.set_offset(bytes_read);
+
+        Ok(response.size)
+    }
+
+    fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let _ = self.do_get_request(&GetRequest {
+                is_first_req: !self.is_started,
+                setting_object_info: !self.object_info_set,
+                ..Default::default()
+            })?;
+        }
+
+        Ok(self.object_info.clone())
+    }
+
+    fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
+        self.curr_offset = offset;
+
+        let mut read_at_req = GetRequest {
+            is_read_op: true,
+            is_read_at: true,
+            did_offset_change: true,
+            been_read: self.been_read,
+            offset,
+            buffer: b.to_vec(),
+            ..Default::default()
+        };
+
+        if !self.is_started {
+            read_at_req.is_first_req = true;
+        }
+
+        let response = self.do_get_request(&read_at_req)?;
+        let bytes_read = response.size;
+        if !self.object_info_set {
+            self.curr_offset += bytes_read;
+        } else {
+            let oerr = self.set_offset(bytes_read);
+        }
+        Ok(response.size)
+    }
+
+    fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
+        if !self.is_started || !self.object_info_set {
+            let seek_req = GetRequest {
+                is_read_op: false,
+                offset: offset,
+                is_first_req: true,
+                ..Default::default()
+            };
+            let _ = self.do_get_request(&seek_req);
+        }
+
+        let mut new_offset = self.curr_offset;
+
+        match whence {
+            0 => {
+                new_offset = offset;
+            }
+            1 => {
+                new_offset += offset;
+            }
+            2 => {
+                new_offset = self.object_info.size as i64 + offset as i64;
+            }
+            _ => {
+                return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence))));
+            }
+        }
+
+        self.seek_data = (new_offset != self.curr_offset) || self.seek_data;
+        self.curr_offset = new_offset;
+
+        Ok(self.curr_offset)
+    }
+
+    fn close(&mut self) -> Result<(), std::io::Error> {
+        self.is_closed = true;
+        Ok(())
+    }
+}

ecstore/src/client/api_get_options.rs

@@ -0,0 +1,127 @@
+#![allow(clippy::map_entry)]
+use http::{HeaderMap, HeaderName, HeaderValue};
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use tracing::warn;
+
+use crate::client::api_error_response::err_invalid_argument;
+
+#[derive(Default)]
+pub struct AdvancedGetOptions {
+    replication_deletemarker: bool,
+    is_replication_ready_for_deletemarker: bool,
+    replication_proxy_request: String,
+}
+
+pub struct GetObjectOptions {
+    pub headers: HashMap<String, String>,
+    pub req_params: HashMap<String, String>,
+    //pub server_side_encryption: encrypt.ServerSide,
+    pub version_id: String,
+    pub part_number: i64,
+    pub checksum: bool,
+    pub internal: AdvancedGetOptions,
+}
+
+type StatObjectOptions = GetObjectOptions;
+
+impl Default for GetObjectOptions {
+    fn default() -> Self {
+        Self {
+            headers: HashMap::new(),
+            req_params: HashMap::new(),
+            //server_side_encryption: encrypt.ServerSide::default(),
+            version_id: "".to_string(),
+            part_number: 0,
+            checksum: false,
+            internal: AdvancedGetOptions::default(),
+        }
+    }
+}
+
+impl GetObjectOptions {
+    pub fn header(&self) -> HeaderMap {
+        let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
+        for (k, v) in &self.headers {
+            if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
+                headers.insert(header_name, v.parse().expect("err"));
+            } else {
+                warn!("Invalid header name: {}", k);
+            }
+        }
+        if self.checksum {
+            headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
+        }
+        headers
+    }
+
+    pub fn set(&self, key: &str, value: &str) {
+        //self.headers[http.CanonicalHeaderKey(key)] = value;
+    }
+
+    pub fn set_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn add_req_param(&mut self, key: &str, value: &str) {
+        self.req_params.insert(key.to_string(), value.to_string());
+    }
+
+    pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-Match", &format!("\"{etag}\""));
+        Ok(())
+    }
+
+    pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
+        self.set("If-None-Match", &format!("\"{etag}\""));
+        Ok(())
+    }
+
+    pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Unmodified-Since", &mod_time.to_string());
+        Ok(())
+    }
+
+    pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
+        if mod_time.unix_timestamp() == 0 {
+            return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
+        }
+        self.set("If-Modified-Since", &mod_time.to_string());
+        Ok(())
+    }
+
+    pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
+        if start == 0 && end < 0 {
+            self.set("Range", &format!("bytes={}", end));
+        } else if 0 < start && end == 0 {
+            self.set("Range", &format!("bytes={}-", start));
+        } else if 0 <= start && start <= end {
+            self.set("Range", &format!("bytes={}-{}", start, end));
+        } else {
+            return Err(std::io::Error::other(err_invalid_argument(&format!(
+                "Invalid range specified: start={} end={}",
+                start, end
+            ))));
+        }
+        Ok(())
+    }
+
+    pub fn to_query_values(&self) -> HashMap<String, String> {
+        let mut url_values = HashMap::new();
+        if self.version_id != "" {
+            url_values.insert("versionId".to_string(), self.version_id.clone());
+        }
+        if self.part_number > 0 {
+            url_values.insert("partNumber".to_string(), self.part_number.to_string());
+        }
+
+        for (key, value) in self.req_params.iter() {
+            url_values.insert(key.to_string(), value.to_string());
+        }
+
+        url_values
+    }
+}

ecstore/src/client/api_list.rs

@@ -0,0 +1,282 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use http::{HeaderMap, StatusCode};
+use std::collections::HashMap;
+
+use crate::client::{
+    api_error_response::http_resp_to_error_response,
+    api_s3_datatypes::{
+        ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
+    },
+    credentials,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+};
+use crate::store_api::BucketInfo;
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+impl TransitionClient {
+    pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn list_objects_v2_query(
+        &self,
+        bucket_name: &str,
+        object_prefix: &str,
+        continuation_token: &str,
+        fetch_owner: bool,
+        metadata: bool,
+        delimiter: &str,
+        start_after: &str,
+        max_keys: i64,
+        headers: HeaderMap,
+    ) -> Result<ListBucketV2Result, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        url_values.insert("list-type".to_string(), "2".to_string());
+        if metadata {
+            url_values.insert("metadata".to_string(), "true".to_string());
+        }
+        if start_after != "" {
+            url_values.insert("start-after".to_string(), start_after.to_string());
+        }
+        url_values.insert("encoding-type".to_string(), "url".to_string());
+        url_values.insert("prefix".to_string(), object_prefix.to_string());
+        url_values.insert("delimiter".to_string(), delimiter.to_string());
+
+        if continuation_token != "" {
+            url_values.insert("continuation-token".to_string(), continuation_token.to_string());
+        }
+
+        if fetch_owner {
+            url_values.insert("fetch-owner".to_string(), "true".to_string());
+        }
+
+        if max_keys > 0 {
+            url_values.insert("max-keys".to_string(), max_keys.to_string());
+        }
+
+        let mut resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: headers,
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
+        }
+
+        //let mut list_bucket_result = ListBucketV2Result::default();
+        let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+        let mut list_bucket_result = match serde_xml_rs::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
+            Ok(result) => result,
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        };
+        //println!("list_bucket_result: {:?}", list_bucket_result);
+
+        if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" {
+            return Err(std::io::Error::other(credentials::ErrorResponse {
+                sts_error: credentials::STSError {
+                    r#type: "".to_string(),
+                    code: "NotImplemented".to_string(),
+                    message: "Truncated response should have continuation token set".to_string(),
+                },
+                request_id: "".to_string(),
+            }));
+        }
+
+        for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() {
+            obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?;
+            //list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond);
+        }
+
+        for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() {
+            obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?;
+        }
+
+        Ok(list_bucket_result)
+    }
+
+    pub fn list_object_versions_query(
+        &self,
+        bucket_name: &str,
+        opts: &ListObjectsOptions,
+        key_marker: &str,
+        version_id_marker: &str,
+        delimiter: &str,
+    ) -> Result<ListVersionsResult, std::io::Error> {
+        /*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
+          return ListVersionsResult{}, err
+        }
+        if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
+          return ListVersionsResult{}, err
+        }
+        urlValues := make(url.Values)
+
+        urlValues.Set("versions", "")
+
+        urlValues.Set("prefix", opts.Prefix)
+
+        urlValues.Set("delimiter", delimiter)
+
+        if keyMarker != "" {
+          urlValues.Set("key-marker", keyMarker)
+        }
+
+        if opts.max_keys > 0 {
+          urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys))
+        }
+
+        if versionIDMarker != "" {
+          urlValues.Set("version-id-marker", versionIDMarker)
+        }
+
+        if opts.WithMetadata {
+          urlValues.Set("metadata", "true")
+        }
+
+        urlValues.Set("encoding-type", "url")
+
+        let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{
+            bucketName:       bucketName,
+            queryValues:      urlValues,
+            contentSHA256Hex: emptySHA256Hex,
+            customHeader:     opts.headers,
+        }).await?;
+        defer closeResponse(resp)
+        if err != nil {
+          return ListVersionsResult{}, err
+        }
+        if resp != nil {
+          if resp.StatusCode != http.StatusOK {
+            return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
+          }
+        }
+
+        listObjectVersionsOutput := ListVersionsResult{}
+        err = xml_decoder(resp.Body, &listObjectVersionsOutput)
+        if err != nil {
+          return ListVersionsResult{}, err
+        }
+
+        for i, obj := range listObjectVersionsOutput.Versions {
+          listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        for i, obj := range listObjectVersionsOutput.CommonPrefixes {
+          listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        if listObjectVersionsOutput.NextKeyMarker != "" {
+          listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
+          if err != nil {
+            return listObjectVersionsOutput, err
+          }
+        }
+
+        Ok(listObjectVersionsOutput)*/
+        todo!();
+    }
+
+    pub fn list_objects_query(
+        &self,
+        bucket_name: &str,
+        object_prefix: &str,
+        object_marker: &str,
+        delimiter: &str,
+        max_keys: i64,
+        headers: HeaderMap,
+    ) -> Result<ListBucketResult, std::io::Error> {
+        todo!();
+    }
+
+    pub fn list_multipart_uploads_query(
+        &self,
+        bucket_name: &str,
+        key_marker: &str,
+        upload_id_marker: &str,
+        prefix: &str,
+        delimiter: &str,
+        max_uploads: i64,
+    ) -> Result<ListMultipartUploadsResult, std::io::Error> {
+        todo!();
+    }
+
+    pub fn list_object_parts(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+    ) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
+        todo!();
+    }
+
+    pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
+        todo!();
+    }
+
+    pub async fn list_object_parts_query(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+        part_number_marker: i64,
+        max_parts: i64,
+    ) -> Result<ListObjectPartsResult, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct ListObjectsOptions {
+    reverse_versions: bool,
+    with_versions: bool,
+    with_metadata: bool,
+    prefix: String,
+    recursive: bool,
+    max_keys: i64,
+    start_after: String,
+    use_v1: bool,
+    headers: HeaderMap,
+}
+
+impl ListObjectsOptions {
+    pub fn set(&mut self, key: &str, value: &str) {
+        todo!();
+    }
+}
+
+fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Error> {
+    match encoding_type {
+        "url" => {
+            //return url::QueryUnescape(name);
+            return Ok(name.to_string());
+        }
+        _ => {
+            return Ok(name.to_string());
+        }
+    }
+}

ecstore/src/client/api_put_object.rs

@@ -0,0 +1,423 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use http::{HeaderMap, HeaderName, HeaderValue};
+use std::{collections::HashMap, sync::Arc};
+use time::{Duration, OffsetDateTime, macros::format_description};
+use tracing::{error, info, warn};
+
+use reader::hasher::Hasher;
+use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
+use s3s::header::{
+    X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
+    X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
+};
+//use crate::disk::{BufferReader, Reader};
+use crate::checksum::ChecksumMode;
+use crate::client::{
+    api_error_response::{err_entity_too_large, err_invalid_argument},
+    api_put_object_common::optimal_part_info,
+    api_put_object_multipart::UploadPartParams,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
+    constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
+    credentials::SignatureType,
+    transition_api::{ReaderImpl, TransitionClient, UploadInfo},
+};
+use rustfs_utils::{
+    crypto::base64_encode,
+    net::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
+};
+
+#[derive(Debug, Clone)]
+pub struct AdvancedPutOptions {
+    pub source_version_id: String,
+    pub source_etag: String,
+    pub replication_status: ReplicationStatus,
+    pub source_mtime: OffsetDateTime,
+    pub replication_request: bool,
+    pub retention_timestamp: OffsetDateTime,
+    pub tagging_timestamp: OffsetDateTime,
+    pub legalhold_timestamp: OffsetDateTime,
+    pub replication_validity_check: bool,
+}
+
+impl Default for AdvancedPutOptions {
+    fn default() -> Self {
+        Self {
+            source_version_id: "".to_string(),
+            source_etag: "".to_string(),
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            source_mtime: OffsetDateTime::now_utc(),
+            replication_request: false,
+            retention_timestamp: OffsetDateTime::now_utc(),
+            tagging_timestamp: OffsetDateTime::now_utc(),
+            legalhold_timestamp: OffsetDateTime::now_utc(),
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PutObjectOptions {
+    pub user_metadata: HashMap<String, String>,
+    pub user_tags: HashMap<String, String>,
+    //pub progress: ReaderImpl,
+    pub content_type: String,
+    pub content_encoding: String,
+    pub content_disposition: String,
+    pub content_language: String,
+    pub cache_control: String,
+    pub expires: OffsetDateTime,
+    pub mode: ObjectLockRetentionMode,
+    pub retain_until_date: OffsetDateTime,
+    //pub server_side_encryption: encrypt.ServerSide,
+    pub num_threads: u64,
+    pub storage_class: String,
+    pub website_redirect_location: String,
+    pub part_size: u64,
+    pub legalhold: ObjectLockLegalHoldStatus,
+    pub send_content_md5: bool,
+    pub disable_content_sha256: bool,
+    pub disable_multipart: bool,
+    pub auto_checksum: ChecksumMode,
+    pub checksum: ChecksumMode,
+    pub concurrent_stream_parts: bool,
+    pub internal: AdvancedPutOptions,
+    pub custom_header: HeaderMap,
+}
+
+impl Default for PutObjectOptions {
+    fn default() -> Self {
+        Self {
+            user_metadata: HashMap::new(),
+            user_tags: HashMap::new(),
+            //progress: ReaderImpl::Body(Bytes::new()),
+            content_type: "".to_string(),
+            content_encoding: "".to_string(),
+            content_disposition: "".to_string(),
+            content_language: "".to_string(),
+            cache_control: "".to_string(),
+            expires: OffsetDateTime::UNIX_EPOCH,
+            mode: ObjectLockRetentionMode::from_static(""),
+            retain_until_date: OffsetDateTime::UNIX_EPOCH,
+            //server_side_encryption: encrypt.ServerSide::default(),
+            num_threads: 0,
+            storage_class: "".to_string(),
+            website_redirect_location: "".to_string(),
+            part_size: 0,
+            legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
+            send_content_md5: false,
+            disable_content_sha256: false,
+            disable_multipart: false,
+            auto_checksum: ChecksumMode::ChecksumNone,
+            checksum: ChecksumMode::ChecksumNone,
+            concurrent_stream_parts: false,
+            internal: AdvancedPutOptions::default(),
+            custom_header: HeaderMap::new(),
+        }
+    }
+}
+
+impl PutObjectOptions {
+    fn set_matche_tag(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header
+                .insert("If-Match", HeaderValue::from_str("*").expect("err"));
+        } else {
+            self.custom_header
+                .insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
+        }
+    }
+
+    fn set_matche_tag_except(&mut self, etag: &str) {
+        if etag == "*" {
+            self.custom_header
+                .insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
+        } else {
+            self.custom_header
+                .insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
+        }
+    }
+
+    pub fn header(&self) -> HeaderMap {
+        let mut header = HeaderMap::new();
+
+        let mut content_type = self.content_type.clone();
+        if content_type == "" {
+            content_type = "application/octet-stream".to_string();
+        }
+        header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
+
+        if self.content_encoding != "" {
+            header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
+        }
+        if self.content_disposition != "" {
+            header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
+        }
+        if self.content_language != "" {
+            header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
+        }
+        if self.cache_control != "" {
+            header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
+        }
+
+        if self.expires.unix_timestamp() != 0 {
+            header.insert(
+                "Expires",
+                HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
+            ); //rustfs invalid heade
+        }
+
+        if self.mode.as_str() != "" {
+            header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
+        }
+
+        if self.retain_until_date.unix_timestamp() != 0 {
+            header.insert(
+                X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
+                HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
+            );
+        }
+
+        if self.legalhold.as_str() != "" {
+            header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
+        }
+
+        if self.storage_class != "" {
+            header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
+        }
+
+        if self.website_redirect_location != "" {
+            header.insert(
+                X_AMZ_WEBSITE_REDIRECT_LOCATION,
+                HeaderValue::from_str(&self.website_redirect_location).expect("err"),
+            );
+        }
+
+        if !self.internal.replication_status.as_str().is_empty() {
+            header.insert(
+                X_AMZ_REPLICATION_STATUS,
+                HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
+            );
+        }
+
+        for (k, v) in &self.user_metadata {
+            if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k)
+            {
+                if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
+                    header.insert(header_name, HeaderValue::from_str(&v).unwrap());
+                }
+            } else {
+                if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) {
+                    header.insert(header_name, HeaderValue::from_str(&v).unwrap());
+                }
+            }
+        }
+
+        for (k, v) in self.custom_header.iter() {
+            header.insert(k.clone(), v.clone());
+        }
+
+        header
+    }
+
+    fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
+        //if self.checksum.is_set() {
+        /*if !self.trailing_header_support {
+            return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled")));
+        }*/
+        /*else if self.override_signer_type == SignatureType::SignatureV2 {
+            return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures")));
+        }*/
+        //}
+
+        Ok(())
+    }
+}
+
+impl TransitionClient {
+    pub async fn put_object(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        object_size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        if object_size < 0 && opts.disable_multipart {
+            return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
+        }
+
+        self.put_object_common(bucket_name, object_name, reader, object_size, opts)
+            .await
+    }
+
+    pub async fn put_object_common(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(
+                size,
+                MAX_MULTIPART_PUT_OBJECT_SIZE,
+                bucket_name,
+                object_name,
+            )));
+        }
+        let mut opts = opts.clone();
+        opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+
+        let mut part_size = opts.part_size as i64;
+        if opts.part_size == 0 {
+            part_size = MIN_PART_SIZE;
+        }
+
+        if SignatureType::SignatureV2 == self.override_signer_type {
+            if size >= 0 && size < part_size || opts.disable_multipart {
+                return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+            }
+            return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        if size < 0 {
+            if opts.disable_multipart {
+                return Err(std::io::Error::other("no length provided and multipart disabled"));
+            }
+            if opts.concurrent_stream_parts && opts.num_threads > 1 {
+                return self
+                    .put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts)
+                    .await;
+            }
+            return self
+                .put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts)
+                .await;
+        }
+
+        if size <= part_size || opts.disable_multipart {
+            return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
+        }
+
+        self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts)
+            .await
+    }
+
+    pub async fn put_object_multipart_stream_no_length(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?;
+
+        let mut opts = opts.clone();
+
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            //add_auto_checksum_headers(&mut opts);
+        }
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+
+        let mut custom_header = HeaderMap::new();
+
+        while part_number <= total_parts_count {
+            buf = match &mut reader {
+                ReaderImpl::Body(content_body) => content_body.to_vec(),
+                ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
+            };
+            let length = buf.len();
+
+            let mut md5_base64: String = "".to_string();
+            if opts.send_content_md5 {
+                let mut md5_hasher = self.md5_hasher.lock().unwrap();
+                let mut hash = md5_hasher.as_mut().expect("err");
+                hash.write(&buf[..length]);
+                md5_base64 = base64_encode(hash.sum().as_bytes());
+            } else {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap());
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = ReaderImpl::Body(Bytes::from(buf));
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: rd,
+                part_number,
+                md5_base64,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: Default::default(),
+                trailer: Default::default(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+}

ecstore/src/client/api_put_object_common.rs

@@ -0,0 +1,97 @@
+#![allow(clippy::map_entry)]
+use crate::client::{
+    api_error_response::{err_entity_too_large, err_invalid_argument},
+    api_put_object::PutObjectOptions,
+    constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
+    transition_api::ReaderImpl,
+    transition_api::TransitionClient,
+};
+
+const NULL_VERSION_ID: &str = "null";
+
+pub fn is_object(reader: &ReaderImpl) -> bool {
+    todo!();
+}
+
+pub fn is_read_at(reader: ReaderImpl) -> bool {
+    todo!();
+}
+
+pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
+    let unknown_size;
+    let mut object_size = object_size;
+    if object_size == -1 {
+        unknown_size = true;
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    } else {
+        unknown_size = false;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other(err_entity_too_large(
+            object_size,
+            MAX_MULTIPART_PUT_OBJECT_SIZE,
+            "",
+            "",
+        )));
+    }
+
+    let mut part_size_flt: f64;
+    if configured_part_size > 0 {
+        if configured_part_size as i64 > object_size {
+            return Err(std::io::Error::other(err_entity_too_large(
+                configured_part_size as i64,
+                object_size,
+                "",
+                "",
+            )));
+        }
+
+        if !unknown_size {
+            if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
+                return Err(std::io::Error::other(err_invalid_argument(
+                    "Part size * max_parts(10000) is lesser than input objectSize.",
+                )));
+            }
+        }
+
+        if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Input part size is smaller than allowed minimum of 5MiB.",
+            )));
+        }
+
+        if configured_part_size as i64 > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Input part size is bigger than allowed maximum of 5GiB.",
+            )));
+        }
+
+        part_size_flt = configured_part_size as f64;
+        if unknown_size {
+            object_size = configured_part_size as i64 * MAX_PARTS_COUNT;
+        }
+    } else {
+        let mut configured_part_size = configured_part_size;
+        configured_part_size = MIN_PART_SIZE as u64;
+        part_size_flt = (object_size / MAX_PARTS_COUNT) as f64;
+        part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64;
+    }
+
+    let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
+    let part_size = part_size_flt.ceil() as i64;
+    let last_part_size = object_size - (total_parts_count - 1) * part_size;
+    Ok((total_parts_count, part_size, last_part_size))
+}
+
+impl TransitionClient {
+    pub async fn new_upload_id(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &PutObjectOptions,
+    ) -> Result<String, std::io::Error> {
+        let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
+        Ok(init_multipart_upload_result.upload_id)
+    }
+}

ecstore/src/client/api_put_object_multipart.rs

@@ -0,0 +1,413 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use s3s::S3ErrorCode;
+use std::io::Read;
+use std::{collections::HashMap, sync::Arc};
+use time::{OffsetDateTime, format_description};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use tracing::{error, info};
+use url::form_urlencoded::Serializer;
+use uuid::Uuid;
+
+use reader::hasher::Hasher;
+use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
+use s3s::{Body, dto::StreamingBlob};
+//use crate::disk::{Reader, BufferReader};
+use crate::client::{
+    api_error_response::{
+        err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
+    },
+    api_put_object::PutObjectOptions,
+    api_put_object_common::optimal_part_info,
+    api_s3_datatypes::{
+        CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart,
+    },
+    constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
+};
+use crate::{
+    checksum::ChecksumMode,
+    disk::DiskAPI,
+    store_api::{GetObjectReader, StorageAPI},
+};
+use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+
+impl TransitionClient {
+    pub async fn put_object_multipart(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info = self
+            .put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts)
+            .await;
+        if let Err(err) = &info {
+            let err_resp = to_error_response(err);
+            if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
+                if size > MAX_SINGLE_PUT_OBJECT_SIZE {
+                    return Err(std::io::Error::other(err_entity_too_large(
+                        size,
+                        MAX_SINGLE_PUT_OBJECT_SIZE,
+                        bucket_name,
+                        object_name,
+                    )));
+                }
+                return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
+            }
+        }
+        Ok(info?)
+    }
+
+    pub async fn put_object_multipart_no_stream(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: &mut ReaderImpl,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut total_uploaded_size: i64 = 0;
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256);
+        let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?;
+        let mut opts = opts.clone();
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut part_number = 1;
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+        let mut custom_header = HeaderMap::new();
+        while part_number <= total_parts_count {
+            match reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            for (k, v) in hash_algos.iter_mut() {
+                v.write(&buf[..length]);
+                hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap());
+            }
+
+            //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
+            let rd = Bytes::from(buf.clone());
+
+            let mut md5_base64: String;
+            let mut sha256_hex: String;
+
+            //if hash_sums["md5"] != nil {
+            md5_base64 = base64_encode(&hash_sums["md5"]);
+            //}
+            //if hash_sums["sha256"] != nil {
+            sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower);
+            //}
+            if hash_sums.len() == 0 {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err"));
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: ReaderImpl::Body(rd),
+                part_number,
+                md5_base64,
+                sha256_hex,
+                size: length as i64,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.insert(part_number, obj_part);
+            total_uploaded_size += length as i64;
+            part_number += 1;
+        }
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        //apply_auto_checksum(&mut opts, all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn initiate_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: &PutObjectOptions,
+    ) -> Result<InitiateMultipartUploadResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploads".to_string(), "".to_string());
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone());
+        }
+
+        let mut custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values: url_values,
+            custom_header,
+            content_body: ReaderImpl::Body(Bytes::new()),
+            content_length: 0,
+            content_md5_base64: "".to_string(),
+            content_sha256_hex: "".to_string(),
+            stream_sha256: false,
+            trailer: HeaderMap::new(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+        //if resp.is_none() {
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+        }
+        //}
+        let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
+        Ok(initiate_multipart_upload_result)
+    }
+
+    pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
+        if p.size > MAX_PART_SIZE {
+            return Err(std::io::Error::other(err_entity_too_large(
+                p.size,
+                MAX_PART_SIZE,
+                &p.bucket_name,
+                &p.object_name,
+            )));
+        }
+        if p.size <= -1 {
+            return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
+        }
+        if p.part_number <= 0 {
+            return Err(std::io::Error::other(err_invalid_argument(
+                "Part number cannot be negative or equal to zero.",
+            )));
+        }
+        if p.upload_id == "" {
+            return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
+        }
+
+        let mut url_values = HashMap::new();
+        url_values.insert("partNumber".to_string(), p.part_number.to_string());
+        url_values.insert("uploadId".to_string(), p.upload_id.clone());
+
+        let buf = match &mut p.reader {
+            ReaderImpl::Body(content_body) => content_body.to_vec(),
+            ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
+        };
+        let mut req_metadata = RequestMetadata {
+            bucket_name: p.bucket_name.clone(),
+            object_name: p.object_name.clone(),
+            query_values: url_values,
+            custom_header: p.custom_header.clone(),
+            content_body: ReaderImpl::Body(Bytes::from(buf)),
+            content_length: p.size,
+            content_md5_base64: p.md5_base64.clone(),
+            content_sha256_hex: p.sha256_hex.clone(),
+            stream_sha256: p.stream_sha256,
+            trailer: p.trailer.clone(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+        //defer closeResponse(resp)
+        //if resp.is_none() {
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp,
+                vec![],
+                &p.bucket_name.clone(),
+                &p.object_name,
+            )));
+        }
+        //}
+        let h = resp.headers();
+        let mut obj_part = ObjectPart {
+            checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
+                h_checksum_crc32.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
+                h_checksum_crc32c.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
+                h_checksum_sha1.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
+                h_checksum_sha256.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
+                h_checksum_crc64nvme.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            ..Default::default()
+        };
+        obj_part.size = p.size;
+        obj_part.part_num = p.part_number;
+        obj_part.etag = if let Some(h_etag) = h.get("ETag") {
+            h_etag.to_str().expect("err").trim_matches('"').to_string()
+        } else {
+            "".to_string()
+        };
+        Ok(obj_part)
+    }
+
+    pub async fn complete_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+        complete: CompleteMultipartUpload,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+        let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec();
+
+        let mut headers = opts.header();
+
+        let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes);
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            query_values: url_values,
+            content_body: ReaderImpl::Body(complete_multipart_upload_buffer),
+            content_length: 100,                //complete_multipart_upload_bytes.len(),
+            content_sha256_hex: "".to_string(), //hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower),
+            custom_header: headers,
+            stream_sha256: Default::default(),
+            trailer: Default::default(),
+            content_md5_base64: "".to_string(),
+            pre_sign_url: Default::default(),
+            add_crc: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            bucket_location: Default::default(),
+            expires: Default::default(),
+        };
+
+        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+
+        let b = resp.body().bytes().expect("err").to_vec();
+        let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();
+
+        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
+            (
+                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
+                "".to_string(),
+            )
+        } else {
+            (OffsetDateTime::now_utc(), "".to_string())
+        };
+
+        let h = resp.headers();
+        Ok(UploadInfo {
+            bucket: complete_multipart_upload_result.bucket,
+            key: complete_multipart_upload_result.key,
+            etag: trim_etag(&complete_multipart_upload_result.etag),
+            version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
+                h_x_amz_version_id.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            location: complete_multipart_upload_result.location,
+            expiration: exp_time,
+            expiration_rule_id: rule_id,
+            checksum_sha256: complete_multipart_upload_result.checksum_sha256,
+            checksum_sha1: complete_multipart_upload_result.checksum_sha1,
+            checksum_crc32: complete_multipart_upload_result.checksum_crc32,
+            checksum_crc32c: complete_multipart_upload_result.checksum_crc32c,
+            checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme,
+            ..Default::default()
+        })
+    }
+}
+
+pub struct UploadPartParams {
+    pub bucket_name: String,
+    pub object_name: String,
+    pub upload_id: String,
+    pub reader: ReaderImpl,
+    pub part_number: i64,
+    pub md5_base64: String,
+    pub sha256_hex: String,
+    pub size: i64,
+    //pub sse: encrypt.ServerSide,
+    pub stream_sha256: bool,
+    pub custom_header: HeaderMap,
+    pub trailer: HeaderMap,
+}

ecstore/src/client/api_put_object_streaming.rs

@@ -0,0 +1,526 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use futures::future::join_all;
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use std::sync::RwLock;
+use std::{collections::HashMap, sync::Arc};
+use time::{OffsetDateTime, format_description};
+use tokio::{select, sync::mpsc};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use uuid::Uuid;
+
+use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
+use crate::client::{
+    api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
+    api_put_object::PutObjectOptions,
+    api_put_object_common::{is_object, optimal_part_info},
+    api_put_object_multipart::UploadPartParams,
+    api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
+    constants::ISO8601_DATEFORMAT,
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
+};
+use reader::hasher::Hasher;
+use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
+
+pub struct UploadedPartRes {
+    pub error: std::io::Error,
+    pub part_num: i64,
+    pub size: i64,
+    pub part: ObjectPart,
+}
+
+pub struct UploadPartReq {
+    pub part_num: i64,
+    pub part: ObjectPart,
+}
+
+impl TransitionClient {
+    pub async fn put_object_multipart_stream(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let info: UploadInfo;
+        if opts.concurrent_stream_parts && opts.num_threads > 1 {
+            info = self
+                .put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts)
+                .await?;
+        } else if !is_object(&reader) && !opts.send_content_md5 {
+            info = self
+                .put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts)
+                .await?;
+        } else {
+            info = self
+                .put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts)
+                .await?;
+        }
+
+        Ok(info)
+    }
+
+    pub async fn put_object_multipart_stream_from_readat(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, part_size, lastpart_size) = ret;
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        let with_checksum = self.trailing_header_support;
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        todo!();
+    }
+
+    pub async fn put_object_multipart_stream_optional_checksum(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.auto_checksum = opts.checksum.clone();
+            opts.send_content_md5 = false;
+        }
+
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(size, opts.part_size)?;
+        let (total_parts_count, mut part_size, lastpart_size) = ret;
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut custom_header = opts.header().clone();
+
+        let mut total_uploaded_size: i64 = 0;
+
+        let mut parts_info = HashMap::<i64, ObjectPart>::new();
+        let mut buf = Vec::<u8>::with_capacity(part_size as usize);
+
+        let mut md5_base64: String = "".to_string();
+        for part_number in 1..=total_parts_count {
+            if part_number == total_parts_count {
+                part_size = lastpart_size;
+            }
+
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            if opts.send_content_md5 {
+                let mut md5_hasher = self.md5_hasher.lock().unwrap();
+                let mut md5_hash = md5_hasher.as_mut().expect("err");
+                md5_hash.reset();
+                md5_hash.write(&buf[..length]);
+                md5_base64 = base64_encode(md5_hash.sum().as_bytes());
+            } else {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) {
+                    custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err"));
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized());
+                }
+            }
+
+            let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress);
+            let mut p = UploadPartParams {
+                bucket_name: bucket_name.to_string(),
+                object_name: object_name.to_string(),
+                upload_id: upload_id.clone(),
+                reader: hooked,
+                part_number,
+                md5_base64: md5_base64.clone(),
+                size: part_size,
+                //sse: opts.server_side_encryption,
+                stream_sha256: !opts.disable_content_sha256,
+                custom_header: custom_header.clone(),
+                sha256_hex: "".to_string(),
+                trailer: HeaderMap::new(),
+            };
+            let obj_part = self.upload_part(&mut p).await?;
+
+            parts_info.entry(part_number).or_insert(obj_part);
+
+            total_uploaded_size += part_size as i64;
+        }
+
+        if size > 0 {
+            if total_uploaded_size != size {
+                return Err(std::io::Error::other(err_unexpected_eof(
+                    total_uploaded_size,
+                    size,
+                    bucket_name,
+                    object_name,
+                )));
+            }
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
+        let part_number = total_parts_count;
+        for i in 1..part_number {
+            let part = parts_info[&i].clone();
+
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_multipart_stream_parallel(
+        self: Arc<Self>,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl, /*GetObjectReader*/
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+            opts.auto_checksum = opts.checksum.clone();
+        }
+        if !opts.send_content_md5 {
+            add_auto_checksum_headers(&mut opts);
+        }
+
+        let ret = optimal_part_info(-1, opts.part_size)?;
+        let (total_parts_count, part_size, _) = ret;
+
+        let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
+        opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
+
+        let mut total_uploaded_size: i64 = 0;
+        let mut parts_info = Arc::new(RwLock::new(HashMap::<i64, ObjectPart>::new()));
+
+        let n_buffers = opts.num_threads;
+        let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize);
+        //let all = Vec::<u8>::with_capacity(n_buffers as usize * part_size as usize);
+        for i in 0..n_buffers {
+            //bufs_tx.send(&all[i * part_size..i * part_size + part_size]);
+            bufs_tx.send(Vec::<u8>::with_capacity(part_size as usize));
+        }
+
+        let mut futures = Vec::with_capacity(total_parts_count as usize);
+        let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize);
+        let cancel_token = CancellationToken::new();
+
+        //reader = newHook(reader, opts.progress);
+
+        for part_number in 1..=total_parts_count {
+            let mut buf = Vec::<u8>::new();
+            select! {
+                buf = bufs_rx.recv() => {}
+                err = err_rx.recv() => {
+                    //cancel_token.cancel();
+                    //wg.Wait()
+                    return Err(err.expect("err"));
+                }
+                else => (),
+            }
+
+            if buf.len() != part_size as usize {
+                return Err(std::io::Error::other(format!(
+                    "read buffer < {} than expected partSize: {}",
+                    buf.len(),
+                    part_size
+                )));
+            }
+
+            match &mut reader {
+                ReaderImpl::Body(content_body) => {
+                    buf = content_body.to_vec();
+                }
+                ReaderImpl::ObjectBody(content_body) => {
+                    buf = content_body.read_all().await?;
+                }
+            }
+            let length = buf.len();
+
+            let mut custom_header = HeaderMap::new();
+            if !opts.send_content_md5 {
+                let csum;
+                {
+                    let mut crc = opts.auto_checksum.hasher()?;
+                    crc.reset();
+                    crc.write(&buf[..length]);
+                    csum = crc.sum();
+                }
+                if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
+                    if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) {
+                        custom_header.insert(header_name, header_value);
+                    }
+                } else {
+                    warn!("Invalid header name: {}", opts.auto_checksum.key());
+                }
+            }
+
+            let clone_bufs_tx = bufs_tx.clone();
+            let clone_parts_info = parts_info.clone();
+            let clone_upload_id = upload_id.clone();
+            let clone_self = self.clone();
+            futures.push(async move {
+                let mut md5_base64: String = "".to_string();
+
+                if opts.send_content_md5 {
+                    let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
+                    let mut md5_hash = md5_hasher.as_mut().expect("err");
+                    md5_hash.write(&buf[..length]);
+                    md5_base64 = base64_encode(md5_hash.sum().as_bytes());
+                }
+
+                //defer wg.Done()
+                let mut p = UploadPartParams {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    upload_id: clone_upload_id,
+                    reader: ReaderImpl::Body(Bytes::from(buf.clone())),
+                    part_number,
+                    md5_base64,
+                    size: length as i64,
+                    //sse:           opts.server_side_encryption,
+                    stream_sha256: !opts.disable_content_sha256,
+                    custom_header,
+                    sha256_hex: "".to_string(),
+                    trailer: HeaderMap::new(),
+                };
+                let obj_part = clone_self.upload_part(&mut p).await.expect("err");
+
+                let mut clone_parts_info = clone_parts_info.write().unwrap();
+                clone_parts_info.entry(part_number).or_insert(obj_part);
+
+                clone_bufs_tx.send(buf);
+            });
+
+            total_uploaded_size += length as i64;
+        }
+
+        let results = join_all(futures).await;
+
+        select! {
+            err = err_rx.recv() => {
+                return Err(err.expect("err"));
+            }
+            else => (),
+        }
+
+        let mut compl_multipart_upload = CompleteMultipartUpload::default();
+
+        let mut part_number: i64 = total_parts_count;
+        let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.read().unwrap().len());
+        for i in 1..part_number {
+            let part = parts_info.read().unwrap()[&i].clone();
+
+            all_parts.push(part.clone());
+            compl_multipart_upload.parts.push(CompletePart {
+                etag: part.etag,
+                part_num: part.part_num,
+                checksum_crc32: part.checksum_crc32,
+                checksum_crc32c: part.checksum_crc32c,
+                checksum_sha1: part.checksum_sha1,
+                checksum_sha256: part.checksum_sha256,
+                checksum_crc64nvme: part.checksum_crc64nvme,
+                ..Default::default()
+            });
+        }
+
+        compl_multipart_upload.parts.sort();
+
+        let mut opts = PutObjectOptions {
+            //server_side_encryption: opts.server_side_encryption,
+            auto_checksum: opts.auto_checksum,
+            ..Default::default()
+        };
+        apply_auto_checksum(&mut opts, &mut all_parts);
+
+        let mut upload_info = self
+            .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
+            .await?;
+
+        upload_info.size = total_uploaded_size;
+        Ok(upload_info)
+    }
+
+    pub async fn put_object_gcs(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        mut reader: ReaderImpl,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            opts.send_content_md5 = false;
+        }
+
+        let mut md5_base64: String = "".to_string();
+        let progress_reader = reader; //newHook(reader, opts.progress);
+
+        self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts)
+            .await
+    }
+
+    pub async fn put_object_do(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        reader: ReaderImpl,
+        md5_base64: &str,
+        sha256_hex: &str,
+        size: i64,
+        opts: &PutObjectOptions,
+    ) -> Result<UploadInfo, std::io::Error> {
+        let custom_header = opts.header();
+
+        let mut req_metadata = RequestMetadata {
+            bucket_name: bucket_name.to_string(),
+            object_name: object_name.to_string(),
+            custom_header,
+            content_body: reader,
+            content_length: size,
+            content_md5_base64: md5_base64.to_string(),
+            content_sha256_hex: sha256_hex.to_string(),
+            stream_sha256: !opts.disable_content_sha256,
+            add_crc: Default::default(),
+            bucket_location: Default::default(),
+            pre_sign_url: Default::default(),
+            query_values: Default::default(),
+            extra_pre_sign_header: Default::default(),
+            expires: Default::default(),
+            trailer: Default::default(),
+        };
+        let mut add_crc = false; //self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure);
+        let mut opts = opts.clone();
+        if opts.checksum.is_set() {
+            req_metadata.add_crc = opts.checksum;
+        } else if add_crc {
+            for (k, _) in opts.user_metadata {
+                if k.to_lowercase().starts_with("x-amz-checksum-") {
+                    add_crc = false;
+                }
+            }
+            if add_crc {
+                opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
+                req_metadata.add_crc = opts.auto_checksum;
+            }
+        }
+
+        if opts.internal.source_version_id != "" {
+            if !opts.internal.source_version_id.is_empty() {
+                if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
+                    return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
+                }
+            }
+            let mut url_values = HashMap::new();
+            url_values.insert("versionId".to_string(), opts.internal.source_version_id);
+            req_metadata.query_values = url_values;
+        }
+
+        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+
+        if resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+        }
+
+        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
+            (
+                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
+                "".to_string(),
+            )
+        } else {
+            (OffsetDateTime::now_utc(), "".to_string())
+        };
+        let h = resp.headers();
+        Ok(UploadInfo {
+            bucket: bucket_name.to_string(),
+            key: object_name.to_string(),
+            etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
+            version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
+                h_x_amz_version_id.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            size: size,
+            expiration: exp_time,
+            expiration_rule_id: rule_id,
+            checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
+                h_checksum_crc32.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
+                h_checksum_crc32c.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
+                h_checksum_sha1.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
+                h_checksum_sha256.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
+                h_checksum_crc64nvme.to_str().expect("err").to_string()
+            } else {
+                "".to_string()
+            },
+            ..Default::default()
+        })
+    }
+}

ecstore/src/client/api_remove.rs

@@ -0,0 +1,472 @@
+#![allow(clippy::map_entry)]
+use bytes::Bytes;
+use http::{HeaderMap, HeaderValue, Method, StatusCode};
+use s3s::S3ErrorCode;
+use s3s::dto::ReplicationStatus;
+use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
+use std::fmt::Display;
+use std::{collections::HashMap, sync::Arc};
+use time::OffsetDateTime;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+
+use crate::client::{
+    api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
+    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
+};
+use crate::{
+    disk::DiskAPI,
+    store_api::{GetObjectReader, ObjectInfo, StorageAPI},
+};
+use reader::hasher::{sum_md5_base64, sum_sha256_hex};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+struct RemoveBucketOptions {
+    forced_elete: bool,
+}
+
+#[derive(Debug)]
+pub struct AdvancedRemoveOptions {
+    replication_delete_marker: bool,
+    replication_status: ReplicationStatus,
+    replication_mtime: OffsetDateTime,
+    replication_request: bool,
+    replication_validity_check: bool,
+}
+
+impl Default for AdvancedRemoveOptions {
+    fn default() -> Self {
+        Self {
+            replication_delete_marker: false,
+            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
+            replication_mtime: OffsetDateTime::now_utc(),
+            replication_request: false,
+            replication_validity_check: false,
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectOptions {
+    pub force_delete: bool,
+    pub governance_bypass: bool,
+    pub version_id: String,
+    pub internal: AdvancedRemoveOptions,
+}
+
+impl TransitionClient {
+    pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> {
+        let mut headers = HeaderMap::new();
+        /*if opts.force_delete {
+            headers.insert(rustFSForceDelete, "true");
+        }*/
+
+        let resp = self
+            .execute_method(
+                Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: headers,
+                    object_name: "".to_string(),
+                    query_values: Default::default(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.delete(bucket_name);
+        }
+        Ok(())
+    }
+
+    pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: Default::default(),
+                    object_name: "".to_string(),
+                    query_values: Default::default(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.delete(bucket_name);
+        }
+
+        Ok(())
+    }
+
+    pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
+        let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err");
+        res.err
+    }
+
+    pub async fn remove_object_inner(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        opts: RemoveObjectOptions,
+    ) -> Result<RemoveObjectResult, std::io::Error> {
+        let mut url_values = HashMap::new();
+
+        if opts.version_id != "" {
+            url_values.insert("versionId".to_string(), opts.version_id.clone());
+        }
+
+        let mut headers = HeaderMap::new();
+
+        if opts.governance_bypass {
+            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
+        }
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    query_values: url_values,
+                    custom_header: headers,
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+
+        Ok(RemoveObjectResult {
+            object_name: object_name.to_string(),
+            object_version_id: opts.version_id,
+            delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
+            delete_marker_version_id: resp
+                .headers()
+                .get("x-amz-version-id")
+                .expect("err")
+                .to_str()
+                .expect("err")
+                .to_string(),
+            ..Default::default()
+        })
+    }
+
+    pub async fn remove_objects_with_result(
+        self: Arc<Self>,
+        bucket_name: &str,
+        objects_rx: Receiver<ObjectInfo>,
+        opts: RemoveObjectsOptions,
+    ) -> Receiver<RemoveObjectResult> {
+        let (result_tx, mut result_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        tokio::spawn(async move {
+            self_clone
+                .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
+                .await;
+        });
+        result_rx
+    }
+
+    pub async fn remove_objects(
+        self: Arc<Self>,
+        bucket_name: &str,
+        objects_rx: Receiver<ObjectInfo>,
+        opts: RemoveObjectsOptions,
+    ) -> Receiver<RemoveObjectError> {
+        let (error_tx, mut error_rx) = mpsc::channel(1);
+
+        let self_clone = Arc::clone(&self);
+        let bucket_name_owned = bucket_name.to_string();
+
+        let (result_tx, mut result_rx) = mpsc::channel(1);
+        tokio::spawn(async move {
+            self_clone
+                .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
+                .await;
+        });
+        tokio::spawn(async move {
+            while let Some(res) = result_rx.recv().await {
+                if res.err.is_none() {
+                    continue;
+                }
+                error_tx
+                    .send(RemoveObjectError {
+                        object_name: res.object_name,
+                        version_id: res.object_version_id,
+                        err: res.err,
+                        ..Default::default()
+                    })
+                    .await;
+            }
+        });
+
+        error_rx
+    }
+
+    pub async fn remove_objects_inner(
+        &self,
+        bucket_name: &str,
+        mut objects_rx: Receiver<ObjectInfo>,
+        result_tx: &Sender<RemoveObjectResult>,
+        opts: RemoveObjectsOptions,
+    ) -> Result<(), std::io::Error> {
+        let max_entries = 1000;
+        let mut finish = false;
+        let mut url_values = HashMap::new();
+        url_values.insert("delete".to_string(), "".to_string());
+
+        loop {
+            if finish {
+                break;
+            }
+            let mut count = 0;
+            let mut batch = Vec::<ObjectInfo>::new();
+
+            while let Some(object) = objects_rx.recv().await {
+                if has_invalid_xml_char(&object.name) {
+                    let remove_result = self
+                        .remove_object_inner(
+                            bucket_name,
+                            &object.name,
+                            RemoveObjectOptions {
+                                version_id: object.version_id.expect("err").to_string(),
+                                governance_bypass: opts.governance_bypass,
+                                ..Default::default()
+                            },
+                        )
+                        .await?;
+                    let remove_result_clone = remove_result.clone();
+                    if !remove_result.err.is_none() {
+                        match to_error_response(&remove_result.err.expect("err")).code {
+                            S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
+                                continue;
+                            }
+                            _ => (),
+                        }
+                        result_tx.send(remove_result_clone.clone()).await;
+                    }
+
+                    result_tx.send(remove_result_clone).await;
+                    continue;
+                }
+
+                batch.push(object);
+                count += 1;
+                if count >= max_entries {
+                    break;
+                }
+            }
+            if count == 0 {
+                break;
+            }
+            if count < max_entries {
+                finish = true;
+            }
+
+            let mut headers = HeaderMap::new();
+            if opts.governance_bypass {
+                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
+            }
+
+            let remove_bytes = generate_remove_multi_objects_request(&batch);
+            let resp = self
+                .execute_method(
+                    http::Method::POST,
+                    &mut RequestMetadata {
+                        bucket_name: bucket_name.to_string(),
+                        query_values: url_values.clone(),
+                        content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
+                        content_length: remove_bytes.len() as i64,
+                        content_md5_base64: sum_md5_base64(&remove_bytes),
+                        content_sha256_hex: sum_sha256_hex(&remove_bytes),
+                        custom_header: headers,
+                        object_name: "".to_string(),
+                        stream_sha256: false,
+                        trailer: HeaderMap::new(),
+                        pre_sign_url: Default::default(),
+                        add_crc: Default::default(),
+                        extra_pre_sign_header: Default::default(),
+                        bucket_location: Default::default(),
+                        expires: Default::default(),
+                    },
+                )
+                .await?;
+
+            let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
+            process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
+        }
+        Ok(())
+    }
+
+    pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> {
+        let upload_ids = self.find_upload_ids(bucket_name, object_name)?;
+        for upload_id in upload_ids {
+            self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn abort_multipart_upload(
+        &self,
+        bucket_name: &str,
+        object_name: &str,
+        upload_id: &str,
+    ) -> Result<(), std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("uploadId".to_string(), upload_id.to_string());
+
+        let resp = self
+            .execute_method(
+                http::Method::DELETE,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: object_name.to_string(),
+                    query_values: url_values,
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    custom_header: HeaderMap::new(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    content_md5_base64: "".to_string(),
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await?;
+        //if resp.is_some() {
+        if resp.status() != StatusCode::NO_CONTENT {
+            let error_response: ErrorResponse;
+            match resp.status() {
+                StatusCode::NOT_FOUND => {
+                    error_response = ErrorResponse {
+                        code: S3ErrorCode::NoSuchUpload,
+                        message: "The specified multipart upload does not exist.".to_string(),
+                        bucket_name: bucket_name.to_string(),
+                        key: object_name.to_string(),
+                        request_id: resp
+                            .headers()
+                            .get("x-amz-request-id")
+                            .expect("err")
+                            .to_str()
+                            .expect("err")
+                            .to_string(),
+                        host_id: resp
+                            .headers()
+                            .get("x-amz-id-2")
+                            .expect("err")
+                            .to_str()
+                            .expect("err")
+                            .to_string(),
+                        region: resp
+                            .headers()
+                            .get("x-amz-bucket-region")
+                            .expect("err")
+                            .to_str()
+                            .expect("err")
+                            .to_string(),
+                        ..Default::default()
+                    };
+                }
+                _ => {
+                    return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
+                }
+            }
+            return Err(std::io::Error::other(error_response));
+        }
+        //}
+        Ok(())
+    }
+}
+
+#[derive(Debug, Default)]
+struct RemoveObjectError {
+    object_name: String,
+    version_id: String,
+    err: Option<std::io::Error>,
+}
+
+impl Display for RemoveObjectError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if self.err.is_none() {
+            return write!(f, "unexpected remove object error result");
+        }
+        write!(f, "{}", self.err.as_ref().expect("err").to_string())
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct RemoveObjectResult {
+    pub object_name: String,
+    pub object_version_id: String,
+    pub delete_marker: bool,
+    pub delete_marker_version_id: String,
+    pub err: Option<std::io::Error>,
+}
+
+impl Clone for RemoveObjectResult {
+    fn clone(&self) -> Self {
+        Self {
+            object_name: self.object_name.clone(),
+            object_version_id: self.object_version_id.clone(),
+            delete_marker: self.delete_marker,
+            delete_marker_version_id: self.delete_marker_version_id.clone(),
+            err: None, //err
+        }
+    }
+}
+
+pub struct RemoveObjectsOptions {
+    pub governance_bypass: bool,
+}
+
+pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
+    todo!();
+}
+
+pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender<RemoveObjectResult>) {
+    todo!();
+}
+
+fn has_invalid_xml_char(str: &str) -> bool {
+    false
+}

ecstore/src/client/api_s3_datatypes.rs

@@ -0,0 +1,336 @@
+#![allow(clippy::map_entry)]
+use s3s::dto::Owner;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use time::OffsetDateTime;
+
+use crate::checksum::ChecksumMode;
+use crate::client::transition_api::ObjectMultipartInfo;
+use rustfs_utils::crypto::base64_decode;
+
+use super::transition_api;
+
+pub struct ListAllMyBucketsResult {
+    pub owner: Owner,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct CommonPrefix {
+    pub prefix: String,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct ListBucketV2Result {
+    pub common_prefixes: Vec<CommonPrefix>,
+    pub contents: Vec<transition_api::ObjectInfo>,
+    pub delimiter: String,
+    pub encoding_type: String,
+    pub is_truncated: bool,
+    pub max_keys: i64,
+    pub name: String,
+    pub next_continuation_token: String,
+    pub continuation_token: String,
+    pub prefix: String,
+    pub fetch_owner: String,
+    pub start_after: String,
+}
+
+pub struct Version {
+    etag: String,
+    is_latest: bool,
+    key: String,
+    last_modified: OffsetDateTime,
+    owner: Owner,
+    size: i64,
+    storage_class: String,
+    version_id: String,
+    user_metadata: HashMap<String, String>,
+    user_tags: HashMap<String, String>,
+    is_delete_marker: bool,
+}
+
+pub struct ListVersionsResult {
+    versions: Vec<Version>,
+    common_prefixes: Vec<CommonPrefix>,
+    name: String,
+    prefix: String,
+    delimiter: String,
+    max_keys: i64,
+    encoding_type: String,
+    is_truncated: bool,
+    key_marker: String,
+    version_id_marker: String,
+    next_key_marker: String,
+    next_version_id_marker: String,
+}
+
+impl ListVersionsResult {
+    fn unmarshal_xml() -> Result<(), std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct ListBucketResult {
+    common_prefixes: Vec<CommonPrefix>,
+    contents: Vec<transition_api::ObjectInfo>,
+    delimiter: String,
+    encoding_type: String,
+    is_truncated: bool,
+    marker: String,
+    max_keys: i64,
+    name: String,
+    next_marker: String,
+    prefix: String,
+}
+
+pub struct ListMultipartUploadsResult {
+    bucket: String,
+    key_marker: String,
+    upload_id_marker: String,
+    next_key_marker: String,
+    next_upload_id_marker: String,
+    encoding_type: String,
+    max_uploads: i64,
+    is_truncated: bool,
+    uploads: Vec<ObjectMultipartInfo>,
+    prefix: String,
+    delimiter: String,
+    common_prefixes: Vec<CommonPrefix>,
+}
+
+pub struct Initiator {
+    id: String,
+    display_name: String,
+}
+
+pub struct CopyObjectResult {
+    pub etag: String,
+    pub last_modified: OffsetDateTime,
+}
+
+#[derive(Debug, Clone)]
+pub struct ObjectPart {
+    pub etag: String,
+    pub part_num: i64,
+    pub last_modified: OffsetDateTime,
+    pub size: i64,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+impl Default for ObjectPart {
+    fn default() -> Self {
+        ObjectPart {
+            etag: Default::default(),
+            part_num: 0,
+            last_modified: OffsetDateTime::now_utc(),
+            size: 0,
+            checksum_crc32: Default::default(),
+            checksum_crc32c: Default::default(),
+            checksum_sha1: Default::default(),
+            checksum_sha256: Default::default(),
+            checksum_crc64nvme: Default::default(),
+        }
+    }
+}
+
+impl ObjectPart {
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn checksum_raw(&self, t: &ChecksumMode) -> Result<Vec<u8>, std::io::Error> {
+        let b = self.checksum(t);
+        if b == "" {
+            return Err(std::io::Error::other("no checksum set"));
+        }
+        let decoded = match base64_decode(b.as_bytes()) {
+            Ok(b) => b,
+            Err(e) => return Err(std::io::Error::other(e)),
+        };
+        if decoded.len() != t.raw_byte_len() as usize {
+            return Err(std::io::Error::other("checksum length mismatch"));
+        }
+        Ok(decoded)
+    }
+}
+
+pub struct ListObjectPartsResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+    pub initiator: Initiator,
+    pub owner: Owner,
+    pub storage_class: String,
+    pub part_number_marker: i32,
+    pub next_part_number_marker: i32,
+    pub max_parts: i32,
+    pub checksum_algorithm: String,
+    pub checksum_type: String,
+    pub is_truncated: bool,
+    pub object_parts: Vec<ObjectPart>,
+    pub encoding_type: String,
+}
+
+#[derive(Debug, Default)]
+pub struct InitiateMultipartUploadResult {
+    pub bucket: String,
+    pub key: String,
+    pub upload_id: String,
+}
+
+#[derive(Debug, Default)]
+pub struct CompleteMultipartUploadResult {
+    pub location: String,
+    pub bucket: String,
+    pub key: String,
+    pub etag: String,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
+pub struct CompletePart {
+    //api has
+    pub etag: String,
+    pub part_num: i64,
+    pub checksum_crc32: String,
+    pub checksum_crc32c: String,
+    pub checksum_sha1: String,
+    pub checksum_sha256: String,
+    pub checksum_crc64nvme: String,
+}
+
+impl CompletePart {
+    fn checksum(&self, t: &ChecksumMode) -> String {
+        match t {
+            ChecksumMode::ChecksumCRC32C => {
+                return self.checksum_crc32c.clone();
+            }
+            ChecksumMode::ChecksumCRC32 => {
+                return self.checksum_crc32.clone();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return self.checksum_sha1.clone();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return self.checksum_sha256.clone();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return self.checksum_crc64nvme.clone();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+}
+
+pub struct CopyObjectPartResult {
+    pub etag: String,
+    pub last_modified: OffsetDateTime,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+pub struct CompleteMultipartUpload {
+    pub parts: Vec<CompletePart>,
+}
+
+impl CompleteMultipartUpload {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match serde_xml_rs::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct CreateBucketConfiguration {
+    pub location: String,
+}
+
+#[derive(serde::Serialize)]
+pub struct DeleteObject {
+    //api has
+    pub key: String,
+    pub version_id: String,
+}
+
+pub struct DeletedObject {
+    //s3s has
+    pub key: String,
+    pub version_id: String,
+    pub deletemarker: bool,
+    pub deletemarker_version_id: String,
+}
+
+pub struct NonDeletedObject {
+    pub key: String,
+    pub code: String,
+    pub message: String,
+    pub version_id: String,
+}
+
+#[derive(serde::Serialize)]
+pub struct DeleteMultiObjects {
+    pub quiet: bool,
+    pub objects: Vec<DeleteObject>,
+}
+
+impl DeleteMultiObjects {
+    pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
+        //let buf = serde_json::to_string(self)?;
+        let buf = match serde_xml_rs::to_string(self) {
+            Ok(buf) => buf,
+            Err(e) => {
+                return Err(std::io::Error::other(e));
+            }
+        };
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
+        todo!();
+    }
+}
+
+pub struct DeleteMultiObjectsResult {
+    pub deleted_objects: Vec<DeletedObject>,
+    pub undeleted_objects: Vec<NonDeletedObject>,
+}

ecstore/src/client/bucket_cache.rs

@@ -0,0 +1,221 @@
+#![allow(clippy::map_entry)]
+use http::Request;
+use hyper::StatusCode;
+use hyper::body::Incoming;
+use std::{collections::HashMap, sync::Arc};
+use tracing::warn;
+use tracing::{debug, error, info};
+
+use crate::client::{
+    api_error_response::{http_resp_to_error_response, to_error_response},
+    transition_api::{Document, TransitionClient},
+};
+use crate::signer;
+use reader::hasher::{Hasher, Sha256};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use s3s::Body;
+use s3s::S3ErrorCode;
+
+use super::constants::UNSIGNED_PAYLOAD;
+use super::credentials::SignatureType;
+
+pub struct BucketLocationCache {
+    items: HashMap<String, String>,
+}
+
+impl BucketLocationCache {
+    pub fn new() -> BucketLocationCache {
+        BucketLocationCache { items: HashMap::new() }
+    }
+
+    pub fn get(&self, bucket_name: &str) -> Option<String> {
+        self.items.get(bucket_name).map(|s| s.clone())
+    }
+
+    pub fn set(&mut self, bucket_name: &str, location: &str) {
+        self.items.insert(bucket_name.to_string(), location.to_string());
+    }
+
+    pub fn delete(&mut self, bucket_name: &str) {
+        self.items.remove(bucket_name);
+    }
+}
+
+impl TransitionClient {
+    pub async fn get_bucket_location(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        Ok(self.get_bucket_location_inner(bucket_name).await?)
+    }
+
+    async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
+        if self.region != "" {
+            return Ok(self.region.clone());
+        }
+
+        let mut location;
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            let ret = bucket_loc_cache.get(bucket_name);
+            if let Some(location) = ret {
+                return Ok(location);
+            }
+            //location = ret?;
+        }
+
+        let req = self.get_bucket_location_request(bucket_name)?;
+
+        let mut resp = self.doit(req).await?;
+        location = process_bucket_location_response(resp, bucket_name).await?;
+        {
+            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
+            bucket_loc_cache.set(bucket_name, &location);
+        }
+        Ok(location)
+    }
+
+    fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<Body>, std::io::Error> {
+        let mut url_values = HashMap::new();
+        url_values.insert("location".to_string(), "".to_string());
+
+        let mut target_url = self.endpoint_url.clone();
+        let scheme = self.endpoint_url.scheme();
+        let h = target_url.host().expect("host is none.");
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let p = target_url.port().unwrap_or(default_port);
+
+        let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
+
+        let mut url_str: String = "".to_string();
+
+        if is_virtual_style {
+            url_str = scheme.to_string();
+            url_str.push_str("://");
+            url_str.push_str(bucket_name);
+            url_str.push_str(".");
+            url_str.push_str(target_url.host_str().expect("err"));
+            url_str.push_str("/?location");
+        } else {
+            let mut path = bucket_name.to_string();
+            path.push_str("/");
+            target_url.set_path(&path);
+            {
+                let mut q = target_url.query_pairs_mut();
+                for (k, v) in url_values {
+                    q.append_pair(&k, &urlencoding::encode(&v));
+                }
+            }
+            url_str = target_url.to_string();
+        }
+
+        let mut req_builder = Request::builder().method(http::Method::GET).uri(url_str);
+
+        self.set_user_agent(&mut req_builder);
+
+        let value;
+        {
+            let mut creds_provider = self.creds_provider.lock().unwrap();
+            value = match creds_provider.get_with_context(Some(self.cred_context())) {
+                Ok(v) => v,
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        let mut signer_type = value.signer_type.clone();
+        let mut access_key_id = value.access_key_id;
+        let mut secret_access_key = value.secret_access_key;
+        let mut session_token = value.session_token;
+
+        if self.override_signer_type != SignatureType::SignatureDefault {
+            signer_type = self.override_signer_type.clone();
+        }
+
+        if value.signer_type == SignatureType::SignatureAnonymous {
+            signer_type = SignatureType::SignatureAnonymous
+        }
+
+        if signer_type == SignatureType::SignatureAnonymous {
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => return Ok(req),
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        if signer_type == SignatureType::SignatureV2 {
+            let req_builder = signer::sign_v2(req_builder, 0, &access_key_id, &secret_access_key, is_virtual_style);
+            let req = match req_builder.body(Body::empty()) {
+                Ok(req) => return Ok(req),
+                Err(err) => {
+                    return Err(std::io::Error::other(err));
+                }
+            };
+        }
+
+        let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string();
+        if self.secure {
+            content_sha256 = UNSIGNED_PAYLOAD.to_string();
+        }
+
+        req_builder
+            .headers_mut()
+            .expect("err")
+            .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
+        let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
+        let req = match req_builder.body(Body::empty()) {
+            Ok(req) => return Ok(req),
+            Err(err) => {
+                return Err(std::io::Error::other(err));
+            }
+        };
+    }
+}
+
+async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
+    //if resp != nil {
+    if resp.status() != StatusCode::OK {
+        let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, "");
+        match err_resp.code {
+                S3ErrorCode::NotImplemented => {
+                    match err_resp.server.as_str() {
+                        "AmazonSnowball" => {
+                            return Ok("snowball".to_string());
+                        }
+                        "cloudflare" => {
+                            return Ok("us-east-1".to_string());
+                        }
+                        _ => {
+                            return Err(std::io::Error::other(err_resp));
+                        }
+                    }
+                }
+                S3ErrorCode::AuthorizationHeaderMalformed |
+                //S3ErrorCode::InvalidRegion |
+                S3ErrorCode::AccessDenied => {
+                    if err_resp.region == "" {
+                      return Ok("us-east-1".to_string());
+                    }
+                    return Ok(err_resp.region);
+                }
+                _ => {
+                    return Err(std::io::Error::other(err_resp));
+                }
+            }
+    }
+    //}
+
+    let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
+    let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
+
+    let mut location = location_constraint;
+    if location == "" {
+        location = "us-east-1".to_string();
+    }
+
+    if location == "EU" {
+        location = "eu-west-1".to_string();
+    }
+
+    Ok(location)
+}

ecstore/src/client/constants.rs

@@ -0,0 +1,25 @@
+#![allow(clippy::map_entry)]
+use lazy_static::lazy_static;
+use std::{collections::HashMap, sync::Arc};
+use time::{format_description::FormatItem, macros::format_description};
+
+pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
+pub const MAX_PARTS_COUNT: i64 = 10000;
+pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16;
+
+pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+
+pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
+pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
+
+pub const TOTAL_WORKERS: i64 = 4;
+
+pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
+pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
+    format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
+
+const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
+const GetObjectAttributesMaxParts: i64 = 1000;
+const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFs-Source-Mtime";

ecstore/src/client/credentials.rs

@@ -0,0 +1,165 @@
+use std::fmt::{Display, Formatter};
+
+use time::OffsetDateTime;
+
+#[derive(Debug, Default, Clone, Eq, PartialEq)]
+pub enum SignatureType {
+    #[default]
+    SignatureDefault,
+    SignatureV4,
+    SignatureV2,
+    SignatureV4Streaming,
+    SignatureAnonymous,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Credentials<P: Provider + Default> {
+    creds: Value,
+    force_refresh: bool,
+    provider: P,
+}
+
+impl<P: Provider + Default> Credentials<P> {
+    pub fn new(provider: P) -> Self {
+        Self {
+            provider: provider,
+            force_refresh: true,
+            ..Default::default()
+        }
+    }
+
+    pub fn get(&mut self) -> Result<Value, std::io::Error> {
+        self.get_with_context(None)
+    }
+
+    pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
+        if self.is_expired() {
+            let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
+            self.creds = creds;
+            self.force_refresh = false;
+        }
+
+        Ok(self.creds.clone())
+    }
+
+    fn expire(&mut self) {
+        self.force_refresh = true;
+    }
+
+    pub fn is_expired(&self) -> bool {
+        self.force_refresh || self.provider.is_expired()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Value {
+    pub access_key_id: String,
+    pub secret_access_key: String,
+    pub session_token: String,
+    pub expiration: OffsetDateTime,
+    pub signer_type: SignatureType,
+}
+
+impl Default for Value {
+    fn default() -> Self {
+        Self {
+            access_key_id: "".to_string(),
+            secret_access_key: "".to_string(),
+            session_token: "".to_string(),
+            expiration: OffsetDateTime::now_utc(),
+            signer_type: SignatureType::SignatureDefault,
+        }
+    }
+}
+
+pub struct CredContext {
+    //pub client: SendRequest,
+    pub endpoint: String,
+}
+
+trait Provider {
+    fn retrieve(&self) -> Value;
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value;
+    fn is_expired(&self) -> bool;
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Static(pub Value);
+
+impl Provider for Static {
+    fn retrieve(&self) -> Value {
+        if self.0.access_key_id == "" || self.0.secret_access_key == "" {
+            return Value {
+                signer_type: SignatureType::SignatureAnonymous,
+                ..Default::default()
+            };
+        }
+        self.0.clone()
+    }
+
+    fn retrieve_with_cred_context(&self, _: CredContext) -> Value {
+        self.retrieve()
+    }
+
+    fn is_expired(&self) -> bool {
+        false
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct STSError {
+    pub r#type: String,
+    pub code: String,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, thiserror::Error)]
+pub struct ErrorResponse {
+    pub sts_error: STSError,
+    pub request_id: String,
+}
+
+impl Display for ErrorResponse {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.error())
+    }
+}
+
+impl ErrorResponse {
+    fn error(&self) -> String {
+        if self.sts_error.message == "" {
+            return format!("Error response code {}.", self.sts_error.code);
+        }
+        return self.sts_error.message.clone();
+    }
+}
+
+struct Error {
+    code: String,
+    message: String,
+    bucket_name: String,
+    key: String,
+    resource: String,
+    request_id: String,
+    host_id: String,
+    region: String,
+    server: String,
+    status_code: i64,
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        if self.message == "" {
+            return write!(f, "{}", format!("Error response code {}.", self.code));
+        }
+        write!(f, "{}", self.message)
+    }
+}
+
+pub fn xml_decoder<T>(body: &[u8]) -> Result<T, std::io::Error> {
+    todo!();
+}
+
+pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error> {
+    todo!();
+}

ecstore/src/client/hook_reader.rs

@@ -0,0 +1,45 @@
+#![allow(clippy::map_entry)]
+use std::{collections::HashMap, sync::Arc};
+
+use crate::{
+    disk::{
+        error::{is_unformatted_disk, DiskError},
+        format::{DistributionAlgoVersion, FormatV3},
+        new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
+    },
+    store_api::{
+        BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
+        ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult,
+        ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
+    },
+    credentials::{Credentials, SignatureType,},
+    api_put_object_multipart::UploadPartParams,
+};
+
+use http::HeaderMap;
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+use tracing::{error, info};
+use url::Url;
+
+struct HookReader {
+    source: GetObjectReader,
+    hook:   GetObjectReader,
+}
+
+impl HookReader {
+    pub fn new(source: GetObjectReader, hook: GetObjectReader) -> HookReader {
+        HookReader {
+            source,
+            hook,
+        }
+    }
+
+    fn seek(&self, offset: i64, whence: i64) -> Result<i64> {
+        todo!();
+    }
+
+    fn read(&self, b: &[u8]) -> Result<i64> {
+        todo!();
+    }
+}

ecstore/src/client/mod.rs

@@ -0,0 +1,18 @@
+pub mod admin_handler_utils;
+pub mod api_bucket_policy;
+pub mod api_error_response;
+pub mod api_get_object;
+pub mod api_get_options;
+pub mod api_list;
+pub mod api_put_object;
+pub mod api_put_object_common;
+pub mod api_put_object_multipart;
+pub mod api_put_object_streaming;
+pub mod api_remove;
+pub mod api_s3_datatypes;
+pub mod bucket_cache;
+pub mod constants;
+pub mod credentials;
+pub mod object_api_utils;
+pub mod object_handlers_common;
+pub mod transition_api;

ecstore/src/client/object_api_utils.rs

@@ -0,0 +1,141 @@
+#![allow(clippy::map_entry)]
+use http::HeaderMap;
+use std::io::Cursor;
+use std::{collections::HashMap, sync::Arc};
+use tokio::io::BufReader;
+
+use crate::error::ErrorResponse;
+use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions};
+use rustfs_filemeta::fileinfo::ObjectPartInfo;
+use rustfs_rio::HashReader;
+use s3s::S3ErrorCode;
+
+//#[derive(Clone)]
+pub struct PutObjReader {
+    pub reader: HashReader,
+    pub raw_reader: HashReader,
+    //pub sealMD5Fn: SealMD5CurrFn,
+}
+
+impl PutObjReader {
+    pub fn new(raw_reader: HashReader) -> Self {
+        todo!();
+    }
+
+    fn size(&self) -> usize {
+        //self.reader.size()
+        todo!();
+    }
+
+    fn md5_current_hex_string(&self) -> String {
+        todo!();
+    }
+
+    fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
+        self.reader = enc_reader;
+
+        Ok(())
+    }
+}
+
+pub type ObjReaderFn = Arc<dyn Fn(BufReader<Cursor<Vec<u8>>>, HeaderMap) -> GetObjectReader + 'static>;
+
+fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRangeSpec> {
+    if oi.size == 0 || oi.parts.len() == 0 {
+        return None;
+    }
+
+    let mut start: i64 = 0;
+    let mut end: i64 = -1;
+    let mut i = 0;
+    while i < oi.parts.len() && i < part_number {
+        start = end + 1;
+        end = start + oi.parts[i].actual_size as i64 - 1;
+        i += 1;
+    }
+
+    Some(HTTPRangeSpec {
+        start: start as usize,
+        end: Some(end as usize),
+        is_suffix_length: false,
+    })
+}
+
+fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
+    let mut skip_length: i64 = 0;
+    let mut cumulative_actual_size: i64 = 0;
+    let mut first_part_idx: i64 = 0;
+    let mut compressed_offset: i64 = 0;
+    let mut part_skip: i64 = 0;
+    let mut decrypt_skip: i64 = 0;
+    let mut seq_num: u64 = 0;
+    for (i, part) in oi.parts.iter().enumerate() {
+        cumulative_actual_size += part.actual_size as i64;
+        if cumulative_actual_size <= offset {
+            compressed_offset += part.size as i64;
+        } else {
+            first_part_idx = i as i64;
+            skip_length = cumulative_actual_size - part.actual_size as i64;
+            break;
+        }
+    }
+    skip_length = offset - skip_length;
+
+    let parts: &[ObjectPartInfo] = &oi.parts;
+    if skip_length > 0
+        && parts.len() > first_part_idx as usize
+        && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0
+    {
+        todo!();
+    }
+
+    (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
+}
+
+pub fn new_getobjectreader(
+    rs: HTTPRangeSpec,
+    oi: &ObjectInfo,
+    opts: &ObjectOptions,
+    h: &HeaderMap,
+) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
+    //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
+    let mut is_encrypted = false;
+    let is_compressed = false; //oi.is_compressed_ok();
+
+    let mut get_fn: ObjReaderFn;
+
+    let (off, length) = match rs.get_offset_length(oi.size) {
+        Ok(x) => x,
+        Err(err) => {
+            return Err(ErrorResponse {
+                code: S3ErrorCode::InvalidRange,
+                message: err.to_string(),
+                key: None,
+                bucket_name: None,
+                region: None,
+                request_id: None,
+                host_id: "".to_string(),
+            });
+        }
+    };
+    get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
+        //Box::pin({
+        /*let r = GetObjectReader {
+            object_info: oi.clone(),
+            stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
+        };
+        r*/
+        todo!();
+        //})
+    });
+
+    Ok((get_fn, off as i64, length as i64))
+}
+
+pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
+    if let Some(etag) = metadata.get("etag") {
+        etag.clone()
+    } else {
+        metadata["md5Sum"].clone()
+    }
+}

ecstore/src/client/object_handlers_common.rs

@@ -0,0 +1,33 @@
+use crate::StorageAPI;
+use crate::bucket::lifecycle::lifecycle;
+use crate::bucket::versioning::VersioningApi;
+use crate::bucket::versioning_sys::BucketVersioningSys;
+use crate::store::ECStore;
+use crate::store_api::{ObjectOptions, ObjectToDelete};
+use lock::local_locker::MAX_DELETE_LIST;
+
+pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) {
+    let mut remaining = to_del;
+    loop {
+        if remaining.len() <= 0 {
+            break;
+        };
+        let mut to_del = remaining;
+        if to_del.len() > MAX_DELETE_LIST {
+            remaining = &to_del[MAX_DELETE_LIST..];
+            to_del = &to_del[..MAX_DELETE_LIST];
+        } else {
+            remaining = &[];
+        }
+        let vc = BucketVersioningSys::get(bucket).await.expect("err!");
+        let deleted_objs = api.delete_objects(
+            bucket,
+            to_del.to_vec(),
+            ObjectOptions {
+                //prefix_enabled_fn:  vc.prefix_enabled(""),
+                version_suspended: vc.suspended(),
+                ..Default::default()
+            },
+        );
+    }
+}

ecstore/src/cmd/bucket_replication.rs

@@ -56,6 +56,7 @@ use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh3::xxh3_64;
 // use bucket_targets::{self, GLOBAL_Bucket_Target_Sys};
+use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject;
 
 #[derive(Serialize, Deserialize, Debug)]
 struct MRFReplicateEntry {
@@ -2013,6 +2014,7 @@ impl ReplicateObjectInfo {
             data_blocks: 0,
             version_id: Uuid::try_parse(&self.version_id).ok(),
             delete_marker: self.delete_marker,
+            transitioned_object: TransitionedObject::default(),
             user_tags: self.user_tags.clone(),
             parts: Vec::new(),
             is_latest: true,

ecstore/src/disk/local.rs

@@ -2149,7 +2149,7 @@ impl DiskAPI for LocalDisk {
         // Check if the current bucket has a configured lifecycle policy
         if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await {
             if lc_has_active_rules(&lc, "") {
-                cache.info.life_cycle = Some(lc);
+                cache.info.lifecycle = Some(lc);
             }
         }
 
@@ -2258,9 +2258,9 @@ impl DiskAPI for LocalDisk {
                         }
                     }
 
-                    for frer_version in fivs.free_versions.iter() {
+                    for free_version in fivs.free_versions.iter() {
                         let _obj_info = ObjectInfo::from_file_info(
-                            frer_version,
+                            free_version,
                             &item.bucket,
                             &item.object_path().to_string_lossy(),
                             versioned,

ecstore/src/error.rs

@@ -1,3 +1,5 @@
+use s3s::{S3Error, S3ErrorCode};
+
 use rustfs_utils::path::decode_dir_object;
 
 use crate::disk::error::DiskError;
@@ -724,6 +726,254 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
     }
 }
 
+pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool {
+    err.contains("Connection closed by foreign host")
+        || err.contains("TLS handshake timeout")
+        || err.contains("i/o timeout")
+        || err.contains("connection timed out")
+        || err.contains("connection reset by peer")
+        || err.contains("broken pipe")
+        || err.to_lowercase().contains("503 service unavailable")
+        || err.contains("use of closed network connection")
+        || err.contains("An existing connection was forcibly closed by the remote host")
+        || err.contains("client error (Connect)")
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct GenericError {
+    pub bucket: String,
+    pub object: String,
+    pub version_id: String,
+    //pub err:       Error,
+}
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum ObjectApiError {
+    #[error("Operation timed out")]
+    OperationTimedOut,
+
+    #[error("etag of the object has changed")]
+    InvalidETag,
+
+    #[error("BackendDown")]
+    BackendDown(String),
+
+    #[error("Unsupported headers in Metadata")]
+    UnsupportedMetadata,
+
+    #[error("Method not allowed: {}/{}", .0.bucket, .0.object)]
+    MethodNotAllowed(GenericError),
+
+    #[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)]
+    InvalidObjectState(GenericError),
+}
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+#[error("{}", .message)]
+pub struct ErrorResponse {
+    pub code: S3ErrorCode,
+    pub message: String,
+    pub key: Option<String>,
+    pub bucket_name: Option<String>,
+    pub region: Option<String>,
+    pub request_id: Option<String>,
+    pub host_id: String,
+}
+
+pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::io::Error {
+    let mut bucket = "";
+    let mut object = "";
+    let mut version_id = "";
+    if params.len() >= 1 {
+        bucket = params[0];
+    }
+    if params.len() >= 2 {
+        object = params[1];
+    }
+    if params.len() >= 3 {
+        version_id = params[2];
+    }
+
+    if is_network_or_host_down(&err.to_string(), false) {
+        return std::io::Error::other(ObjectApiError::BackendDown(format!("{}", err)));
+    }
+
+    let mut err_ = std::io::Error::other(err.to_string());
+    let r_err = err;
+    let mut err = err_;
+    let bucket = bucket.to_string();
+    let object = object.to_string();
+    let version_id = version_id.to_string();
+
+    match r_err.code {
+        /*S3ErrorCode::SlowDownWrite => {
+            err = StorageError::InsufficientWriteQuorum;//{bucket: bucket, object: object};
+        }*/
+        /*S3ErrorCode::SlowDown/* SlowDownRead */ => {
+            err = std::io::Error::other(StorageError::InsufficientReadQuorum.to_string());
+        }*/
+        /*S3ErrorCode::PreconditionFailed => {
+            err = Error::from(StorageError::PreConditionFailed);
+        }*/
+        /*S3ErrorCode::InvalidRange => {
+            err = Error::from(StorageError::InvalidRange);
+        }*/
+        /*S3ErrorCode::BucketAlreadyOwnedByYou => {
+            err = Error::from(StorageError::BucketAlreadyOwnedByYou);
+        }*/
+        S3ErrorCode::BucketNotEmpty => {
+            err = std::io::Error::other(StorageError::BucketNotEmpty("".to_string()).to_string());
+        }
+        /*S3ErrorCode::NoSuchBucketPolicy => {
+            err = Error::from(StorageError::BucketPolicyNotFound);
+        }*/
+        /*S3ErrorCode::NoSuchLifecycleConfiguration => {
+            err = Error::from(StorageError::BucketLifecycleNotFound);
+        }*/
+        S3ErrorCode::InvalidBucketName => {
+            err = std::io::Error::other(StorageError::BucketNameInvalid(bucket));
+        }
+        S3ErrorCode::InvalidPart => {
+            err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object /* , version_id */));
+        }
+        S3ErrorCode::NoSuchBucket => {
+            err = std::io::Error::other(StorageError::BucketNotFound(bucket));
+        }
+        S3ErrorCode::NoSuchKey => {
+            if object != "" {
+                err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object));
+            } else {
+                err = std::io::Error::other(StorageError::BucketNotFound(bucket));
+            }
+        }
+        S3ErrorCode::NoSuchVersion => {
+            if object != "" {
+                err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); //, version_id);
+            } else {
+                err = std::io::Error::other(StorageError::BucketNotFound(bucket));
+            }
+        }
+        /*S3ErrorCode::XRustFsInvalidObjectName => {
+            err = Error::from(StorageError::ObjectNameInvalid(bucket, object));
+        }*/
+        S3ErrorCode::AccessDenied => {
+            err = std::io::Error::other(StorageError::PrefixAccessDenied(bucket, object));
+        }
+        /*S3ErrorCode::XAmzContentSHA256Mismatch => {
+            err = hash.SHA256Mismatch{};
+        }*/
+        S3ErrorCode::NoSuchUpload => {
+            err = std::io::Error::other(StorageError::InvalidUploadID(bucket, object, version_id));
+        }
+        /*S3ErrorCode::EntityTooSmall => {
+            err = std::io::Error::other(StorageError::PartTooSmall);
+        }*/
+        /*S3ErrorCode::ReplicationPermissionCheck => {
+            err = std::io::Error::other(StorageError::ReplicationPermissionCheck);
+        }*/
+        _ => {
+            err = std::io::Error::other("err");
+        }
+    }
+
+    err
+}
+
+pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error {
+    let storage_err = &err;
+    let mut bucket: String = "".to_string();
+    let mut object: String = "".to_string();
+    if params.len() >= 1 {
+        bucket = params[0].to_string();
+    }
+    if params.len() >= 2 {
+        object = decode_dir_object(params[1]);
+    }
+    return match storage_err {
+        /*StorageError::NotImplemented => s3_error!(NotImplemented),
+        StorageError::InvalidArgument(bucket, object, version_id) => {
+            s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id)
+        }*/
+        StorageError::MethodNotAllowed => S3Error::with_message(
+            S3ErrorCode::MethodNotAllowed,
+            ObjectApiError::MethodNotAllowed(GenericError {
+                bucket: bucket,
+                object: object,
+                ..Default::default()
+            })
+            .to_string(),
+        ),
+        /*StorageError::BucketNotFound(bucket) => {
+            s3_error!(NoSuchBucket, "bucket not found {}", bucket)
+        }
+        StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket),
+        StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket),
+        StorageError::ObjectNameInvalid(bucket, object) => {
+            s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object)
+        }
+        StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket),
+        StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."),
+        StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"),
+        StorageError::PrefixAccessDenied(bucket, object) => {
+            s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object)
+        }
+        StorageError::InvalidUploadIDKeyCombination(bucket, object) => {
+            s3_error!(InvalidArgument, "Invalid UploadID KeyCombination:  {}/{}", bucket, object)
+        }
+        StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket),
+        StorageError::ObjectNameTooLong(bucket, object) => {
+            s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object)
+        }
+        StorageError::ObjectNamePrefixAsSlash(bucket, object) => {
+            s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object)
+        }
+        StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object),
+        StorageError::VersionNotFound(bucket, object, version_id) => {
+            s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id)
+        }
+        StorageError::InvalidUploadID(bucket, object, version_id) => {
+            s3_error!(InvalidPart, "Invalid upload id:  {}/{}-{}", bucket, object, version_id)
+        }
+        StorageError::InvalidVersionID(bucket, object, version_id) => {
+            s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id)
+        }
+        // extended
+        StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!(
+            InvalidArgument,
+            "invalid data movement operation, source and destination pool are the same for : {}/{}-{}",
+            bucket,
+            object,
+            version_id
+        ),
+        // extended
+        StorageError::ObjectExistsAsDirectory(bucket, object) => {
+            s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object)
+        }
+        StorageError::InsufficientReadQuorum => {
+            s3_error!(SlowDown, "Storage resources are insufficient for the read operation")
+        }
+        StorageError::InsufficientWriteQuorum => {
+            s3_error!(SlowDown, "Storage resources are insufficient for the write operation")
+        }
+        StorageError::DecommissionNotStarted => s3_error!(InvalidArgument, "Decommission Not Started"),
+
+        StorageError::VolumeNotFound(bucket) => {
+            s3_error!(NoSuchBucket, "bucket not found {}", bucket)
+        }
+        StorageError::InvalidPart(bucket, object, version_id) => {
+            s3_error!(
+                InvalidPart,
+                "Specified part could not be found. PartNumber {}, Expected {}, got {}",
+                bucket,
+                object,
+                version_id
+            )
+        }
+        StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),*/
+        _ => s3s::S3Error::with_message(S3ErrorCode::Custom("err".into()), err.to_string()),
+    };
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

ecstore/src/event/mod.rs

@@ -0,0 +1,3 @@
+pub mod name;
+pub mod targetid;
+pub mod targetlist;

ecstore/src/event/name.rs

@@ -0,0 +1,147 @@
+#[derive(Default)]
+pub enum EventName {
+    ObjectAccessedGet,
+    ObjectAccessedGetRetention,
+    ObjectAccessedGetLegalHold,
+    ObjectAccessedHead,
+    ObjectAccessedAttributes,
+    ObjectCreatedCompleteMultipartUpload,
+    ObjectCreatedCopy,
+    ObjectCreatedPost,
+    ObjectCreatedPut,
+    ObjectCreatedPutRetention,
+    ObjectCreatedPutLegalHold,
+    ObjectCreatedPutTagging,
+    ObjectCreatedDeleteTagging,
+    ObjectRemovedDelete,
+    ObjectRemovedDeleteMarkerCreated,
+    ObjectRemovedDeleteAllVersions,
+    ObjectRemovedNoOP,
+    BucketCreated,
+    BucketRemoved,
+    ObjectReplicationFailed,
+    ObjectReplicationComplete,
+    ObjectReplicationMissedThreshold,
+    ObjectReplicationReplicatedAfterThreshold,
+    ObjectReplicationNotTracked,
+    ObjectRestorePost,
+    ObjectRestoreCompleted,
+    ObjectTransitionFailed,
+    ObjectTransitionComplete,
+    ObjectManyVersions,
+    ObjectLargeVersions,
+    PrefixManyFolders,
+    ILMDelMarkerExpirationDelete,
+    objectSingleTypesEnd,
+    ObjectAccessedAll,
+    ObjectCreatedAll,
+    ObjectRemovedAll,
+    ObjectReplicationAll,
+    ObjectRestoreAll,
+    ObjectTransitionAll,
+    ObjectScannerAll,
+    #[default]
+    Everything,
+}
+
+impl EventName {
+    fn expand(&self) -> Vec<EventName> {
+        todo!();
+    }
+
+    fn mask(&self) -> u64 {
+        todo!();
+    }
+}
+
+impl AsRef<str> for EventName {
+    fn as_ref(&self) -> &str {
+        match self {
+            EventName::BucketCreated => "s3:BucketCreated:*",
+            EventName::BucketRemoved => "s3:BucketRemoved:*",
+            EventName::ObjectAccessedAll => "s3:ObjectAccessed:*",
+            EventName::ObjectAccessedGet => "s3:ObjectAccessed:Get",
+            EventName::ObjectAccessedGetRetention => "s3:ObjectAccessed:GetRetention",
+            EventName::ObjectAccessedGetLegalHold => "s3:ObjectAccessed:GetLegalHold",
+            EventName::ObjectAccessedHead => "s3:ObjectAccessed:Head",
+            EventName::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes",
+            EventName::ObjectCreatedAll => "s3:ObjectCreated:*",
+            EventName::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload",
+            EventName::ObjectCreatedCopy => "s3:ObjectCreated:Copy",
+            EventName::ObjectCreatedPost => "s3:ObjectCreated:Post",
+            EventName::ObjectCreatedPut => "s3:ObjectCreated:Put",
+            EventName::ObjectCreatedPutTagging => "s3:ObjectCreated:PutTagging",
+            EventName::ObjectCreatedDeleteTagging => "s3:ObjectCreated:DeleteTagging",
+            EventName::ObjectCreatedPutRetention => "s3:ObjectCreated:PutRetention",
+            EventName::ObjectCreatedPutLegalHold => "s3:ObjectCreated:PutLegalHold",
+            EventName::ObjectRemovedAll => "s3:ObjectRemoved:*",
+            EventName::ObjectRemovedDelete => "s3:ObjectRemoved:Delete",
+            EventName::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated",
+            EventName::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP",
+            EventName::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions",
+            EventName::ILMDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete",
+            EventName::ObjectReplicationAll => "s3:Replication:*",
+            EventName::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication",
+            EventName::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication",
+            EventName::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked",
+            EventName::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold",
+            EventName::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold",
+            EventName::ObjectRestoreAll => "s3:ObjectRestore:*",
+            EventName::ObjectRestorePost => "s3:ObjectRestore:Post",
+            EventName::ObjectRestoreCompleted => "s3:ObjectRestore:Completed",
+            EventName::ObjectTransitionAll => "s3:ObjectTransition:*",
+            EventName::ObjectTransitionFailed => "s3:ObjectTransition:Failed",
+            EventName::ObjectTransitionComplete => "s3:ObjectTransition:Complete",
+            EventName::ObjectManyVersions => "s3:Scanner:ManyVersions",
+            EventName::ObjectLargeVersions => "s3:Scanner:LargeVersions",
+            EventName::PrefixManyFolders => "s3:Scanner:BigPrefix",
+            _ => "",
+        }
+    }
+}
+
+impl From<&str> for EventName {
+    fn from(s: &str) -> Self {
+        match s {
+            "s3:BucketCreated:*" => EventName::BucketCreated,
+            "s3:BucketRemoved:*" => EventName::BucketRemoved,
+            "s3:ObjectAccessed:*" => EventName::ObjectAccessedAll,
+            "s3:ObjectAccessed:Get" => EventName::ObjectAccessedGet,
+            "s3:ObjectAccessed:GetRetention" => EventName::ObjectAccessedGetRetention,
+            "s3:ObjectAccessed:GetLegalHold" => EventName::ObjectAccessedGetLegalHold,
+            "s3:ObjectAccessed:Head" => EventName::ObjectAccessedHead,
+            "s3:ObjectAccessed:Attributes" => EventName::ObjectAccessedAttributes,
+            "s3:ObjectCreated:*" => EventName::ObjectCreatedAll,
+            "s3:ObjectCreated:CompleteMultipartUpload" => EventName::ObjectCreatedCompleteMultipartUpload,
+            "s3:ObjectCreated:Copy" => EventName::ObjectCreatedCopy,
+            "s3:ObjectCreated:Post" => EventName::ObjectCreatedPost,
+            "s3:ObjectCreated:Put" => EventName::ObjectCreatedPut,
+            "s3:ObjectCreated:PutRetention" => EventName::ObjectCreatedPutRetention,
+            "s3:ObjectCreated:PutLegalHold" => EventName::ObjectCreatedPutLegalHold,
+            "s3:ObjectCreated:PutTagging" => EventName::ObjectCreatedPutTagging,
+            "s3:ObjectCreated:DeleteTagging" => EventName::ObjectCreatedDeleteTagging,
+            "s3:ObjectRemoved:*" => EventName::ObjectRemovedAll,
+            "s3:ObjectRemoved:Delete" => EventName::ObjectRemovedDelete,
+            "s3:ObjectRemoved:DeleteMarkerCreated" => EventName::ObjectRemovedDeleteMarkerCreated,
+            "s3:ObjectRemoved:NoOP" => EventName::ObjectRemovedNoOP,
+            "s3:ObjectRemoved:DeleteAllVersions" => EventName::ObjectRemovedDeleteAllVersions,
+            "s3:LifecycleDelMarkerExpiration:Delete" => EventName::ILMDelMarkerExpirationDelete,
+            "s3:Replication:*" => EventName::ObjectReplicationAll,
+            "s3:Replication:OperationFailedReplication" => EventName::ObjectReplicationFailed,
+            "s3:Replication:OperationCompletedReplication" => EventName::ObjectReplicationComplete,
+            "s3:Replication:OperationMissedThreshold" => EventName::ObjectReplicationMissedThreshold,
+            "s3:Replication:OperationReplicatedAfterThreshold" => EventName::ObjectReplicationReplicatedAfterThreshold,
+            "s3:Replication:OperationNotTracked" => EventName::ObjectReplicationNotTracked,
+            "s3:ObjectRestore:*" => EventName::ObjectRestoreAll,
+            "s3:ObjectRestore:Post" => EventName::ObjectRestorePost,
+            "s3:ObjectRestore:Completed" => EventName::ObjectRestoreCompleted,
+            "s3:ObjectTransition:Failed" => EventName::ObjectTransitionFailed,
+            "s3:ObjectTransition:Complete" => EventName::ObjectTransitionComplete,
+            "s3:ObjectTransition:*" => EventName::ObjectTransitionAll,
+            "s3:Scanner:ManyVersions" => EventName::ObjectManyVersions,
+            "s3:Scanner:LargeVersions" => EventName::ObjectLargeVersions,
+            "s3:Scanner:BigPrefix" => EventName::PrefixManyFolders,
+            _ => EventName::Everything,
+        }
+    }
+}

ecstore/src/event/targetid.rs

@@ -0,0 +1,10 @@
+pub struct TargetID {
+    id: String,
+    name: String,
+}
+
+impl TargetID {
+    fn to_string(&self) -> String {
+        format!("{}:{}", self.id, self.name)
+    }
+}

ecstore/src/event/targetlist.rs

@@ -0,0 +1,31 @@
+use std::sync::atomic::AtomicI64;
+
+use super::targetid::TargetID;
+
+#[derive(Default)]
+pub struct TargetList {
+    pub current_send_calls: AtomicI64,
+    pub total_events: AtomicI64,
+    pub events_skipped: AtomicI64,
+    pub events_errors_total: AtomicI64,
+    //pub targets: HashMap<TargetID, Target>,
+    //pub queue:   AsyncEvent,
+    //pub targetStats: HashMap<TargetID, TargetStat>,
+}
+
+impl TargetList {
+    pub fn new() -> TargetList {
+        TargetList::default()
+    }
+}
+
+struct TargetStat {
+    current_send_calls: i64,
+    total_events: i64,
+    failed_events: i64,
+}
+
+struct TargetIDResult {
+    id: TargetID,
+    err: std::io::Error,
+}

ecstore/src/event_notification.rs

@@ -0,0 +1,59 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+use crate::bucket::metadata::BucketMetadata;
+use crate::event::name::EventName;
+use crate::event::targetlist::TargetList;
+use crate::store::ECStore;
+use crate::store_api::ObjectInfo;
+
+pub struct EventNotifier {
+    target_list: TargetList,
+    //bucket_rules_map: HashMap<String , HashMap<EventName, Rules>>,
+}
+
+impl EventNotifier {
+    pub fn new() -> Arc<RwLock<Self>> {
+        Arc::new(RwLock::new(Self {
+            target_list: TargetList::new(),
+            //bucket_rules_map: HashMap::new(),
+        }))
+    }
+
+    fn get_arn_list(&self) -> Vec<String> {
+        todo!();
+    }
+
+    fn set(&self, bucket: &str, meta: BucketMetadata) {
+        todo!();
+    }
+
+    fn init_bucket_targets(&self, api: ECStore) -> Result<(), std::io::Error> {
+        /*if err := self.target_list.Add(globalNotifyTargetList.Targets()...); err != nil {
+          return err
+        }
+        self.target_list = self.target_list.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4)
+        nil*/
+        todo!();
+    }
+
+    fn send(&self, args: EventArgs) {
+        todo!();
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct EventArgs {
+    pub event_name: String,
+    pub bucket_name: String,
+    pub object: ObjectInfo,
+    pub req_params: HashMap<String, String>,
+    pub resp_elements: HashMap<String, String>,
+    pub host: String,
+    pub user_agent: String,
+}
+
+impl EventArgs {}
+
+pub fn send_event(args: EventArgs) {}

ecstore/src/global.rs

@@ -1,9 +1,12 @@
 use crate::heal::mrf::MRFState;
 use crate::{
+    bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
     disk::DiskStore,
     endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
+    event_notification::EventNotifier,
     heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
     store::ECStore,
+    tier::tier::TierConfigMgr,
 };
 use lazy_static::lazy_static;
 use policy::auth::Credentials;
@@ -20,24 +23,32 @@ pub const DISK_MIN_INODES: u64 = 1000;
 pub const DISK_FILL_FRACTION: f64 = 0.99;
 pub const DISK_RESERVE_FRACTION: f64 = 0.15;
 
+pub const DEFAULT_PORT: u16 = 9000;
+
 lazy_static! {
-    static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
-    pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
-    pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
-    pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
-    pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
-    pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
-    pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
-    pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
-    pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
-    pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
-    pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
-    pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
-    pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
-    pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
-    static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
-    pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
-}
+static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
+pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
+pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
+pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
+pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
+pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
+pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
+pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
+pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new();
+pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
+pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
+pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
+pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
+pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
+pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
+//pub static ref GLOBAL_RemoteTargetTransport
+pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
+pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
+static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
+pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
+pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
+pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
+pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();}
 
 static GLOBAL_ACTIVE_CRED: OnceLock<Credentials> = OnceLock::new();
 

ecstore/src/heal/data_scanner.rs

@@ -12,13 +12,35 @@ use std::{
     time::{Duration, SystemTime},
 };
 
+use common::defer;
+use time::{self, OffsetDateTime};
+
 use super::{
     data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics},
     data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend},
     data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash},
     heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode},
 };
-use crate::{bucket::metadata_sys, cmd::bucket_replication::queue_replication_heal};
+use crate::bucket::{
+    object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion},
+    utils::is_meta_bucketname,
+};
+use crate::cmd::bucket_replication::queue_replication_heal;
+use crate::event::name::EventName;
+use crate::{
+    bucket::{
+        lifecycle::{
+            bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc},
+            bucket_lifecycle_ops::{self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, expire_transitioned_object},
+            lifecycle::{self, ExpirationOptions, Lifecycle},
+        },
+        metadata_sys,
+    },
+    event_notification::{EventArgs, send_event},
+    global::GLOBAL_LocalNodeName,
+    heal::data_scanner,
+    store_api::{ObjectOptions, ObjectToDelete, StorageAPI},
+};
 use crate::{
     bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys},
     cmd::bucket_replication::ReplicationStatusType,
@@ -55,8 +77,12 @@ use lazy_static::lazy_static;
 use rand::Rng;
 use rmp_serde::{Deserializer, Serializer};
 use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
+use rustfs_utils::path::encode_dir_object;
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
-use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus};
+use s3s::dto::{
+    BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration,
+    ReplicationRuleStatus, VersioningConfiguration,
+};
 use serde::{Deserialize, Serialize};
 use tokio::{
     sync::{
@@ -515,8 +541,68 @@ impl ScannerItem {
         path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())])
     }
 
-    pub async fn apply_versions_actions(&self, fives: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
-        let obj_infos = self.apply_newer_noncurrent_version_limit(fives).await?;
+    async fn apply_lifecycle(&self, oi: &ObjectInfo) -> (lifecycle::IlmAction, i64) {
+        let mut size = oi.get_actual_size().expect("err!");
+        if self.debug {
+            info!("apply_lifecycle debug");
+        }
+        if self.lifecycle.is_none() {
+            return (lifecycle::IlmAction::NoneAction, size as i64);
+        }
+
+        let version_id = oi.version_id;
+
+        let mut vc = None;
+        let mut lr = None;
+        let mut rcfg = None;
+        if !is_meta_bucketname(&self.bucket) {
+            vc = Some(BucketVersioningSys::get(&self.bucket).await.unwrap());
+            lr = BucketObjectLockSys::get(&self.bucket).await;
+            rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await {
+                Some(replication_config)
+            } else {
+                None
+            };
+        }
+
+        let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await;
+        if self.debug {
+            if !version_id.is_none() {
+                info!(
+                    "lifecycle: {} (version-id={}), Initial scan: {}",
+                    self.object_path().to_string_lossy().to_string(),
+                    version_id.expect("err"),
+                    lc_evt.action
+                );
+            } else {
+                info!(
+                    "lifecycle: {} Initial scan: {}",
+                    self.object_path().to_string_lossy().to_string(),
+                    lc_evt.action
+                );
+            }
+        }
+
+        match lc_evt.action {
+            lifecycle::IlmAction::DeleteVersionAction
+            | lifecycle::IlmAction::DeleteAllVersionsAction
+            | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
+                size = 0;
+            }
+            lifecycle::IlmAction::DeleteAction => {
+                if !vc.unwrap().prefix_enabled(&oi.name) {
+                    size = 0
+                }
+            }
+            _ => (),
+        }
+
+        apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await;
+        (lc_evt.action, size as i64)
+    }
+
+    pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
+        let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?;
         if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize {
             // todo
         }
@@ -533,16 +619,24 @@ impl ScannerItem {
         Ok(obj_infos)
     }
 
-    pub async fn apply_newer_noncurrent_version_limit(&self, fives: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
+    pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
         // let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent);
+
+        let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await {
+            rcfg.mode.is_some()
+        } else {
+            false
+        };
+        let vcfg = BucketVersioningSys::get(&self.bucket).await?;
+
         let versioned = match BucketVersioningSys::get(&self.bucket).await {
             Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()),
             Err(_) => false,
         };
-        let mut object_infos = Vec::with_capacity(fives.len());
+        let mut object_infos = Vec::with_capacity(fivs.len());
 
         if self.lifecycle.is_none() {
-            for info in fives.iter() {
+            for info in fivs.iter() {
                 object_infos.push(ObjectInfo::from_file_info(
                     info,
                     &self.bucket,
@@ -553,6 +647,72 @@ impl ScannerItem {
             return Ok(object_infos);
         }
 
+        let event = self
+            .lifecycle
+            .as_ref()
+            .expect("lifecycle err.")
+            .noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts {
+                name: self.object_path().to_string_lossy().to_string(),
+                ..Default::default()
+            })
+            .await;
+        let lim = event.newer_noncurrent_versions;
+        if lim == 0 || fivs.len() <= lim + 1 {
+            for fi in fivs.iter() {
+                object_infos.push(ObjectInfo::from_file_info(
+                    fi,
+                    &self.bucket,
+                    &self.object_path().to_string_lossy(),
+                    versioned,
+                ));
+            }
+            return Ok(object_infos);
+        }
+
+        let overflow_versions = &fivs[lim + 1..];
+        for fi in fivs[..lim + 1].iter() {
+            object_infos.push(ObjectInfo::from_file_info(
+                fi,
+                &self.bucket,
+                &self.object_path().to_string_lossy(),
+                versioned,
+            ));
+        }
+
+        let mut to_del = Vec::<ObjectToDelete>::with_capacity(overflow_versions.len());
+        for fi in overflow_versions.iter() {
+            let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned);
+            if lock_enabled && enforce_retention_for_deletion(&obj) {
+                //if enforce_retention_for_deletion(&obj) {
+                if self.debug {
+                    if obj.version_id.is_some() {
+                        info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err"));
+                    } else {
+                        info!("lifecycle: {} is locked, not deleting\n", obj.name);
+                    }
+                }
+                object_infos.push(obj);
+                continue;
+            }
+
+            if OffsetDateTime::now_utc().unix_timestamp()
+                < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32)
+                    .unix_timestamp()
+            {
+                object_infos.push(obj);
+                continue;
+            }
+
+            to_del.push(ObjectToDelete {
+                object_name: obj.name,
+                version_id: obj.version_id,
+            });
+        }
+
+        if to_del.len() > 0 {
+            let mut expiry_state = GLOBAL_ExpiryState.write().await;
+            expiry_state.enqueue_by_newer_noncurrent(&self.bucket, to_del, event).await;
+        }
         // done().await;
 
         Ok(object_infos)
@@ -560,7 +720,9 @@ impl ScannerItem {
 
     pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, i64) {
         let done = ScannerMetrics::time(ScannerMetric::Ilm);
-        //todo: lifecycle
+
+        let (action, size) = self.apply_lifecycle(oi).await;
+
         info!(
             "apply_actions {} {} {:?} {:?}",
             oi.bucket.clone(),
@@ -580,6 +742,10 @@ impl ScannerItem {
         self.heal_replication(&oi, _size_s).await;
         done();
 
+        if action.delete_all() {
+            return (true, 0);
+        }
+
         (false, oi.size)
     }
 
@@ -756,9 +922,9 @@ impl FolderScanner {
 
             let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name);
             // Todo: lifeCycle
-            let active_life_cycle = if let Some(lc) = self.old_cache.info.life_cycle.as_ref() {
+            let active_life_cycle = if let Some(lc) = self.old_cache.info.lifecycle.as_ref() {
                 if lc_has_active_rules(lc, &prefix) {
-                    self.old_cache.info.life_cycle.clone()
+                    self.old_cache.info.lifecycle.clone()
                 } else {
                     None
                 }
@@ -1400,7 +1566,165 @@ pub async fn scan_data_folder(
     Ok(s.new_cache)
 }
 
-// pub fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: &ObjectLockConfiguration, rcfg: &ReplicationConfiguration， obj: &ObjectInfo)
+pub async fn eval_action_from_lifecycle(
+    lc: &BucketLifecycleConfiguration,
+    lr: Option<DefaultRetention>,
+    rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>,
+    oi: &ObjectInfo,
+) -> lifecycle::Event {
+    let event = lc.eval(&oi.to_lifecycle_opts()).await;
+    //if serverDebugLog {
+    info!("lifecycle: Secondary scan: {}", event.action);
+    //}
+
+    let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
+
+    match event.action {
+        lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
+            if lock_enabled {
+                return lifecycle::Event::default();
+            }
+        }
+        lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => {
+            if oi.version_id.is_none() {
+                return lifecycle::Event::default();
+            }
+            if lock_enabled && enforce_retention_for_deletion(oi) {
+                //if serverDebugLog {
+                if !oi.version_id.is_none() {
+                    info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err"));
+                } else {
+                    info!("lifecycle: {} is locked, not deleting", oi.name);
+                }
+                //}
+                return lifecycle::Event::default();
+            }
+            if let Some(rcfg) = rcfg {
+                if rep_has_active_rules(&rcfg.0, &oi.name, true) {
+                    return lifecycle::Event::default();
+                }
+            }
+        }
+        _ => (),
+    }
+
+    event
+}
+
+async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
+    if oi.delete_marker || oi.is_dir {
+        return false;
+    }
+    GLOBAL_TransitionState.queue_transition_task(oi, event, src).await;
+    true
+}
+
+pub async fn apply_expiry_on_transitioned_object(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    src: &LcEventSrc,
+) -> bool {
+    let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+    if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await {
+        return false;
+    }
+    time_ilm(1);
+
+    true
+}
+
+pub async fn apply_expiry_on_non_transitioned_objects(
+    api: Arc<ECStore>,
+    oi: &ObjectInfo,
+    lc_event: &lifecycle::Event,
+    src: &LcEventSrc,
+) -> bool {
+    let mut opts = ObjectOptions {
+        expiration: ExpirationOptions { expire: true },
+        ..Default::default()
+    };
+
+    if lc_event.action.delete_versioned() {
+        opts.version_id = Some(oi.version_id.expect("err").to_string());
+    }
+
+    opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await;
+    opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await;
+
+    if lc_event.action.delete_all() {
+        opts.delete_prefix = true;
+        opts.delete_prefix_object = true;
+    }
+    let dobj: ObjectInfo;
+    //let  err: Error;
+
+    let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone());
+
+    let mut dobj = api
+        .delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts)
+        .await
+        .unwrap();
+    if dobj.name == "" {
+        dobj = oi.clone();
+    }
+
+    //let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags();
+    //tags["version-id"] = dobj.version_id;
+
+    let mut event_name = EventName::ObjectRemovedDelete;
+    if oi.delete_marker {
+        event_name = EventName::ObjectRemovedDeleteMarkerCreated;
+    }
+    match lc_event.action {
+        lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions,
+        lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete,
+        _ => (),
+    }
+    send_event(EventArgs {
+        event_name: event_name.as_ref().to_string(),
+        bucket_name: dobj.bucket.clone(),
+        object: dobj,
+        user_agent: "Internal: [ILM-Expiry]".to_string(),
+        host: GLOBAL_LocalNodeName.to_string(),
+        ..Default::default()
+    });
+
+    if lc_event.action != lifecycle::IlmAction::NoneAction {
+        let mut num_versions = 1_u64;
+        if lc_event.action.delete_all() {
+            num_versions = oi.num_versions as u64;
+        }
+        time_ilm(num_versions);
+    }
+
+    true
+}
+
+async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
+    let mut expiry_state = GLOBAL_ExpiryState.write().await;
+    expiry_state.enqueue_by_days(oi, event, src).await;
+    true
+}
+
+pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
+    let mut success = false;
+    match event.action {
+        lifecycle::IlmAction::DeleteVersionAction
+        | lifecycle::IlmAction::DeleteAction
+        | lifecycle::IlmAction::DeleteRestoredAction
+        | lifecycle::IlmAction::DeleteRestoredVersionAction
+        | lifecycle::IlmAction::DeleteAllVersionsAction
+        | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => {
+            success = apply_expiry_rule(event, src, oi).await;
+        }
+        lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => {
+            success = apply_transition_rule(event, src, oi).await;
+        }
+        _ => (),
+    }
+    success
+}
 
 #[cfg(test)]
 mod tests {

ecstore/src/heal/data_scanner_metric.rs

@@ -15,14 +15,13 @@ use tokio::sync::{Mutex, RwLock};
 use tracing::debug;
 
 use super::data_scanner::CurrentScannerCycle;
+use crate::bucket::lifecycle::lifecycle;
 
 lazy_static! {
     pub static ref globalScannerMetrics: Arc<ScannerMetrics> = Arc::new(ScannerMetrics::new());
 }
 
-/// Scanner metric types, matching the Go version exactly
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-#[repr(u8)]
+#[derive(Clone, Debug, PartialEq, PartialOrd)]
 pub enum ScannerMetric {
     // START Realtime metrics, that only records
     // last minute latencies and total operation count.
@@ -196,7 +195,8 @@ pub struct ScannerMetrics {
     // All fields must be accessed atomically and aligned.
     operations: Vec<AtomicU64>,
     latency: Vec<LockedLastMinuteLatency>,
-
+    actions: Vec<AtomicU64>,
+    actions_latency: Vec<LockedLastMinuteLatency>,
     // Current paths contains disk -> tracker mappings
     current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
 
@@ -215,6 +215,8 @@ impl ScannerMetrics {
         Self {
             operations,
             latency,
+            actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
+            actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize],
             current_paths: Arc::new(RwLock::new(HashMap::new())),
             cycle_info: Arc::new(RwLock::new(None)),
         }
@@ -222,16 +224,17 @@ impl ScannerMetrics {
 
     /// Log scanner action with custom metadata - compatible with existing usage
     pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap<String, String>) {
+        let metric = metric as usize;
         let start_time = SystemTime::now();
         move |_custom: &HashMap<String, String>| {
             let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
             // Update operation count
-            globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
+            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
             // Update latency for realtime metrics (spawn async task for this)
-            if (metric as usize) < ScannerMetric::LastRealtime as usize {
-                let metric_index = metric as usize;
+            if (metric) < ScannerMetric::LastRealtime as usize {
+                let metric_index = metric;
                 tokio::spawn(async move {
                     globalScannerMetrics.latency[metric_index].add(duration).await;
                 });
@@ -239,23 +242,24 @@ impl ScannerMetrics {
 
             // Log trace metrics
             if metric as u8 > ScannerMetric::StartTrace as u8 {
-                debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
+                //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
             }
         }
     }
 
     /// Time scanner action with size - returns function that takes size
     pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) {
+        let metric = metric as usize;
         let start_time = SystemTime::now();
         move |size: u64| {
             let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
             // Update operation count
-            globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
+            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
             // Update latency for realtime metrics with size (spawn async task)
-            if (metric as usize) < ScannerMetric::LastRealtime as usize {
-                let metric_index = metric as usize;
+            if (metric) < ScannerMetric::LastRealtime as usize {
+                let metric_index = metric;
                 tokio::spawn(async move {
                     globalScannerMetrics.latency[metric_index].add_size(duration, size).await;
                 });
@@ -265,16 +269,17 @@ impl ScannerMetrics {
 
     /// Time a scanner action - returns a closure to call when done
     pub fn time(metric: ScannerMetric) -> impl Fn() {
+        let metric = metric as usize;
         let start_time = SystemTime::now();
         move || {
             let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
             // Update operation count
-            globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
+            globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
             // Update latency for realtime metrics (spawn async task)
-            if (metric as usize) < ScannerMetric::LastRealtime as usize {
-                let metric_index = metric as usize;
+            if (metric) < ScannerMetric::LastRealtime as usize {
+                let metric_index = metric;
                 tokio::spawn(async move {
                     globalScannerMetrics.latency[metric_index].add(duration).await;
                 });
@@ -284,17 +289,18 @@ impl ScannerMetrics {
 
     /// Time N scanner actions - returns function that takes count, then returns completion function
     pub fn time_n(metric: ScannerMetric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
+        let metric = metric as usize;
         let start_time = SystemTime::now();
         Box::new(move |count: usize| {
             Box::new(move || {
                 let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
 
                 // Update operation count
-                globalScannerMetrics.operations[metric as usize].fetch_add(count as u64, Ordering::Relaxed);
+                globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed);
 
                 // Update latency for realtime metrics (spawn async task)
-                if (metric as usize) < ScannerMetric::LastRealtime as usize {
-                    let metric_index = metric as usize;
+                if (metric) < ScannerMetric::LastRealtime as usize {
+                    let metric_index = metric;
                     tokio::spawn(async move {
                         globalScannerMetrics.latency[metric_index].add(duration).await;
                     });
@@ -303,31 +309,51 @@ impl ScannerMetrics {
         })
     }
 
+    pub fn time_ilm(a: lifecycle::IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
+        let a_clone = a as usize;
+        if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize {
+            return Box::new(move |_: u64| Box::new(move || {}));
+        }
+        let start = SystemTime::now();
+        Box::new(move |versions: u64| {
+            Box::new(move || {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                tokio::spawn(async move {
+                    globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed);
+                    globalScannerMetrics.actions_latency[a_clone].add(duration).await;
+                });
+            })
+        })
+    }
+
     /// Increment time with specific duration
     pub async fn inc_time(metric: ScannerMetric, duration: Duration) {
+        let metric = metric as usize;
         // Update operation count
-        globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed);
+        globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed);
 
         // Update latency for realtime metrics
-        if (metric as usize) < ScannerMetric::LastRealtime as usize {
-            globalScannerMetrics.latency[metric as usize].add(duration).await;
+        if (metric) < ScannerMetric::LastRealtime as usize {
+            globalScannerMetrics.latency[metric].add(duration).await;
         }
     }
 
     /// Get lifetime operation count for a metric
     pub fn lifetime(&self, metric: ScannerMetric) -> u64 {
-        if (metric as usize) >= ScannerMetric::Last as usize {
+        let metric = metric as usize;
+        if (metric) >= ScannerMetric::Last as usize {
             return 0;
         }
-        self.operations[metric as usize].load(Ordering::Relaxed)
+        self.operations[metric].load(Ordering::Relaxed)
     }
 
     /// Get last minute statistics for a metric
     pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem {
-        if (metric as usize) >= ScannerMetric::LastRealtime as usize {
+        let metric = metric as usize;
+        if (metric) >= ScannerMetric::LastRealtime as usize {
             return AccElem::default();
         }
-        self.latency[metric as usize].total().await
+        self.latency[metric].total().await
     }
 
     /// Set current cycle information

ecstore/src/heal/data_usage_cache.rs

@@ -129,6 +129,59 @@ const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERS
     },
 ];
 
+#[derive(Clone, Copy, Default)]
+pub struct TierStats {
+    pub total_size: u64,
+    pub num_versions: i32,
+    pub num_objects: i32,
+}
+
+impl TierStats {
+    pub fn add(&self, u: &TierStats) -> TierStats {
+        TierStats {
+            total_size: self.total_size + u.total_size,
+            num_versions: self.num_versions + u.num_versions,
+            num_objects: self.num_objects + u.num_objects,
+        }
+    }
+}
+
+#[derive(Clone)]
+struct AllTierStats {
+    tiers: HashMap<String, TierStats>,
+}
+
+impl AllTierStats {
+    pub fn new() -> Self {
+        Self { tiers: HashMap::new() }
+    }
+
+    fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
+        for (tier, st) in tiers {
+            self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st));
+        }
+    }
+
+    fn merge(&mut self, other: AllTierStats) {
+        for (tier, st) in other.tiers {
+            self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st));
+        }
+    }
+
+    fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
+        for (tier, st) in &self.tiers {
+            stats.insert(
+                tier.clone(),
+                TierStats {
+                    total_size: st.total_size.clone(),
+                    num_versions: st.num_versions.clone(),
+                    num_objects: st.num_objects.clone(),
+                },
+            );
+        }
+    }
+}
+
 // sizeHistogram is a size histogram.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct SizeHistogram(Vec<u64>);
@@ -346,8 +399,7 @@ pub struct DataUsageCacheInfo {
     pub last_update: Option<SystemTime>,
     pub skip_healing: bool,
     #[serde(skip)]
-    pub life_cycle: Option<BucketLifecycleConfiguration>,
-    // pub life_cycle:
+    pub lifecycle: Option<BucketLifecycleConfiguration>,
     #[serde(skip)]
     pub updates: Option<Sender<DataUsageEntry>>,
     #[serde(skip)]

ecstore/src/lib.rs

@@ -28,6 +28,13 @@ mod store_init;
 pub mod store_list_objects;
 pub mod store_utils;
 
+pub mod checksum;
+pub mod client;
+pub mod event;
+pub mod event_notification;
+pub mod signer;
+pub mod tier;
+
 pub use global::new_object_layer_fn;
 pub use global::set_global_endpoints;
 pub use global::update_erasure_type;

ecstore/src/set_disk.rs

@@ -1,4 +1,6 @@
 use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
+use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
+use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl};
 use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
 use crate::disk::{
     self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS,
@@ -6,12 +8,15 @@ use crate::disk::{
 };
 use crate::erasure_coding;
 use crate::erasure_coding::bitrot_verify;
+use crate::error::ObjectApiError;
 use crate::error::{Error, Result};
 use crate::global::GLOBAL_MRFState;
+use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
 use crate::heal::data_usage_cache::DataUsageCache;
 use crate::heal::heal_ops::{HealEntryFn, HealSequence};
 use crate::store_api::ObjectToDelete;
 use crate::{
+    bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, get_transitioned_object_reader, put_restore_opts},
     cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
     config::{GLOBAL_StorageClass, storageclass},
     disk::{
@@ -20,6 +25,8 @@ use crate::{
         UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk,
     },
     error::{StorageError, to_object_err},
+    event::name::EventName,
+    event_notification::{EventArgs, send_event},
     global::{
         GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id,
         is_dist_erasure,
@@ -68,7 +75,8 @@ use rustfs_utils::{
     crypto::{base64_decode, base64_encode, hex},
     path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
 };
-use sha2::Sha256;
+use s3s::header::X_AMZ_RESTORE;
+use sha2::{Digest, Sha256};
 use std::hash::Hash;
 use std::mem;
 use std::time::SystemTime;
@@ -3040,33 +3048,33 @@ impl SetDisks {
 
         let task = if !skip_background_task {
             Some(tokio::spawn(async move {
-                let last_save = Some(SystemTime::now());
-                let mut need_loop = true;
-                while need_loop {
-                    select! {
-                        _ = ticker.tick() => {
-                            if !cache.info.last_update.eq(&last_save) {
+            let last_save = Some(SystemTime::now());
+            let mut need_loop = true;
+            while need_loop {
+                select! {
+                    _ = ticker.tick() => {
+                        if !cache.info.last_update.eq(&last_save) {
+                            let _ = cache.save(DATA_USAGE_CACHE_NAME).await;
+                            let _ = updates.send(cache.clone()).await;
+                        }
+                    }
+                    result = buckets_results_rx.recv() => {
+                        match result {
+                            Some(result) => {
+                                cache.replace(&result.name, &result.parent, result.entry);
+                                cache.info.last_update = Some(SystemTime::now());
+                            },
+                            None => {
+                                need_loop = false;
+                                cache.info.next_cycle = want_cycle;
+                                cache.info.last_update = Some(SystemTime::now());
                                 let _ = cache.save(DATA_USAGE_CACHE_NAME).await;
                                 let _ = updates.send(cache.clone()).await;
                             }
                         }
-                        result = buckets_results_rx.recv() => {
-                            match result {
-                                Some(result) => {
-                                    cache.replace(&result.name, &result.parent, result.entry);
-                                    cache.info.last_update = Some(SystemTime::now());
-                                },
-                                None => {
-                                    need_loop = false;
-                                    cache.info.next_cycle = want_cycle;
-                                    cache.info.last_update = Some(SystemTime::now());
-                                    let _ = cache.save(DATA_USAGE_CACHE_NAME).await;
-                                    let _ = updates.send(cache.clone()).await;
-                                }
-                            }
-                        }
                     }
                 }
+            }
             }))
         } else {
             None
@@ -3175,7 +3183,7 @@ impl SetDisks {
         info!("ns_scanner start");
         let _ = join_all(futures).await;
         if let Some(task) = task {
-            let _ = task.await;
+        let _ = task.await;
         }
         info!("ns_scanner completed");
         Ok(())
@@ -3702,6 +3710,45 @@ impl SetDisks {
 
         Ok(())
     }
+
+    pub async fn update_restore_metadata(
+        &self,
+        bucket: &str,
+        object: &str,
+        obj_info: &ObjectInfo,
+        opts: &ObjectOptions,
+    ) -> Result<()> {
+        let mut oi = obj_info.clone();
+        oi.metadata_only = true;
+
+        if let Some(user_defined) = &mut oi.user_defined {
+            user_defined.remove(X_AMZ_RESTORE.as_str());
+        }
+
+        let version_id = oi.version_id.clone().map(|v| v.to_string());
+        let obj = self
+            .copy_object(
+                bucket,
+                object,
+                bucket,
+                object,
+                &mut oi,
+                &ObjectOptions {
+                    version_id: version_id.clone(),
+                    ..Default::default()
+                },
+                &ObjectOptions {
+                    version_id: version_id,
+                    ..Default::default()
+                },
+            )
+            .await;
+        if let Err(err) = obj {
+            //storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
+            return Err(err);
+        }
+        Ok(())
+    }
 }
 
 #[async_trait::async_trait]
@@ -3997,7 +4044,7 @@ impl ObjectIO for SetDisks {
 
             if opts.data_movement {
                 fi.set_data_moved();
-            }
+        }
         }
 
         let (online_disks, _, op_old_dir) = Self::rename_data(
@@ -4188,6 +4235,48 @@ impl StorageAPI for SetDisks {
             src_opts.versioned || src_opts.version_suspended,
         ))
     }
+    #[tracing::instrument(skip(self))]
+    async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
+        let disks = self.get_disks(0, 0).await?;
+        let write_quorum = disks.len() / 2 + 1;
+
+        let mut futures = Vec::with_capacity(disks.len());
+        let mut errs = Vec::with_capacity(disks.len());
+
+        for disk in disks.iter() {
+            futures.push(async move {
+                if let Some(disk) = disk {
+                    match disk
+                        .delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default())
+                        .await
+                    {
+                        Ok(r) => Ok(r),
+                        Err(e) => Err(e),
+                    }
+                } else {
+                    Err(DiskError::DiskNotFound)
+                }
+            });
+        }
+
+        let results = join_all(futures).await;
+        for result in results {
+            match result {
+                Ok(_) => {
+                    errs.push(None);
+                }
+                Err(e) => {
+                    errs.push(Some(e));
+                }
+            }
+        }
+
+        if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
+            return Err(err.into());
+        }
+        Ok(())
+    }
+
     #[tracing::instrument(skip(self))]
     async fn delete_objects(
         &self,
@@ -4381,6 +4470,22 @@ impl StorageAPI for SetDisks {
         Ok(oi)
     }
 
+    #[tracing::instrument(skip(self))]
+    async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
+        GLOBAL_MRFState
+            .add_partial(PartialOperation {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                version_id: Some(version_id.to_string()),
+                queued: Utc::now(),
+                set_index: self.set_index,
+                pool_index: self.pool_index,
+                ..Default::default()
+            })
+            .await;
+        Ok(())
+    }
+
     #[tracing::instrument(skip(self))]
     async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         // TODO: nslock
@@ -4465,6 +4570,218 @@ impl StorageAPI for SetDisks {
         Ok(oi.user_tags)
     }
 
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
+        let tgt_client = match tier_config_mgr.get_driver(&opts.transition.tier).await {
+            Ok(client) => client,
+            Err(err) => {
+                return Err(Error::other(err.to_string()));
+            }
+        };
+
+        /*if !opts.no_lock {
+            let lk = self.new_ns_lock(bucket, object);
+            let lkctx = lk.get_lock(globalDeleteOperationTimeout)?;
+            //ctx = lkctx.Context()
+            //defer lk.Unlock(lkctx)
+        }*/
+
+        let (mut fi, mut meta_arr, mut online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true).await?;
+        /*if err != nil {
+            return Err(to_object_err(err, vec![bucket, object]));
+        }*/
+        /*if fi.deleted {
+            if opts.version_id.is_none() {
+                return Err(to_object_err(DiskError::FileNotFound, vec![bucket, object]));
+            }
+            return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object]));
+        }*/
+        if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp()
+            || !(opts.transition.etag == extract_etag(&fi.metadata))
+        {
+            return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object]));
+        }
+        if fi.transition_status == TRANSITION_COMPLETE {
+            return Ok(());
+        }
+
+        /*if fi.xlv1 {
+            if let Err(err) = self.heal_object(bucket, object, "", &HealOpts {no_lock: true, ..Default::default()}) {
+                return err.expect("err");
+            }
+            (fi, meta_arr, online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true);
+            if err != nil {
+                return to_object_err(err, vec![bucket, object]);
+            }
+        }*/
+        //let traceFn = GLOBAL_LifecycleSys.trace(fi.to_object_info(bucket, object, opts.Versioned || opts.VersionSuspended));
+
+        let dest_obj = gen_transition_objname(bucket);
+        if let Err(err) = dest_obj {
+            //traceFn(ILMTransition, nil, err)
+            return Err(to_object_err(err, vec![]));
+        }
+        let dest_obj = dest_obj.unwrap();
+
+        let oi = ObjectInfo::from_file_info(&fi, &bucket, &object, opts.versioned || opts.version_suspended);
+
+        let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size);
+        //let h = HeaderMap::new();
+        //let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi});
+        let reader = ReaderImpl::ObjectBody(GetObjectReader {
+            stream: Box::new(pr),
+            object_info: oi,
+        });
+
+        let cloned_bucket = bucket.to_string();
+        let cloned_object = object.to_string();
+        let cloned_fi = fi.clone();
+        let set_index = self.set_index;
+        let pool_index = self.pool_index;
+        tokio::spawn(async move {
+            if let Err(e) = Self::get_object_with_fileinfo(
+                &cloned_bucket,
+                &cloned_object,
+                0,
+                cloned_fi.size,
+                &mut pw,
+                cloned_fi,
+                meta_arr,
+                &online_disks,
+                set_index,
+                pool_index,
+            )
+            .await
+            {
+                error!("get_object_with_fileinfo err {:?}", e);
+            };
+        });
+
+        let rv = tgt_client
+            .put_with_meta(&dest_obj, reader, fi.size as i64, {
+                let mut m = HashMap::<String, String>::new();
+                m.insert("name".to_string(), object.to_string());
+                m
+            })
+            .await;
+        //pr.CloseWithError(err);
+        if let Err(err) = rv {
+            //traceFn(ILMTransition, nil, err)
+            return Err(StorageError::Io(err));
+        }
+        let rv = rv.unwrap();
+        fi.transition_status = TRANSITION_COMPLETE.to_string();
+        fi.transitioned_objname = dest_obj;
+        fi.transition_tier = opts.transition.tier.clone();
+        fi.transition_version_id = if rv == "" { None } else { Some(Uuid::parse_str(&rv)?) };
+        let mut event_name = EventName::ObjectTransitionComplete.as_ref();
+
+        let disks = self.get_disks(0, 0).await?;
+
+        if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await {
+            event_name = EventName::ObjectTransitionFailed.as_ref();
+        }
+
+        for disk in disks.iter() {
+            if let Some(disk) = disk {
+                if disk.is_online().await {
+                    continue;
+                }
+            }
+            self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err"))
+                .await;
+            break;
+        }
+
+        let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
+        send_event(EventArgs {
+            event_name: event_name.to_string(),
+            bucket_name: bucket.to_string(),
+            object: obj_info,
+            user_agent: "Internal: [ILM-Transition]".to_string(),
+            host: GLOBAL_LocalNodeName.to_string(),
+            ..Default::default()
+        });
+        //let tags = opts.lifecycle_audit_event.tags();
+        //auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn)
+        Ok(())
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        let set_restore_header_fn = async move |oi: &mut ObjectInfo, rerr: Option<Error>| -> Result<()> {
+            if rerr.is_none() {
+                return Ok(());
+            }
+            self.update_restore_metadata(bucket, object, oi, opts).await?;
+            Err(rerr.unwrap())
+        };
+        let mut oi = ObjectInfo::default();
+        let fi = self.get_object_fileinfo(&bucket, &object, &opts, true).await;
+        if let Err(err) = fi {
+            return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
+        }
+        let (mut actual_fi, _, _) = fi.unwrap();
+
+        oi = ObjectInfo::from_file_info(&actual_fi, bucket, object, opts.versioned || opts.version_suspended);
+        let ropts = put_restore_opts(bucket, object, &opts.transition.restore_request, &oi);
+        /*if oi.parts.len() == 1 {
+            let mut rs: HTTPRangeSpec;
+            let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts);
+            //if err != nil {
+            //    return set_restore_header_fn(&mut oi, Some(toObjectErr(err, bucket, object)));
+            //}
+            //defer gr.Close()
+            let hash_reader = HashReader::new(gr, gr.obj_info.size, "", "", gr.obj_info.size);
+            let p_reader = PutObjReader::new(StreamingBlob::from(Box::pin(hash_reader)), hash_reader.size());
+            if let Err(err) = self.put_object(bucket, object, &mut p_reader, &ropts).await {
+                return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object])));
+            } else {
+                return Ok(());
+            }
+        }
+
+        let res = self.new_multipart_upload(bucket, object, &ropts).await?;
+        //if err != nil {
+        //    return set_restore_header_fn(&mut oi, err);
+        //}
+
+        let mut uploaded_parts: Vec<CompletePart> = vec![];
+        let mut rs: HTTPRangeSpec;
+        let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts).await?;
+        //if err != nil {
+        //    return set_restore_header_fn(&mut oi, err);
+        //}
+
+        for part_info in oi.parts {
+            //let hr = HashReader::new(LimitReader(gr, part_info.size), part_info.size, "", "", part_info.size);
+            let hr = HashReader::new(gr, part_info.size as i64, part_info.size as i64, None, false);
+            //if err != nil {
+            //    return set_restore_header_fn(&mut oi, err);
+            //}
+            let mut p_reader = PutObjReader::new(hr, hr.size());
+            let p_info = self.put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()).await?;
+            //if let Err(err) = p_info {
+            //    return set_restore_header_fn(&mut oi, err);
+            //}
+            if p_info.size != part_info.size {
+                return set_restore_header_fn(&mut oi, Some(Error::from(ObjectApiError::InvalidObjectState(GenericError{bucket: bucket.to_string(), object: object.to_string(), ..Default::default()}))));
+            }
+            uploaded_parts.push(CompletePart {
+                part_num: p_info.part_num,
+                etag: p_info.etag,
+            });
+        }
+        if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions {
+            mod_time: oi.mod_time,
+            ..Default::default()
+        }).await {
+            set_restore_header_fn(&mut oi, Some(err));
+        }*/
+        Ok(())
+    }
+
     #[tracing::instrument(level = "debug", skip(self))]
     async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         let (mut fi, _, disks) = self.get_object_fileinfo(bucket, object, opts, false).await?;
@@ -5846,7 +6163,7 @@ mod tests {
         // Test that all CHECK_PART constants have expected values
         assert_eq!(CHECK_PART_UNKNOWN, 0);
         assert_eq!(CHECK_PART_SUCCESS, 1);
-        assert_eq!(CHECK_PART_FILE_NOT_FOUND, 4); // 实际值是 4，不是 2
+        assert_eq!(CHECK_PART_FILE_NOT_FOUND, 4); // 实际值是4，不是2
         assert_eq!(CHECK_PART_VOLUME_NOT_FOUND, 3);
         assert_eq!(CHECK_PART_FILE_CORRUPT, 5);
     }
@@ -6111,7 +6428,7 @@ mod tests {
         assert_eq!(conv_part_err_to_int(&Some(disk_err)), CHECK_PART_FILE_NOT_FOUND);
 
         let other_err = DiskError::other("other error");
-        assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // other 错误应该返回 UNKNOWN，不是 SUCCESS
+        assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // other错误应该返回UNKNOWN，不是SUCCESS
     }
 
     #[test]

ecstore/src/sets.rs

@@ -29,6 +29,7 @@ use futures::future::join_all;
 use http::HeaderMap;
 use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api};
 use madmin::heal_commands::{HealDriveInfo, HealResultItem};
+use rustfs_filemeta::FileInfo;
 use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
 use tokio::sync::RwLock;
 use uuid::Uuid;
@@ -459,6 +460,11 @@ impl StorageAPI for Sets {
         ))
     }
 
+    #[tracing::instrument(skip(self))]
+    async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
+        unimplemented!()
+    }
+
     #[tracing::instrument(skip(self))]
     async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         if opts.delete_prefix && !opts.delete_prefix_object {
@@ -572,6 +578,24 @@ impl StorageAPI for Sets {
         self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await
     }
 
+    #[tracing::instrument(skip(self))]
+    async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        self.get_disks_by_key(object).transition_object(bucket, object, opts).await
+    }
+
+    #[tracing::instrument(skip(self))]
+    async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
+        self.get_disks_by_key(object).add_partial(bucket, object, version_id).await;
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        self.get_disks_by_key(object)
+            .restore_transitioned_object(bucket, object, opts)
+            .await
+    }
+
     #[tracing::instrument(skip(self))]
     async fn copy_object_part(
         &self,

ecstore/src/signer/mod.rs

@@ -0,0 +1,13 @@
+pub mod ordered_qs;
+pub mod request_signature_streaming;
+pub mod request_signature_streaming_unsigned_trailer;
+pub mod request_signature_v2;
+pub mod request_signature_v4;
+pub mod utils;
+
+pub use request_signature_streaming::streaming_sign_v4;
+pub use request_signature_v2::pre_sign_v2;
+pub use request_signature_v2::sign_v2;
+pub use request_signature_v4::pre_sign_v4;
+pub use request_signature_v4::sign_v4;
+pub use request_signature_v4::sign_v4_trailer;

ecstore/src/signer/ordered_qs.rs

@@ -0,0 +1,109 @@
+//! Ordered query strings
+
+use crate::signer::utils::stable_sort_by_first;
+
+/// Immutable query string container
+#[derive(Debug, Default, Clone)]
+pub struct OrderedQs {
+    /// Ascending query strings
+    qs: Vec<(String, String)>,
+}
+
+/// [`OrderedQs`]
+#[derive(Debug, thiserror::Error)]
+#[error("ParseOrderedQsError: {inner}")]
+pub struct ParseOrderedQsError {
+    /// url decode error
+    inner: serde_urlencoded::de::Error,
+}
+
+impl OrderedQs {
+    /// Constructs [`OrderedQs`] from vec
+    ///
+    /// + strings must be url-decoded
+    #[cfg(test)]
+    #[must_use]
+    pub fn from_vec_unchecked(mut v: Vec<(String, String)>) -> Self {
+        stable_sort_by_first(&mut v);
+        Self { qs: v }
+    }
+
+    /// Parses [`OrderedQs`] from query
+    ///
+    /// # Errors
+    /// Returns [`ParseOrderedQsError`] if query cannot be decoded
+    pub fn parse(query: &str) -> Result<Self, ParseOrderedQsError> {
+        let result = serde_urlencoded::from_str::<Vec<(String, String)>>(query);
+        let mut v = result.map_err(|e| ParseOrderedQsError { inner: e })?;
+        stable_sort_by_first(&mut v);
+        Ok(Self { qs: v })
+    }
+
+    #[must_use]
+    pub fn has(&self, name: &str) -> bool {
+        self.qs.binary_search_by_key(&name, |x| x.0.as_str()).is_ok()
+    }
+
+    /// Gets query values by name. Time `O(logn)`
+    pub fn get_all(&self, name: &str) -> impl Iterator<Item = &str> + use<'_> {
+        let qs = self.qs.as_slice();
+
+        let lower_bound = qs.partition_point(|x| x.0.as_str() < name);
+        let upper_bound = qs.partition_point(|x| x.0.as_str() <= name);
+
+        qs[lower_bound..upper_bound].iter().map(|x| x.1.as_str())
+    }
+
+    pub fn get_unique(&self, name: &str) -> Option<&str> {
+        let qs = self.qs.as_slice();
+        let lower_bound = qs.partition_point(|x| x.0.as_str() < name);
+
+        let mut iter = qs[lower_bound..].iter();
+        let pair = iter.next()?;
+
+        if let Some(following) = iter.next() {
+            if following.0 == name {
+                return None;
+            }
+        }
+
+        (pair.0.as_str() == name).then_some(pair.1.as_str())
+    }
+}
+
+impl AsRef<[(String, String)]> for OrderedQs {
+    fn as_ref(&self) -> &[(String, String)] {
+        self.qs.as_ref()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tag() {
+        {
+            let query = "tagging";
+            let qs = OrderedQs::parse(query).unwrap();
+            assert_eq!(qs.as_ref(), &[("tagging".to_owned(), String::new())]);
+
+            assert_eq!(qs.get_unique("taggin"), None);
+            assert_eq!(qs.get_unique("tagging"), Some(""));
+            assert_eq!(qs.get_unique("taggingg"), None);
+        }
+
+        {
+            let query = "tagging&tagging";
+            let qs = OrderedQs::parse(query).unwrap();
+            assert_eq!(
+                qs.as_ref(),
+                &[("tagging".to_owned(), String::new()), ("tagging".to_owned(), String::new())]
+            );
+
+            assert_eq!(qs.get_unique("taggin"), None);
+            assert_eq!(qs.get_unique("tagging"), None);
+            assert_eq!(qs.get_unique("taggingg"), None);
+        }
+    }
+}

ecstore/src/signer/request_signature_streaming.rs

@@ -0,0 +1,80 @@
+use bytes::{Bytes, BytesMut};
+use futures::prelude::*;
+use futures::task;
+use http::HeaderMap;
+use http::Uri;
+use http::header::TRAILER;
+use http::request::{self, Request};
+use hyper::Method;
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+use std::fmt::Write;
+use std::pin::Pin;
+use std::sync::Mutex;
+use stdx::str::StrExt;
+use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description};
+use tracing::{debug, error, info, warn};
+
+use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key};
+use crate::client::constants::UNSIGNED_PAYLOAD;
+use rustfs_utils::{
+    crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256},
+    hash::EMPTY_STRING_SHA256_HASH,
+};
+
+const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
+const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD";
+const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER";
+const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024;
+const CHUNK_SIGCONST_LEN: i64 = 17;
+const SIGNATURESTR_LEN: i64 = 64;
+const CRLF_LEN: i64 = 2;
+const TRAILER_KV_SEPARATOR: &str = ":";
+const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature";
+
+lazy_static! {
+    static ref ignored_streaming_headers: HashMap<String, bool> = {
+        let mut m = <HashMap<String, bool>>::new();
+        m.insert("authorization".to_string(), true);
+        m.insert("user-agent".to_string(), true);
+        m.insert("content-type".to_string(), true);
+        m
+    };
+}
+
+fn build_chunk_string_to_sign(t: OffsetDateTime, region: &str, previous_sig: &str, chunk_check_sum: &str) -> String {
+    let mut string_to_sign_parts = <Vec<String>>::new();
+    string_to_sign_parts.push(STREAMING_PAYLOAD_HDR.to_string());
+    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
+    string_to_sign_parts.push(t.format(&format).unwrap());
+    string_to_sign_parts.push(get_scope(region, t, SERVICE_TYPE_S3));
+    string_to_sign_parts.push(previous_sig.to_string());
+    string_to_sign_parts.push(EMPTY_STRING_SHA256_HASH.to_string());
+    string_to_sign_parts.push(chunk_check_sum.to_string());
+    string_to_sign_parts.join("\n")
+}
+
+fn build_chunk_signature(
+    chunk_check_sum: &str,
+    req_time: OffsetDateTime,
+    region: &str,
+    previous_signature: &str,
+    secret_access_key: &str,
+) -> String {
+    let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, previous_signature, chunk_check_sum);
+    let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3);
+    get_signature(signing_key, &chunk_string_to_sign)
+}
+
+pub fn streaming_sign_v4(
+    req: request::Builder,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    region: &str,
+    data_len: i64,
+    req_time: OffsetDateTime, /*, sh256: md5simd.Hasher*/
+) -> request::Builder {
+    todo!();
+}

ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs

@@ -0,0 +1,11 @@
+use http::request;
+use time::OffsetDateTime;
+
+pub fn streaming_unsigned_v4(
+    mut req: request::Builder,
+    session_token: &str,
+    data_len: i64,
+    req_time: OffsetDateTime,
+) -> request::Builder {
+    todo!();
+}

ecstore/src/signer/request_signature_v2.rs

@@ -0,0 +1,231 @@
+use bytes::{Bytes, BytesMut};
+use http::request;
+use hyper::Uri;
+use std::collections::HashMap;
+use std::fmt::Write;
+use time::{OffsetDateTime, format_description, macros::format_description};
+
+use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1};
+
+use super::utils::get_host_addr;
+
+const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
+const SIGN_V2_ALGORITHM: &str = "AWS";
+
+fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String {
+    let mut path = "".to_string();
+
+    //path = serde_urlencoded::to_string(req.uri_ref().unwrap().path().unwrap()).unwrap();
+    path = req.uri_ref().unwrap().path().to_string();
+    path
+}
+
+pub fn pre_sign_v2(
+    mut req: request::Builder,
+    access_key_id: &str,
+    secret_access_key: &str,
+    expires: i64,
+    virtual_host: bool,
+) -> request::Builder {
+    if access_key_id == "" || secret_access_key == "" {
+        return req;
+    }
+
+    let d = OffsetDateTime::now_utc();
+    let d = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
+    let epoch_expires = d.unix_timestamp() + expires;
+
+    let mut headers = req.headers_mut().expect("err");
+    let expires_str = headers.get("Expires");
+    if expires_str.is_none() {
+        headers.insert("Expires", format!("{:010}", epoch_expires).parse().unwrap());
+    }
+
+    let string_to_sign = pre_string_to_sign_v2(&req, virtual_host);
+    let signature = hex(hmac_sha1(secret_access_key, string_to_sign));
+
+    let result = serde_urlencoded::from_str::<HashMap<String, String>>(req.uri_ref().unwrap().query().unwrap());
+    let mut query = result.unwrap_or_default();
+    if get_host_addr(&req).contains(".storage.googleapis.com") {
+        query.insert("GoogleAccessId".to_string(), access_key_id.to_string());
+    } else {
+        query.insert("AWSAccessKeyId".to_string(), access_key_id.to_string());
+    }
+
+    query.insert("Expires".to_string(), format!("{:010}", epoch_expires));
+
+    let uri = req.uri_ref().unwrap().clone();
+    let mut parts = req.uri_ref().unwrap().clone().into_parts();
+    parts.path_and_query = Some(
+        format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature)
+            .parse()
+            .unwrap(),
+    );
+    let req = req.uri(Uri::from_parts(parts).unwrap());
+
+    req
+}
+
+fn post_pre_sign_signature_v2(policy_base64: &str, secret_access_key: &str) -> String {
+    let signature = hex(hmac_sha1(secret_access_key, policy_base64));
+    signature
+}
+
+pub fn sign_v2(
+    mut req: request::Builder,
+    content_len: i64,
+    access_key_id: &str,
+    secret_access_key: &str,
+    virtual_host: bool,
+) -> request::Builder {
+    if access_key_id == "" || secret_access_key == "" {
+        return req;
+    }
+
+    let d = OffsetDateTime::now_utc();
+    let d2 = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
+
+    let string_to_sign = string_to_sign_v2(&req, virtual_host);
+    let mut headers = req.headers_mut().expect("err");
+
+    let date = headers.get("Date").unwrap();
+    if date.to_str().unwrap() == "" {
+        headers.insert(
+            "Date",
+            d2.format(&format_description::well_known::Rfc2822)
+                .unwrap()
+                .to_string()
+                .parse()
+                .unwrap(),
+        );
+    }
+
+    let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id);
+    let auth_header = format!("{}{}", auth_header, base64_encode(&hmac_sha1(secret_access_key, string_to_sign)));
+
+    headers.insert("Authorization", auth_header.parse().unwrap());
+
+    req
+}
+
+fn pre_string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String {
+    let mut buf = BytesMut::new();
+    write_pre_sign_v2_headers(&mut buf, &req);
+    write_canonicalized_headers(&mut buf, &req);
+    write_canonicalized_resource(&mut buf, &req, virtual_host);
+    String::from_utf8(buf.to_vec()).unwrap()
+}
+
+fn write_pre_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) {
+    buf.write_str(req.method_ref().unwrap().as_str());
+    buf.write_char('\n');
+    buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap());
+    buf.write_char('\n');
+    buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap());
+    buf.write_char('\n');
+    buf.write_str(req.headers_ref().unwrap().get("Expires").unwrap().to_str().unwrap());
+    buf.write_char('\n');
+}
+
+fn string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String {
+    let mut buf = BytesMut::new();
+    write_sign_v2_headers(&mut buf, &req);
+    write_canonicalized_headers(&mut buf, &req);
+    write_canonicalized_resource(&mut buf, &req, virtual_host);
+    String::from_utf8(buf.to_vec()).unwrap()
+}
+
+fn write_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) {
+    buf.write_str(req.method_ref().unwrap().as_str());
+    buf.write_char('\n');
+    buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap());
+    buf.write_char('\n');
+    buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap());
+    buf.write_char('\n');
+    buf.write_str(req.headers_ref().unwrap().get("Date").unwrap().to_str().unwrap());
+    buf.write_char('\n');
+}
+
+fn write_canonicalized_headers(buf: &mut BytesMut, req: &request::Builder) {
+    let mut proto_headers = Vec::<String>::new();
+    let mut vals = HashMap::<String, Vec<String>>::new();
+    for k in req.headers_ref().expect("err").keys() {
+        let lk = k.as_str().to_lowercase();
+        if lk.starts_with("x-amz") {
+            proto_headers.push(lk.clone());
+            let vv = req
+                .headers_ref()
+                .expect("err")
+                .get_all(k)
+                .iter()
+                .map(|e| e.to_str().unwrap().to_string())
+                .collect();
+            vals.insert(lk, vv);
+        }
+    }
+    proto_headers.sort();
+    for k in proto_headers {
+        buf.write_str(&k);
+        buf.write_char(':');
+        for (idx, v) in vals[&k].iter().enumerate() {
+            if idx > 0 {
+                buf.write_char(',');
+            }
+            buf.write_str(v);
+        }
+        buf.write_char('\n');
+    }
+}
+
+const INCLUDED_QUERY: &[&str] = &[
+    "acl",
+    "delete",
+    "lifecycle",
+    "location",
+    "logging",
+    "notification",
+    "partNumber",
+    "policy",
+    "requestPayment",
+    "response-cache-control",
+    "response-content-disposition",
+    "response-content-encoding",
+    "response-content-language",
+    "response-content-type",
+    "response-expires",
+    "uploadId",
+    "uploads",
+    "versionId",
+    "versioning",
+    "versions",
+    "website",
+];
+
+fn write_canonicalized_resource(buf: &mut BytesMut, req: &request::Builder, virtual_host: bool) {
+    let request_url = req.uri_ref().unwrap();
+    buf.write_str(&encode_url2path(req, virtual_host));
+    if request_url.query().unwrap() != "" {
+        let mut n: i64 = 0;
+        let result = serde_urlencoded::from_str::<HashMap<String, Vec<String>>>(req.uri_ref().unwrap().query().unwrap());
+        let mut vals = result.unwrap_or_default();
+        for resource in INCLUDED_QUERY {
+            let vv = &vals[*resource];
+            if vv.len() > 0 {
+                n += 1;
+                match n {
+                    1 => {
+                        buf.write_char('?');
+                    }
+                    _ => {
+                        buf.write_char('&');
+                        buf.write_str(resource);
+                        if vv[0].len() > 0 {
+                            buf.write_char('=');
+                            buf.write_str(&vv[0]);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

ecstore/src/signer/request_signature_v4.rs

@@ -0,0 +1,716 @@
+use bytes::{Bytes, BytesMut};
+use http::HeaderMap;
+use http::Uri;
+use http::header::TRAILER;
+use http::request::{self, Request};
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+use std::fmt::Write;
+use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description};
+use tracing::{debug, error, info, warn};
+
+use super::ordered_qs::OrderedQs;
+use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4;
+use super::utils::stable_sort_by_first;
+use super::utils::{get_host_addr, sign_v4_trim_all};
+use crate::client::constants::UNSIGNED_PAYLOAD;
+use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
+use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+
+pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
+pub const SERVICE_TYPE_S3: &str = "s3";
+pub const SERVICE_TYPE_STS: &str = "sts";
+
+lazy_static! {
+    static ref v4_ignored_headers: HashMap<String, bool> = {
+        let mut m = <HashMap<String, bool>>::new();
+        m.insert("accept-encoding".to_string(), true);
+        m.insert("authorization".to_string(), true);
+        m.insert("user-agent".to_string(), true);
+        m
+    };
+}
+
+pub fn get_signing_key(secret: &str, loc: &str, t: OffsetDateTime, service_type: &str) -> [u8; 32] {
+    let mut s = "AWS4".to_string();
+    s.push_str(secret);
+    let format = format_description!("[year][month][day]");
+    let date = hmac_sha256(s.into_bytes(), t.format(&format).unwrap().into_bytes());
+    let location = hmac_sha256(date, loc);
+    let service = hmac_sha256(location, service_type);
+    let signing_key = hmac_sha256(service, "aws4_request");
+    signing_key
+}
+
+pub fn get_signature(signing_key: [u8; 32], string_to_sign: &str) -> String {
+    hex(hmac_sha256(signing_key, string_to_sign))
+}
+
+pub fn get_scope(location: &str, t: OffsetDateTime, service_type: &str) -> String {
+    let format = format_description!("[year][month][day]");
+    let mut ans = String::from("");
+    ans.push_str(&t.format(&format).unwrap().to_string());
+    ans.push('/');
+    ans.push_str(location); // TODO: use a `Region` type
+    ans.push('/');
+    ans.push_str(service_type);
+    ans.push_str("/aws4_request");
+    ans
+}
+
+fn get_credential(access_key_id: &str, location: &str, t: OffsetDateTime, service_type: &str) -> String {
+    let scope = get_scope(location, t, service_type);
+    let mut s = access_key_id.to_string();
+    s.push_str("/");
+    s.push_str(&scope);
+    s
+}
+
+fn get_hashed_payload(req: &request::Builder) -> String {
+    let headers = req.headers_ref().unwrap();
+    let mut hashed_payload = "";
+    if let Some(payload) = headers.get("X-Amz-Content-Sha256") {
+        hashed_payload = payload.to_str().unwrap();
+    }
+    if hashed_payload == "" {
+        hashed_payload = UNSIGNED_PAYLOAD;
+    }
+    hashed_payload.to_string()
+}
+
+fn get_canonical_headers(req: &request::Builder, ignored_headers: &HashMap<String, bool>) -> String {
+    let mut headers = Vec::<String>::new();
+    let mut vals = HashMap::<String, Vec<String>>::new();
+    for k in req.headers_ref().expect("err").keys() {
+        if ignored_headers.get(&k.to_string()).is_some() {
+            continue;
+        }
+        headers.push(k.as_str().to_lowercase());
+        let vv = req
+            .headers_ref()
+            .expect("err")
+            .get_all(k)
+            .iter()
+            .map(|e| e.to_str().unwrap().to_string())
+            .collect();
+        vals.insert(k.as_str().to_lowercase(), vv);
+    }
+    if !header_exists("host", &headers) {
+        headers.push("host".to_string());
+    }
+    headers.sort();
+
+    debug!("get_canonical_headers vals: {:?}", vals);
+    debug!("get_canonical_headers headers: {:?}", headers);
+
+    let mut buf = BytesMut::new();
+    for k in headers {
+        buf.write_str(&k);
+        buf.write_char(':');
+        let k: &str = &k;
+        match k {
+            "host" => {
+                buf.write_str(&get_host_addr(&req));
+                buf.write_char('\n');
+            }
+            _ => {
+                for (idx, v) in vals[k].iter().enumerate() {
+                    if idx > 0 {
+                        buf.write_char(',');
+                    }
+                    buf.write_str(&sign_v4_trim_all(v));
+                }
+                buf.write_char('\n');
+            }
+        }
+    }
+    String::from_utf8(buf.to_vec()).unwrap()
+}
+
+fn header_exists(key: &str, headers: &[String]) -> bool {
+    for k in headers {
+        if k == key {
+            return true;
+        }
+    }
+    false
+}
+
+fn get_signed_headers(req: &request::Builder, ignored_headers: &HashMap<String, bool>) -> String {
+    let mut headers = Vec::<String>::new();
+    let headers_ref = req.headers_ref().expect("err");
+    debug!("get_signed_headers headers: {:?}", headers_ref);
+    for (k, _) in headers_ref {
+        if ignored_headers.get(&k.to_string()).is_some() {
+            continue;
+        }
+        headers.push(k.as_str().to_lowercase());
+    }
+    if !header_exists("host", &headers) {
+        headers.push("host".to_string());
+    }
+    headers.sort();
+    headers.join(";")
+}
+
+fn get_canonical_request(req: &request::Builder, ignored_headers: &HashMap<String, bool>, hashed_payload: &str) -> String {
+    let mut canonical_query_string = "".to_string();
+    if let Some(q) = req.uri_ref().unwrap().query() {
+        let mut query = q.split('&').map(|h| h.to_string()).collect::<Vec<String>>();
+        query.sort();
+        canonical_query_string = query.join("&");
+        canonical_query_string = canonical_query_string.replace("+", "%20");
+    }
+
+    let mut canonical_request = <Vec<String>>::new();
+    canonical_request.push(req.method_ref().unwrap().to_string());
+    canonical_request.push(req.uri_ref().unwrap().path().to_string());
+    canonical_request.push(canonical_query_string);
+    canonical_request.push(get_canonical_headers(&req, ignored_headers));
+    canonical_request.push(get_signed_headers(&req, ignored_headers));
+    canonical_request.push(hashed_payload.to_string());
+    canonical_request.join("\n")
+}
+
+fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: &str, service_type: &str) -> String {
+    let mut string_to_sign = SIGN_V4_ALGORITHM.to_string();
+    string_to_sign.push_str("\n");
+    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
+    string_to_sign.push_str(&t.format(&format).unwrap());
+    string_to_sign.push_str("\n");
+    string_to_sign.push_str(&get_scope(location, t, service_type));
+    string_to_sign.push_str("\n");
+    string_to_sign.push_str(&hex_sha256(canonical_request.as_bytes(), |s| s.to_string()));
+    string_to_sign
+}
+
+pub fn pre_sign_v4(
+    req: request::Builder,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+    expires: i64,
+    t: OffsetDateTime,
+) -> request::Builder {
+    if access_key_id == "" || secret_access_key == "" {
+        return req;
+    }
+
+    //let t = OffsetDateTime::now_utc();
+    //let date = AmzDate::parse(timestamp).unwrap();
+    let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
+
+    //let credential = get_scope(location, t, SERVICE_TYPE_S3);
+    let credential = get_credential(access_key_id, location, t, SERVICE_TYPE_S3);
+    let signed_headers = get_signed_headers(&req, &v4_ignored_headers);
+
+    let mut query = <Vec<(String, String)>>::new();
+    if let Some(q) = req.uri_ref().unwrap().query() {
+        let result = serde_urlencoded::from_str::<Vec<(String, String)>>(q);
+        query = result.unwrap_or_default();
+    }
+    query.push(("X-Amz-Algorithm".to_string(), SIGN_V4_ALGORITHM.to_string()));
+    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
+    query.push(("X-Amz-Date".to_string(), t.format(&format).unwrap().to_string()));
+    query.push(("X-Amz-Expires".to_string(), format!("{:010}", expires)));
+    query.push(("X-Amz-SignedHeaders".to_string(), signed_headers));
+    query.push(("X-Amz-Credential".to_string(), credential));
+    if session_token != "" {
+        query.push(("X-Amz-Security-Token".to_string(), session_token.to_string()));
+    }
+
+    let uri = req.uri_ref().unwrap().clone();
+    let mut parts = req.uri_ref().unwrap().clone().into_parts();
+    parts.path_and_query = Some(
+        format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
+            .parse()
+            .unwrap(),
+    );
+    let req = req.uri(Uri::from_parts(parts).unwrap());
+
+    let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+    let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, SERVICE_TYPE_S3);
+    //println!("canonical_request: \n{}\n", canonical_request);
+    //println!("string_to_sign: \n{}\n", string_to_sign);
+    let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3);
+    let signature = get_signature(signing_key, &string_to_sign);
+
+    let uri = req.uri_ref().unwrap().clone();
+    let mut parts = req.uri_ref().unwrap().clone().into_parts();
+    parts.path_and_query = Some(
+        format!(
+            "{}?{}&X-Amz-Signature={}",
+            uri.path(),
+            serde_urlencoded::to_string(&query).unwrap(),
+            signature
+        )
+        .parse()
+        .unwrap(),
+    );
+    let req = req.uri(Uri::from_parts(parts).unwrap());
+
+    req
+}
+
+fn post_pre_sign_signature_v4(policy_base64: &str, t: OffsetDateTime, secret_access_key: &str, location: &str) -> String {
+    let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3);
+    let signature = get_signature(signing_key, policy_base64);
+    signature
+}
+
+fn sign_v4_sts(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, location: &str) -> request::Builder {
+    sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new())
+}
+
+fn sign_v4_inner(
+    mut req: request::Builder,
+    content_len: i64,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+    service_type: &str,
+    trailer: HeaderMap,
+) -> request::Builder {
+    if access_key_id == "" || secret_access_key == "" {
+        return req;
+    }
+
+    let t = OffsetDateTime::now_utc();
+    let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
+
+    let mut headers = req.headers_mut().expect("err");
+    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
+    headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap());
+
+    if session_token != "" {
+        headers.insert("X-Amz-Security-Token", session_token.parse().unwrap());
+    }
+
+    if trailer.len() > 0 {
+        for (k, _) in &trailer {
+            headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap());
+        }
+
+        headers.insert("Content-Encoding", "aws-chunked".parse().unwrap());
+        headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap());
+    }
+
+    if service_type == SERVICE_TYPE_STS {
+        headers.remove("X-Amz-Content-Sha256");
+    }
+
+    let hashed_payload = get_hashed_payload(&req);
+    let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &hashed_payload);
+    let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, service_type);
+    let signing_key = get_signing_key(secret_access_key, location, t, service_type);
+    let credential = get_credential(access_key_id, location, t2, service_type);
+    let signed_headers = get_signed_headers(&req, &v4_ignored_headers);
+    let signature = get_signature(signing_key, &string_to_sign);
+    //debug!("\n\ncanonical_request: \n{}\nstring_to_sign: \n{}\nsignature: \n{}\n\n", &canonical_request, &string_to_sign, &signature);
+
+    let mut headers = req.headers_mut().expect("err");
+
+    let auth = format!(
+        "{} Credential={}, SignedHeaders={}, Signature={}",
+        SIGN_V4_ALGORITHM, credential, signed_headers, signature
+    );
+    headers.insert("Authorization", auth.parse().unwrap());
+
+    if trailer.len() > 0 {
+        //req.Trailer = trailer;
+        for (_, v) in &trailer {
+            headers.append(http::header::TRAILER, v.clone());
+        }
+        return streaming_unsigned_v4(req, &session_token, content_len, t);
+    }
+    req
+}
+
+fn unsigned_trailer(mut req: request::Builder, content_len: i64, trailer: HeaderMap) {
+    if trailer.len() > 0 {
+        return;
+    }
+    let t = OffsetDateTime::now_utc();
+    let t = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
+
+    let mut headers = req.headers_mut().expect("err");
+    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
+    headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap());
+
+    for (k, _) in &trailer {
+        headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap());
+    }
+
+    headers.insert("Content-Encoding", "aws-chunked".parse().unwrap());
+    headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap());
+
+    if trailer.len() > 0 {
+        for (_, v) in &trailer {
+            headers.append(http::header::TRAILER, v.clone());
+        }
+    }
+    streaming_unsigned_v4(req, "", content_len, t);
+}
+
+pub fn sign_v4(
+    mut req: request::Builder,
+    content_len: i64,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+) -> request::Builder {
+    sign_v4_inner(
+        req,
+        content_len,
+        access_key_id,
+        secret_access_key,
+        session_token,
+        location,
+        SERVICE_TYPE_S3,
+        HeaderMap::new(),
+    )
+}
+
+pub fn sign_v4_trailer(
+    req: request::Builder,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+    trailer: HeaderMap,
+) -> request::Builder {
+    sign_v4_inner(
+        req,
+        0,
+        access_key_id,
+        secret_access_key,
+        session_token,
+        location,
+        SERVICE_TYPE_S3,
+        trailer,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn example_list_objects() {
+        // let access_key_id = "AKIAIOSFODNN7EXAMPLE";
+        let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
+        let timestamp = "20130524T000000Z";
+        let t = datetime!(2013-05-24 0:00 UTC);
+        // let bucket = "examplebucket";
+        let region = "us-east-1";
+        let service = "s3";
+        let path = "/";
+
+        let mut req = Request::builder()
+            .method(http::Method::GET)
+            .uri("http://examplebucket.s3.amazonaws.com/?");
+        let mut headers = req.headers_mut().expect("err");
+        headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
+        headers.insert(
+            "x-amz-content-sha256",
+            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+                .parse()
+                .unwrap(),
+        );
+        headers.insert("x-amz-date", timestamp.parse().unwrap());
+
+        let mut query = <Vec<(String, String)>>::new();
+        query.push(("max-keys".to_string(), "2".to_string()));
+        query.push(("prefix".to_string(), "J".to_string()));
+        let uri = req.uri_ref().unwrap().clone();
+        let mut parts = req.uri_ref().unwrap().clone().into_parts();
+        parts.path_and_query = Some(
+            format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
+                .parse()
+                .unwrap(),
+        );
+        let req = req.uri(Uri::from_parts(parts).unwrap());
+
+        let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+        assert_eq!(
+            canonical_request,
+            concat!(
+                "GET\n",
+                "/\n",
+                "max-keys=2&prefix=J\n",
+                "host:examplebucket.s3.amazonaws.com\n",
+                "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
+                "x-amz-date:",
+                "20130524T000000Z",
+                "\n",
+                "\n",
+                "host;x-amz-content-sha256;x-amz-date\n",
+                "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+            )
+        );
+
+        let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
+        assert_eq!(
+            string_to_sign,
+            concat!(
+                "AWS4-HMAC-SHA256\n",
+                "20130524T000000Z",
+                "\n",
+                "20130524/us-east-1/s3/aws4_request\n",
+                "df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7",
+            )
+        );
+
+        let signing_key = get_signing_key(secret_access_key, region, t, service);
+        let signature = get_signature(signing_key, &string_to_sign);
+
+        assert_eq!(signature, "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7");
+    }
+
+    #[test]
+    fn example_signature() {
+        // let access_key_id = "rustfsadmin";
+        let secret_access_key = "rustfsadmin";
+        let timestamp = "20250505T011054Z";
+        let t = datetime!(2025-05-05 01:10:54 UTC);
+        // let bucket = "mblock2";
+        let region = "us-east-1";
+        let service = "s3";
+        let path = "/mblock2/";
+
+        let mut req = Request::builder()
+            .method(http::Method::GET)
+            .uri("http://192.168.1.11:9020/mblock2/?");
+
+        let mut headers = req.headers_mut().expect("err");
+        headers.insert("host", "192.168.1.11:9020".parse().unwrap());
+        headers.insert(
+            "x-amz-content-sha256",
+            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+                .parse()
+                .unwrap(),
+        );
+        headers.insert("x-amz-date", timestamp.parse().unwrap());
+
+        let mut query: Vec<(String, String)> = Vec::new();
+        let uri = req.uri_ref().unwrap().clone();
+        let mut parts = req.uri_ref().unwrap().clone().into_parts();
+        parts.path_and_query = Some(
+            format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
+                .parse()
+                .unwrap(),
+        );
+        let req = req.uri(Uri::from_parts(parts).unwrap());
+
+        let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+        println!("canonical_request: \n{}\n", canonical_request);
+        assert_eq!(
+            canonical_request,
+            concat!(
+                "GET\n",
+                "/mblock2/\n",
+                "\n",
+                "host:192.168.1.11:9020\n",
+                "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
+                "x-amz-date:",
+                "20250505T011054Z",
+                "\n",
+                "\n",
+                "host;x-amz-content-sha256;x-amz-date\n",
+                "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+            )
+        );
+
+        let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
+        println!("string_to_sign: \n{}\n", string_to_sign);
+        assert_eq!(
+            string_to_sign,
+            concat!(
+                "AWS4-HMAC-SHA256\n",
+                "20250505T011054Z",
+                "\n",
+                "20250505/us-east-1/s3/aws4_request\n",
+                "c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0",
+            )
+        );
+
+        let signing_key = get_signing_key(secret_access_key, region, t, service);
+        let signature = get_signature(signing_key, &string_to_sign);
+        println!("signature: \n{}\n", signature);
+        assert_eq!(signature, "df4116595e27b0dfd1103358947d9199378cc6386c4657abd8c5f0b11ebb4931");
+    }
+
+    #[test]
+    fn example_signature2() {
+        // let access_key_id = "rustfsadmin";
+        let secret_access_key = "rustfsadmin";
+        let timestamp = "20250507T051030Z";
+        let t = datetime!(2025-05-07 05:10:30 UTC);
+        // let bucket = "mblock2";
+        let region = "us-east-1";
+        let service = "s3";
+        let path = "/mblock2/";
+
+        let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?list-type=2&encoding-type=url&prefix=mypre&delimiter=%2F&fetch-owner=true&max-keys=1");
+
+        let mut headers = req.headers_mut().expect("err");
+        headers.insert("host", "192.168.1.11:9020".parse().unwrap());
+        headers.insert(
+            "x-amz-content-sha256",
+            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+                .parse()
+                .unwrap(),
+        );
+        headers.insert("x-amz-date", timestamp.parse().unwrap());
+
+        /*let uri = req.uri_ref().unwrap().clone();
+        println!("{:?}", uri);
+        let mut canonical_query_string = "".to_string();
+        if let Some(q) = uri.query() {
+            let result = serde_urlencoded::from_str::<Vec<String>>(q);
+            let mut query = result.unwrap_or_default();
+            query.sort();
+            canonical_query_string = query.join("&");
+            canonical_query_string.replace("+", "%20");
+        }
+        let mut parts = req.uri_ref().unwrap().clone().into_parts();
+        parts.path_and_query = Some(format!("{}?{}", uri.path(), canonical_query_string).parse().unwrap());
+        let req = req.uri(Uri::from_parts(parts).unwrap());*/
+        println!("{:?}", req.uri_ref().unwrap().query());
+        let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+        println!("canonical_request: \n{}\n", canonical_request);
+        assert_eq!(
+            canonical_request,
+            concat!(
+                "GET\n",
+                "/mblock2/\n",
+                "delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n",
+                "host:192.168.1.11:9020\n",
+                "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n",
+                "x-amz-date:",
+                "20250507T051030Z",
+                "\n",
+                "\n",
+                "host;x-amz-content-sha256;x-amz-date\n",
+                "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+            )
+        );
+
+        let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
+        println!("string_to_sign: \n{}\n", string_to_sign);
+        assert_eq!(
+            string_to_sign,
+            concat!(
+                "AWS4-HMAC-SHA256\n",
+                "20250507T051030Z",
+                "\n",
+                "20250507/us-east-1/s3/aws4_request\n",
+                "e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999",
+            )
+        );
+
+        let signing_key = get_signing_key(secret_access_key, region, t, service);
+        let signature = get_signature(signing_key, &string_to_sign);
+        println!("signature: \n{}\n", signature);
+        assert_eq!(signature, "760278c9a77d5c245ac83d85917bddc3e3b14343091e8f4ad8edbbf73107d685");
+    }
+
+    #[test]
+    fn example_presigned_url() {
+        use hyper::Uri;
+
+        let access_key_id = "AKIAIOSFODNN7EXAMPLE";
+        let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
+        let timestamp = "20130524T000000Z";
+        let t = datetime!(2013-05-24 0:00 UTC);
+        // let bucket = "mblock2";
+        let region = "us-east-1";
+        let service = "s3";
+        let path = "/";
+        let session_token = "";
+
+        let mut req = Request::builder()
+            .method(http::Method::GET)
+            .uri("http://examplebucket.s3.amazonaws.com/test.txt");
+
+        let mut headers = req.headers_mut().expect("err");
+        headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap());
+
+        req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", &region, 86400, t);
+
+        let mut canonical_request = req.method_ref().unwrap().as_str().to_string();
+        canonical_request.push_str("\n");
+        canonical_request.push_str(req.uri_ref().unwrap().path());
+        canonical_request.push_str("\n");
+        canonical_request.push_str(req.uri_ref().unwrap().query().unwrap());
+        canonical_request.push_str("\n");
+        canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers));
+        canonical_request.push_str("\n");
+        canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers));
+        canonical_request.push_str("\n");
+        canonical_request.push_str(&get_hashed_payload(&req));
+        //println!("canonical_request: \n{}\n", canonical_request);
+        assert_eq!(
+            canonical_request,
+            concat!(
+                "GET\n",
+                "/test.txt\n",
+                "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404\n",
+                "host:examplebucket.s3.amazonaws.com\n",
+                "\n",
+                "host\n",
+                "UNSIGNED-PAYLOAD",
+            )
+        );
+    }
+
+    #[test]
+    fn example_presigned_url2() {
+        use hyper::Uri;
+
+        let access_key_id = "rustfsadmin";
+        let secret_access_key = "rustfsadmin";
+        let timestamp = "20130524T000000Z";
+        let t = datetime!(2013-05-24 0:00 UTC);
+        // let bucket = "mblock2";
+        let region = "us-east-1";
+        let service = "s3";
+        let path = "/mblock2/";
+        let session_token = "";
+
+        let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/test.txt?delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2");
+
+        let mut headers = req.headers_mut().expect("err");
+        headers.insert("host", "192.168.1.11:9020".parse().unwrap());
+
+        req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", &region, 86400, t);
+
+        let mut canonical_request = req.method_ref().unwrap().as_str().to_string();
+        canonical_request.push_str("\n");
+        canonical_request.push_str(req.uri_ref().unwrap().path());
+        canonical_request.push_str("\n");
+        canonical_request.push_str(req.uri_ref().unwrap().query().unwrap());
+        canonical_request.push_str("\n");
+        canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers));
+        canonical_request.push_str("\n");
+        canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers));
+        canonical_request.push_str("\n");
+        canonical_request.push_str(&get_hashed_payload(&req));
+        //println!("canonical_request: \n{}\n", canonical_request);
+        assert_eq!(
+            canonical_request,
+            concat!(
+                "GET\n",
+                "/mblock2/test.txt\n",
+                "delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=rustfsadmin%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=e4af975e4a7e2c0449451740c7e9a425123681d2a8830bfb188789ea19618b20\n",
+                "host:192.168.1.11:9020\n",
+                "\n",
+                "host\n",
+                "UNSIGNED-PAYLOAD",
+            )
+        );
+    }
+}

ecstore/src/signer/utils.rs

@@ -0,0 +1,31 @@
+use http::request;
+
+pub fn get_host_addr(req: &request::Builder) -> String {
+    let host = req.headers_ref().expect("err").get("host");
+    let uri = req.uri_ref().unwrap();
+    let req_host;
+    if let Some(port) = uri.port() {
+        req_host = format!("{}:{}", uri.host().unwrap(), port);
+    } else {
+        req_host = uri.host().unwrap().to_string();
+    }
+    if host.is_some() && req_host != host.unwrap().to_str().unwrap().to_string() {
+        return host.unwrap().to_str().unwrap().to_string();
+    }
+    /*if req.uri_ref().unwrap().host().is_some() {
+        return req.uri_ref().unwrap().host().unwrap();
+    }*/
+    req_host
+}
+
+pub fn sign_v4_trim_all(input: &str) -> String {
+    let ss = input.split_whitespace().collect::<Vec<_>>();
+    ss.join(" ")
+}
+
+pub fn stable_sort_by_first<T>(v: &mut [(T, T)])
+where
+    T: Ord,
+{
+    v.sort_by(|lhs, rhs| lhs.0.cmp(&rhs.0));
+}

ecstore/src/store.rs

@@ -1,19 +1,21 @@
 #![allow(clippy::map_entry)]
 
+use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry;
 use crate::bucket::metadata_sys::{self, set_bucket_metadata};
 use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
 use crate::config::GLOBAL_StorageClass;
 use crate::config::storageclass;
 use crate::disk::endpoint::{Endpoint, EndpointType};
 use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
+use crate::error::{Error, Result};
 use crate::error::{
     StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
     is_err_version_not_found, to_object_err,
 };
 use crate::global::{
     DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME,
-    GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_endpoints, is_dist_erasure, is_erasure_sd,
-    set_global_deployment_id, set_object_layer,
+    GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_endpoints, is_dist_erasure,
+    is_erasure_sd, set_global_deployment_id, set_object_layer,
 };
 use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo};
 use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo};
@@ -26,7 +28,7 @@ use crate::rebalance::RebalanceMeta;
 use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO};
 use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
 use crate::{
-    bucket::metadata::BucketMetadata,
+    bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
     disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
     endpoints::EndpointServerPools,
     rpc::S3PeerSys,
@@ -38,10 +40,6 @@ use crate::{
     },
     store_init,
 };
-use rustfs_utils::crypto::base64_decode;
-use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
-
-use crate::error::{Error, Result};
 use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port};
 use futures::future::join_all;
 use glob::Pattern;
@@ -49,7 +47,10 @@ use http::HeaderMap;
 use lazy_static::lazy_static;
 use madmin::heal_commands::HealResultItem;
 use rand::Rng as _;
+use rustfs_filemeta::FileInfo;
 use rustfs_filemeta::MetaCacheEntry;
+use rustfs_utils::crypto::base64_decode;
+use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
 use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
 use std::cmp::Ordering;
 use std::net::SocketAddr;
@@ -330,6 +331,14 @@ impl ECStore {
             }
         }
 
+        init_background_expiry(self.clone()).await;
+
+        TransitionState::init(self.clone()).await;
+
+        if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await {
+            info!("TierConfigMgr init error: {}", err);
+        }
+
         Ok(())
     }
 
@@ -1857,7 +1866,6 @@ impl StorageAPI for ECStore {
                 return self.pools[idx].new_multipart_upload(bucket, object, opts).await;
             }
         }
-
         let idx = self.get_pool_idx(bucket, object, -1).await?;
         if opts.data_movement && idx == opts.src_pool_idx {
             return Err(StorageError::DataMovementOverwriteErr(
@@ -1870,6 +1878,49 @@ impl StorageAPI for ECStore {
         self.pools[idx].new_multipart_upload(bucket, object, opts).await
     }
 
+    #[tracing::instrument(skip(self))]
+    async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
+        let object = encode_dir_object(object);
+
+        if self.single_pool() {
+            self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
+        }
+
+        let idx = self
+            .get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default())
+            .await?;
+
+        self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
+        Ok(())
+    }
+    #[tracing::instrument(skip(self))]
+    async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        let object = encode_dir_object(object);
+        if self.single_pool() {
+            return self.pools[0].transition_object(bucket, &object, opts).await;
+        }
+
+        //opts.skip_decommissioned = true;
+        //opts.no_lock = true;
+        let idx = self.get_pool_idx_existing_with_opts(bucket, &object, opts).await?;
+
+        self.pools[idx].transition_object(bucket, &object, opts).await
+    }
+
+    #[tracing::instrument(skip(self))]
+    async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+        let object = encode_dir_object(object);
+        if self.single_pool() {
+            return self.pools[0].restore_transitioned_object(bucket, &object, opts).await;
+        }
+
+        //opts.skip_decommissioned = true;
+        //opts.nolock = true;
+        let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?;
+
+        self.pools[idx].restore_transitioned_object(bucket, &object, opts).await
+    }
+
     #[tracing::instrument(skip(self))]
     async fn copy_object_part(
         &self,
@@ -2098,6 +2149,20 @@ impl StorageAPI for ECStore {
         self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
     }
 
+    #[tracing::instrument(skip(self))]
+    async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
+        check_del_obj_args(bucket, object)?;
+
+        let object = rustfs_utils::path::encode_dir_object(object);
+
+        if self.single_pool() {
+            return self.pools[0]
+                .delete_object_version(bucket, object.as_str(), fi, force_del_marker)
+                .await;
+        }
+        Ok(())
+    }
+
     #[tracing::instrument(skip(self))]
     async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
         let object = encode_dir_object(object);

ecstore/src/store_api.rs

@@ -4,6 +4,11 @@ use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusTy
 use crate::error::{Error, Result};
 use crate::heal::heal_ops::HealSequence;
 use crate::store_utils::clean_metadata;
+use crate::{
+    bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent,
+    bucket::lifecycle::lifecycle::ExpirationOptions,
+    bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions},
+};
 use crate::{disk::DiskStore, heal::heal_commands::HealOpts};
 use http::{HeaderMap, HeaderValue};
 use madmin::heal_commands::HealResultItem;
@@ -290,6 +295,10 @@ pub struct ObjectOptions {
     pub replication_request: bool,
     pub delete_marker: bool,
 
+    pub transition: TransitionOptions,
+    pub expiration: ExpirationOptions,
+    pub lifecycle_audit_event: LcAuditEvent,
+
     pub eval_metadata: Option<HashMap<String, String>>,
 }
 
@@ -362,6 +371,7 @@ pub struct ObjectInfo {
     pub data_blocks: usize,
     pub version_id: Option<Uuid>,
     pub delete_marker: bool,
+    pub transitioned_object: TransitionedObject,
     pub user_tags: String,
     pub parts: Vec<ObjectPartInfo>,
     pub is_latest: bool,
@@ -395,6 +405,7 @@ impl Clone for ObjectInfo {
             data_blocks: self.data_blocks,
             version_id: self.version_id,
             delete_marker: self.delete_marker,
+            transitioned_object: self.transitioned_object.clone(),
             user_tags: self.user_tags.clone(),
             parts: self.parts.clone(),
             is_latest: self.is_latest,
@@ -497,7 +508,18 @@ impl ObjectInfo {
 
         // TODO:expires
         // TODO:ReplicationState
-        // TODO:TransitionedObject
+
+        let transitioned_object = TransitionedObject {
+            name: fi.transitioned_objname.clone(),
+            version_id: if let Some(transition_version_id) = fi.transition_version_id {
+                transition_version_id.to_string()
+            } else {
+                "".to_string()
+            },
+            status: fi.transition_status.clone(),
+            free_version: fi.tier_free_version(),
+            tier: fi.transition_tier.clone(),
+        };
 
         let metadata = {
             let mut v = fi.metadata.clone();
@@ -540,6 +562,7 @@ impl ObjectInfo {
             etag,
             inlined,
             user_defined: metadata,
+            transitioned_object,
             ..Default::default()
         }
     }
@@ -813,6 +836,7 @@ pub trait StorageAPI: ObjectIO {
         src_opts: &ObjectOptions,
         dst_opts: &ObjectOptions,
     ) -> Result<ObjectInfo>;
+    async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()>;
     async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo>;
     async fn delete_objects(
         &self,
@@ -887,6 +911,9 @@ pub trait StorageAPI: ObjectIO {
     async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
     // DecomTieredObject
     async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String>;
+    async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>;
+    async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
+    async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
     async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
     async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
 

ecstore/src/tier/mod.rs

@@ -0,0 +1,9 @@
+pub mod tier;
+pub mod tier_admin;
+pub mod tier_config;
+pub mod tier_gen;
+pub mod tier_handlers;
+pub mod warm_backend;
+pub mod warm_backend_minio;
+pub mod warm_backend_rustfs;
+pub mod warm_backend_s3;

ecstore/src/tier/tier.rs

@@ -0,0 +1,455 @@
+use bytes::Bytes;
+use http::status::StatusCode;
+use lazy_static::lazy_static;
+use rand::Rng;
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::{HashMap, hash_map::Entry},
+    io::Cursor,
+    sync::Arc,
+    time::Duration,
+};
+use time::OffsetDateTime;
+use tokio::io::BufReader;
+use tokio::{select, sync::RwLock, time::interval};
+use tracing::{debug, error, info, warn};
+
+use crate::client::admin_handler_utils::AdminError;
+use crate::error::{Error, Result, StorageError};
+use crate::new_object_layer_fn;
+use crate::tier::{
+    tier_admin::TierCreds,
+    tier_config::{TierConfig, TierType},
+    tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND},
+    warm_backend::{check_warm_backend, new_warm_backend},
+};
+use crate::{
+    StorageAPI,
+    config::com::{CONFIG_PREFIX, read_config},
+    disk::RUSTFS_META_BUCKET,
+    store::ECStore,
+    store_api::{ObjectOptions, PutObjReader},
+};
+use rustfs_rio::HashReader;
+use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
+use s3s::S3ErrorCode;
+
+use super::{
+    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
+    warm_backend::WarmBackendImpl,
+};
+
+const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
+
+pub const TIER_CONFIG_FILE: &str = "tier-config.json";
+pub const TIER_CONFIG_FORMAT: u16 = 1;
+pub const TIER_CONFIG_V1: u16 = 1;
+pub const TIER_CONFIG_VERSION: u16 = 1;
+
+lazy_static! {
+    //pub static ref TIER_CONFIG_PATH: PathBuf = path_join(&[PathBuf::from(RUSTFS_CONFIG_PREFIX), PathBuf::from(TIER_CONFIG_FILE)]);
+}
+
+const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
+
+pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError {
+    code: "XRustFSAdminTierMissingCredentials",
+    message: "Specified remote credentials are empty",
+    status_code: StatusCode::FORBIDDEN,
+};
+
+pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError {
+    code: "XRustFSAdminTierBackendInUse",
+    message: "Specified remote tier is already in use",
+    status_code: StatusCode::CONFLICT,
+};
+
+pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError {
+    code: "XRustFSAdminTierTypeUnsupported",
+    message: "Specified tier type is unsupported",
+    status_code: StatusCode::BAD_REQUEST,
+};
+
+pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError {
+    code: "XRustFSAdminTierBackendNotEmpty",
+    message: "Specified remote backend is not empty",
+    status_code: StatusCode::BAD_REQUEST,
+};
+
+pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError {
+    code: "XRustFSAdminTierInvalidConfig",
+    message: "Unable to setup remote tier, check tier configuration",
+    status_code: StatusCode::BAD_REQUEST,
+};
+
+#[derive(Serialize, Deserialize)]
+pub struct TierConfigMgr {
+    #[serde(skip)]
+    pub driver_cache: HashMap<String, WarmBackendImpl>,
+    pub tiers: HashMap<String, TierConfig>,
+    pub last_refreshed_at: OffsetDateTime,
+}
+
+impl TierConfigMgr {
+    pub fn new() -> Arc<RwLock<Self>> {
+        Arc::new(RwLock::new(Self {
+            driver_cache: HashMap::new(),
+            tiers: HashMap::new(),
+            last_refreshed_at: OffsetDateTime::now_utc(),
+        }))
+    }
+
+    pub fn unmarshal(data: &[u8]) -> std::result::Result<TierConfigMgr, std::io::Error> {
+        let cfg: TierConfigMgr = serde_json::from_slice(data)?;
+        //let mut cfg = TierConfigMgr(m);
+        //let mut cfg = m;
+        Ok(cfg)
+    }
+
+    pub fn marshal(&self) -> std::result::Result<Bytes, std::io::Error> {
+        let data = serde_json::to_vec(&self)?;
+
+        //let mut data = Vec<u8>::with_capacity(self.msg_size()+4);
+        let mut data = Bytes::from(data);
+
+        //LittleEndian::write_u16(&mut data[0..2], TIER_CONFIG_FORMAT);
+        //LittleEndian::write_u16(&mut data[2..4], TIER_CONFIG_VERSION);
+
+        Ok(data)
+    }
+
+    pub fn refreshed_at(&self) -> OffsetDateTime {
+        self.last_refreshed_at
+    }
+
+    pub fn is_tier_valid(&self, tier_name: &str) -> bool {
+        let (_, valid) = self.is_tier_name_in_use(tier_name);
+        valid
+    }
+
+    pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) {
+        if let Some(t) = self.tiers.get(tier_name) {
+            return (t.tier_type.clone(), true);
+        }
+        (TierType::Unsupported, false)
+    }
+
+    pub async fn add(&mut self, tier: TierConfig, force: bool) -> std::result::Result<(), AdminError> {
+        let tier_name = &tier.name;
+        if tier_name != tier_name.to_uppercase().as_str() {
+            return Err(ERR_TIER_NAME_NOT_UPPERCASE);
+        }
+
+        let (_, b) = self.is_tier_name_in_use(tier_name);
+        if b {
+            return Err(ERR_TIER_ALREADY_EXISTS);
+        }
+
+        let d = new_warm_backend(&tier, true).await?;
+
+        if !force {
+            let in_use = d.in_use().await;
+            match in_use {
+                Ok(b) => {
+                    if b {
+                        return Err(ERR_TIER_BACKEND_IN_USE);
+                    }
+                }
+                Err(err) => {
+                    warn!("tier add failed, err: {:?}", err);
+                    if err.to_string().contains("connect") {
+                        return Err(ERR_TIER_CONNECT_ERR);
+                    } else if err.to_string().contains("authorization") {
+                        return Err(ERR_TIER_INVALID_CREDENTIALS);
+                    } else if err.to_string().contains("bucket") {
+                        return Err(ERR_TIER_BUCKET_NOT_FOUND);
+                    }
+                    return Err(ERR_TIER_PERM_ERR);
+                }
+            }
+        }
+
+        self.driver_cache.insert(tier_name.to_string(), d);
+        self.tiers.insert(tier_name.to_string(), tier);
+
+        Ok(())
+    }
+
+    pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> {
+        let d = self.get_driver(tier_name).await;
+        if let Err(err) = d {
+            match err {
+                ERR_TIER_NOT_FOUND => {
+                    return Ok(());
+                }
+                _ => {
+                    return Err(err);
+                }
+            }
+        }
+        if !force {
+            let inuse = d.expect("err").in_use().await;
+            if let Err(err) = inuse {
+                return Err(ERR_TIER_PERM_ERR);
+            } else if inuse.expect("err") {
+                return Err(ERR_TIER_BACKEND_NOT_EMPTY);
+            }
+        }
+        self.tiers.remove(tier_name);
+        self.driver_cache.remove(tier_name);
+        Ok(())
+    }
+
+    pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> {
+        let d = match self.get_driver(tier_name).await {
+            Ok(d) => d,
+            Err(err) => {
+                return Err(std::io::Error::other(err));
+            }
+        };
+        if let Err(err) = check_warm_backend(Some(d)).await {
+            return Err(std::io::Error::other(err));
+        } else {
+            return Ok(());
+        }
+    }
+
+    pub fn empty(&self) -> bool {
+        self.list_tiers().len() == 0
+    }
+
+    pub fn tier_type(&self, tier_name: &str) -> String {
+        let cfg = self.tiers.get(tier_name);
+        if cfg.is_none() {
+            return "internal".to_string();
+        }
+        cfg.expect("err").tier_type.to_string()
+    }
+
+    pub fn list_tiers(&self) -> Vec<TierConfig> {
+        let mut tier_cfgs = Vec::<TierConfig>::new();
+        for (_, tier) in self.tiers.iter() {
+            let tier = tier.clone();
+            tier_cfgs.push(tier);
+        }
+        tier_cfgs
+    }
+
+    pub fn get(&self, tier_name: &str) -> Option<TierConfig> {
+        for (tier_name2, tier) in self.tiers.iter() {
+            if tier_name == tier_name2 {
+                return Some(tier.clone());
+            }
+        }
+        None
+    }
+
+    pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> {
+        let (tier_type, exists) = self.is_tier_name_in_use(tier_name);
+        if !exists {
+            return Err(ERR_TIER_NOT_FOUND);
+        }
+
+        let mut cfg = self.tiers[tier_name].clone();
+        match tier_type {
+            TierType::S3 => {
+                let mut s3 = cfg.s3.as_mut().expect("err");
+                if creds.aws_role {
+                    s3.aws_role = true
+                }
+                if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
+                    s3.aws_role_arn = creds.aws_role_arn;
+                    s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
+                }
+                if creds.access_key != "" && creds.secret_key != "" {
+                    s3.access_key = creds.access_key;
+                    s3.secret_key = creds.secret_key;
+                }
+            }
+            TierType::RustFS => {
+                let mut rustfs = cfg.rustfs.as_mut().expect("err");
+                if creds.access_key == "" || creds.secret_key == "" {
+                    return Err(ERR_TIER_MISSING_CREDENTIALS);
+                }
+                rustfs.access_key = creds.access_key;
+                rustfs.secret_key = creds.secret_key;
+            }
+            TierType::MinIO => {
+                let mut minio = cfg.minio.as_mut().expect("err");
+                if creds.access_key == "" || creds.secret_key == "" {
+                    return Err(ERR_TIER_MISSING_CREDENTIALS);
+                }
+                minio.access_key = creds.access_key;
+                minio.secret_key = creds.secret_key;
+            }
+            _ => (),
+        }
+
+        let d = new_warm_backend(&cfg, true).await?;
+        self.tiers.insert(tier_name.to_string(), cfg);
+        self.driver_cache.insert(tier_name.to_string(), d);
+        Ok(())
+    }
+
+    pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
+        Ok(match self.driver_cache.entry(tier_name.to_string()) {
+            Entry::Occupied(e) => e.into_mut(),
+            Entry::Vacant(e) => {
+                let t = self.tiers.get(tier_name);
+                if t.is_none() {
+                    return Err(ERR_TIER_NOT_FOUND);
+                }
+                let d = new_warm_backend(t.expect("err"), false).await?;
+                e.insert(d)
+            }
+        })
+    }
+
+    pub async fn reload(&mut self, api: Arc<ECStore>) -> std::result::Result<(), std::io::Error> {
+        //let Some(api) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) };
+        let new_config = load_tier_config(api).await;
+
+        match &new_config {
+            Ok(_c) => {}
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        }
+        self.driver_cache.clear();
+        self.tiers.clear();
+        let new_config = new_config.expect("err");
+        for (tier, cfg) in new_config.tiers {
+            self.tiers.insert(tier, cfg);
+        }
+        self.last_refreshed_at = OffsetDateTime::now_utc();
+        Ok(())
+    }
+
+    #[tracing::instrument(level = "debug", name = "tier_save", skip(self))]
+    pub async fn save(&self) -> std::result::Result<(), std::io::Error> {
+        let Some(api) = new_object_layer_fn() else {
+            return Err(std::io::Error::other("errServerNotInitialized"));
+        };
+        //let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?;
+
+        self.save_tiering_config(api).await
+    }
+
+    pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
+        let data = self.marshal()?;
+
+        let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
+
+        self.save_config(api, &config_file, data).await
+    }
+
+    pub async fn save_config<S: StorageAPI>(
+        &self,
+        api: Arc<S>,
+        file: &str,
+        data: Bytes,
+    ) -> std::result::Result<(), std::io::Error> {
+        self.save_config_with_opts(
+            api,
+            file,
+            data,
+            &ObjectOptions {
+                max_parity: true,
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
+    pub async fn save_config_with_opts<S: StorageAPI>(
+        &self,
+        api: Arc<S>,
+        file: &str,
+        data: Bytes,
+        opts: &ObjectOptions,
+    ) -> std::result::Result<(), std::io::Error> {
+        debug!("save tier config:{}", file);
+        let _ = api
+            .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn refresh_tier_config(&mut self, api: Arc<ECStore>) {
+        //let r = rand.New(rand.NewSource(time.Now().UnixNano()));
+        let mut rng = rand::rng();
+        let r = rng.random_range(0.0..1.0);
+        let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64);
+
+        let mut t = interval(TIER_CFG_REFRESH + rand_interval());
+        loop {
+            select! {
+                _ = t.tick() => {
+                    if let Err(err) = self.reload(api.clone()).await {
+                      info!("{}", err);
+                    }
+                }
+                else => ()
+            }
+            t.reset();
+        }
+    }
+
+    pub async fn init(&mut self, api: Arc<ECStore>) -> Result<()> {
+        self.reload(api).await?;
+        //if globalIsDistErasure {
+        //    self.refresh_tier_config(api).await;
+        //}
+        Ok(())
+    }
+}
+
+async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierConfigMgr> {
+    let mut cfg = TierConfigMgr {
+        driver_cache: HashMap::new(),
+        tiers: HashMap::new(),
+        last_refreshed_at: OffsetDateTime::now_utc(),
+    };
+    //lookup_configs(&mut cfg, api.clone()).await;
+    cfg.save_tiering_config(api).await?;
+
+    Ok(cfg)
+}
+
+#[tracing::instrument(level = "debug")]
+async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
+    let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
+    let data = read_config(api.clone(), config_file.as_str()).await;
+    if let Err(err) = data {
+        if is_err_config_not_found(&err) {
+            warn!("config not found, start to init");
+            let cfg = new_and_save_tiering_config(api).await?;
+            return Ok(cfg);
+        } else {
+            error!("read config err {:?}", &err);
+            return Err(std::io::Error::other(err));
+        }
+    }
+
+    let cfg;
+    let version = 1; //LittleEndian::read_u16(&data[2..4]);
+    match version {
+        TIER_CONFIG_V1/*  | TIER_CONFIG_VERSION */ => {
+            cfg = match TierConfigMgr::unmarshal(&data.unwrap()) {
+                Ok(cfg) => cfg,
+                Err(err) => {
+                    return Err(std::io::Error::other(err.to_string()));
+                }
+            };
+        }
+        _ => {
+            return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version)));
+        }
+    }
+
+    Ok(cfg)
+}
+
+pub fn is_err_config_not_found(err: &StorageError) -> bool {
+    matches!(err, StorageError::ObjectNotFound(_, _))
+}

ecstore/src/tier/tier_admin.rs

@@ -0,0 +1,26 @@
+use http::status::StatusCode;
+use rand::Rng;
+use serde::{Deserialize, Serialize};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tracing::warn;
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierCreds {
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+
+    #[serde(rename = "awsRole")]
+    pub aws_role: bool,
+    #[serde(rename = "awsRoleWebIdentityTokenFile")]
+    pub aws_role_web_identity_token_file: String,
+    #[serde(rename = "awsRoleArn")]
+    pub aws_role_arn: String,
+
+    //azsp: ServicePrincipalAuth,
+
+    //#[serde(rename = "credsJson")]
+    pub creds_json: Vec<u8>,
+}

ecstore/src/tier/tier_config.rs

@@ -0,0 +1,316 @@
+use serde::{Deserialize, Serialize};
+use std::fmt::Display;
+use tracing::info;
+
+const C_TierConfigVer: &str = "v1";
+
+const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
+const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config";
+const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version";
+const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type";
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+pub enum TierType {
+    #[default]
+    Unsupported,
+    #[serde(rename = "s3")]
+    S3,
+    #[serde(rename = "azure")]
+    Azure,
+    #[serde(rename = "gcs")]
+    GCS,
+    #[serde(rename = "rustfs")]
+    RustFS,
+    #[serde(rename = "minio")]
+    MinIO,
+}
+
+impl Display for TierType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TierType::S3 => {
+                write!(f, "S3")
+            }
+            TierType::RustFS => {
+                write!(f, "RustFS")
+            }
+            TierType::MinIO => {
+                write!(f, "MinIO")
+            }
+            _ => {
+                write!(f, "Unsupported")
+            }
+        }
+    }
+}
+
+impl TierType {
+    pub fn new(sc_type: &str) -> Self {
+        match sc_type {
+            "S3" => TierType::S3,
+            "RustFS" => TierType::RustFS,
+            "MinIO" => TierType::MinIO,
+            _ => TierType::Unsupported,
+        }
+    }
+
+    pub fn to_string(&self) -> String {
+        match self {
+            TierType::S3 => "s3".to_string(),
+            TierType::RustFS => "rustfs".to_string(),
+            TierType::MinIO => "minio".to_string(),
+            _ => "unsupported".to_string(),
+        }
+    }
+}
+
+#[derive(Default, Debug, Serialize, Deserialize)]
+#[serde(default)]
+pub struct TierConfig {
+    #[serde(skip)]
+    pub version: String,
+    #[serde(rename = "type")]
+    pub tier_type: TierType,
+    #[serde(skip)]
+    pub name: String,
+    #[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
+    pub s3: Option<TierS3>,
+    //TODO: azure: Option<TierAzure>,
+    //TODO: gcs: Option<TierGCS>,
+    #[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")]
+    pub rustfs: Option<TierRustFS>,
+    #[serde(rename = "minio", skip_serializing_if = "Option::is_none")]
+    pub minio: Option<TierMinIO>,
+}
+
+impl Clone for TierConfig {
+    fn clone(&self) -> TierConfig {
+        let mut s3 = None;
+        //az  TierAzure
+        //gcs TierGCS
+        let mut r = None;
+        let mut m = None;
+        match self.tier_type {
+            TierType::S3 => {
+                let mut s3_ = self.s3.as_ref().expect("err").clone();
+                s3_.secret_key = "REDACTED".to_string();
+                s3 = Some(s3_);
+            }
+            TierType::RustFS => {
+                let mut r_ = self.rustfs.as_ref().expect("err").clone();
+                r_.secret_key = "REDACTED".to_string();
+                r = Some(r_);
+            }
+            TierType::MinIO => {
+                let mut m_ = self.minio.as_ref().expect("err").clone();
+                m_.secret_key = "REDACTED".to_string();
+                m = Some(m_);
+            }
+            _ => (),
+        }
+        TierConfig {
+            version: self.version.clone(),
+            tier_type: self.tier_type.clone(),
+            name: self.name.clone(),
+            s3: s3,
+            //azure:   az,
+            //gcs:     gcs,
+            rustfs: r,
+            minio: m,
+        }
+    }
+}
+
+impl TierConfig {
+    pub fn unmarshal(data: &[u8]) -> Result<TierConfig, std::io::Error> {
+        /*let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
+        let mut cfg = TierConfig(m);
+        cfg.set_defaults();
+        Ok(cfg)*/
+        todo!();
+    }
+
+    pub fn marshal(&self) -> Result<Vec<u8>, std::io::Error> {
+        let data = serde_json::to_vec(&self)?;
+        Ok(data)
+    }
+
+    fn endpoint(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(),
+            TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(),
+            TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+
+    fn bucket(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(),
+            TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(),
+            TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+
+    fn prefix(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(),
+            TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(),
+            TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+
+    fn region(&self) -> String {
+        match self.tier_type {
+            TierType::S3 => self.s3.as_ref().expect("err").region.clone(),
+            TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(),
+            TierType::MinIO => self.minio.as_ref().expect("err").region.clone(),
+            _ => {
+                info!("unexpected tier type {}", self.tier_type);
+                "".to_string()
+            }
+        }
+    }
+}
+
+//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierS3 {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accesskey")]
+    pub access_key: String,
+    #[serde(rename = "secretkey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+    #[serde(rename = "storageclass")]
+    pub storage_class: String,
+    #[serde(skip)]
+    pub aws_role: bool,
+    #[serde(skip)]
+    pub aws_role_web_identity_token_file: String,
+    #[serde(skip)]
+    pub aws_role_arn: String,
+    #[serde(skip)]
+    pub aws_role_session_name: String,
+    #[serde(skip)]
+    pub aws_role_duration_seconds: i32,
+}
+
+impl TierS3 {
+    fn new<F>(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec<F>) -> Result<TierConfig, std::io::Error>
+    where
+        F: Fn(TierS3) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
+    {
+        if name == "" {
+            return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
+        }
+        let sc = TierS3 {
+            access_key: access_key.to_string(),
+            secret_key: secret_key.to_string(),
+            bucket: bucket.to_string(),
+            endpoint: "https://s3.amazonaws.com".to_string(),
+            region: "".to_string(),
+            storage_class: "".to_string(),
+            ..Default::default()
+        };
+
+        for option in options {
+            let option = option(sc.clone());
+            let option = *option;
+            option?;
+        }
+
+        Ok(TierConfig {
+            version: C_TierConfigVer.to_string(),
+            tier_type: TierType::S3,
+            name: name.to_string(),
+            s3: Some(sc),
+            ..Default::default()
+        })
+    }
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierRustFS {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accesskey")]
+    pub access_key: String,
+    #[serde(rename = "secretkey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+    #[serde(rename = "storageclass")]
+    pub storage_class: String,
+}
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[serde(default)]
+pub struct TierMinIO {
+    pub name: String,
+    pub endpoint: String,
+    #[serde(rename = "accesskey")]
+    pub access_key: String,
+    #[serde(rename = "secretkey")]
+    pub secret_key: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub region: String,
+}
+
+impl TierMinIO {
+    fn new<F>(
+        name: &str,
+        endpoint: &str,
+        access_key: &str,
+        secret_key: &str,
+        bucket: &str,
+        options: Vec<F>,
+    ) -> Result<TierConfig, std::io::Error>
+    where
+        F: Fn(TierMinIO) -> Box<Result<(), std::io::Error>> + Send + Sync + 'static,
+    {
+        if name == "" {
+            return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
+        }
+        let m = TierMinIO {
+            access_key: access_key.to_string(),
+            secret_key: secret_key.to_string(),
+            bucket: bucket.to_string(),
+            endpoint: endpoint.to_string(),
+            ..Default::default()
+        };
+
+        for option in options {
+            let option = option(m.clone());
+            let option = *option;
+            option?;
+        }
+
+        Ok(TierConfig {
+            version: C_TierConfigVer.to_string(),
+            tier_type: TierType::MinIO,
+            name: name.to_string(),
+            minio: Some(m),
+            ..Default::default()
+        })
+    }
+}

ecstore/src/tier/tier_config_gen.rs

@@ -0,0 +1,48 @@
+use std::{fmt::Display, pin::Pin, sync::Arc};
+use tracing::info;
+use common::error::{Error, Result};
+use crate::bucket::tier_config::{TierType, TierConfig,};
+
+impl TierType {
+    fn decode_msg(&self/*, dc *msgp.Reader*/) -> Result<()> {
+        todo!();
+    }
+
+    fn encode_msg(&self/*, en *msgp.Writer*/) -> Result<()> {
+        todo!();
+    }
+
+    pub fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>> {
+        todo!();
+    }
+
+    pub fn unmarshal_msg(&self, bts: &[u8]) -> Result<Vec<u8>> {
+        todo!();
+    }
+
+    pub fn msg_size() -> usize {
+        todo!();
+    }
+}
+
+impl TierConfig {
+    fn decode_msg(&self, dc *msgp.Reader) -> Result<()> {
+        todo!();
+    }
+
+    pub fn encode_msg(&self, en *msgp.Writer) -> Result<()> {
+        todo!();
+    }
+
+    pub fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>> {
+        todo!();
+    }
+
+    pub fn unmarshal_msg(&self, bts: &[u8]) -> Result<Vec<u8>> {
+        todo!();
+    }
+
+    fn msg_size(&self) -> usize {
+        todo!();
+    }
+}

ecstore/src/tier/tier_gen.rs

@@ -0,0 +1,23 @@
+use crate::tier::tier::TierConfigMgr;
+
+impl TierConfigMgr {
+    fn decode_msg(/*dc *msgp.Reader*/) -> Result<(), std::io::Error> {
+        todo!();
+    }
+
+    fn encode_msg(/*en *msgp.Writer*/) -> Result<(), std::io::Error> {
+        todo!();
+    }
+
+    pub fn marshal_msg(&self, b: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+        todo!();
+    }
+
+    pub fn unmarshal_msg(buf: &[u8]) -> Result<Self, std::io::Error> {
+        todo!();
+    }
+
+    pub fn msg_size(&self) -> usize {
+        100
+    }
+}

ecstore/src/tier/tier_handlers.rs

@@ -0,0 +1,51 @@
+use crate::client::admin_handler_utils::AdminError;
+use http::status::StatusCode;
+use tracing::warn;
+
+pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError {
+    code: "XRustFSAdminTierAlreadyExists",
+    message: "Specified remote tier already exists",
+    status_code: StatusCode::CONFLICT,
+};
+
+pub const ERR_TIER_NOT_FOUND: AdminError = AdminError {
+    code: "XRustFSAdminTierNotFound",
+    message: "Specified remote tier was not found",
+    status_code: StatusCode::NOT_FOUND,
+};
+
+pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError {
+    code: "XRustFSAdminTierNameNotUpperCase",
+    message: "Tier name must be in uppercase",
+    status_code: StatusCode::BAD_REQUEST,
+};
+
+pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError {
+    code: "XRustFSAdminTierBucketNotFound",
+    message: "Remote tier bucket not found",
+    status_code: StatusCode::BAD_REQUEST,
+};
+
+pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError {
+    code: "XRustFSAdminTierInvalidCredentials",
+    message: "Invalid remote tier credentials",
+    status_code: StatusCode::BAD_REQUEST,
+};
+
+pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError {
+    code: "XRustFSAdminTierReserved",
+    message: "Cannot use reserved tier name",
+    status_code: StatusCode::BAD_REQUEST,
+};
+
+pub const ERR_TIER_PERM_ERR: AdminError = AdminError {
+    code: "TierPermErr",
+    message: "Tier Perm Err",
+    status_code: StatusCode::OK,
+};
+
+pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError {
+    code: "TierConnectErr",
+    message: "Tier Connect Err",
+    status_code: StatusCode::OK,
+};

ecstore/src/tier/warm_backend.rs

@@ -0,0 +1,105 @@
+use bytes::Bytes;
+use std::collections::HashMap;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    transition_api::{ReadCloser, ReaderImpl},
+};
+use crate::error::is_err_bucket_not_found;
+use crate::tier::{
+    tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
+    tier_config::{TierConfig, TierType},
+    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
+    warm_backend_minio::WarmBackendMinIO,
+    warm_backend_rustfs::WarmBackendRustFS,
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::{info, warn};
+
+pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
+
+const PROBE_OBJECT: &str = "probeobject";
+
+#[derive(Default)]
+pub struct WarmBackendGetOpts {
+    pub start_offset: i64,
+    pub length: i64,
+}
+
+#[async_trait::async_trait]
+pub trait WarmBackend {
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error>;
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error>;
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
+    async fn in_use(&self) -> Result<bool, std::io::Error>;
+}
+
+pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
+    let w = w.expect("err");
+    let remote_version_id = w
+        .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
+        .await;
+    if let Err(err) = remote_version_id {
+        return Err(ERR_TIER_PERM_ERR);
+    }
+
+    let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await;
+    //xhttp.DrainBody(r);
+    if let Err(err) = r {
+        //if is_err_bucket_not_found(&err) {
+        //    return Err(ERR_TIER_BUCKET_NOT_FOUND);
+        //}
+        /*else if is_err_signature_does_not_match(err) {
+            return Err(ERR_TIER_MISSING_CREDENTIALS);
+        }*/
+        //else {
+        return Err(ERR_TIER_PERM_ERR);
+        //}
+    }
+    if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await {
+        return Err(ERR_TIER_PERM_ERR);
+    };
+    Ok(())
+}
+
+pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
+    let mut d: Option<WarmBackendImpl> = None;
+    match tier.tier_type {
+        TierType::S3 => {
+            let dd = WarmBackendS3::new(tier.s3.as_ref().expect("err"), &tier.name).await;
+            if let Err(err) = dd {
+                info!("{}", err);
+                return Err(ERR_TIER_INVALID_CONFIG);
+            }
+            d = Some(Box::new(dd.expect("err")));
+        }
+        TierType::RustFS => {
+            let dd = WarmBackendRustFS::new(tier.rustfs.as_ref().expect("err"), &tier.name).await;
+            if let Err(err) = dd {
+                warn!("{}", err);
+                return Err(ERR_TIER_INVALID_CONFIG);
+            }
+            d = Some(Box::new(dd.expect("err")));
+        }
+        TierType::MinIO => {
+            let dd = WarmBackendMinIO::new(tier.minio.as_ref().expect("err"), &tier.name).await;
+            if let Err(err) = dd {
+                warn!("{}", err);
+                return Err(ERR_TIER_INVALID_CONFIG);
+            }
+            d = Some(Box::new(dd.expect("err")));
+        }
+        _ => {
+            return Err(ERR_TIER_TYPE_UNSUPPORTED);
+        }
+    }
+
+    Ok(d.expect("err"))
+}

ecstore/src/tier/warm_backend_minio.rs

@@ -0,0 +1,139 @@
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::tier::{
+    tier_config::TierMinIO,
+    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend_s3::WarmBackendS3,
+};
+use tracing::warn;
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendMinIO(WarmBackendS3);
+
+impl WarmBackendMinIO {
+    pub async fn new(conf: &TierMinIO, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => {
+                return Err(std::io::Error::other(e.to_string()));
+            }
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds: creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let client =
+            TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
+        //client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag);
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendMinIO {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(
+                &self.0.bucket,
+                &self.0.get_dest(object),
+                r,
+                length,
+                &PutObjectOptions {
+                    storage_class: self.0.storage_class.clone(),
+                    part_size: part_size as u64,
+                    disable_content_sha256: true,
+                    user_metadata: meta,
+                    ..Default::default()
+                },
+            )
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}

ecstore/src/tier/warm_backend_rustfs.rs

@@ -0,0 +1,137 @@
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use tracing::warn;
+
+use crate::client::{
+    admin_handler_utils::AdminError,
+    api_put_object::PutObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
+};
+use crate::tier::{
+    tier_config::TierRustFS,
+    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend_s3::WarmBackendS3,
+};
+
+const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
+const MAX_PARTS_COUNT: i64 = 10000;
+const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
+
+pub struct WarmBackendRustFS(WarmBackendS3);
+
+impl WarmBackendRustFS {
+    pub async fn new(conf: &TierRustFS, tier: &str) -> Result<Self, std::io::Error> {
+        if conf.access_key == "" || conf.secret_key == "" {
+            return Err(std::io::Error::other("both access and secret keys are required"));
+        }
+
+        if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let u = match url::Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(e) => return Err(std::io::Error::other(e)),
+        };
+
+        let creds = Credentials::new(Static(Value {
+            access_key_id: conf.access_key.clone(),
+            secret_access_key: conf.secret_key.clone(),
+            session_token: "".to_string(),
+            signer_type: SignatureType::SignatureV4,
+            ..Default::default()
+        }));
+        let opts = Options {
+            creds: creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            trailing_headers: true,
+            ..Default::default()
+        };
+        let scheme = u.scheme();
+        let default_port = if scheme == "https" { 443 } else { 80 };
+        let client =
+            TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?;
+        //client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag);
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self(WarmBackendS3 {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(),
+            storage_class: "".to_string(),
+        }))
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendRustFS {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let part_size = optimal_part_size(length)?;
+        let client = self.0.client.clone();
+        let res = client
+            .put_object(
+                &self.0.bucket,
+                &self.0.get_dest(object),
+                r,
+                length,
+                &PutObjectOptions {
+                    storage_class: self.0.storage_class.clone(),
+                    part_size: part_size as u64,
+                    disable_content_sha256: true,
+                    user_metadata: meta,
+                    ..Default::default()
+                },
+            )
+            .await?;
+        //self.ToObjectError(err, object)
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        self.0.get(object, rv, opts).await
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        self.0.remove(object, rv).await
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        self.0.in_use().await
+    }
+}
+
+fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
+    let mut object_size = object_size;
+    if object_size == -1 {
+        object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
+    }
+
+    if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
+        return Err(std::io::Error::other("entity too large"));
+    }
+
+    let configured_part_size = MIN_PART_SIZE;
+    let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64;
+    part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64;
+
+    let part_size = part_size_flt as i64;
+    if part_size == 0 {
+        return Ok(MIN_PART_SIZE);
+    }
+    Ok(part_size)
+}

ecstore/src/tier/warm_backend_s3.rs

@@ -0,0 +1,175 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use url::Url;
+
+use crate::client::{
+    api_get_options::GetObjectOptions,
+    api_put_object::PutObjectOptions,
+    api_remove::RemoveObjectOptions,
+    credentials::{Credentials, SignatureType, Static, Value},
+    transition_api::{Options, TransitionClient, TransitionCore},
+    transition_api::{ReadCloser, ReaderImpl},
+};
+use crate::error::ErrorResponse;
+use crate::error::error_resp_to_object_err;
+use crate::tier::{
+    tier_config::TierS3,
+    warm_backend::{WarmBackend, WarmBackendGetOpts},
+};
+use rustfs_utils::path::SLASH_SEPARATOR;
+
+pub struct WarmBackendS3 {
+    pub client: Arc<TransitionClient>,
+    pub core: TransitionCore,
+    pub bucket: String,
+    pub prefix: String,
+    pub storage_class: String,
+}
+
+impl WarmBackendS3 {
+    pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
+        let u = match Url::parse(&conf.endpoint) {
+            Ok(u) => u,
+            Err(err) => {
+                return Err(std::io::Error::other(err.to_string()));
+            }
+        };
+
+        if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
+            || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
+        {
+            return Err(std::io::Error::other("both the token file and the role ARN are required"));
+        } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" {
+            return Err(std::io::Error::other("both the access and secret keys are required"));
+        } else if conf.aws_role
+            && (conf.aws_role_web_identity_token_file != ""
+                || conf.aws_role_arn != ""
+                || conf.access_key != ""
+                || conf.secret_key != "")
+        {
+            return Err(std::io::Error::other(
+                "AWS Role cannot be activated with static credentials or the web identity token file",
+            ));
+        } else if conf.bucket == "" {
+            return Err(std::io::Error::other("no bucket name was provided"));
+        }
+
+        let mut creds: Credentials<Static>;
+
+        if conf.access_key != "" && conf.secret_key != "" {
+            //creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
+            creds = Credentials::new(Static(Value {
+                access_key_id: conf.access_key.clone(),
+                secret_access_key: conf.secret_key.clone(),
+                session_token: "".to_string(),
+                signer_type: SignatureType::SignatureV4,
+                ..Default::default()
+            }));
+        } else {
+            return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
+        }
+        let opts = Options {
+            creds: creds,
+            secure: u.scheme() == "https",
+            //transport: GLOBAL_RemoteTargetTransport,
+            ..Default::default()
+        };
+        let client = TransitionClient::new(&u.host().expect("err").to_string(), opts).await?;
+        //client.set_appinfo(format!("s3-tier-{}", tier), ReleaseTag);
+
+        let client = Arc::new(client);
+        let core = TransitionCore(Arc::clone(&client));
+        Ok(Self {
+            client,
+            core,
+            bucket: conf.bucket.clone(),
+            prefix: conf.prefix.clone().trim_matches('/').to_string(),
+            storage_class: conf.storage_class.clone(),
+        })
+    }
+
+    fn to_object_err(&self, err: ErrorResponse, params: Vec<&str>) -> std::io::Error {
+        let mut object = "";
+        if params.len() >= 1 {
+            object = params.first().cloned().unwrap_or_default();
+        }
+
+        error_resp_to_object_err(err, vec![&self.bucket, &self.get_dest(object)])
+    }
+
+    pub fn get_dest(&self, object: &str) -> String {
+        let mut dest_obj = object.to_string();
+        if self.prefix != "" {
+            dest_obj = format!("{}/{}", &self.prefix, object);
+        }
+        return dest_obj;
+    }
+}
+
+#[async_trait::async_trait]
+impl WarmBackend for WarmBackendS3 {
+    async fn put_with_meta(
+        &self,
+        object: &str,
+        r: ReaderImpl,
+        length: i64,
+        meta: HashMap<String, String>,
+    ) -> Result<String, std::io::Error> {
+        let client = self.client.clone();
+        let res = client
+            .put_object(
+                &self.bucket,
+                &self.get_dest(object),
+                r,
+                length,
+                &PutObjectOptions {
+                    send_content_md5: true,
+                    storage_class: self.storage_class.clone(),
+                    user_metadata: meta,
+                    ..Default::default()
+                },
+            )
+            .await?;
+        Ok(res.version_id)
+    }
+
+    async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result<String, std::io::Error> {
+        self.put_with_meta(object, r, length, HashMap::new()).await
+    }
+
+    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+        let mut gopts = GetObjectOptions::default();
+
+        if rv != "" {
+            gopts.version_id = rv.to_string();
+        }
+        if opts.start_offset >= 0 && opts.length > 0 {
+            if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) {
+                return Err(std::io::Error::other(err));
+            }
+        }
+        let c = TransitionCore(Arc::clone(&self.client));
+        let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
+
+        Ok(r)
+    }
+
+    async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+        let mut ropts = RemoveObjectOptions::default();
+        if rv != "" {
+            ropts.version_id = rv.to_string();
+        }
+        let client = self.client.clone();
+        let err = client.remove_object(&self.bucket, &self.get_dest(object), ropts).await;
+        Err(std::io::Error::other(err.expect("err")))
+    }
+
+    async fn in_use(&self) -> Result<bool, std::io::Error> {
+        let result = self
+            .core
+            .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
+            .await?;
+
+        Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
+    }
+}

reader/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "reader"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+tracing.workspace = true
+s3s.workspace = true
+thiserror.workspace = true
+bytes.workspace = true
+pin-project-lite.workspace = true
+hex-simd = "0.8.0"
+base64-simd = "0.8.0"
+md-5.workspace = true
+sha2 = { version = "0.11.0-pre.4" }
+futures.workspace = true
+async-trait.workspace = true
+common.workspace = true
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

reader/src/error.rs

@@ -0,0 +1,12 @@
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum ReaderError {
+    #[error("stream input error {0}")]
+    StreamInput(String),
+    //
+    #[error("etag: expected ETag {0} does not match computed ETag {1}")]
+    VerifyError(String, String),
+    #[error("Bad checksum: Want {0} does not match calculated {1}")]
+    ChecksumMismatch(String, String),
+    #[error("Bad sha256: Expected {0} does not match calculated {1}")]
+    SHA256Mismatch(String, String),
+}

reader/src/hasher.rs

@@ -0,0 +1,184 @@
+use md5::{Digest as Md5Digest, Md5};
+use sha2::{
+    Digest, Sha256 as sha_sha256,
+    digest::{Reset, Update},
+};
+pub trait Hasher {
+    fn write(&mut self, bytes: &[u8]);
+    fn reset(&mut self);
+    fn sum(&mut self) -> String;
+    fn size(&self) -> usize;
+    fn block_size(&self) -> usize;
+}
+
+#[derive(Default)]
+pub enum HashType {
+    #[default]
+    Undefined,
+    Uuid(Uuid),
+    Md5(MD5),
+    Sha256(Sha256),
+}
+
+impl Hasher for HashType {
+    fn write(&mut self, bytes: &[u8]) {
+        match self {
+            HashType::Md5(md5) => md5.write(bytes),
+            HashType::Sha256(sha256) => sha256.write(bytes),
+            HashType::Uuid(uuid) => uuid.write(bytes),
+            HashType::Undefined => (),
+        }
+    }
+
+    fn reset(&mut self) {
+        match self {
+            HashType::Md5(md5) => md5.reset(),
+            HashType::Sha256(sha256) => sha256.reset(),
+            HashType::Uuid(uuid) => uuid.reset(),
+            HashType::Undefined => (),
+        }
+    }
+
+    fn sum(&mut self) -> String {
+        match self {
+            HashType::Md5(md5) => md5.sum(),
+            HashType::Sha256(sha256) => sha256.sum(),
+            HashType::Uuid(uuid) => uuid.sum(),
+            HashType::Undefined => "".to_owned(),
+        }
+    }
+
+    fn size(&self) -> usize {
+        match self {
+            HashType::Md5(md5) => md5.size(),
+            HashType::Sha256(sha256) => sha256.size(),
+            HashType::Uuid(uuid) => uuid.size(),
+            HashType::Undefined => 0,
+        }
+    }
+
+    fn block_size(&self) -> usize {
+        match self {
+            HashType::Md5(md5) => md5.block_size(),
+            HashType::Sha256(sha256) => sha256.block_size(),
+            HashType::Uuid(uuid) => uuid.block_size(),
+            HashType::Undefined => 64,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Sha256 {
+    hasher: sha_sha256,
+}
+
+impl Sha256 {
+    pub fn new() -> Self {
+        Self {
+            hasher: sha_sha256::new(),
+        }
+    }
+}
+impl Default for Sha256 {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Hasher for Sha256 {
+    fn write(&mut self, bytes: &[u8]) {
+        Update::update(&mut self.hasher, bytes);
+    }
+
+    fn reset(&mut self) {
+        Reset::reset(&mut self.hasher);
+    }
+
+    fn sum(&mut self) -> String {
+        hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower)
+    }
+
+    fn size(&self) -> usize {
+        32
+    }
+
+    fn block_size(&self) -> usize {
+        64
+    }
+}
+
+#[derive(Debug)]
+pub struct MD5 {
+    hasher: Md5,
+}
+
+impl MD5 {
+    pub fn new() -> Self {
+        Self { hasher: Md5::new() }
+    }
+}
+impl Default for MD5 {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Hasher for MD5 {
+    fn write(&mut self, bytes: &[u8]) {
+        self.hasher.update(bytes);
+    }
+
+    fn reset(&mut self) {}
+
+    fn sum(&mut self) -> String {
+        hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower)
+    }
+
+    fn size(&self) -> usize {
+        32
+    }
+
+    fn block_size(&self) -> usize {
+        64
+    }
+}
+
+pub struct Uuid {
+    id: String,
+}
+
+impl Uuid {
+    pub fn new(id: String) -> Self {
+        Self { id }
+    }
+}
+
+impl Hasher for Uuid {
+    fn write(&mut self, _bytes: &[u8]) {}
+
+    fn reset(&mut self) {}
+
+    fn sum(&mut self) -> String {
+        self.id.clone()
+    }
+
+    fn size(&self) -> usize {
+        self.id.len()
+    }
+
+    fn block_size(&self) -> usize {
+        64
+    }
+}
+
+pub fn sum_sha256_hex(data: &[u8]) -> String {
+    let mut hash = Sha256::new();
+    hash.write(data);
+    base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum())
+}
+
+pub fn sum_md5_base64(data: &[u8]) -> String {
+    let mut hash = MD5::new();
+    hash.write(data);
+    base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum())
+}

reader/src/lib.rs

@@ -0,0 +1,7 @@
+pub mod error;
+pub mod hasher;
+pub mod reader;
+
+pub fn hex(data: impl AsRef<[u8]>) -> String {
+    hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
+}

reader/src/reader.rs

@@ -0,0 +1,547 @@
+use bytes::Bytes;
+use s3s::StdError;
+use std::any::Any;
+use std::io::Read;
+use std::{collections::VecDeque, io::Cursor};
+
+use std::pin::Pin;
+use std::task::Poll;
+
+use crate::{
+    error::ReaderError,
+    hasher::{HashType, Uuid},
+};
+
+// use futures::stream::Stream;
+use super::hasher::{Hasher, MD5, Sha256};
+use futures::Stream;
+use std::io::{Error, Result};
+
+pin_project_lite::pin_project! {
+    #[derive(Default)]
+    pub struct EtagReader<S> {
+        #[pin]
+        inner: S,
+        md5: HashType,
+        checksum:Option<String>,
+        bytes_read:usize,
+    }
+}
+
+impl<S> EtagReader<S> {
+    pub fn new(inner: S, etag: Option<String>, force_md5: Option<String>) -> Self {
+        let md5 = {
+            if let Some(m) = force_md5 {
+                HashType::Uuid(Uuid::new(m))
+            } else {
+                HashType::Md5(MD5::new())
+            }
+        };
+        Self {
+            inner,
+            md5,
+            checksum: etag,
+            bytes_read: 0,
+        }
+    }
+
+    pub fn etag(&mut self) -> String {
+        self.md5.sum()
+    }
+}
+
+impl<S> Stream for EtagReader<S>
+where
+    S: Stream<Item = std::result::Result<Bytes, StdError>>,
+{
+    type Item = std::result::Result<Bytes, StdError>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.project();
+        let poll = this.inner.poll_next(cx);
+
+        if let Poll::Ready(ref res) = poll {
+            match res {
+                Some(Ok(bytes)) => {
+                    *this.bytes_read += bytes.len();
+                    this.md5.write(bytes);
+                }
+                Some(Err(err)) => {
+                    return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string())))));
+                }
+                None => {
+                    if let Some(etag) = this.checksum {
+                        let got = this.md5.sum();
+                        if got.as_str() != etag.as_str() {
+                            return Poll::Ready(Some(Err(Box::new(ReaderError::VerifyError(etag.to_owned(), got)))));
+                        }
+                    }
+                }
+            }
+        }
+
+        poll
+    }
+}
+
+pin_project_lite::pin_project! {
+    #[derive(Default)]
+    pub struct HashReader<S> {
+        #[pin]
+        inner: S,
+        sha256: Option<Sha256>,
+        md5: Option<MD5>,
+        md5_hex:Option<String>,
+        sha256_hex:Option<String>,
+        size:usize,
+        actual_size: usize,
+        bytes_read:usize,
+    }
+}
+
+impl<S> HashReader<S> {
+    pub fn new(inner: S, size: usize, md5_hex: Option<String>, sha256_hex: Option<String>, actual_size: usize) -> Self {
+        let md5 = { if md5_hex.is_some() { Some(MD5::new()) } else { None } };
+        let sha256 = { if sha256_hex.is_some() { Some(Sha256::new()) } else { None } };
+        Self {
+            inner,
+            size,
+            actual_size,
+            md5_hex,
+            sha256_hex,
+            bytes_read: 0,
+            md5,
+            sha256,
+        }
+    }
+}
+
+impl<S> Stream for HashReader<S>
+where
+    S: Stream<Item = std::result::Result<Bytes, StdError>>,
+{
+    type Item = std::result::Result<Bytes, StdError>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.project();
+        let poll = this.inner.poll_next(cx);
+
+        if let Poll::Ready(ref res) = poll {
+            match res {
+                Some(Ok(bytes)) => {
+                    *this.bytes_read += bytes.len();
+                    if let Some(sha) = this.sha256 {
+                        sha.write(bytes);
+                    }
+
+                    if let Some(md5) = this.md5 {
+                        md5.write(bytes);
+                    }
+                }
+                Some(Err(err)) => {
+                    return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string())))));
+                }
+                None => {
+                    if let Some(hash) = this.sha256 {
+                        if let Some(hex) = this.sha256_hex {
+                            let got = hash.sum();
+                            let src = hex.as_str();
+                            if src != got.as_str() {
+                                println!("sha256 err src:{},got:{}", src, got);
+                                return Poll::Ready(Some(Err(Box::new(ReaderError::SHA256Mismatch(src.to_string(), got)))));
+                            }
+                        }
+                    }
+
+                    if let Some(hash) = this.md5 {
+                        if let Some(hex) = this.md5_hex {
+                            let got = hash.sum();
+                            let src = hex.as_str();
+                            if src != got.as_str() {
+                                // TODO: ERR
+                                println!("md5 err src:{},got:{}", src, got);
+                                return Poll::Ready(Some(Err(Box::new(ReaderError::ChecksumMismatch(src.to_string(), got)))));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // println!("poll {:?}", poll);
+
+        poll
+    }
+}
+
+#[async_trait::async_trait]
+pub trait Reader {
+    async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result<usize>;
+    async fn seek(&mut self, offset: usize) -> Result<()>;
+    async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize>;
+    async fn read_all(&mut self) -> Result<Vec<u8>> {
+        let mut data = Vec::new();
+
+        Ok(data)
+    }
+    fn as_any(&self) -> &dyn Any;
+}
+
+#[derive(Debug)]
+pub struct BufferReader {
+    pub inner: Cursor<Vec<u8>>,
+    pos: usize,
+}
+
+impl BufferReader {
+    pub fn new(inner: Vec<u8>) -> Self {
+        Self {
+            inner: Cursor::new(inner),
+            pos: 0,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl Reader for BufferReader {
+    #[tracing::instrument(level = "debug", skip(self, buf))]
+    async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result<usize> {
+        self.seek(offset).await?;
+        self.read_exact(buf).await
+    }
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn seek(&mut self, offset: usize) -> Result<()> {
+        if self.pos != offset {
+            self.inner.set_position(offset as u64);
+        }
+
+        Ok(())
+    }
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize> {
+        let bytes_read = self.inner.read_exact(buf)?;
+        self.pos += buf.len();
+        //Ok(bytes_read)
+        Ok(0)
+    }
+
+    async fn read_all(&mut self) -> Result<Vec<u8>> {
+        let mut data = Vec::new();
+        self.inner.read_to_end(&mut data)?;
+
+        Ok(data)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+pin_project_lite::pin_project! {
+    pub struct ChunkedStream<S> {
+        #[pin]
+        inner: S,
+        chuck_size: usize,
+        streams: VecDeque<Bytes>,
+        remaining:Vec<u8>,
+    }
+}
+
+impl<S> ChunkedStream<S> {
+    pub fn new(inner: S, chuck_size: usize) -> Self {
+        Self {
+            inner,
+            chuck_size,
+            streams: VecDeque::new(),
+            remaining: Vec::new(),
+        }
+    }
+}
+
+impl<S> Stream for ChunkedStream<S>
+where
+    S: Stream<Item = std::result::Result<Bytes, StdError>> + Send + Sync,
+    // E: std::error::Error + Send + Sync,
+{
+    type Item = std::result::Result<Bytes, StdError>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Option<Self::Item>> {
+        let (items, op_items) = self.inner.size_hint();
+        let this = self.project();
+
+        if let Some(b) = this.streams.pop_front() {
+            return Poll::Ready(Some(Ok(b)));
+        }
+
+        let poll = this.inner.poll_next(cx);
+
+        match poll {
+            Poll::Ready(res_op) => match res_op {
+                Some(res) => match res {
+                    Ok(bytes) => {
+                        let chuck_size = *this.chuck_size;
+                        let mut bytes = bytes;
+
+                        // println!("get len {}", bytes.len());
+                        // 如果有剩余
+                        if !this.remaining.is_empty() {
+                            let need_size = chuck_size - this.remaining.len();
+                            // 传入的数据大小需要补齐的大小，使用传入数据补齐
+                            if bytes.len() >= need_size {
+                                let add_bytes = bytes.split_to(need_size);
+                                this.remaining.extend_from_slice(&add_bytes);
+                                this.streams.push_back(Bytes::from(this.remaining.clone()));
+                                this.remaining.clear();
+                            } else {
+                                // 不够，直接追加
+                                let need_size = bytes.len();
+                                let add_bytes = bytes.split_to(need_size);
+                                this.remaining.extend_from_slice(&add_bytes);
+                            }
+                        }
+
+                        loop {
+                            if bytes.len() < chuck_size {
+                                break;
+                            }
+                            let chuck = bytes.split_to(chuck_size);
+                            this.streams.push_back(chuck);
+                        }
+
+                        if !bytes.is_empty() {
+                            this.remaining.extend_from_slice(&bytes);
+                        }
+
+                        if let Some(b) = this.streams.pop_front() {
+                            return Poll::Ready(Some(Ok(b)));
+                        }
+
+                        if items > 0 || op_items.is_some() {
+                            return Poll::Pending;
+                        }
+
+                        if !this.remaining.is_empty() {
+                            let b = this.remaining.clone();
+                            this.remaining.clear();
+                            return Poll::Ready(Some(Ok(Bytes::from(b))));
+                        }
+                        Poll::Ready(None)
+                    }
+                    Err(err) => Poll::Ready(Some(Err(err))),
+                },
+                None => {
+                    // println!("get empty");
+                    if let Some(b) = this.streams.pop_front() {
+                        return Poll::Ready(Some(Ok(b)));
+                    }
+                    if !this.remaining.is_empty() {
+                        let b = this.remaining.clone();
+                        this.remaining.clear();
+                        return Poll::Ready(Some(Ok(Bytes::from(b))));
+                    }
+                    Poll::Ready(None)
+                }
+            },
+            Poll::Pending => {
+                // println!("get Pending");
+                Poll::Pending
+            }
+        }
+
+        // if let Poll::Ready(Some(res)) = poll {
+        //     warn!("poll res ...");
+        //     match res {
+        //         Ok(bytes) => {
+        //             let chuck_size = *this.chuck_size;
+        //             let mut bytes = bytes;
+        //             if this.remaining.len() > 0 {
+        //                 let need_size = chuck_size - this.remaining.len();
+        //                 let add_bytes = bytes.split_to(need_size);
+        //                 this.remaining.extend_from_slice(&add_bytes);
+        //                 warn!("poll push_back remaining ...1");
+        //                 this.streams.push_back(Bytes::from(this.remaining.clone()));
+        //                 this.remaining.clear();
+        //             }
+
+        //             loop {
+        //                 if bytes.len() < chuck_size {
+        //                     break;
+        //                 }
+        //                 let chuck = bytes.split_to(chuck_size);
+        //                 warn!("poll push_back ...1");
+        //                 this.streams.push_back(chuck);
+        //             }
+
+        //             warn!("poll remaining extend_from_slice...1");
+        //             this.remaining.extend_from_slice(&bytes);
+        //         }
+        //         Err(err) => return Poll::Ready(Some(Err(err))),
+        //     }
+        // }
+
+        // if let Some(b) = this.streams.pop_front() {
+        //     warn!("poll pop_front ...");
+        //     return Poll::Ready(Some(Ok(b)));
+        // }
+
+        // if this.remaining.len() > 0 {
+        //     let b = this.remaining.clone();
+        //     this.remaining.clear();
+
+        //     warn!("poll remaining ...1");
+        //     return Poll::Ready(Some(Ok(Bytes::from(b))));
+        // }
+        // Poll::Pending
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let mut items = self.streams.len();
+        if !self.remaining.is_empty() {
+            items += 1;
+        }
+        (items, Some(items))
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+    use futures::StreamExt;
+
+    #[tokio::test]
+    async fn test_etag_reader() {
+        let data1 = vec![1u8; 60]; // 65536
+        let data2 = vec![0u8; 32]; // 65536
+        let chunk1 = Bytes::from(data1);
+        let chunk2 = Bytes::from(data2);
+
+        let chunk_results: Vec<std::result::Result<Bytes, StdError>> = vec![Ok(chunk1), Ok(chunk2)];
+
+        let mut stream = futures::stream::iter(chunk_results);
+
+        let mut hash_reader = EtagReader::new(&mut stream, None, None);
+
+        // let chunk_size = 8;
+
+        // let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size);
+
+        loop {
+            match hash_reader.next().await {
+                Some(res) => match res {
+                    Ok(bytes) => {
+                        println!("bytes: {}, {:?}", bytes.len(), bytes);
+                    }
+                    Err(err) => {
+                        println!("err:{:?}", err);
+                        break;
+                    }
+                },
+                None => {
+                    println!("next none");
+                    break;
+                }
+            }
+        }
+
+        println!("etag:{}", hash_reader.etag());
+
+        // 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45
+        // println!("md5: {:?}", hash_reader.hex());
+    }
+
+    #[tokio::test]
+    async fn test_hash_reader() {
+        let data1 = vec![1u8; 60]; // 65536
+        let data2 = vec![0u8; 32]; // 65536
+        let size = data1.len() + data2.len();
+        let chunk1 = Bytes::from(data1);
+        let chunk2 = Bytes::from(data2);
+
+        let chunk_results: Vec<std::result::Result<Bytes, StdError>> = vec![Ok(chunk1), Ok(chunk2)];
+
+        let mut stream = futures::stream::iter(chunk_results);
+
+        let mut hash_reader = HashReader::new(
+            &mut stream,
+            size,
+            Some("d94c485610a7a00a574df55e45d3cc0c".to_string()),
+            Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()),
+            0,
+        );
+
+        // let chunk_size = 8;
+
+        // let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size);
+
+        loop {
+            match hash_reader.next().await {
+                Some(res) => match res {
+                    Ok(bytes) => {
+                        println!("bytes: {}, {:?}", bytes.len(), bytes);
+                    }
+                    Err(err) => {
+                        println!("err:{:?}", err);
+                        break;
+                    }
+                },
+                None => {
+                    println!("next none");
+                    break;
+                }
+            }
+        }
+
+        // BUG: borrow of moved value: `md5_stream`
+
+        // 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45
+        // println!("md5: {:?}", hash_reader.hex());
+    }
+
+    #[tokio::test]
+    async fn test_chunked_stream() {
+        let data1 = vec![1u8; 60]; // 65536
+        let data2 = vec![0u8; 33]; // 65536
+        let data3 = vec![4u8; 5]; // 65536
+        let chunk1 = Bytes::from(data1);
+        let chunk2 = Bytes::from(data2);
+        let chunk3 = Bytes::from(data3);
+
+        let chunk_results: Vec<std::result::Result<Bytes, StdError>> = vec![Ok(chunk1), Ok(chunk2), Ok(chunk3)];
+
+        let mut stream = futures::stream::iter(chunk_results);
+        // let mut hash_reader = HashReader::new(
+        //     &mut stream,
+        //     size,
+        //     Some("d94c485610a7a00a574df55e45d3cc0c".to_string()),
+        //     Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()),
+        //     0,
+        // );
+
+        let chunk_size = 8;
+
+        let mut etag_reader = EtagReader::new(&mut stream, None, None);
+
+        let mut chunked_stream = ChunkedStream::new(&mut etag_reader, chunk_size);
+
+        loop {
+            match chunked_stream.next().await {
+                Some(res) => match res {
+                    Ok(bytes) => {
+                        println!("bytes: {}, {:?}", bytes.len(), bytes);
+                    }
+                    Err(err) => {
+                        println!("err:{:?}", err);
+                        break;
+                    }
+                },
+                None => {
+                    println!("next none");
+                    break;
+                }
+            }
+        }
+
+        println!("etag:{}", etag_reader.etag());
+    }
+}

rustfs/src/admin/handlers.rs

@@ -63,6 +63,7 @@ pub mod pools;
 pub mod rebalance;
 pub mod service_account;
 pub mod sts;
+pub mod tier;
 pub mod trace;
 pub mod user;
 use urlencoding::decode;

rustfs/src/admin/handlers/tier.rs

@@ -0,0 +1,385 @@
+use std::str::from_utf8;
+
+use http::{HeaderMap, StatusCode};
+use iam::get_global_action_cred;
+use matchit::Params;
+use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
+use serde::Deserialize;
+use serde_urlencoded::from_bytes;
+use tracing::{debug, info, warn};
+
+use crate::{
+    admin::{router::Operation, utils::has_space_be},
+    auth::{check_key_valid, get_session_token},
+};
+use crypto::{decrypt_data, encrypt_data};
+use ecstore::{
+    client::admin_handler_utils::AdminError,
+    config::storageclass,
+    global::GLOBAL_TierConfigMgr,
+    tier::{
+        tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS},
+        tier_admin::TierCreds,
+        tier_config::{TierConfig, TierType},
+        tier_handlers::{
+            ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
+            ERR_TIER_NOT_FOUND,
+        },
+    },
+};
+
+#[derive(Debug, Deserialize, Default)]
+pub struct AddTierQuery {
+    #[serde(rename = "accessKey")]
+    pub access_key: Option<String>,
+    pub status: Option<String>,
+    pub tier: Option<String>,
+    pub force: String,
+}
+
+pub struct AddTier {}
+#[async_trait::async_trait]
+impl Operation for AddTier {
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let query = {
+            if let Some(query) = req.uri.query() {
+                let input: AddTierQuery =
+                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
+                input
+            } else {
+                AddTierQuery::default()
+            }
+        };
+
+        let Some(input_cred) = req.credentials else {
+            return Err(s3_error!(InvalidRequest, "get cred failed"));
+        };
+
+        let (cred, _owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+        let mut input = req.input;
+        let body = match input.store_all_unlimited().await {
+            Ok(b) => b,
+            Err(e) => {
+                warn!("get body failed, e: {:?}", e);
+                return Err(s3_error!(InvalidRequest, "get body failed"));
+            }
+        };
+
+        let mut args: TierConfig = serde_json::from_slice(&body)
+            .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?;
+
+        match args.tier_type {
+            TierType::S3 => {
+                args.name = args.s3.clone().unwrap().name;
+            }
+            TierType::RustFS => {
+                args.name = args.rustfs.clone().unwrap().name;
+            }
+            TierType::MinIO => {
+                args.name = args.minio.clone().unwrap().name;
+            }
+            _ => (),
+        }
+        debug!("add tier args {:?}", args);
+
+        let mut force: bool = false;
+        let force_str = query.force;
+        if force_str != "" {
+            force = force_str.parse().unwrap();
+        }
+        match args.name.as_str() {
+            storageclass::STANDARD | storageclass::RRS => {
+                warn!("tier reserved name, args.name: {}", args.name);
+                return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name"));
+            }
+            &_ => (),
+        }
+
+        let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
+        //tier_config_mgr.reload(api);
+        match tier_config_mgr.add(args, force).await {
+            Err(ERR_TIER_ALREADY_EXISTS) => {
+                return Err(S3Error::with_message(
+                    S3ErrorCode::Custom("TierNameAlreadyExist".into()),
+                    "tier name already exists!",
+                ));
+            }
+            Err(ERR_TIER_NAME_NOT_UPPERCASE) => {
+                return Err(S3Error::with_message(
+                    S3ErrorCode::Custom("TierNameNotUppercase".into()),
+                    "tier name not uppercase!",
+                ));
+            }
+            Err(ERR_TIER_BACKEND_IN_USE) => {
+                return Err(S3Error::with_message(
+                    S3ErrorCode::Custom("TierNameBackendInUse!".into()),
+                    "tier name backend in use!",
+                ));
+            }
+            Err(ERR_TIER_CONNECT_ERR) => {
+                return Err(S3Error::with_message(
+                    S3ErrorCode::Custom("TierConnectError".into()),
+                    "tier connect error!",
+                ));
+            }
+            Err(ERR_TIER_INVALID_CREDENTIALS) => {
+                return Err(S3Error::with_message(
+                    S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()),
+                    ERR_TIER_INVALID_CREDENTIALS.message,
+                ));
+            }
+            Err(e) => {
+                warn!("tier_config_mgr add failed, e: {:?}", e);
+                return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier add failed"));
+            }
+            Ok(_) => (),
+        }
+        if let Err(e) = tier_config_mgr.save().await {
+            warn!("tier_config_mgr save failed, e: {:?}", e);
+            return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed"));
+        }
+        //globalNotificationSys.LoadTransitionTierConfig(ctx);
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
+    }
+}
+
+pub struct EditTier {}
+#[async_trait::async_trait]
+impl Operation for EditTier {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let query = {
+            if let Some(query) = req.uri.query() {
+                let input: AddTierQuery =
+                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?;
+                input
+            } else {
+                AddTierQuery::default()
+            }
+        };
+
+        let Some(input_cred) = req.credentials else {
+            return Err(s3_error!(InvalidRequest, "get cred failed"));
+        };
+
+        let (cred, _owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+        //{"accesskey":"gggggg","secretkey":"jjjjjjj"}
+        //{"detailedMessage":"failed to perform PUT: The Access Key Id you provided does not exist in our records.","message":"an error occurred, please try again"}
+        //200 OK
+        let mut input = req.input;
+        let body = match input.store_all_unlimited().await {
+            Ok(b) => b,
+            Err(e) => {
+                warn!("get body failed, e: {:?}", e);
+                return Err(s3_error!(InvalidRequest, "get body failed"));
+            }
+        };
+
+        let creds: TierCreds = serde_json::from_slice(&body)
+            .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?;
+
+        debug!("edit tier args {:?}", creds);
+
+        let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
+
+        let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
+        //tier_config_mgr.reload(api);
+        match tier_config_mgr.edit(&tier_name, creds).await {
+            Err(ERR_TIER_NOT_FOUND) => {
+                return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!"));
+            }
+            Err(ERR_TIER_MISSING_CREDENTIALS) => {
+                return Err(S3Error::with_message(
+                    S3ErrorCode::Custom("TierMissingCredentials".into()),
+                    "tier missing credentials!",
+                ));
+            }
+            Err(e) => {
+                warn!("tier_config_mgr edit failed, e: {:?}", e);
+                return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier edit failed"));
+            }
+            Ok(_) => (),
+        }
+        if let Err(e) = tier_config_mgr.save().await {
+            warn!("tier_config_mgr save failed, e: {:?}", e);
+            return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed"));
+        }
+        //globalNotificationSys.LoadTransitionTierConfig(ctx);
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
+    }
+}
+
+#[derive(Debug, Deserialize, Default)]
+pub struct BucketQuery {
+    #[serde(rename = "bucket")]
+    pub bucket: String,
+}
+pub struct ListTiers {}
+#[async_trait::async_trait]
+impl Operation for ListTiers {
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let query = {
+            if let Some(query) = req.uri.query() {
+                let input: BucketQuery =
+                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
+                input
+            } else {
+                BucketQuery::default()
+            }
+        };
+
+        //{"items":[{"minio":{"accesskey":"minioadmin","bucket":"mblock2","endpoint":"http://192.168.1.11:9020","name":"COLDTIER","objects":"0","prefix":"mypre/","secretkey":"REDACTED","usage":"0 B","versions":"0"},"status":true,"type":"minio"}]}
+        let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await;
+        let tiers = tier_config_mgr.list_tiers();
+
+        let data = serde_json::to_vec(&tiers)
+            .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tiers err {}", e)))?;
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
+    }
+}
+
+pub struct RemoveTier {}
+#[async_trait::async_trait]
+impl Operation for RemoveTier {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        warn!("handle RemoveTier");
+
+        let query = {
+            if let Some(query) = req.uri.query() {
+                let input: AddTierQuery =
+                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
+                input
+            } else {
+                AddTierQuery::default()
+            }
+        };
+
+        let Some(input_cred) = req.credentials else {
+            return Err(s3_error!(InvalidRequest, "get cred failed"));
+        };
+
+        let (cred, _owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+        let sys_cred = get_global_action_cred()
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?;
+
+        let mut force: bool = false;
+        let force_str = query.force;
+        if force_str != "" {
+            force = force_str.parse().unwrap();
+        }
+
+        let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default();
+
+        let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
+        //tier_config_mgr.reload(api);
+        match tier_config_mgr.remove(&tier_name, force).await {
+            Err(ERR_TIER_NOT_FOUND) => {
+                return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found."));
+            }
+            Err(ERR_TIER_BACKEND_NOT_EMPTY) => {
+                return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used."));
+            }
+            Err(e) => {
+                warn!("tier_config_mgr remove failed, e: {:?}", e);
+                return Err(S3Error::with_message(
+                    S3ErrorCode::Custom("TierRemoveFailed".into()),
+                    "tier remove failed",
+                ));
+            }
+            Ok(_) => (),
+        }
+        if let Err(e) = tier_config_mgr.save().await {
+            warn!("tier_config_mgr save failed, e: {:?}", e);
+            return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed"));
+        }
+        //globalNotificationSys.LoadTransitionTierConfig(ctx);
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
+    }
+}
+
+pub struct VerifyTier {}
+#[async_trait::async_trait]
+impl Operation for VerifyTier {
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        warn!("handle RemoveTier");
+
+        let query = {
+            if let Some(query) = req.uri.query() {
+                let input: AddTierQuery =
+                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
+                input
+            } else {
+                AddTierQuery::default()
+            }
+        };
+
+        let Some(input_cred) = req.credentials else {
+            return Err(s3_error!(InvalidRequest, "get cred failed"));
+        };
+
+        let (cred, _owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+        let sys_cred = get_global_action_cred()
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?;
+
+        let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await;
+        tier_config_mgr.verify(&query.tier.unwrap());
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
+    }
+}
+
+pub struct GetTierInfo {}
+#[async_trait::async_trait]
+impl Operation for GetTierInfo {
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        warn!("handle GetTierInfo");
+
+        let query = {
+            if let Some(query) = req.uri.query() {
+                let input: AddTierQuery =
+                    from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?;
+                input
+            } else {
+                AddTierQuery::default()
+            }
+        };
+
+        let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await;
+        let info = tier_config_mgr.get(&query.tier.unwrap());
+
+        let data = serde_json::to_vec(&info)
+            .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tier err {}", e)))?;
+
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
+    }
+}

rustfs/src/admin/mod.rs

@@ -7,7 +7,7 @@ pub mod utils;
 use handlers::{
     group, policys, pools, rebalance,
     service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount},
-    sts, user,
+    sts, tier, user,
 };
 
 use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler};
@@ -311,5 +311,38 @@ fn register_user_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()>
         AdminOperation(&policys::SetPolicyForUserOrGroup {}),
     )?;
 
+    // ?
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(),
+        AdminOperation(&tier::ListTiers {}),
+    )?;
+    // ?
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/tier-stats").as_str(),
+        AdminOperation(&tier::GetTierInfo {}),
+    )?;
+    // ?force=xxx
+    r.insert(
+        Method::DELETE,
+        format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(),
+        AdminOperation(&tier::RemoveTier {}),
+    )?;
+    // ?force=xxx
+    // body: AddOrUpdateTierReq
+    r.insert(
+        Method::PUT,
+        format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(),
+        AdminOperation(&tier::AddTier {}),
+    )?;
+    // ?
+    // body: AddOrUpdateTierReq
+    r.insert(
+        Method::POST,
+        format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(),
+        AdminOperation(&tier::EditTier {}),
+    )?;
+
     Ok(())
 }

rustfs/src/storage/ecfs.rs

@@ -18,6 +18,7 @@ use chrono::Utc;
 use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder;
 use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder;
 use datafusion::arrow::json::writer::JsonArray;
+use ecstore::bucket::error::BucketMetadataError;
 use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
 use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG;
 use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG;
@@ -54,7 +55,13 @@ use ecstore::store_api::ObjectToDelete;
 use ecstore::store_api::PutObjReader;
 use ecstore::store_api::StorageAPI;
 // use ecstore::store_api::RESERVED_METADATA_PREFIX;
-use futures::StreamExt;
+use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier;
+use ecstore::bucket::utils::serialize;
+use ecstore::cmd::bucket_replication::ReplicationStatusType;
+use ecstore::cmd::bucket_replication::ReplicationType;
+use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER;
+use futures::pin_mut;
+use futures::{Stream, StreamExt};
 use http::HeaderMap;
 use lazy_static::lazy_static;
 use policy::auth;
@@ -99,6 +106,11 @@ use tracing::info;
 use tracing::warn;
 use uuid::Uuid;
 
+use ecstore::bucket::{
+    lifecycle::{bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS, lifecycle::Lifecycle},
+    object_lock::objectlock_sys::BucketObjectLockSys,
+};
+
 macro_rules! try_ {
     ($result:expr) => {
         match $result {
@@ -1710,12 +1722,35 @@ impl S3 for FS {
             ..
         } = req.input;
 
-        // warn!("lifecycle_configuration {:?}", &lifecycle_configuration);
+        let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
+        if rcfg.is_err() {
+            return Err(S3Error::with_message(
+                S3ErrorCode::Custom("BucketLockIsNotExist".into()),
+                "bucket lock is not exist.",
+            ));
+        }
+        let rcfg = rcfg.expect("get_lifecycle_config err!").0;
 
-        // TODO: objcetLock
+        //info!("lifecycle_configuration: {:?}", &lifecycle_configuration);
 
         let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) };
 
+        if let Err(err) = input_cfg.validate(&rcfg).await {
+            //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed."));
+            return Err(S3Error::with_message(
+                S3ErrorCode::Custom("ValidateFailed".into()),
+                format!("{}", err.to_string()),
+            ));
+        }
+
+        if let Err(err) = validate_transition_tier(&input_cfg).await {
+            //warn!("lifecycle_configuration add failed, err: {:?}", err);
+            return Err(S3Error::with_message(
+                S3ErrorCode::Custom("CustomError".into()),
+                format!("{}", err.to_string()),
+            ));
+        }
+
         let data = try_!(serialize(&input_cfg));
         metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data)
             .await
@@ -2156,6 +2191,30 @@ impl S3 for FS {
         }))
     }
 
+    async fn get_object_attributes(
+        &self,
+        req: S3Request<GetObjectAttributesInput>,
+    ) -> S3Result<S3Response<GetObjectAttributesOutput>> {
+        let GetObjectAttributesInput { bucket, key, .. } = req.input;
+
+        let Some(store) = new_object_layer_fn() else {
+            return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
+        };
+
+        if let Err(e) = store
+            .get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+        {
+            return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
+        }
+
+        Ok(S3Response::new(GetObjectAttributesOutput {
+            delete_marker: None,
+            object_parts: None,
+            ..Default::default()
+        }))
+    }
+
     async fn put_object_acl(&self, req: S3Request<PutObjectAclInput>) -> S3Result<S3Response<PutObjectAclOutput>> {
         let PutObjectAclInput {
             bucket,
@@ -2399,7 +2458,7 @@ impl S3 for FS {
         let retain_until_date = object_info
             .user_defined
             .get("x-amz-object-lock-retain-until-date")
-            .and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok())
+                .and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok())
             .map(Timestamp::from);
 
         Ok(S3Response::new(GetObjectRetentionOutput {