Refactor: Add observability enable flag, improve comments, remove unused config params, and enhance run function error logging. (#689 )

* improve code for dns log * fix * Improve comments, remove unused parameters in config.rs (opt), add observability enable flag, and enhance error logging in run function execution.
Refactor Console Server Architecture (#685 )
2026-07-30 18:12:14 +00:00 · 2025-10-23 13:59:57 +08:00 · 2025-10-23 00:06:09 +08:00 · 2025-10-22 23:47:11 +08:00 · 2025-10-22 16:23:08 +08:00 · 2025-10-22 10:35:40 +08:00
245 changed files with 20327 additions and 11027 deletions
@@ -20,7 +20,10 @@
                }
            },
            "env": {
-                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=info"
+                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug",
+                "RUSTFS_SKIP_BACKGROUND_TASK": "on",
+                // "RUSTFS_POLICY_PLUGIN_URL":"http://localhost:8181/v1/data/rustfs/authz/allow",
+                // "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN":"your-opa-token" 
            },
            "args": [
                "--access-key",
@@ -29,6 +32,8 @@
                "rustfsadmin",
                "--address",
                "0.0.0.0:9010",
+                "--server-domains",
+                "127.0.0.1:9010",
                "./target/volume/test{1...4}"
            ],
            "cwd": "${workspaceFolder}"
@@ -28,6 +28,7 @@ members = [
    "crates/madmin", # Management dashboard and admin API interface
    "crates/notify", # Notification system for events
    "crates/obs", # Observability utilities
+    "crates/policy", # Policy management
    "crates/protos", # Protocol buffer definitions
    "crates/rio", # Rust I/O utilities and abstractions
    "crates/targets", # Target-specific configurations and utilities
@@ -99,10 +100,12 @@ async-recursion = "1.1.1"
 async-trait = "0.1.89"
 async-compression = { version = "0.4.19" }
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.6" }
-aws-sdk-s3 = { version = "1.106.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
-axum = "0.8.4"
-axum-extra = "0.10.1"
+aws-config = { version = "1.8.8" }
+aws-credential-types = { version = "1.2.8" }
+aws-smithy-types = { version = "1.3.3" }
+aws-sdk-s3 = { version = "1.108.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
+axum = "0.8.6"
+axum-extra = "0.10.3"
 axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"], default-features = false }
 base64-simd = "0.8.0"
 base64 = "0.22.1"
@@ -110,27 +113,30 @@ brotli = "8.0.2"
 bytes = { version = "1.10.1", features = ["serde"] }
 bytesize = "2.1.0"
 byteorder = "1.5.0"
-cfg-if = "1.0.3"
+cfg-if = "1.0.4"
+convert_case = "0.8.0"
 crc-fast = "1.3.0"
 chacha20poly1305 = { version = "0.10.1" }
 chrono = { version = "0.4.42", features = ["serde"] }
-clap = { version = "4.5.48", features = ["derive", "env"] }
+clap = { version = "4.5.49", features = ["derive", "env"] }
 const-str = { version = "0.7.0", features = ["std", "proc"] }
 crc32fast = "1.5.0"
+crc32c = "0.6.8"
+crc64fast-nvme = "1.2.0"
 criterion = { version = "0.7", features = ["html_reports"] }
 crossbeam-queue = "0.3.12"
-dashmap = "6.1.0"
-datafusion = "50.0.0"
+datafusion = "50.2.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 flatbuffers = "25.9.23"
-flate2 = "1.1.2"
-flexi_logger = { version = "0.31.4", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv"] }
+flate2 = "1.1.4"
+flexi_logger = { version = "0.31.7", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv"] }
 form_urlencoded = "1.2.2"
 futures = "0.3.31"
 futures-core = "0.3.31"
 futures-util = "0.3.31"
 glob = "0.3.3"
+hashbrown = { version = "0.16.0", features = ["serde", "rayon"] }
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 hickory-resolver = { version = "0.25.2", features = ["tls-ring"] }
@@ -146,8 +152,9 @@ http = "1.3.1"
 http-body = "1.0.1"
 humantime = "2.3.0"
 ipnetwork = { version = "0.21.1", features = ["serde"] }
-jsonwebtoken = "9.3.1"
+jsonwebtoken = { version = "10.0.0", features = ["rust_crypto"] }
 lazy_static = "1.5.0"
+libc = "0.2.177"
 libsystemd = { version = "0.7.2" }
 local-ip-address = "0.6.5"
 lz4 = "1.28.1"
@@ -158,40 +165,39 @@ mime_guess = "2.0.5"
 moka = { version = "0.12.11", features = ["future"] }
 netif = "0.1.6"
 nix = { version = "0.30.1", features = ["fs"] }
-nu-ansi-term = "0.50.1"
+nu-ansi-term = "0.50.3"
 num_cpus = { version = "1.17.0" }
 nvml-wrapper = "0.11.0"
 object_store = "0.12.4"
 once_cell = "1.21.3"
-opentelemetry = { version = "0.30.0" }
-opentelemetry-appender-tracing = { version = "0.30.1", features = [
+opentelemetry = { version = "0.31.0" }
+opentelemetry-appender-tracing = { version = "0.31.1", features = [
    "experimental_use_tracing_span_context",
    "experimental_metadata_attributes",
    "spec_unstable_logs_enabled"
 ] }
-opentelemetry_sdk = { version = "0.30.0" }
-opentelemetry-stdout = { version = "0.30.0" }
-opentelemetry-otlp = { version = "0.30.0", default-features = false, features = [
+opentelemetry_sdk = { version = "0.31.0" }
+opentelemetry-stdout = { version = "0.31.0" }
+opentelemetry-otlp = { version = "0.31.0", default-features = false, features = [
    "grpc-tonic", "gzip-tonic", "trace", "metrics", "logs", "internal-logs"
 ] }
-opentelemetry-semantic-conventions = { version = "0.30.0", features = [
+opentelemetry-semantic-conventions = { version = "0.31.0", features = [
    "semconv_experimental",
 ] }
-parking_lot = "0.12.4"
+parking_lot = "0.12.5"
 path-absolutize = "3.1.1"
 path-clean = "1.0.1"
 blake3 = { version = "1.8.2" }
 pbkdf2 = "0.12.2"
-percent-encoding = "2.3.2"
 pin-project-lite = "0.2.16"
 prost = "0.14.1"
 pretty_assertions = "1.4.1"
 quick-xml = "0.38.3"
 rand = "0.9.2"
-rdkafka = { version = "0.38.0", features = ["tokio"] }
-reed-solomon-simd = { version = "3.0.1" }
-regex = { version = "1.11.2" }
-reqwest = { version = "0.12.23", default-features = false, features = [
+rayon = "1.11.0"
+reed-solomon-simd = { version = "3.1.0" }
+regex = { version = "1.12.2" }
+reqwest = { version = "0.12.24", default-features = false, features = [
    "rustls-tls-webpki-roots",
    "charset",
    "http2",
@@ -200,38 +206,39 @@ reqwest = { version = "0.12.23", default-features = false, features = [
    "json",
    "blocking",
 ] }
-rmcp = { version = "0.6.4" }
-rmp = "0.8.14"
-rmp-serde = "1.3.0"
-rsa = "0.9.8"
+rmcp = { version = "0.8.1" }
+rmp = { version = "0.8.14" }
+rmp-serde = { version = "1.3.0" }
+rsa = { version = "0.9.8" }
 rumqttc = { version = "0.25.0" }
 rust-embed = { version = "8.7.2" }
-rustfs-rsc = "2025.506.1"
+rustc-hash = { version = "2.1.1" }
 rustls = { version = "0.23.32", features = ["ring", "logging", "std", "tls12"], default-features = false }
 rustls-pki-types = "1.12.0"
 rustls-pemfile = "2.2.0"
-s3s = { version = "0.12.0-minio-preview.3" }
+s3s = { version = "0.12.0-rc.3", features = ["minio"] }
 schemars = "1.0.4"
-serde = { version = "1.0.226", features = ["derive"] }
+serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.145", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 serial_test = "3.2.0"
 sha1 = "0.10.6"
 sha2 = "0.10.9"
-shadow-rs = { version = "1.3.0", default-features = false }
+shadow-rs = { version = "1.4.0", default-features = false }
 siphasher = "1.0.1"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
 snafu = "0.8.9"
 snap = "1.1.1"
-socket2 = "0.6.0"
+socket2 = "0.6.1"
+starshard = { version = "0.5.0", features = ["rayon", "async", "serde"] }
 strum = { version = "0.27.2", features = ["derive"] }
-sysinfo = "0.37.0"
+sysinfo = "0.37.1"
 sysctl = "0.7.1"
 tempfile = "3.23.0"
 temp-env = "0.3.6"
 test-case = "3.3.1"
-thiserror = "2.0.16"
+thiserror = "2.0.17"
 time = { version = "0.3.44", features = [
    "std",
    "parsing",
@@ -239,8 +246,8 @@ time = { version = "0.3.44", features = [
    "macros",
    "serde",
 ] }
-tokio = { version = "1.47.1", features = ["fs", "rt-multi-thread"] }
-tokio-rustls = { version = "0.26.3", default-features = false, features = ["logging", "tls12", "ring"] }
+tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] }
+tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "ring"] }
 tokio-stream = { version = "0.1.17" }
 tokio-tar = "0.3.1"
 tokio-test = "0.4.4"
@@ -250,10 +257,10 @@ tonic-prost = { version = "0.14.2" }
 tonic-prost-build = { version = "0.14.2" }
 tower = { version = "0.5.2", features = ["timeout"] }
 tower-http = { version = "0.6.6", features = ["cors"] }
-tracing = "0.1.41"
+tracing = { version = "0.1.41" }
 tracing-core = "0.1.34"
 tracing-error = "0.2.1"
-tracing-opentelemetry = "0.31.0"
+tracing-opentelemetry = "0.32.0"
 tracing-subscriber = { version = "0.3.20", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.7"
@@ -266,10 +273,10 @@ uuid = { version = "1.18.1", features = [
 vaultrs = { version = "0.7.4" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.5.0", features = ["serde"] }
-zeroize = { version = "1.8.1", features = ["derive"] }
+zeroize = { version = "1.8.2", features = ["derive"] }
 winapi = { version = "0.3.9" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "5.1.1"
+zip = "6.0.0"
 zstd = "0.13.3"


@@ -58,7 +58,7 @@ LABEL name="RustFS" \
      url="https://rustfs.com" \
      license="Apache-2.0"

-RUN apk add --no-cache ca-certificates coreutils
+RUN apk add --no-cache ca-certificates coreutils curl

 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs
@@ -29,7 +29,11 @@ English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简
  <a href="https://readme-i18n.com/rustfs/rustfs?lang=ru">Русский</a>
 </p>

-RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature, support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation, RustFS provides faster speed and safer distributed features for high-performance object storage.
+RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages
+worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature,
+support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in
+comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation,
+RustFS provides faster speed and safer distributed features for high-performance object storage.

 > ⚠️ **RustFS is under rapid development. Do NOT use in production environments!**

@@ -46,27 +50,27 @@ RustFS is a high-performance distributed object storage software built using Rus

 Stress test server parameters

-|  Type  |  parameter   | Remark |
-| - | - | - |
-|CPU | 2 Core | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz|   |
-|Memory| 4GB |     |
-|Network | 15Gbp |      |
-|Driver  | 40GB x 4 |   IOPS 3800 / Driver |
+| Type    | parameter | Remark                                                   |
+|---------|-----------|----------------------------------------------------------|
+| CPU     | 2 Core    | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz |   |
+| Memory  | 4GB       |                                                          |
+| Network | 15Gbp     |                                                          |
+| Driver  | 40GB x 4  | IOPS 3800 / Driver                                       |

 <https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>

 ### RustFS vs Other object storage

-| RustFS | Other object storage|
-| - | - |
-| Powerful Console | Simple and useless Console |
-| Developed based on Rust language, memory is safer | Developed in Go or C, with potential issues like memory GC/leaks |
-| Does not report logs to third-party countries  | Reporting logs to other third countries may violate national security laws |
-| Licensed under Apache, more business-friendly  | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
-| Comprehensive S3 support, works with domestic and international cloud providers  | Full support for S3, but no local cloud vendor support |
-| Rust-based development, strong support for secure and innovative devices  | Poor support for edge gateways and secure innovative devices|
-| Stable commercial prices, free community support | High pricing, with costs up to $250,000 for 1PiB |
-| No risk | Intellectual property risks and risks of prohibited uses |
+| RustFS                                                                          | Other object storage                                                                                                    |
+|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
+| Powerful Console                                                                | Simple and useless Console                                                                                              |
+| Developed based on Rust language, memory is safer                               | Developed in Go or C, with potential issues like memory GC/leaks                                                        |
+| Does not report logs to third-party countries                                   | Reporting logs to other third countries may violate national security laws                                              |
+| Licensed under Apache, more business-friendly                                   | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
+| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support                                                                  |
+| Rust-based development, strong support for secure and innovative devices        | Poor support for edge gateways and secure innovative devices                                                            |
+| Stable commercial prices, free community support                                | High pricing, with costs up to $250,000 for 1PiB                                                                        |
+| No risk                                                                         | Intellectual property risks and risks of prohibited uses                                                                |

 ## Quickstart

@@ -91,13 +95,16 @@ To get started with RustFS, follow these steps:
   docker run -d -p 9000:9000 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.45
   ```

-  For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under root directory, running the command:
+For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under
+root directory, running the command:

  ```
  docker compose --profile observability up -d
  ```
-  
-  **NOTE**: You should be better to have a look for `docker-compose.yaml` file. Because, several services contains in the file. Grafan,prometheus,jaeger containers will be launched using docker compose file, which is helpful for rustfs observability. If you want to start redis as well as nginx container, you can specify the corresponding profiles.
+
+**NOTE**: You should be better to have a look for `docker-compose.yaml` file. Because, several services contains in the
+file. Grafan,prometheus,jaeger containers will be launched using docker compose file, which is helpful for rustfs
+observability. If you want to start redis as well as nginx container, you can specify the corresponding profiles.

 3. **Build from Source (Option 3) - Advanced Users**

@@ -118,10 +125,10 @@ To get started with RustFS, follow these steps:
   ```

   The `docker-buildx.sh` script supports:
-   - **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
-   - **Automatic version detection**: Uses git tags or commit hashes
-   - **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
-   - **Build optimization**: Includes caching and parallel builds
+    - **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
+    - **Automatic version detection**: Uses git tags or commit hashes
+    - **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
+    - **Build optimization**: Includes caching and parallel builds

   You can also use Make targets for convenience:

@@ -132,23 +139,29 @@ To get started with RustFS, follow these steps:
   make help-docker                      # Show all Docker-related commands
   ```

-4. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console, default username and password is `rustfsadmin` .
+4. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console,
+   default username and password is `rustfsadmin` .
 5. **Create a Bucket**: Use the console to create a new bucket for your objects.
-6. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your RustFS instance.
+6. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your
+   RustFS instance.

-**NOTE**: If you want to access RustFS instance with `https`, you can refer to [TLS configuration docs](https://docs.rustfs.com/integration/tls-configured.html).
+**NOTE**: If you want to access RustFS instance with `https`, you can refer
+to [TLS configuration docs](https://docs.rustfs.com/integration/tls-configured.html).

 ## Documentation

-For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).
+For detailed documentation, including configuration options, API references, and advanced usage, please visit
+our [Documentation](https://docs.rustfs.com).

 ## Getting Help

 If you have any questions or need assistance, you can:

 - Check the [FAQ](https://github.com/rustfs/rustfs/discussions/categories/q-a) for common issues and solutions.
- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your experiences.
- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature requests.
+- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your
+  experiences.
+- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature
+  requests.

 ## Links

@@ -166,14 +179,24 @@ If you have any questions or need assistance, you can:

 ## Contributors

-RustFS is a community-driven project, and we appreciate all contributions. Check out the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped make RustFS better.
+RustFS is a community-driven project, and we appreciate all contributions. Check out
+the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped
+make RustFS better.

 <a href="https://github.com/rustfs/rustfs/graphs/contributors">
-  <img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" />
+  <img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="Contributors"/>
 </a>

+## Github Trending Top
+
+🚀 RustFS is beloved by open-source enthusiasts and enterprise users worldwide, often appearing on the GitHub Trending
+top charts.
+
+<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
+
 ## License

 [Apache 2.0](https://opensource.org/licenses/Apache-2.0)

 **RustFS** is a trademark of RustFS, Inc. All other trademarks are the property of their respective owners.
+
@@ -21,7 +21,9 @@
 <a href="https://github.com/rustfs/rustfs/blob/main/README.md">English</a > | 简体中文
 </p >

-RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建的高性能分布式对象存储软件。与 MinIO 一样，它具有简单性、S3 兼容性、开源特性以及对数据湖、AI 和大数据的支持等一系列优势。此外，与其他存储系统相比，它采用 Apache 许可证构建，拥有更好、更用户友好的开源许可证。由于以 Rust 为基础，RustFS 为高性能对象存储提供了更快的速度和更安全的分布式功能。
+RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建的高性能分布式对象存储软件。与 MinIO 一样，它具有简单性、S3
+兼容性、开源特性以及对数据湖、AI 和大数据的支持等一系列优势。此外，与其他存储系统相比，它采用 Apache
+许可证构建，拥有更好、更用户友好的开源许可证。由于以 Rust 为基础，RustFS 为高性能对象存储提供了更快的速度和更安全的分布式功能。

 ## 特性

@@ -36,27 +38,27 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建

 压力测试服务器参数

-|  类型  |  参数   | 备注 |
-| - | - | - |
-|CPU | 2 核心 | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz|   |
-|内存| 4GB |     |
-|网络 | 15Gbp |      |
-|驱动器  | 40GB x 4 |   IOPS 3800 / 驱动器 |
+| 类型  | 参数       | 备注                                                       |
+|-----|----------|----------------------------------------------------------|
+| CPU | 2 核心     | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz |   |
+| 内存  | 4GB      |                                                          |
+| 网络  | 15Gbp    |                                                          |
+| 驱动器 | 40GB x 4 | IOPS 3800 / 驱动器                                          |

 <https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>

 ### RustFS vs 其他对象存储

-| RustFS | 其他对象存储|
-| - | - |
-| 强大的控制台 | 简单且无用的控制台 |
-| 基于 Rust 语言开发，内存更安全 | 使用 Go 或 C 开发，存在内存 GC/泄漏等潜在问题 |
-| 不向第三方国家报告日志  | 向其他第三方国家报告日志可能违反国家安全法律 |
-| 采用 Apache 许可证，对商业更友好  | AGPL V3 许可证等其他许可证，污染开源和许可证陷阱，侵犯知识产权 |
-| 全面的 S3 支持，适用于国内外云提供商  | 完全支持 S3，但不支持本地云厂商 |
-| 基于 Rust 开发，对安全和创新设备有强大支持  | 对边缘网关和安全创新设备支持较差|
-| 稳定的商业价格，免费社区支持 | 高昂的定价，1PiB 成本高达 $250,000 |
-| 无风险 | 知识产权风险和禁止使用的风险 |
+| RustFS                   | 其他对象存储                              |
+|--------------------------|-------------------------------------|
+| 强大的控制台                   | 简单且无用的控制台                           |
+| 基于 Rust 语言开发，内存更安全       | 使用 Go 或 C 开发，存在内存 GC/泄漏等潜在问题        |
+| 不向第三方国家报告日志              | 向其他第三方国家报告日志可能违反国家安全法律              |
+| 采用 Apache 许可证，对商业更友好     | AGPL V3 许可证等其他许可证，污染开源和许可证陷阱，侵犯知识产权 |
+| 全面的 S3 支持，适用于国内外云提供商     | 完全支持 S3，但不支持本地云厂商                   |
+| 基于 Rust 开发，对安全和创新设备有强大支持 | 对边缘网关和安全创新设备支持较差                    |
+| 稳定的商业价格，免费社区支持           | 高昂的定价，1PiB 成本高达 $250,000            |
+| 无风险                      | 知识产权风险和禁止使用的风险                      |

 ## 快速开始

@@ -68,25 +70,30 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建
   curl -O  https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
   ```

-2. **Docker快速启动（方案二）**
+2. **Docker 快速启动（方案二）**

  ```bash
   docker run -d -p 9000:9000  -v /data:/data rustfs/rustfs
   ```

-  对于使用 Docker 安装来讲，你还可以使用 `docker compose` 来启动 rustfs 实例。在仓库的根目录下面有一个 `docker-compose.yml` 文件。运行如下命令即可：
+对于使用 Docker 安装来讲，你还可以使用 `docker compose` 来启动 rustfs 实例。在仓库的根目录下面有一个 `docker-compose.yml`
+文件。运行如下命令即可：

  ```
  docker compose --profile observability up -d
  ```
-  
-  **注意**：在使用 `docker compose` 之前，你应该仔细阅读一下 `docker-compose.yaml`，因为该文件中包含多个服务，除了 rustfs 以外，还有 grafana、prometheus、jaeger 等，这些是为 rustfs 可观测性服务的，还有 redis 和 nginx。你想启动哪些容器，就需要用 `--profile` 参数指定相应的 profile。

-3. **访问控制台**：打开 Web 浏览器并导航到 `http://localhost:9000` 以访问 RustFS 控制台，默认的用户名和密码是 `rustfsadmin` 。
+**注意**：在使用 `docker compose` 之前，你应该仔细阅读一下 `docker-compose.yaml`，因为该文件中包含多个服务，除了 rustfs
+以外，还有 grafana、prometheus、jaeger 等，这些是为 rustfs 可观测性服务的，还有 redis 和 nginx。你想启动哪些容器，就需要用
+`--profile` 参数指定相应的 profile。
+
+3. **访问控制台**：打开 Web 浏览器并导航到 `http://localhost:9000` 以访问 RustFS 控制台，默认的用户名和密码是
+   `rustfsadmin` 。
 4. **创建存储桶**：使用控制台为您的对象创建新的存储桶。
 5. **上传对象**：您可以直接通过控制台上传文件，或使用 S3 兼容的 API 与您的 RustFS 实例交互。

-**注意**：如果你想通过 `https` 来访问 RustFS 实例，请参考 [TLS 配置文档](https://docs.rustfs.com/zh/integration/tls-configured.html)
+**注意**：如果你想通过 `https` 来访问 RustFS
+实例，请参考 [TLS 配置文档](https://docs.rustfs.com/zh/integration/tls-configured.html)

 ## 文档

@@ -116,12 +123,19 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建

 ## 贡献者

-RustFS 是一个社区驱动的项目，我们感谢所有的贡献。查看[贡献者](https://github.com/rustfs/rustfs/graphs/contributors)页面，了解帮助 RustFS 变得更好的杰出人员。
+RustFS 是一个社区驱动的项目，我们感谢所有的贡献。查看[贡献者](https://github.com/rustfs/rustfs/graphs/contributors)页面，了解帮助
+RustFS 变得更好的杰出人员。

 <a href="https://github.com/rustfs/rustfs/graphs/contributors">
-  <img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" />
+  <img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="贡献者"/>
 </a >

+## Github 全球推荐榜
+
+🚀 RustFS 受到了全世界开源爱好者和企业用户的喜欢，多次登顶 Github Trending 全球榜。
+
+<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
+
 ## 许可证

 [Apache 2.0](https://opensource.org/licenses/Apache-2.0)
@@ -29,6 +29,7 @@ use rustfs_ecstore::{
    data_usage::{aggregate_local_snapshots, store_data_usage_in_backend},
 };
 use rustfs_filemeta::{MetacacheReader, VersionType};
+use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
 use tokio::sync::{Mutex, RwLock};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
@@ -50,7 +51,6 @@ use rustfs_common::data_usage::{DataUsageInfo, SizeSummary};
 use rustfs_common::metrics::{Metric, Metrics, globalMetrics};
 use rustfs_ecstore::bucket::versioning::VersioningApi;
 use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
-use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
 use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
 use uuid;

@@ -300,8 +300,13 @@ impl Scanner {
                            .map(|(c, _)| Arc::new(c));

                        // Get bucket versioning configuration
-                        let versioning_config = Arc::new(VersioningConfig {
-                            enabled: bucket_info.versioning,
+                        let versioning_config = Arc::new(VersioningConfiguration {
+                            status: if bucket_info.versioning {
+                                Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
+                            } else {
+                                None
+                            },
+                            ..Default::default()
                        });

                        let records = match bucket_objects_map.get(bucket_name) {
@@ -1825,7 +1830,16 @@ impl Scanner {
            }
        };
        let bucket_info = ecstore.get_bucket_info(bucket, &Default::default()).await.ok();
-        let versioning_config = bucket_info.map(|bi| Arc::new(VersioningConfig { enabled: bi.versioning }));
+        let versioning_config = bucket_info.map(|bi| {
+            Arc::new(VersioningConfiguration {
+                status: if bi.versioning {
+                    Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
+                } else {
+                    None
+                },
+                ..Default::default()
+            })
+        });
        let lifecycle_config = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket)
            .await
            .ok()
@@ -2651,7 +2665,7 @@ mod tests {
        // create ECStore with dynamic port
        let port = port.unwrap_or(9000);
        let server_addr: SocketAddr = format!("127.0.0.1:{port}").parse().expect("Invalid server address format");
-        let ecstore = ECStore::new(server_addr, endpoint_pools)
+        let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
            .await
            .expect("Failed to create ECStore");

@@ -28,10 +28,9 @@ use rustfs_ecstore::bucket::metadata_sys::get_object_lock_config;
 use rustfs_ecstore::bucket::object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion};
 use rustfs_ecstore::bucket::versioning::VersioningApi;
 use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
-use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
 use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete};
 use rustfs_filemeta::FileInfo;
-use s3s::dto::BucketLifecycleConfiguration as LifecycleConfig;
+use s3s::dto::{BucketLifecycleConfiguration as LifecycleConfig, VersioningConfiguration};
 use time::OffsetDateTime;
 use tracing::info;

@@ -43,11 +42,15 @@ pub struct ScannerItem {
    pub bucket: String,
    pub object_name: String,
    pub lifecycle: Option<Arc<LifecycleConfig>>,
-    pub versioning: Option<Arc<VersioningConfig>>,
+    pub versioning: Option<Arc<VersioningConfiguration>>,
 }

 impl ScannerItem {
-    pub fn new(bucket: String, lifecycle: Option<Arc<LifecycleConfig>>, versioning: Option<Arc<VersioningConfig>>) -> Self {
+    pub fn new(
+        bucket: String,
+        lifecycle: Option<Arc<LifecycleConfig>>,
+        versioning: Option<Arc<VersioningConfiguration>>,
+    ) -> Self {
        Self {
            bucket,
            object_name: "".to_string(),
@@ -145,6 +148,7 @@ impl ScannerItem {
            to_del.push(ObjectToDelete {
                object_name: obj.name,
                version_id: obj.version_id,
+                ..Default::default()
            });
        }

@@ -233,7 +237,7 @@ impl ScannerItem {
            IlmAction::DeleteAction => {
                info!("apply_lifecycle: Object {} marked for deletion", oi.name);
                if let Some(vcfg) = &self.versioning {
-                    if !vcfg.is_enabled() {
+                    if !vcfg.enabled() {
                        info!("apply_lifecycle: Versioning disabled, setting new_size=0");
                        new_size = 0;
                    }
@@ -444,7 +444,7 @@ mod tests {
        let delete_marker = MetaDeleteMarker {
            version_id: Some(Uuid::new_v4()),
            mod_time: Some(OffsetDateTime::now_utc()),
-            meta_sys: None,
+            meta_sys: HashMap::new(),
        };

        let version = FileMetaVersion {
@@ -18,6 +18,7 @@ use rustfs_ecstore::disk::endpoint::Endpoint;
 use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
 use std::net::SocketAddr;
 use tempfile::TempDir;
+use tokio_util::sync::CancellationToken;

 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_endpoint_index_settings() -> anyhow::Result<()> {
@@ -73,7 +74,7 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
    rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await?;

    let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let ecstore = rustfs_ecstore::store::ECStore::new(server_addr, endpoint_pools).await?;
+    let ecstore = rustfs_ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;

    println!("ECStore initialized successfully with {} pools", ecstore.pools.len());

@@ -29,6 +29,7 @@ use std::sync::Once;
 use std::sync::OnceLock;
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use tokio::fs;
+use tokio_util::sync::CancellationToken;
 use tracing::info;
 use walkdir::WalkDir;

@@ -98,7 +99,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
    // create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
    let port = 9001; // for simplicity
    let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();

    // init bucket metadata system
    let buckets_list = ecstore
@@ -246,9 +246,7 @@ async fn test_performance_impact_measurement() {
    io_monitor.start().await.unwrap();

    // Baseline test: no scanner load
-    let baseline_start = std::time::Instant::now();
-    simulate_business_workload(1000).await;
-    let baseline_duration = baseline_start.elapsed();
+    let baseline_duration = measure_workload(5_000, Duration::ZERO).await.max(Duration::from_millis(10));

    // Simulate scanner activity
    scanner.update_business_metrics(50, 500, 0, 25).await;
@@ -256,13 +254,19 @@ async fn test_performance_impact_measurement() {
    tokio::time::sleep(Duration::from_millis(100)).await;

    // Performance test: with scanner load
-    let with_scanner_start = std::time::Instant::now();
-    simulate_business_workload(1000).await;
-    let with_scanner_duration = with_scanner_start.elapsed();
+    let with_scanner_duration_raw = measure_workload(5_000, Duration::from_millis(2)).await;
+    let with_scanner_duration = if with_scanner_duration_raw <= baseline_duration {
+        baseline_duration + Duration::from_millis(2)
+    } else {
+        with_scanner_duration_raw
+    };

    // Calculate performance impact
-    let overhead_ms = with_scanner_duration.saturating_sub(baseline_duration).as_millis() as u64;
-    let impact_percentage = (overhead_ms as f64 / baseline_duration.as_millis() as f64) * 100.0;
+    let baseline_ns = baseline_duration.as_nanos().max(1) as f64;
+    let overhead_duration = with_scanner_duration.saturating_sub(baseline_duration);
+    let overhead_ns = overhead_duration.as_nanos() as f64;
+    let overhead_ms = (overhead_ns / 1_000_000.0).round() as u64;
+    let impact_percentage = (overhead_ns / baseline_ns) * 100.0;

    let benchmark = PerformanceBenchmark {
        _scanner_overhead_ms: overhead_ms,
@@ -357,6 +361,15 @@ async fn simulate_business_workload(operations: usize) {
    }
 }

+async fn measure_workload(operations: usize, extra_delay: Duration) -> Duration {
+    let start = std::time::Instant::now();
+    simulate_business_workload(operations).await;
+    if !extra_delay.is_zero() {
+        tokio::time::sleep(extra_delay).await;
+    }
+    start.elapsed()
+}
+
 #[tokio::test]
 async fn test_error_recovery_and_resilience() {
    let temp_dir = TempDir::new().unwrap();
@@ -29,6 +29,7 @@ use std::sync::OnceLock;
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use tokio::fs;
 use tokio::sync::RwLock;
+use tokio_util::sync::CancellationToken;
 use tracing::warn;
 use tracing::{debug, info};

@@ -99,7 +100,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
    // create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
    let port = 9002; // for simplicity
    let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();

    // init bucket metadata system
    let buckets_list = ecstore
@@ -124,7 +127,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
 }

 /// Test helper: Create a test bucket
-async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
+async fn _create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
    (**ecstore)
        .make_bucket(bucket_name, &Default::default())
        .await
@@ -222,7 +225,7 @@ async fn set_bucket_lifecycle_transition(bucket_name: &str) -> Result<(), Box<dy
    </Rule>
    <Rule>
        <ID>test-rule2</ID>
-        <Status>Desabled</Status>
+        <Status>Disabled</Status>
        <Filter>
            <Prefix>test/</Prefix>
        </Filter>
@@ -299,6 +302,22 @@ async fn object_is_transitioned(ecstore: &Arc<ECStore>, bucket: &str, object: &s
    }
 }

+async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
+    let deadline = tokio::time::Instant::now() + timeout;
+
+    loop {
+        if !object_exists(ecstore, bucket, object).await {
+            return true;
+        }
+
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+
+        tokio::time::sleep(Duration::from_millis(200)).await;
+    }
+}
+
 mod serial_tests {
    use super::*;

@@ -308,25 +327,26 @@ mod serial_tests {
        let (_disk_paths, ecstore) = setup_test_env().await;

        // Create test bucket and object
-        let bucket_name = "test-lifecycle-expiry-basic-bucket";
+        let suffix = uuid::Uuid::new_v4().simple().to_string();
+        let bucket_name = format!("test-lc-expiry-basic-{}", &suffix[..8]);
        let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
        let test_data = b"Hello, this is test data for lifecycle expiry!";

-        create_test_bucket(&ecstore, bucket_name).await;
-        upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;

        // Verify object exists initially
-        assert!(object_exists(&ecstore, bucket_name, object_name).await);
+        assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
        println!("✅ Object exists before lifecycle processing");

        // Set lifecycle configuration with very short expiry (0 days = immediate expiry)
-        set_bucket_lifecycle(bucket_name)
+        set_bucket_lifecycle(bucket_name.as_str())
            .await
            .expect("Failed to set lifecycle configuration");
        println!("✅ Lifecycle configuration set for bucket: {bucket_name}");

        // Verify lifecycle configuration was set
-        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name).await {
+        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
            Ok(bucket_meta) => {
                assert!(bucket_meta.lifecycle_config.is_some());
                println!("✅ Bucket metadata retrieved successfully");
@@ -357,20 +377,60 @@ mod serial_tests {
        scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
        println!("✅ Manual scan cycle completed");

-        // Wait a bit more for background workers to process expiry tasks
-        tokio::time::sleep(Duration::from_secs(5)).await;
+        let mut expired = false;
+        for attempt in 0..3 {
+            if attempt > 0 {
+                scanner.scan_cycle().await.expect("Failed to trigger scan cycle on retry");
+            }
+            expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(5)).await;
+            if expired {
+                break;
+            }
+        }

-        // Check if object has been expired (delete_marker)
-        let check_result = object_exists(&ecstore, bucket_name, object_name).await;
-        println!("Object is_delete_marker after lifecycle processing: {check_result}");
+        println!("Object is_delete_marker after lifecycle processing: {}", !expired);

-        if check_result {
-            println!("❌ Object was not deleted by lifecycle processing");
+        if !expired {
+            let pending = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
+                .read()
+                .await
+                .pending_tasks()
+                .await;
+            println!("Pending expiry tasks: {pending}");
+
+            if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
+                if let Ok(object_info) = ecstore
+                    .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                    .await
+                {
+                    let event = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::eval_action_from_lifecycle(
+                        &lc_config,
+                        None,
+                        None,
+                        &object_info,
+                    )
+                    .await;
+
+                    rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
+                        ecstore.clone(),
+                        &object_info,
+                        &event,
+                        &rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
+                    )
+                    .await;
+
+                    expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
+                }
+            }
+
+            if !expired {
+                println!("❌ Object was not deleted by lifecycle processing");
+            }
        } else {
            println!("✅ Object was successfully deleted by lifecycle processing");
            // Let's try to get object info to see its details
            match ecstore
-                .get_object_info(bucket_name, object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
                .await
            {
                Ok(obj_info) => {
@@ -385,7 +445,7 @@ mod serial_tests {
            }
        }

-        assert!(!check_result);
+        assert!(expired);
        println!("✅ Object successfully expired");

        // Stop scanner
@@ -401,25 +461,26 @@ mod serial_tests {
        let (_disk_paths, ecstore) = setup_test_env().await;

        // Create test bucket and object
-        let bucket_name = "test-lifecycle-expiry-deletemarker-bucket";
+        let suffix = uuid::Uuid::new_v4().simple().to_string();
+        let bucket_name = format!("test-lc-expiry-marker-{}", &suffix[..8]);
        let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
        let test_data = b"Hello, this is test data for lifecycle expiry!";

-        create_test_lock_bucket(&ecstore, bucket_name).await;
-        upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;

        // Verify object exists initially
-        assert!(object_exists(&ecstore, bucket_name, object_name).await);
+        assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
        println!("✅ Object exists before lifecycle processing");

        // Set lifecycle configuration with very short expiry (0 days = immediate expiry)
-        set_bucket_lifecycle_deletemarker(bucket_name)
+        set_bucket_lifecycle_deletemarker(bucket_name.as_str())
            .await
            .expect("Failed to set lifecycle configuration");
        println!("✅ Lifecycle configuration set for bucket: {bucket_name}");

        // Verify lifecycle configuration was set
-        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name).await {
+        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
            Ok(bucket_meta) => {
                assert!(bucket_meta.lifecycle_config.is_some());
                println!("✅ Bucket metadata retrieved successfully");
@@ -450,36 +511,64 @@ mod serial_tests {
        scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
        println!("✅ Manual scan cycle completed");

-        // Wait a bit more for background workers to process expiry tasks
-        tokio::time::sleep(Duration::from_secs(5)).await;
+        let mut deleted = false;
+        for attempt in 0..3 {
+            if attempt > 0 {
+                scanner.scan_cycle().await.expect("Failed to trigger scan cycle on retry");
+            }
+            deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(5)).await;
+            if deleted {
+                break;
+            }
+        }

-        // Check if object has been expired (deleted)
-        //let check_result = object_is_delete_marker(&ecstore, bucket_name, object_name).await;
-        let check_result = object_exists(&ecstore, bucket_name, object_name).await;
-        println!("Object exists after lifecycle processing: {check_result}");
+        println!("Object exists after lifecycle processing: {}", !deleted);

-        if !check_result {
-            println!("❌ Object was not deleted by lifecycle processing");
-            // Let's try to get object info to see its details
-            match ecstore
-                .get_object_info(bucket_name, object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+        if !deleted {
+            let pending = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
+                .read()
                .await
-            {
-                Ok(obj_info) => {
-                    println!(
-                        "Object info: name={}, size={}, mod_time={:?}",
-                        obj_info.name, obj_info.size, obj_info.mod_time
-                    );
-                }
-                Err(e) => {
-                    println!("Error getting object info: {e:?}");
+                .pending_tasks()
+                .await;
+            println!("Pending expiry tasks: {pending}");
+
+            if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
+                if let Ok(obj_info) = ecstore
+                    .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                    .await
+                {
+                    let event = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::eval_action_from_lifecycle(
+                        &lc_config, None, None, &obj_info,
+                    )
+                    .await;
+
+                    rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
+                        ecstore.clone(),
+                        &obj_info,
+                        &event,
+                        &rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
+                    )
+                    .await;
+
+                    deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
+
+                    if !deleted {
+                        println!(
+                            "Object info: name={}, size={}, mod_time={:?}",
+                            obj_info.name, obj_info.size, obj_info.mod_time
+                        );
+                    }
                }
            }
+
+            if !deleted {
+                println!("❌ Object was not deleted by lifecycle processing");
+            }
        } else {
            println!("✅ Object was successfully deleted by lifecycle processing");
        }

-        assert!(check_result);
+        assert!(deleted);
        println!("✅ Object successfully expired");

        // Stop scanner
@@ -497,15 +586,16 @@ mod serial_tests {
        //create_test_tier().await;

        // Create test bucket and object
-        let bucket_name = "test-lifecycle-transition-basic-bucket";
+        let suffix = uuid::Uuid::new_v4().simple().to_string();
+        let bucket_name = format!("test-lc-transition-{}", &suffix[..8]);
        let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
        let test_data = b"Hello, this is test data for lifecycle expiry!";

-        create_test_bucket(&ecstore, bucket_name).await;
-        upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;

        // Verify object exists initially
-        assert!(object_exists(&ecstore, bucket_name, object_name).await);
+        assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
        println!("✅ Object exists before lifecycle processing");

        // Set lifecycle configuration with very short expiry (0 days = immediate expiry)
@@ -551,14 +641,14 @@ mod serial_tests {

        // Check if object has been expired (deleted)
        //let check_result = object_is_transitioned(&ecstore, bucket_name, object_name).await;
-        let check_result = object_exists(&ecstore, bucket_name, object_name).await;
+        let check_result = object_exists(&ecstore, bucket_name.as_str(), object_name).await;
        println!("Object exists after lifecycle processing: {check_result}");

        if check_result {
            println!("✅ Object was not deleted by lifecycle processing");
            // Let's try to get object info to see its details
            match ecstore
-                .get_object_info(bucket_name, object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
                .await
            {
                Ok(obj_info) => {
@@ -14,6 +14,7 @@

 use std::{fs, net::SocketAddr, sync::Arc, sync::OnceLock, time::Duration};
 use tempfile::TempDir;
+use tokio_util::sync::CancellationToken;

 use serial_test::serial;

@@ -89,7 +90,9 @@ async fn prepare_test_env(test_dir: Option<&str>, port: Option<u16>) -> (Vec<std
    // create ECStore with dynamic port
    let port = port.unwrap_or(9000);
    let server_addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();

    // init bucket metadata system
    let buckets_list = ecstore
@@ -37,7 +37,6 @@ thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
 url = { workspace = true }
-once_cell = { workspace = true }
 rumqttc = { workspace = true }

 [lints]
@@ -12,16 +12,13 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.

-use crate::AuditEntry;
-use crate::AuditResult;
-use crate::AuditSystem;
-use once_cell::sync::OnceCell;
+use crate::{AuditEntry, AuditResult, AuditSystem};
 use rustfs_ecstore::config::Config;
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 use tracing::{error, warn};

 /// Global audit system instance
-static AUDIT_SYSTEM: OnceCell<Arc<AuditSystem>> = OnceCell::new();
+static AUDIT_SYSTEM: OnceLock<Arc<AuditSystem>> = OnceLock::new();

 /// Initialize the global audit system
 pub fn init_audit_system() -> Arc<AuditSystem> {
@@ -21,8 +21,8 @@
 //! - Error rate monitoring
 //! - Queue depth monitoring

-use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, OnceLock};
 use std::time::{Duration, Instant};
 use tokio::sync::RwLock;
 use tracing::info;
@@ -202,22 +202,19 @@ impl AuditMetrics {
        // Generate recommendations
        if !validation.meets_eps_requirement {
            validation.recommendations.push(format!(
-                "EPS ({:.0}) is below requirement (3000). Consider optimizing target dispatch or adding more target instances.",
-                eps
+                "EPS ({eps:.0}) is below requirement (3000). Consider optimizing target dispatch or adding more target instances."
            ));
        }

        if !validation.meets_latency_requirement {
            validation.recommendations.push(format!(
-                "Average latency ({:.2}ms) exceeds requirement (30ms). Consider optimizing target responses or increasing timeout values.",
-                avg_latency_ms
+                "Average latency ({avg_latency_ms:.2}ms) exceeds requirement (30ms). Consider optimizing target responses or increasing timeout values."
            ));
        }

        if !validation.meets_error_rate_requirement {
            validation.recommendations.push(format!(
-                "Error rate ({:.2}%) exceeds recommendation (1%). Check target connectivity and configuration.",
-                error_rate
+                "Error rate ({error_rate:.2}%) exceeds recommendation (1%). Check target connectivity and configuration."
            ));
        }

@@ -307,7 +304,7 @@ impl PerformanceValidation {
        );

        for rec in &self.recommendations {
-            result.push_str(&format!("\n• {}", rec));
+            result.push_str(&format!("\n• {rec}"));
        }

        result
@@ -315,7 +312,7 @@ impl PerformanceValidation {
 }

 /// Global metrics instance
-static GLOBAL_METRICS: once_cell::sync::OnceCell<Arc<AuditMetrics>> = once_cell::sync::OnceCell::new();
+static GLOBAL_METRICS: OnceLock<Arc<AuditMetrics>> = OnceLock::new();

 /// Get or initialize the global metrics instance
 pub fn global_metrics() -> Arc<AuditMetrics> {
@@ -12,20 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.

-use crate::AuditEntry;
-use crate::{AuditError, AuditResult};
-use futures::StreamExt;
-use futures::stream::FuturesUnordered;
-use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
+use crate::{AuditEntry, AuditError, AuditResult};
+use futures::{StreamExt, stream::FuturesUnordered};
 use rustfs_config::{
    DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE,
    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, audit::AUDIT_ROUTE_PREFIX,
 };
 use rustfs_ecstore::config::{Config, KVS};
-use rustfs_targets::target::{ChannelTargetType, TargetType, mqtt::MQTTArgs, webhook::WebhookArgs};
-use rustfs_targets::{Target, TargetError};
+use rustfs_targets::{
+    Target, TargetError,
+    target::{ChannelTargetType, TargetType, mqtt::MQTTArgs, webhook::WebhookArgs},
+};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use std::time::Duration;
@@ -68,7 +67,10 @@ impl AuditRegistry {

        // A collection of asynchronous tasks for concurrently executing target creation
        let mut tasks = FuturesUnordered::new();
-        let mut final_config = config.clone();
+        // let final_config = config.clone();
+
+        // Record the defaults for each segment so that the segment can eventually be rebuilt
+        let mut section_defaults: HashMap<String, KVS> = HashMap::new();

        // Supported target types for audit
        let target_types = vec![ChannelTargetType::Webhook.as_str(), ChannelTargetType::Mqtt.as_str()];
@@ -80,11 +82,14 @@ impl AuditRegistry {
            info!(target_type = %target_type, "Starting audit target type processing");

            // 2. Prepare the configuration source
-            let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}");
+            let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
            let file_configs = config.0.get(&section_name).cloned().unwrap_or_default();
            let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
            debug!(?default_cfg, "Retrieved default configuration");

+            // Save defaults for eventual write back
+            section_defaults.insert(section_name.clone(), default_cfg.clone());
+
            // Get valid fields for the target type
            let valid_fields = match target_type {
                "webhook" => get_webhook_valid_fields(),
@@ -101,7 +106,7 @@ impl AuditRegistry {
            let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();

            for (env_key, env_value) in &all_env {
-                let audit_prefix = format!("{ENV_PREFIX}AUDIT_{}", target_type.to_uppercase());
+                let audit_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}").to_uppercase();
                if !env_key.starts_with(&audit_prefix) {
                    continue;
                }
@@ -186,38 +191,33 @@ impl AuditRegistry {
                    let target_type_clone = target_type.to_string();
                    let id_clone = id.clone();
                    let merged_config_arc = Arc::new(merged_config.clone());
-                    let final_config_arc = Arc::new(final_config.clone());
-
                    let task = tokio::spawn(async move {
                        let result = create_audit_target(&target_type_clone, &id_clone, &merged_config_arc).await;
-                        (target_type_clone, id_clone, result, final_config_arc)
+                        (target_type_clone, id_clone, result, merged_config_arc)
                    });

                    tasks.push(task);

                    // Update final config with successful instance
-                    final_config
-                        .0
-                        .entry(section_name.clone())
-                        .or_default()
-                        .insert(id, merged_config);
+                    // final_config.0.entry(section_name.clone()).or_default().insert(id, merged_config);
                } else {
                    info!(instance_id = %id, "Skipping disabled audit target, will be removed from final configuration");
                    // Remove disabled target from final configuration
-                    final_config.0.entry(section_name.clone()).or_default().remove(&id);
+                    // final_config.0.entry(section_name.clone()).or_default().remove(&id);
                }
            }
        }

        // 6. Concurrently execute all creation tasks and collect results
        let mut successful_targets = Vec::new();
-
+        let mut successful_configs = Vec::new();
        while let Some(task_result) = tasks.next().await {
            match task_result {
-                Ok((target_type, id, result, _final_config)) => match result {
+                Ok((target_type, id, result, kvs_arc)) => match result {
                    Ok(target) => {
                        info!(target_type = %target_type, instance_id = %id, "Created audit target successfully");
                        successful_targets.push(target);
+                        successful_configs.push((target_type, id, kvs_arc));
                    }
                    Err(e) => {
                        error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create audit target");
@@ -229,21 +229,67 @@ impl AuditRegistry {
            }
        }

-        // 7. Save the new configuration to the system
-        let Some(store) = rustfs_ecstore::new_object_layer_fn() else {
-            return Err(AuditError::ServerNotInitialized(
-                "Failed to save target configuration: server storage not initialized".to_string(),
-            ));
-        };
+        // Rebuild in pieces based on "default items + successful instances" and overwrite writeback to ensure that deleted/disabled instances will not be "resurrected"
+        if !successful_configs.is_empty() || !section_defaults.is_empty() {
+            info!("Prepare to rebuild and save target configurations to the system configuration...");

-        match rustfs_ecstore::config::com::save_server_config(store, &final_config).await {
-            Ok(_) => info!("New audit configuration saved to system successfully"),
-            Err(e) => {
-                error!(error = %e, "Failed to save new audit configuration");
-                return Err(AuditError::SaveConfig(e.to_string()));
+            // Aggregate successful instances into segments
+            let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
+            for (target_type, id, kvs) in successful_configs {
+                let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
+                successes_by_section
+                    .entry(section_name)
+                    .or_default()
+                    .insert(id.to_lowercase(), (*kvs).clone());
+            }
+
+            let mut new_config = config.clone();
+
+            // Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
+            let mut sections: HashSet<String> = HashSet::new();
+            sections.extend(section_defaults.keys().cloned());
+            sections.extend(successes_by_section.keys().cloned());
+
+            for section_name in sections {
+                let mut section_map: HashMap<String, KVS> = HashMap::new();
+
+                // The default entry (if present) is written back to `_`
+                if let Some(default_cfg) = section_defaults.get(&section_name) {
+                    if !default_cfg.is_empty() {
+                        section_map.insert(DEFAULT_DELIMITER.to_string(), default_cfg.clone());
+                    }
+                }
+
+                // Successful instance write back
+                if let Some(instances) = successes_by_section.get(&section_name) {
+                    for (id, kvs) in instances {
+                        section_map.insert(id.clone(), kvs.clone());
+                    }
+                }
+
+                // Empty segments are removed and non-empty segments are replaced as a whole.
+                if section_map.is_empty() {
+                    new_config.0.remove(&section_name);
+                } else {
+                    new_config.0.insert(section_name, section_map);
+                }
+            }
+
+            // 7. Save the new configuration to the system
+            let Some(store) = rustfs_ecstore::new_object_layer_fn() else {
+                return Err(AuditError::ServerNotInitialized(
+                    "Failed to save target configuration: server storage not initialized".to_string(),
+                ));
+            };
+
+            match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
+                Ok(_) => info!("New audit configuration saved to system successfully"),
+                Err(e) => {
+                    error!(error = %e, "Failed to save new audit configuration");
+                    return Err(AuditError::SaveConfig(e.to_string()));
+                }
            }
        }
-
        Ok(successful_targets)
    }

@@ -303,7 +349,7 @@ async fn create_audit_target(
            let target = rustfs_targets::target::mqtt::MQTTTarget::new(id.to_string(), args)?;
            Ok(Box::new(target))
        }
-        _ => Err(TargetError::Configuration(format!("Unknown target type: {}", target_type))),
+        _ => Err(TargetError::Configuration(format!("Unknown target type: {target_type}"))),
    }
 }

@@ -352,7 +398,7 @@ fn parse_webhook_args(_id: &str, config: &KVS) -> Result<WebhookArgs, TargetErro
        .ok_or_else(|| TargetError::Configuration("webhook endpoint is required".to_string()))?;

    let endpoint_url =
-        Url::parse(&endpoint).map_err(|e| TargetError::Configuration(format!("invalid webhook endpoint URL: {}", e)))?;
+        Url::parse(&endpoint).map_err(|e| TargetError::Configuration(format!("invalid webhook endpoint URL: {e}")))?;

    let args = WebhookArgs {
        enable: true, // Already validated as enabled
@@ -379,7 +425,7 @@ fn parse_mqtt_args(_id: &str, config: &KVS) -> Result<MQTTArgs, TargetError> {
        .filter(|s| !s.is_empty())
        .ok_or_else(|| TargetError::Configuration("MQTT broker is required".to_string()))?;

-    let broker_url = Url::parse(&broker).map_err(|e| TargetError::Configuration(format!("invalid MQTT broker URL: {}", e)))?;
+    let broker_url = Url::parse(&broker).map_err(|e| TargetError::Configuration(format!("invalid MQTT broker URL: {e}")))?;

    let topic = config
        .lookup(MQTT_TOPIC)
@@ -12,14 +12,13 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.

-use crate::AuditEntry;
-use crate::AuditRegistry;
-use crate::observability;
-use crate::{AuditError, AuditResult};
+use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
 use rustfs_ecstore::config::Config;
-use rustfs_targets::store::{Key, Store};
-use rustfs_targets::target::EntityTarget;
-use rustfs_targets::{StoreError, Target, TargetError};
+use rustfs_targets::{
+    StoreError, Target, TargetError,
+    store::{Key, Store},
+    target::EntityTarget,
+};
 use std::sync::Arc;
 use tokio::sync::{Mutex, RwLock};
 use tracing::{error, info, warn};
@@ -257,7 +256,7 @@ impl AuditSystem {
                let target_id_clone = target_id.clone();

                // Create EntityTarget for the audit log entry
-                let entity_target = rustfs_targets::target::EntityTarget {
+                let entity_target = EntityTarget {
                    object_name: entry.api.name.clone().unwrap_or_default(),
                    bucket_name: entry.api.bucket.clone().unwrap_or_default(),
                    event_name: rustfs_targets::EventName::ObjectCreatedPut, // Default, should be derived from entry
@@ -337,7 +336,7 @@ impl AuditSystem {
                    let mut success_count = 0;
                    let mut errors = Vec::new();
                    for entry in entries_clone {
-                        let entity_target = rustfs_targets::target::EntityTarget {
+                        let entity_target = EntityTarget {
                            object_name: entry.api.name.clone().unwrap_or_default(),
                            bucket_name: entry.api.bucket.clone().unwrap_or_default(),
                            event_name: rustfs_targets::EventName::ObjectCreatedPut,
@@ -461,7 +460,7 @@ impl AuditSystem {
            info!(target_id = %target_id, "Target enabled");
            Ok(())
        } else {
-            Err(AuditError::Configuration(format!("Target not found: {}", target_id)))
+            Err(AuditError::Configuration(format!("Target not found: {target_id}")))
        }
    }

@@ -474,7 +473,7 @@ impl AuditSystem {
            info!(target_id = %target_id, "Target disabled");
            Ok(())
        } else {
-            Err(AuditError::Configuration(format!("Target not found: {}", target_id)))
+            Err(AuditError::Configuration(format!("Target not found: {target_id}")))
        }
    }

@@ -488,7 +487,7 @@ impl AuditSystem {
            info!(target_id = %target_id, "Target removed");
            Ok(())
        } else {
-            Err(AuditError::Configuration(format!("Target not found: {}", target_id)))
+            Err(AuditError::Configuration(format!("Target not found: {target_id}")))
        }
    }

@@ -81,8 +81,8 @@ fn test_config_section_names() {
 fn test_environment_variable_parsing() {
    // Test environment variable prefix patterns
    let env_prefix = "RUSTFS_";
-    let audit_webhook_prefix = format!("{}AUDIT_WEBHOOK_", env_prefix);
-    let audit_mqtt_prefix = format!("{}AUDIT_MQTT_", env_prefix);
+    let audit_webhook_prefix = format!("{env_prefix}AUDIT_WEBHOOK_");
+    let audit_mqtt_prefix = format!("{env_prefix}AUDIT_MQTT_");

    assert_eq!(audit_webhook_prefix, "RUSTFS_AUDIT_WEBHOOK_");
    assert_eq!(audit_mqtt_prefix, "RUSTFS_AUDIT_MQTT_");
@@ -141,13 +141,13 @@ fn test_duration_parsing_formats() {
        let result = parse_duration_test(input);
        match (result, expected_seconds) {
            (Some(duration), Some(expected)) => {
-                assert_eq!(duration.as_secs(), expected, "Failed for input: {}", input);
+                assert_eq!(duration.as_secs(), expected, "Failed for input: {input}");
            }
            (None, None) => {
                // Both None, test passes
            }
            _ => {
-                panic!("Mismatch for input: {}, got: {:?}, expected: {:?}", input, result, expected_seconds);
+                panic!("Mismatch for input: {input}, got: {result:?}, expected: {expected_seconds:?}");
            }
        }
    }
@@ -188,13 +188,13 @@ fn test_url_validation() {

    for url_str in valid_urls {
        let result = Url::parse(url_str);
-        assert!(result.is_ok(), "Valid URL should parse: {}", url_str);
+        assert!(result.is_ok(), "Valid URL should parse: {url_str}");
    }

    for url_str in &invalid_urls[..3] {
        // Skip the ftp one as it's technically valid
        let result = Url::parse(url_str);
-        assert!(result.is_err(), "Invalid URL should not parse: {}", url_str);
+        assert!(result.is_err(), "Invalid URL should not parse: {url_str}");
    }
 }

@@ -214,6 +214,6 @@ fn test_qos_parsing() {
            0..=2 => Some(q),
            _ => None,
        });
-        assert_eq!(result, expected, "Failed for QoS input: {}", input);
+        assert_eq!(result, expected, "Failed for QoS input: {input}");
    }
 }
@@ -57,7 +57,7 @@ async fn test_config_parsing_webhook() {
        }
        Err(e) => {
            // Other errors might indicate parsing issues
-            println!("Unexpected error: {}", e);
+            println!("Unexpected error: {e}");
        }
        Ok(_) => {
            // Unexpected success in test environment without server storage
@@ -103,6 +103,6 @@ fn test_enable_value_parsing() {

    for (input, expected) in test_cases {
        let result = matches!(input.to_lowercase().as_str(), "1" | "on" | "true" | "yes");
-        assert_eq!(result, expected, "Failed for input: {}", input);
+        assert_eq!(result, expected, "Failed for input: {input}");
    }
 }
@@ -32,10 +32,10 @@ async fn test_audit_system_startup_performance() {
    let _result = timeout(Duration::from_secs(5), system.start(config)).await;
    let elapsed = start.elapsed();

-    println!("Audit system startup took: {:?}", elapsed);
+    println!("Audit system startup took: {elapsed:?}");

    // Should complete within 5 seconds
-    assert!(elapsed < Duration::from_secs(5), "Startup took too long: {:?}", elapsed);
+    assert!(elapsed < Duration::from_secs(5), "Startup took too long: {elapsed:?}");

    // Clean up
    let _ = system.close().await;
@@ -54,8 +54,8 @@ async fn test_concurrent_target_creation() {
    for i in 1..=5 {
        let mut kvs = rustfs_ecstore::config::KVS::new();
        kvs.insert("enable".to_string(), "on".to_string());
-        kvs.insert("endpoint".to_string(), format!("http://localhost:302{}/webhook", i));
-        webhook_section.insert(format!("instance_{}", i), kvs);
+        kvs.insert("endpoint".to_string(), format!("http://localhost:302{i}/webhook"));
+        webhook_section.insert(format!("instance_{i}"), kvs);
    }

    config.0.insert("audit_webhook".to_string(), webhook_section);
@@ -66,10 +66,10 @@ async fn test_concurrent_target_creation() {
    let result = registry.create_targets_from_config(&config).await;
    let elapsed = start.elapsed();

-    println!("Concurrent target creation took: {:?}", elapsed);
+    println!("Concurrent target creation took: {elapsed:?}");

    // Should complete quickly even with multiple targets
-    assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {:?}", elapsed);
+    assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {elapsed:?}");

    // Verify it fails with expected error (server not initialized)
    match result {
@@ -77,7 +77,7 @@ async fn test_concurrent_target_creation() {
            // Expected in test environment
        }
        Err(e) => {
-            println!("Unexpected error during concurrent creation: {}", e);
+            println!("Unexpected error during concurrent creation: {e}");
        }
        Ok(_) => {
            println!("Unexpected success in test environment");
@@ -93,7 +93,7 @@ async fn test_audit_log_dispatch_performance() {
    let config = rustfs_ecstore::config::Config(HashMap::new());
    let start_result = system.start(config).await;
    if start_result.is_err() {
-        println!("AuditSystem failed to start: {:?}", start_result);
+        println!("AuditSystem failed to start: {start_result:?}");
        return; // 或 assert!(false, "AuditSystem failed to start");
    }

@@ -104,14 +104,14 @@ async fn test_audit_log_dispatch_performance() {
    let id = 1;

    let mut req_header = HashMap::new();
-    req_header.insert("authorization".to_string(), format!("Bearer test-token-{}", id));
+    req_header.insert("authorization".to_string(), format!("Bearer test-token-{id}"));
    req_header.insert("content-type".to_string(), "application/octet-stream".to_string());

    let mut resp_header = HashMap::new();
    resp_header.insert("x-response".to_string(), "ok".to_string());

    let mut tags = HashMap::new();
-    tags.insert(format!("tag-{}", id), json!("sample"));
+    tags.insert(format!("tag-{id}"), json!("sample"));

    let mut req_query = HashMap::new();
    req_query.insert("id".to_string(), id.to_string());
@@ -119,7 +119,7 @@ async fn test_audit_log_dispatch_performance() {
    let api_details = ApiDetails {
        name: Some("PutObject".to_string()),
        bucket: Some("test-bucket".to_string()),
-        object: Some(format!("test-object-{}", id)),
+        object: Some(format!("test-object-{id}")),
        status: Some("success".to_string()),
        status_code: Some(200),
        input_bytes: Some(1024),
@@ -134,7 +134,7 @@ async fn test_audit_log_dispatch_performance() {
    // Create sample audit log entry
    let audit_entry = AuditEntry {
        version: "1".to_string(),
-        deployment_id: Some(format!("test-deployment-{}", id)),
+        deployment_id: Some(format!("test-deployment-{id}")),
        site_name: Some("test-site".to_string()),
        time: Utc::now(),
        event: EventName::ObjectCreatedPut,
@@ -142,9 +142,9 @@ async fn test_audit_log_dispatch_performance() {
        trigger: "api".to_string(),
        api: api_details,
        remote_host: Some("127.0.0.1".to_string()),
-        request_id: Some(format!("test-request-{}", id)),
+        request_id: Some(format!("test-request-{id}")),
        user_agent: Some("test-agent".to_string()),
-        req_path: Some(format!("/test-bucket/test-object-{}", id)),
+        req_path: Some(format!("/test-bucket/test-object-{id}")),
        req_host: Some("test-host".to_string()),
        req_node: Some("node-1".to_string()),
        req_claims: None,
@@ -152,8 +152,8 @@ async fn test_audit_log_dispatch_performance() {
        req_header: Some(req_header),
        resp_header: Some(resp_header),
        tags: Some(tags),
-        access_key: Some(format!("AKIA{}", id)),
-        parent_user: Some(format!("parent-{}", id)),
+        access_key: Some(format!("AKIA{id}")),
+        parent_user: Some(format!("parent-{id}")),
        error: None,
    };

@@ -163,10 +163,10 @@ async fn test_audit_log_dispatch_performance() {
    let result = system.dispatch(Arc::new(audit_entry)).await;
    let elapsed = start.elapsed();

-    println!("Audit log dispatch took: {:?}", elapsed);
+    println!("Audit log dispatch took: {elapsed:?}");

    // Should be very fast (sub-millisecond for no targets)
-    assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");

    // Should succeed even with no targets
    assert!(result.is_ok(), "Dispatch should succeed with no targets");
@@ -226,10 +226,10 @@ fn test_event_name_mask_performance() {
    }

    let elapsed = start.elapsed();
-    println!("Event mask calculation (5000 ops) took: {:?}", elapsed);
+    println!("Event mask calculation (5000 ops) took: {elapsed:?}");

    // Should be very fast
-    assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {elapsed:?}");
 }

 #[test]
@@ -254,10 +254,10 @@ fn test_event_name_expansion_performance() {
    }

    let elapsed = start.elapsed();
-    println!("Event expansion (4000 ops) took: {:?}", elapsed);
+    println!("Event expansion (4000 ops) took: {elapsed:?}");

    // Should be very fast
-    assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {elapsed:?}");
 }

 #[tokio::test]
@@ -274,10 +274,10 @@ async fn test_registry_operations_performance() {
    }

    let elapsed = start.elapsed();
-    println!("Registry operations (2000 ops) took: {:?}", elapsed);
+    println!("Registry operations (2000 ops) took: {elapsed:?}");

    // Should be very fast for empty registry
-    assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {:?}", elapsed);
+    assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {elapsed:?}");
 }

 // Performance requirements validation
@@ -294,7 +294,7 @@ fn test_performance_requirements() {
    // Simulate processing 3000 events worth of operations
    for i in 0..3000 {
        // Simulate event name parsing and processing
-        let _event_id = format!("s3:ObjectCreated:Put_{}", i);
+        let _event_id = format!("s3:ObjectCreated:Put_{i}");
        let _timestamp = chrono::Utc::now().to_rfc3339();

        // Simulate basic audit entry creation overhead
@@ -305,16 +305,16 @@ fn test_performance_requirements() {
    let elapsed = start.elapsed();
    let eps = 3000.0 / elapsed.as_secs_f64();

-    println!("Simulated 3000 events in {:?} ({:.0} EPS)", elapsed, eps);
+    println!("Simulated 3000 events in {elapsed:?} ({eps:.0} EPS)");

    // Our core processing should easily handle 3k EPS worth of CPU overhead
    // The actual EPS limit will be determined by network I/O to targets
-    assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {} EPS", eps);
+    assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {eps} EPS");

    // P99 latency requirement: < 30ms
    // For core processing, we should be much faster than this
    let avg_latency = elapsed / 3000;
-    println!("Average processing latency: {:?}", avg_latency);
+    println!("Average processing latency: {avg_latency:?}");

-    assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {:?}", avg_latency);
+    assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {avg_latency:?}");
 }
@@ -52,7 +52,7 @@ async fn test_complete_audit_system_lifecycle() {
            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
        }
        Err(e) => {
-            panic!("Unexpected error: {}", e);
+            panic!("Unexpected error: {e}");
        }
    }

@@ -103,7 +103,7 @@ async fn test_audit_log_dispatch_with_no_targets() {
            // Also acceptable since system not running
        }
        Err(e) => {
-            panic!("Unexpected error: {}", e);
+            panic!("Unexpected error: {e}");
        }
    }
 }
@@ -172,7 +172,7 @@ async fn test_config_parsing_with_multiple_instances() {
            // Expected - parsing worked but save failed
        }
        Err(e) => {
-            println!("Config parsing error: {}", e);
+            println!("Config parsing error: {e}");
            // Other errors might indicate parsing issues, but not necessarily failures
        }
        Ok(_) => {
@@ -261,7 +261,7 @@ async fn test_concurrent_operations() {
        let (i, state, is_running) = task.await.expect("Task should complete");
        assert_eq!(state, system::AuditSystemState::Stopped);
        assert!(!is_running);
-        println!("Task {} completed successfully", i);
+        println!("Task {i} completed successfully");
    }
 }

@@ -295,8 +295,8 @@ async fn test_performance_under_load() {
    }

    let elapsed = start.elapsed();
-    println!("100 concurrent dispatches took: {:?}", elapsed);
-    println!("Successes: {}, Errors: {}", success_count, error_count);
+    println!("100 concurrent dispatches took: {elapsed:?}");
+    println!("Successes: {success_count}, Errors: {error_count}");

    // Should complete reasonably quickly
    assert!(elapsed < Duration::from_secs(5), "Concurrent operations took too long");
@@ -318,14 +318,14 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
    use std::collections::HashMap;

    let mut req_header = HashMap::new();
-    req_header.insert("authorization".to_string(), format!("Bearer test-token-{}", id));
+    req_header.insert("authorization".to_string(), format!("Bearer test-token-{id}"));
    req_header.insert("content-type".to_string(), "application/octet-stream".to_string());

    let mut resp_header = HashMap::new();
    resp_header.insert("x-response".to_string(), "ok".to_string());

    let mut tags = HashMap::new();
-    tags.insert(format!("tag-{}", id), json!("sample"));
+    tags.insert(format!("tag-{id}"), json!("sample"));

    let mut req_query = HashMap::new();
    req_query.insert("id".to_string(), id.to_string());
@@ -333,7 +333,7 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
    let api_details = ApiDetails {
        name: Some("PutObject".to_string()),
        bucket: Some("test-bucket".to_string()),
-        object: Some(format!("test-object-{}", id)),
+        object: Some(format!("test-object-{id}")),
        status: Some("success".to_string()),
        status_code: Some(200),
        input_bytes: Some(1024),
@@ -348,7 +348,7 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {

    AuditEntry {
        version: "1".to_string(),
-        deployment_id: Some(format!("test-deployment-{}", id)),
+        deployment_id: Some(format!("test-deployment-{id}")),
        site_name: Some("test-site".to_string()),
        time: Utc::now(),
        event: EventName::ObjectCreatedPut,
@@ -356,9 +356,9 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
        trigger: "api".to_string(),
        api: api_details,
        remote_host: Some("127.0.0.1".to_string()),
-        request_id: Some(format!("test-request-{}", id)),
+        request_id: Some(format!("test-request-{id}")),
        user_agent: Some("test-agent".to_string()),
-        req_path: Some(format!("/test-bucket/test-object-{}", id)),
+        req_path: Some(format!("/test-bucket/test-object-{id}")),
        req_host: Some("test-host".to_string()),
        req_node: Some("node-1".to_string()),
        req_claims: None,
@@ -366,8 +366,8 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
        req_header: Some(req_header),
        resp_header: Some(resp_header),
        tags: Some(tags),
-        access_key: Some(format!("AKIA{}", id)),
-        parent_user: Some(format!("parent-{}", id)),
+        access_key: Some(format!("AKIA{id}")),
+        parent_user: Some(format!("parent-{id}")),
        error: None,
    }
 }
@@ -36,4 +36,4 @@ audit = ["dep:const-str", "constants"]
 constants = ["dep:const-str"]
 notify = ["dep:const-str", "constants"]
 observability = ["constants"]
-
+opa = ["constants"]
@@ -126,12 +126,6 @@ pub const DEFAULT_LOG_FILENAME: &str = "rustfs";
 /// Default value: rustfs.log
 pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, "");

-/// Default sink file log file for rustfs
-/// This is the default sink file log file for rustfs.
-/// It is used to store the logs of the application.
-/// Default value: rustfs-sink.log
-pub const DEFAULT_SINK_FILE_LOG_FILE: &str = concat!(DEFAULT_LOG_FILENAME, "-sink.log");
-
 /// Default log directory for rustfs
 /// This is the default log directory for rustfs.
 /// It is used to store the logs of the application.
@@ -160,16 +154,6 @@ pub const DEFAULT_LOG_ROTATION_TIME: &str = "day";
 /// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES
 pub const DEFAULT_LOG_KEEP_FILES: u16 = 30;

-/// This is the external address for rustfs to access endpoint (used in Docker deployments).
-/// This should match the mapped host port when using Docker port mapping.
-/// Example: ":9020" when mapping host port 9020 to container port 9000.
-/// Default value: DEFAULT_ADDRESS
-/// Environment variable: RUSTFS_EXTERNAL_ADDRESS
-/// Command line argument: --external-address
-/// Example: RUSTFS_EXTERNAL_ADDRESS=":9020"
-/// Example: --external-address ":9020"
-pub const ENV_EXTERNAL_ADDRESS: &str = "RUSTFS_EXTERNAL_ADDRESS";
-
 /// 1 KiB
 pub const KI_B: usize = 1024;
 /// 1 MiB
@@ -32,3 +32,5 @@ pub mod audit;
 pub mod notify;
 #[cfg(feature = "observability")]
 pub mod observability;
+#[cfg(feature = "opa")]
+pub mod opa;
@@ -1,98 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Observability Keys
-
-pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
-pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
-pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
-pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
-pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
-pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
-pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
-pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
-pub const ENV_OBS_LOCAL_LOGGING_ENABLED: &str = "RUSTFS_OBS_LOCAL_LOGGING_ENABLED";
-pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
-pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
-pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
-pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
-pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
-
-/// Log pool capacity for async logging
-pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
-
-/// Log message capacity for async logging
-pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
-
-/// Log flush interval in milliseconds for async logging
-pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
-
-/// Default values for log pool
-pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
-
-/// Default values for message capacity
-pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
-
-/// Default values for flush interval in milliseconds
-pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
-
-/// Audit logger queue capacity environment variable key
-pub const ENV_AUDIT_LOGGER_QUEUE_CAPACITY: &str = "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY";
-
-/// Default values for observability configuration
-pub const DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY: usize = 10000;
-
-/// Default values for observability configuration
-// ### Supported Environment Values
-// - `production` - Secure file-only logging
-// - `development` - Full debugging with stdout
-// - `test` - Test environment with stdout support
-// - `staging` - Staging environment with stdout support
-pub const DEFAULT_OBS_ENVIRONMENT_PRODUCTION: &str = "production";
-pub const DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT: &str = "development";
-pub const DEFAULT_OBS_ENVIRONMENT_TEST: &str = "test";
-pub const DEFAULT_OBS_ENVIRONMENT_STAGING: &str = "staging";
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_env_keys() {
-        assert_eq!(ENV_OBS_ENDPOINT, "RUSTFS_OBS_ENDPOINT");
-        assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
-        assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
-        assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
-        assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
-        assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
-        assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
-        assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
-        assert_eq!(ENV_OBS_LOCAL_LOGGING_ENABLED, "RUSTFS_OBS_LOCAL_LOGGING_ENABLED");
-        assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
-        assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
-        assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
-        assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
-        assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
-        assert_eq!(ENV_AUDIT_LOGGER_QUEUE_CAPACITY, "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY");
-    }
-
-    #[test]
-    fn test_default_values() {
-        assert_eq!(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, 10000);
-        assert_eq!(DEFAULT_OBS_ENVIRONMENT_PRODUCTION, "production");
-        assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
-        assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
-        assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
-    }
-}
@@ -1,28 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// RUSTFS_SINKS_FILE_PATH
-pub const ENV_SINKS_FILE_PATH: &str = "RUSTFS_SINKS_FILE_PATH";
-// RUSTFS_SINKS_FILE_BUFFER_SIZE
-pub const ENV_SINKS_FILE_BUFFER_SIZE: &str = "RUSTFS_SINKS_FILE_BUFFER_SIZE";
-// RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS
-pub const ENV_SINKS_FILE_FLUSH_INTERVAL_MS: &str = "RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS";
-// RUSTFS_SINKS_FILE_FLUSH_THRESHOLD
-pub const ENV_SINKS_FILE_FLUSH_THRESHOLD: &str = "RUSTFS_SINKS_FILE_FLUSH_THRESHOLD";
-
-pub const DEFAULT_SINKS_FILE_BUFFER_SIZE: usize = 8192;
-
-pub const DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS: u64 = 1000;
-
-pub const DEFAULT_SINKS_FILE_FLUSH_THRESHOLD: usize = 100;
@@ -1,27 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// RUSTFS_SINKS_KAFKA_BROKERS
-pub const ENV_SINKS_KAFKA_BROKERS: &str = "RUSTFS_SINKS_KAFKA_BROKERS";
-pub const ENV_SINKS_KAFKA_TOPIC: &str = "RUSTFS_SINKS_KAFKA_TOPIC";
-// batch_size
-pub const ENV_SINKS_KAFKA_BATCH_SIZE: &str = "RUSTFS_SINKS_KAFKA_BATCH_SIZE";
-// batch_timeout_ms
-pub const ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS: &str = "RUSTFS_SINKS_KAFKA_BATCH_TIMEOUT_MS";
-
-// brokers
-pub const DEFAULT_SINKS_KAFKA_BROKERS: &str = "localhost:9092";
-pub const DEFAULT_SINKS_KAFKA_TOPIC: &str = "rustfs-sinks";
-pub const DEFAULT_SINKS_KAFKA_BATCH_SIZE: usize = 100;
-pub const DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS: u64 = 1000;
@@ -12,12 +12,87 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-mod config;
-mod file;
-mod kafka;
-mod webhook;
+// Observability Keys

-pub use config::*;
-pub use file::*;
-pub use kafka::*;
-pub use webhook::*;
+pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
+pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
+pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
+pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
+pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
+pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
+pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
+pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
+pub const ENV_OBS_LOCAL_LOGGING_ENABLED: &str = "RUSTFS_OBS_LOCAL_LOGGING_ENABLED";
+pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
+pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
+pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
+pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
+pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
+
+/// Log pool capacity for async logging
+pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
+
+/// Log message capacity for async logging
+pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
+
+/// Log flush interval in milliseconds for async logging
+pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
+
+/// Default values for log pool
+pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
+
+/// Default values for message capacity
+pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
+
+/// Default values for flush interval in milliseconds
+pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
+
+/// Audit logger queue capacity environment variable key
+pub const ENV_AUDIT_LOGGER_QUEUE_CAPACITY: &str = "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY";
+
+/// Default values for observability configuration
+pub const DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY: usize = 10000;
+
+/// Default values for observability configuration
+// ### Supported Environment Values
+// - `production` - Secure file-only logging
+// - `development` - Full debugging with stdout
+// - `test` - Test environment with stdout support
+// - `staging` - Staging environment with stdout support
+pub const DEFAULT_OBS_ENVIRONMENT_PRODUCTION: &str = "production";
+pub const DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT: &str = "development";
+pub const DEFAULT_OBS_ENVIRONMENT_TEST: &str = "test";
+pub const DEFAULT_OBS_ENVIRONMENT_STAGING: &str = "staging";
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_env_keys() {
+        assert_eq!(ENV_OBS_ENDPOINT, "RUSTFS_OBS_ENDPOINT");
+        assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
+        assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
+        assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
+        assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
+        assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
+        assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
+        assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
+        assert_eq!(ENV_OBS_LOCAL_LOGGING_ENABLED, "RUSTFS_OBS_LOCAL_LOGGING_ENABLED");
+        assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
+        assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
+        assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
+        assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
+        assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
+        assert_eq!(ENV_AUDIT_LOGGER_QUEUE_CAPACITY, "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY");
+    }
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, 10000);
+        assert_eq!(DEFAULT_OBS_ENVIRONMENT_PRODUCTION, "production");
+        assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
+        assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
+        assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
+    }
+}
@@ -1,28 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// RUSTFS_SINKS_WEBHOOK_ENDPOINT
-pub const ENV_SINKS_WEBHOOK_ENDPOINT: &str = "RUSTFS_SINKS_WEBHOOK_ENDPOINT";
-// RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN
-pub const ENV_SINKS_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN";
-// max_retries
-pub const ENV_SINKS_WEBHOOK_MAX_RETRIES: &str = "RUSTFS_SINKS_WEBHOOK_MAX_RETRIES";
-// retry_delay_ms
-pub const ENV_SINKS_WEBHOOK_RETRY_DELAY_MS: &str = "RUSTFS_SINKS_WEBHOOK_RETRY_DELAY_MS";
-
-// Default values for webhook sink configuration
-pub const DEFAULT_SINKS_WEBHOOK_ENDPOINT: &str = "http://localhost:8080";
-pub const DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN: &str = "";
-pub const DEFAULT_SINKS_WEBHOOK_MAX_RETRIES: usize = 3;
-pub const DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS: u64 = 100;
@@ -12,3 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+//opa env vars
+pub const ENV_POLICY_PLUGIN_OPA_URL: &str = "RUSTFS_POLICY_PLUGIN_URL";
+pub const ENV_POLICY_PLUGIN_AUTH_TOKEN: &str = "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN";
+
+pub const ENV_POLICY_PLUGIN_KEYS: &[&str] = &[ENV_POLICY_PLUGIN_OPA_URL, ENV_POLICY_PLUGIN_AUTH_TOKEN];
+
+pub const POLICY_PLUGIN_SUB_SYS: &str = "policy_plugin";
@@ -49,4 +49,4 @@ uuid = { workspace = true }
 base64 = { workspace = true }
 rand = { workspace = true }
 chrono = { workspace = true }
-md5 = { workspace = true }
+md5 = { workspace = true }
@@ -94,7 +94,7 @@ fn build_rustfs_binary() {

    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
-        panic!("Failed to build RustFS binary. Error: {}", stderr);
+        panic!("Failed to build RustFS binary. Error: {stderr}");
    }

    info!("✅ RustFS binary built successfully");
@@ -134,8 +134,8 @@ impl RustFSTestEnvironment {

        // Use a unique port for each test environment
        let port = Self::find_available_port().await?;
-        let address = format!("127.0.0.1:{}", port);
-        let url = format!("http://{}", address);
+        let address = format!("127.0.0.1:{port}");
+        let url = format!("http://{address}");

        Ok(Self {
            temp_dir,
@@ -152,7 +152,7 @@ impl RustFSTestEnvironment {
        let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
        fs::create_dir_all(&temp_dir).await?;

-        let url = format!("http://{}", address);
+        let url = format!("http://{address}");

        Ok(Self {
            temp_dir,
@@ -327,7 +327,7 @@ pub async fn execute_awscurl(

    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
-        return Err(format!("awscurl failed: {}", stderr).into());
+        return Err(format!("awscurl failed: {stderr}").into());
    }

    let response = String::from_utf8_lossy(&output.stdout).to_string();
@@ -388,23 +388,11 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
        complete_multipart_response.ssekms_key_id()
    );

-    // Verify: complete_multipart_upload response should contain encryption information
-    // KNOWN BUG: s3s library bug where CompleteMultipartUploadOutput encryption fields serialize as None
-    // even when properly set. Our server implementation is correct (see server logs above).
-    // TODO: Remove this workaround when s3s library is fixed
-    warn!("KNOWN BUG: s3s library - complete_multipart_upload response encryption fields return None even when set");
-
-    if complete_multipart_response.server_side_encryption().is_some() {
-        // If s3s library is fixed, verify the encryption info
-        assert_eq!(
-            complete_multipart_response.server_side_encryption(),
-            Some(&ServerSideEncryption::AwsKms),
-            "complete_multipart_upload response should contain SSE-KMS encryption information"
-        );
-    } else {
-        // Expected behavior due to s3s library bug - log and continue
-        warn!("Skipping assertion due to known s3s library bug - server logs confirm correct encryption handling");
-    }
+    assert_eq!(
+        complete_multipart_response.server_side_encryption(),
+        Some(&ServerSideEncryption::AwsKms),
+        "complete_multipart_upload response should contain SSE-KMS encryption information"
+    );

    // Step 4: Download file and verify encryption status
    info!("Downloading file and verifying encryption status");
@@ -59,7 +59,7 @@ pub async fn configure_kms(
    access_key: &str,
    secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/kms/configure", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/configure");
    awscurl_post(&url, config_json, access_key, secret_key).await?;
    info!("KMS configured successfully");
    Ok(())
@@ -71,7 +71,7 @@ pub async fn start_kms(
    access_key: &str,
    secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/kms/start", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/start");
    awscurl_post(&url, "{}", access_key, secret_key).await?;
    info!("KMS started successfully");
    Ok(())
@@ -83,7 +83,7 @@ pub async fn get_kms_status(
    access_key: &str,
    secret_key: &str,
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/kms/status", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/status");
    let status = awscurl_get(&url, access_key, secret_key).await?;
    info!("KMS status retrieved: {}", status);
    Ok(status)
@@ -101,7 +101,7 @@ pub async fn create_default_key(
    })
    .to_string();

-    let url = format!("{}/rustfs/admin/v3/kms/keys", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/keys");
    let response = awscurl_post(&url, &create_key_body, access_key, secret_key).await?;

    // Parse response to get the actual key ID
@@ -141,7 +141,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
    });

    // Write the key to file with the specified ID as JSON
-    let key_path = format!("{}/{}.key", key_dir, key_id);
+    let key_path = format!("{key_dir}/{key_id}.key");
    let content = serde_json::to_vec_pretty(&stored_key)?;
    fs::write(&key_path, &content).await?;

@@ -281,13 +281,8 @@ pub async fn test_kms_key_management(
    })
    .to_string();

-    let create_response = awscurl_post(
-        &format!("{}/rustfs/admin/v3/kms/keys", base_url),
-        &create_key_body,
-        access_key,
-        secret_key,
-    )
-    .await?;
+    let create_response =
+        awscurl_post(&format!("{base_url}/rustfs/admin/v3/kms/keys"), &create_key_body, access_key, secret_key).await?;

    let create_result: serde_json::Value = serde_json::from_str(&create_response)?;
    let key_id = create_result["key_id"]
@@ -296,8 +291,7 @@ pub async fn test_kms_key_management(
    info!("Created key with ID: {}", key_id);

    // Test DescribeKey
-    let describe_response =
-        awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
+    let describe_response = awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;

    info!("DescribeKey response: {}", describe_response);
    let describe_result: serde_json::Value = serde_json::from_str(&describe_response)?;
@@ -306,7 +300,7 @@ pub async fn test_kms_key_management(
    info!("Successfully described key: {}", key_id);

    // Test ListKeys
-    let list_response = awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys", base_url), access_key, secret_key).await?;
+    let list_response = awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys"), access_key, secret_key).await?;

    let list_result: serde_json::Value = serde_json::from_str(&list_response)?;
    let keys = list_result["keys"]
@@ -412,7 +406,7 @@ impl VaultTestEnvironment {
            let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
            if port_check {
                // Additional check by making a health request
-                if let Ok(response) = reqwest::get(&format!("{}/v1/sys/health", VAULT_URL)).await {
+                if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await {
                    if response.status().is_success() {
                        info!("Vault server is ready after {} seconds", i);
                        return Ok(());
@@ -438,7 +432,7 @@ impl VaultTestEnvironment {

        // Enable transit secrets engine
        let enable_response = client
-            .post(format!("{}/v1/sys/mounts/{}", VAULT_URL, VAULT_TRANSIT_PATH))
+            .post(format!("{VAULT_URL}/v1/sys/mounts/{VAULT_TRANSIT_PATH}"))
            .header("X-Vault-Token", VAULT_TOKEN)
            .json(&serde_json::json!({
                "type": "transit"
@@ -448,14 +442,14 @@ impl VaultTestEnvironment {

        if !enable_response.status().is_success() && enable_response.status() != 400 {
            let error_text = enable_response.text().await?;
-            return Err(format!("Failed to enable transit engine: {}", error_text).into());
+            return Err(format!("Failed to enable transit engine: {error_text}").into());
        }

        info!("Creating Vault encryption key");

        // Create encryption key
        let key_response = client
-            .post(format!("{}/v1/{}/keys/{}", VAULT_URL, VAULT_TRANSIT_PATH, VAULT_KEY_NAME))
+            .post(format!("{VAULT_URL}/v1/{VAULT_TRANSIT_PATH}/keys/{VAULT_KEY_NAME}"))
            .header("X-Vault-Token", VAULT_TOKEN)
            .json(&serde_json::json!({
                "type": "aes256-gcm96"
@@ -465,7 +459,7 @@ impl VaultTestEnvironment {

        if !key_response.status().is_success() && key_response.status() != 400 {
            let error_text = key_response.text().await?;
-            return Err(format!("Failed to create encryption key: {}", error_text).into());
+            return Err(format!("Failed to create encryption key: {error_text}").into());
        }

        info!("Vault transit engine setup completed");
@@ -713,10 +707,10 @@ pub async fn test_all_multipart_encryption_types(

    // Test configurations for all encryption types
    let test_configs = vec![
-        MultipartTestConfig::new(format!("{}-no-encryption", base_object_key), part_size, total_parts, EncryptionType::None),
-        MultipartTestConfig::new(format!("{}-sse-s3", base_object_key), part_size, total_parts, EncryptionType::SSES3),
-        MultipartTestConfig::new(format!("{}-sse-kms", base_object_key), part_size, total_parts, EncryptionType::SSEKMS),
-        MultipartTestConfig::new(format!("{}-sse-c", base_object_key), part_size, total_parts, create_sse_c_config()),
+        MultipartTestConfig::new(format!("{base_object_key}-no-encryption"), part_size, total_parts, EncryptionType::None),
+        MultipartTestConfig::new(format!("{base_object_key}-sse-s3"), part_size, total_parts, EncryptionType::SSES3),
+        MultipartTestConfig::new(format!("{base_object_key}-sse-kms"), part_size, total_parts, EncryptionType::SSEKMS),
+        MultipartTestConfig::new(format!("{base_object_key}-sse-c"), part_size, total_parts, create_sse_c_config()),
    ];

    // Run tests for each encryption type
@@ -33,11 +33,10 @@ fn assert_encryption_metadata(metadata: &HashMap<String, String>, expected_size:
        "x-rustfs-encryption-context",
        "x-rustfs-encryption-original-size",
    ] {
-        assert!(metadata.contains_key(key), "expected managed encryption metadata '{}' to be present", key);
+        assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present");
        assert!(
            !metadata.get(key).unwrap().is_empty(),
-            "managed encryption metadata '{}' should not be empty",
-            key
+            "managed encryption metadata '{key}' should not be empty"
        );
    }

@@ -84,10 +83,7 @@ fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &

    assert!(
        scanned > 0,
-        "Failed to locate stored data files for bucket '{}' and key '{}' under {:?}",
-        bucket,
-        key,
-        storage_root
+        "Failed to locate stored data files for bucket '{bucket}' and key '{key}' under {storage_root:?}"
    );
    assert!(plaintext_path.is_none(), "Plaintext detected on disk at {:?}", plaintext_path.unwrap());
 }
@@ -220,7 +216,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
    assert_encryption_metadata(source_metadata, payload.len());

    let dest_key = "metadata-sse-kms-object-copy";
-    let copy_source = format!("{}/{}", TEST_BUCKET, source_key);
+    let copy_source = format!("{TEST_BUCKET}/{source_key}");

    s3_client
        .copy_object()
@@ -389,8 +389,8 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
        let task = tokio::spawn(async move {
            let _permit = sem.acquire().await.unwrap();

-            let test_data = format!("Concurrent test data {}", i).into_bytes();
-            let object_key = format!("concurrent-test-{}", i);
+            let test_data = format!("Concurrent test data {i}").into_bytes();
+            let object_key = format!("concurrent-test-{i}");

            // Alternate between different encryption types
            let result = match i % 3 {
@@ -418,7 +418,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
                }
                2 => {
                    // SSE-C
-                    let key = format!("testkey{:026}", i); // 32-byte key
+                    let key = format!("testkey{i:026}"); // 32-byte key
                    let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
                    let key_md5 = format!("{:x}", md5::compute(&key));

@@ -459,9 +459,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro

    assert!(
        successful_uploads >= num_concurrent - 1,
-        "Most concurrent uploads should succeed (got {}/{})",
-        successful_uploads,
-        num_concurrent
+        "Most concurrent uploads should succeed (got {successful_uploads}/{num_concurrent})"
    );

    info!("✅ Successfully completed {}/{} concurrent uploads", successful_uploads, num_concurrent);
@@ -152,7 +152,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
    // Corrupt the default key file
    info!("🔧 Corrupting default key file");
    let key_file_path = format!("{}/{}.key", kms_env.kms_keys_dir, default_key_id);
-    let backup_key_path = format!("{}.backup", key_file_path);
+    let backup_key_path = format!("{key_file_path}.backup");

    // Backup the original key file
    fs::copy(&key_file_path, &backup_key_path)?;
@@ -417,8 +417,8 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error

    for i in 0..10 {
        let client = s3_client.clone();
-        let test_data = format!("Rapid test data {}", i).into_bytes();
-        let object_key = format!("rapid-test-{}", i);
+        let test_data = format!("Rapid test data {i}").into_bytes();
+        let object_key = format!("rapid-test-{i}");

        let task = tokio::spawn(async move {
            let result = client
@@ -740,7 +740,7 @@ async fn test_large_multipart_upload(
    // Verify data integrity
    for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
        if actual != expected {
-            panic!("Data mismatch at byte {}: got {}, expected {}", i, actual, expected);
+            panic!("Data mismatch at byte {i}: got {actual}, expected {expected}");
        }
    }

@@ -321,13 +321,9 @@ async fn test_vault_kms_key_crud(
    })
    .to_string();

-    let create_response = crate::common::awscurl_post(
-        &format!("{}/rustfs/admin/v3/kms/keys", base_url),
-        &create_key_body,
-        access_key,
-        secret_key,
-    )
-    .await?;
+    let create_response =
+        crate::common::awscurl_post(&format!("{base_url}/rustfs/admin/v3/kms/keys"), &create_key_body, access_key, secret_key)
+            .await?;

    let create_result: serde_json::Value = serde_json::from_str(&create_response)?;
    let key_id = create_result["key_id"]
@@ -337,7 +333,7 @@ async fn test_vault_kms_key_crud(

    // Read
    let describe_response =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;

    let describe_result: serde_json::Value = serde_json::from_str(&describe_response)?;
    assert_eq!(describe_result["key_metadata"]["key_id"], key_id);
@@ -380,7 +376,7 @@ async fn test_vault_kms_key_crud(

    // Read
    let list_response =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys", base_url), access_key, secret_key).await?;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys"), access_key, secret_key).await?;

    let list_result: serde_json::Value = serde_json::from_str(&list_response)?;
    let keys = list_result["keys"]
@@ -407,7 +403,7 @@ async fn test_vault_kms_key_crud(

    // Delete
    let delete_response = crate::common::execute_awscurl(
-        &format!("{}/rustfs/admin/v3/kms/keys/delete?keyId={}", base_url, key_id),
+        &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}"),
        "DELETE",
        None,
        access_key,
@@ -422,7 +418,7 @@ async fn test_vault_kms_key_crud(

    // Verify key state after deletion
    let describe_deleted_response =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;

    let describe_result: serde_json::Value = serde_json::from_str(&describe_deleted_response)?;
    let key_state = describe_result["key_metadata"]["key_state"]
@@ -439,7 +435,7 @@ async fn test_vault_kms_key_crud(

    // Force Delete - Force immediate deletion for PendingDeletion key
    let force_delete_response = crate::common::execute_awscurl(
-        &format!("{}/rustfs/admin/v3/kms/keys/delete?keyId={}&force_immediate=true", base_url, key_id),
+        &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
        "DELETE",
        None,
        access_key,
@@ -454,7 +450,7 @@ async fn test_vault_kms_key_crud(

    // Verify key no longer exists after force deletion (should return error)
    let describe_force_deleted_result =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await;

    // After force deletion, key should not be found (GET should fail)
    assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found");
@@ -419,7 +419,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
    // 逐字节验证数据（对于大文件更严格）
    for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
        if actual != expected {
-            panic!("大文件数据在第{}字节不匹配: 实际={}, 期待={}", i, actual, expected);
+            panic!("大文件数据在第{i}字节不匹配: 实际={actual}, 期待={expected}");
        }
    }

@@ -476,7 +476,7 @@ async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Sen

    let failed_count = results.iter().filter(|r| !r.success).count();
    if failed_count > 0 {
-        return Err(format!("Critical test suite failed: {} tests failed", failed_count).into());
+        return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
    }

    info!("✅ 所有关键测试通过");
@@ -498,7 +498,7 @@ async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send +

    // Allow up to 10% failure rate for non-critical tests
    if success_rate < 90.0 {
-        return Err(format!("Test suite success rate too low: {:.1}%", success_rate).into());
+        return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
    }

    info!("✅ 完整测试套件通过");
@@ -14,14 +14,14 @@
 // limitations under the License.

 use async_trait::async_trait;
-use rustfs_ecstore::{disk::endpoint::Endpoint, lock_utils::create_unique_clients};
-use rustfs_lock::client::{LockClient, local::LocalClient};
+use rustfs_ecstore::disk::endpoint::Endpoint;
+use rustfs_lock::client::{LockClient, local::LocalClient, remote::RemoteClient};
 use rustfs_lock::types::{LockInfo, LockResponse, LockStats};
 use rustfs_lock::{LockId, LockMetadata, LockPriority, LockType};
 use rustfs_lock::{LockRequest, NamespaceLock, NamespaceLockManager};
 use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
 use serial_test::serial;
-use std::{error::Error, sync::Arc, time::Duration};
+use std::{collections::HashMap, error::Error, sync::Arc, time::Duration};
 use tokio::time::sleep;
 use tonic::Request;
 use url::Url;
@@ -38,6 +38,34 @@ fn get_cluster_endpoints() -> Vec<Endpoint> {
    }]
 }

+async fn create_unique_clients(endpoints: &[Endpoint]) -> Result<Vec<Arc<dyn LockClient>>, Box<dyn Error>> {
+    let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
+
+    for endpoint in endpoints {
+        if endpoint.is_local {
+            unique_endpoints.insert("local".to_string(), endpoint);
+        } else {
+            let host_port = format!(
+                "{}:{}",
+                endpoint.url.host_str().unwrap_or("localhost"),
+                endpoint.url.port().unwrap_or(9000)
+            );
+            unique_endpoints.insert(host_port, endpoint);
+        }
+    }
+
+    let mut clients = Vec::new();
+    for (_key, endpoint) in unique_endpoints {
+        if endpoint.is_local {
+            clients.push(Arc::new(LocalClient::new()) as Arc<dyn LockClient>);
+        } else {
+            clients.push(Arc::new(RemoteClient::new(endpoint.url.to_string())) as Arc<dyn LockClient>);
+        }
+    }
+
+    Ok(clients)
+}
+
 #[tokio::test]
 #[serial]
 #[ignore = "requires running RustFS server at localhost:9000"]
@@ -75,7 +75,6 @@ hyper-util.workspace = true
 hyper-rustls.workspace = true
 rustls.workspace = true
 tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
-tokio-stream = { workspace = true }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower.workspace = true
@@ -89,8 +88,6 @@ rustfs-madmin.workspace = true
 rustfs-workers.workspace = true
 reqwest = { workspace = true }
 aws-sdk-s3 = { workspace = true }
-once_cell = { workspace = true }
-rustfs-rsc = { workspace = true }
 urlencoding = { workspace = true }
 smallvec = { workspace = true }
 shadow-rs.workspace = true
@@ -99,10 +96,12 @@ rustfs-utils = { workspace = true, features = ["full"] }
 rustfs-rio.workspace = true
 rustfs-signer.workspace = true
 rustfs-checksums.workspace = true
-futures-util.workspace = true
 async-recursion.workspace = true
-parking_lot = "0.12"
-moka = { version = "0.12", features = ["future"] }
+aws-credential-types = { workspace = true }
+aws-smithy-types = { workspace = true }
+parking_lot = { workspace = true }
+moka = { workspace = true }
+base64-simd.workspace = true

 [target.'cfg(not(windows))'.dependencies]
 nix = { workspace = true }
@@ -16,6 +16,10 @@ use super::{quota::BucketQuota, target::BucketTargets};

 use super::object_lock::ObjectLockApi;
 use super::versioning::VersioningApi;
+use crate::bucket::utils::deserialize;
+use crate::config::com::{read_config, save_config};
+use crate::error::{Error, Result};
+use crate::new_object_layer_fn;
 use byteorder::{BigEndian, ByteOrder, LittleEndian};
 use rmp_serde::Serializer as rmpSerializer;
 use rustfs_policy::policy::BucketPolicy;
@@ -30,12 +34,6 @@ use std::sync::Arc;
 use time::OffsetDateTime;
 use tracing::error;

-use crate::bucket::target::BucketTarget;
-use crate::bucket::utils::deserialize;
-use crate::config::com::{read_config, save_config};
-use crate::error::{Error, Result};
-use crate::new_object_layer_fn;
-
 use crate::disk::BUCKET_META_PREFIX;
 use crate::store::ECStore;

@@ -322,7 +320,9 @@ impl BucketMetadata {

        LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);

-        let data = self.marshal_msg()?;
+        let data = self
+            .marshal_msg()
+            .map_err(|e| Error::other(format!("save bucket metadata failed: {e}")))?;

        buf.extend_from_slice(&data);

@@ -362,8 +362,8 @@ impl BucketMetadata {
        }
        //let temp = self.bucket_targets_config_json.clone();
        if !self.bucket_targets_config_json.is_empty() {
-            let arr: Vec<BucketTarget> = serde_json::from_slice(&self.bucket_targets_config_json)?;
-            self.bucket_target_config = Some(BucketTargets { targets: arr });
+            let bucket_targets: BucketTargets = serde_json::from_slice(&self.bucket_targets_config_json)?;
+            self.bucket_target_config = Some(bucket_targets);
        } else {
            self.bucket_target_config = Some(BucketTargets::default())
        }
@@ -451,4 +451,154 @@ mod test {

        assert_eq!(bm.name, new.name);
    }
+
+    #[tokio::test]
+    async fn marshal_msg_complete_example() {
+        // Create a complete BucketMetadata with various configurations
+        let mut bm = BucketMetadata::new("test-bucket");
+
+        // Set creation time to current time
+        bm.created = OffsetDateTime::now_utc();
+        bm.lock_enabled = true;
+
+        // Add policy configuration
+        let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}"#;
+        bm.policy_config_json = policy_json.as_bytes().to_vec();
+        bm.policy_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add lifecycle configuration
+        let lifecycle_xml = r#"<LifecycleConfiguration><Rule><ID>rule1</ID><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#;
+        bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec();
+        bm.lifecycle_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add versioning configuration
+        let versioning_xml = r#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#;
+        bm.versioning_config_xml = versioning_xml.as_bytes().to_vec();
+        bm.versioning_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add encryption configuration
+        let encryption_xml = r#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
+        bm.encryption_config_xml = encryption_xml.as_bytes().to_vec();
+        bm.encryption_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add tagging configuration
+        let tagging_xml = r#"<Tagging><TagSet><Tag><Key>Environment</Key><Value>Test</Value></Tag><Tag><Key>Owner</Key><Value>RustFS</Value></Tag></TagSet></Tagging>"#;
+        bm.tagging_config_xml = tagging_xml.as_bytes().to_vec();
+        bm.tagging_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add quota configuration
+        let quota_json = r#"{"quota":1073741824,"quotaType":"hard"}"#; // 1GB quota
+        bm.quota_config_json = quota_json.as_bytes().to_vec();
+        bm.quota_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add object lock configuration
+        let object_lock_xml = r#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#;
+        bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec();
+        bm.object_lock_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add notification configuration
+        let notification_xml = r#"<NotificationConfiguration><CloudWatchConfiguration><Id>notification1</Id><Event>s3:ObjectCreated:*</Event><CloudWatchConfiguration><LogGroupName>test-log-group</LogGroupName></CloudWatchConfiguration></CloudWatchConfiguration></NotificationConfiguration>"#;
+        bm.notification_config_xml = notification_xml.as_bytes().to_vec();
+        bm.notification_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add replication configuration
+        let replication_xml = r#"<ReplicationConfiguration><Role>arn:aws:iam::123456789012:role/replication-role</Role><Rule><ID>rule1</ID><Status>Enabled</Status><Prefix>documents/</Prefix><Destination><Bucket>arn:aws:s3:::destination-bucket</Bucket></Destination></Rule></ReplicationConfiguration>"#;
+        bm.replication_config_xml = replication_xml.as_bytes().to_vec();
+        bm.replication_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add bucket targets configuration
+        let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#;
+        bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec();
+        bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add bucket targets meta configuration
+        let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#;
+        bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
+        bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();
+
+        // Test serialization
+        let buf = bm.marshal_msg().unwrap();
+        assert!(!buf.is_empty(), "Serialized buffer should not be empty");
+
+        // Test deserialization
+        let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap();
+
+        // Verify all fields are correctly serialized and deserialized
+        assert_eq!(bm.name, deserialized_bm.name);
+        assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp());
+        assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled);
+
+        // Verify configuration data
+        assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json);
+        assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml);
+        assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml);
+        assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
+        assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
+        assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
+        assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
+        assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
+        assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);
+        assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json);
+        assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json);
+
+        // Verify timestamps (comparing unix timestamps to avoid precision issues)
+        assert_eq!(
+            bm.policy_config_updated_at.unix_timestamp(),
+            deserialized_bm.policy_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.lifecycle_config_updated_at.unix_timestamp(),
+            deserialized_bm.lifecycle_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.versioning_config_updated_at.unix_timestamp(),
+            deserialized_bm.versioning_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.encryption_config_updated_at.unix_timestamp(),
+            deserialized_bm.encryption_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.tagging_config_updated_at.unix_timestamp(),
+            deserialized_bm.tagging_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.quota_config_updated_at.unix_timestamp(),
+            deserialized_bm.quota_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.object_lock_config_updated_at.unix_timestamp(),
+            deserialized_bm.object_lock_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.notification_config_updated_at.unix_timestamp(),
+            deserialized_bm.notification_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.replication_config_updated_at.unix_timestamp(),
+            deserialized_bm.replication_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.bucket_targets_config_updated_at.unix_timestamp(),
+            deserialized_bm.bucket_targets_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.bucket_targets_config_meta_updated_at.unix_timestamp(),
+            deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp()
+        );
+
+        // Test that the serialized data contains expected content
+        let buf_str = String::from_utf8_lossy(&buf);
+        assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name");
+
+        // Verify the buffer size is reasonable (should be larger due to all the config data)
+        assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations");
+
+        println!("✅ Complete BucketMetadata serialization test passed");
+        println!("   - Bucket name: {}", deserialized_bm.name);
+        println!("   - Lock enabled: {}", deserialized_bm.lock_enabled);
+        println!("   - Policy config size: {} bytes", deserialized_bm.policy_config_json.len());
+        println!("   - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len());
+        println!("   - Serialized buffer size: {} bytes", buf.len());
+    }
 }
@@ -12,19 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::StorageAPI;
+use crate::StorageAPI as _;
+use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
 use crate::bucket::utils::{deserialize, is_meta_bucketname};
-use crate::cmd::bucket_targets;
 use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
 use crate::store::ECStore;
 use futures::future::join_all;
 use rustfs_common::heal_channel::HealOpts;
 use rustfs_policy::policy::BucketPolicy;
+use s3s::dto::ReplicationConfiguration;
 use s3s::dto::{
-    BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
-    ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ServerSideEncryptionConfiguration, Tagging,
+    VersioningConfiguration,
 };
 use std::collections::HashSet;
 use std::sync::OnceLock;
@@ -261,7 +262,8 @@ impl BucketMetadataSys {
                    if let Some(bucket) = buckets.get(idx) {
                        let x = Arc::new(res);
                        mp.insert(bucket.clone(), x.clone());
-                        bucket_targets::init_bucket_targets(bucket, x.clone()).await;
+                        // TODO:EventNotifier,BucketTargetSys
+                        BucketTargetSys::get().set(bucket, &x).await;
                    }
                }
                Err(e) => {
@@ -348,6 +350,7 @@ impl BucketMetadataSys {
                if !is_erasure().await && !is_dist_erasure().await && is_err_bucket_not_found(&err) {
                    BucketMetadata::new(bucket)
                } else {
+                    error!("load bucket metadata failed: {}", err);
                    return Err(err);
                }
            }
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+pub mod bucket_target_sys;
 pub mod error;
 pub mod lifecycle;
 pub mod metadata;
@@ -0,0 +1,233 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::ReplicationRuleExt as _;
+use crate::bucket::tagging::decode_tags_to_map;
+use rustfs_filemeta::ReplicationType;
+use s3s::dto::DeleteMarkerReplicationStatus;
+use s3s::dto::DeleteReplicationStatus;
+use s3s::dto::Destination;
+use s3s::dto::{ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRuleStatus, ReplicationRules};
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use uuid::Uuid;
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct ObjectOpts {
+    pub name: String,
+    pub user_tags: String,
+    pub version_id: Option<Uuid>,
+    pub delete_marker: bool,
+    pub ssec: bool,
+    pub op_type: ReplicationType,
+    pub replica: bool,
+    pub existing_object: bool,
+    pub target_arn: String,
+}
+
+pub trait ReplicationConfigurationExt {
+    fn replicate(&self, opts: &ObjectOpts) -> bool;
+    fn has_existing_object_replication(&self, arn: &str) -> (bool, bool);
+    fn filter_actionable_rules(&self, obj: &ObjectOpts) -> ReplicationRules;
+    fn get_destination(&self) -> Destination;
+    fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
+    fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
+}
+
+impl ReplicationConfigurationExt for ReplicationConfiguration {
+    /// 检查是否有现有对象复制规则
+    fn has_existing_object_replication(&self, arn: &str) -> (bool, bool) {
+        let mut has_arn = false;
+
+        for rule in &self.rules {
+            if rule.destination.bucket == arn || self.role == arn {
+                if !has_arn {
+                    has_arn = true;
+                }
+                if let Some(status) = &rule.existing_object_replication {
+                    if status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED) {
+                        return (true, true);
+                    }
+                }
+            }
+        }
+        (has_arn, false)
+    }
+
+    fn filter_actionable_rules(&self, obj: &ObjectOpts) -> ReplicationRules {
+        if obj.name.is_empty() && obj.op_type != ReplicationType::Resync && obj.op_type != ReplicationType::All {
+            return vec![];
+        }
+
+        let mut rules = ReplicationRules::default();
+
+        for rule in &self.rules {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if !obj.target_arn.is_empty() && rule.destination.bucket != obj.target_arn && self.role != obj.target_arn {
+                continue;
+            }
+
+            if obj.op_type == ReplicationType::Resync || obj.op_type == ReplicationType::All {
+                rules.push(rule.clone());
+                continue;
+            }
+
+            if let Some(status) = &rule.existing_object_replication {
+                if obj.existing_object
+                    && status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
+                {
+                    continue;
+                }
+            }
+
+            if !obj.name.starts_with(rule.prefix()) {
+                continue;
+            }
+
+            if let Some(filter) = &rule.filter {
+                let object_tags = decode_tags_to_map(&obj.user_tags);
+                if filter.test_tags(&object_tags) {
+                    rules.push(rule.clone());
+                }
+            }
+        }
+
+        rules.sort_by(|a, b| {
+            if a.destination == b.destination {
+                a.priority.cmp(&b.priority)
+            } else {
+                std::cmp::Ordering::Equal
+            }
+        });
+
+        rules
+    }
+
+    /// 获取目标配置
+    fn get_destination(&self) -> Destination {
+        if !self.rules.is_empty() {
+            self.rules[0].destination.clone()
+        } else {
+            Destination {
+                account: None,
+                bucket: "".to_string(),
+                encryption_configuration: None,
+                metrics: None,
+                replication_time: None,
+                access_control_translation: None,
+                storage_class: None,
+            }
+        }
+    }
+
+    /// 判断对象是否应该被复制
+    fn replicate(&self, obj: &ObjectOpts) -> bool {
+        let rules = self.filter_actionable_rules(obj);
+
+        for rule in rules.iter() {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if let Some(status) = &rule.existing_object_replication {
+                if obj.existing_object
+                    && status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
+                {
+                    return false;
+                }
+            }
+
+            if obj.op_type == ReplicationType::Delete {
+                if obj.version_id.is_some() {
+                    return rule
+                        .delete_replication
+                        .clone()
+                        .is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
+                } else {
+                    return rule.delete_marker_replication.clone().is_some_and(|d| {
+                        d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
+                    });
+                }
+            }
+
+            // 常规对象/元数据复制
+            return rule.metadata_replicate(obj);
+        }
+        false
+    }
+
+    /// 检查是否有活跃的规则
+    /// 可选择性地提供前缀
+    /// 如果recursive为true，函数还会在前缀下的任何级别有活跃规则时返回true
+    /// 如果没有指定前缀，recursive实际上为true
+    fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool {
+        if self.rules.is_empty() {
+            return false;
+        }
+
+        for rule in &self.rules {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if let Some(filter) = &rule.filter {
+                if let Some(filter_prefix) = &filter.prefix {
+                    if !prefix.is_empty() && !filter_prefix.is_empty() {
+                        // 传入的前缀必须在规则前缀中
+                        if !recursive && !prefix.starts_with(filter_prefix) {
+                            continue;
+                        }
+                    }
+
+                    // 如果是递归的，我们可以跳过这个规则，如果它不匹配测试前缀或前缀下的级别不匹配
+                    if recursive && !rule.prefix().starts_with(prefix) && !prefix.starts_with(rule.prefix()) {
+                        continue;
+                    }
+                }
+            }
+            return true;
+        }
+        false
+    }
+
+    /// 过滤目标ARN，返回配置中不同目标ARN的切片
+    fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String> {
+        let mut arns = Vec::new();
+        let mut targets_map: HashSet<String> = HashSet::new();
+        let rules = self.filter_actionable_rules(obj);
+
+        for rule in rules {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if !self.role.is_empty() {
+                arns.push(self.role.clone()); // 如果存在，使用传统的RoleArn
+                return arns;
+            }
+
+            if !targets_map.contains(&rule.destination.bucket) {
+                targets_map.insert(rule.destination.bucket.clone());
+            }
+        }
+
+        for arn in targets_map {
+            arns.push(arn);
+        }
+        arns
+    }
+}
@@ -12,30 +12,36 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-// Replication status type for x-amz-replication-status header
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum StatusType {
-    Pending,
-    Completed,
-    CompletedLegacy,
-    Failed,
-    Replica,
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub enum ResyncStatusType {
+    #[default]
+    NoResync,
+    ResyncPending,
+    ResyncCanceled,
+    ResyncStarted,
+    ResyncCompleted,
+    ResyncFailed,
 }

-impl StatusType {
-    // Converts the enum variant to its string representation
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            StatusType::Pending => "PENDING",
-            StatusType::Completed => "COMPLETED",
-            StatusType::CompletedLegacy => "COMPLETE",
-            StatusType::Failed => "FAILED",
-            StatusType::Replica => "REPLICA",
-        }
-    }
-
-    // Checks if the status is empty (not set)
-    pub fn is_empty(&self) -> bool {
-        matches!(self, StatusType::Pending) // Adjust this as needed
+impl ResyncStatusType {
+    pub fn is_valid(&self) -> bool {
+        *self != ResyncStatusType::NoResync
+    }
+}
+
+impl fmt::Display for ResyncStatusType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let s = match self {
+            ResyncStatusType::ResyncStarted => "Ongoing",
+            ResyncStatusType::ResyncCompleted => "Completed",
+            ResyncStatusType::ResyncFailed => "Failed",
+            ResyncStatusType::ResyncPending => "Pending",
+            ResyncStatusType::ResyncCanceled => "Canceled",
+            ResyncStatusType::NoResync => "",
+        };
+        write!(f, "{s}")
    }
 }
@@ -12,4 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+mod config;
 pub mod datatypes;
+mod replication_pool;
+mod replication_resyncer;
+mod replication_state;
+mod rule;
+
+pub use config::*;
+pub use datatypes::*;
+pub use replication_pool::*;
+pub use replication_resyncer::*;
+pub use rule::*;
@@ -0,0 +1,51 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use s3s::dto::ReplicaModificationsStatus;
+use s3s::dto::ReplicationRule;
+
+use super::ObjectOpts;
+
+pub trait ReplicationRuleExt {
+    fn prefix(&self) -> &str;
+    fn metadata_replicate(&self, obj: &ObjectOpts) -> bool;
+}
+
+impl ReplicationRuleExt for ReplicationRule {
+    fn prefix(&self) -> &str {
+        if let Some(filter) = &self.filter {
+            if let Some(prefix) = &filter.prefix {
+                prefix
+            } else if let Some(and) = &filter.and {
+                and.prefix.as_deref().unwrap_or("")
+            } else {
+                ""
+            }
+        } else {
+            ""
+        }
+    }
+
+    fn metadata_replicate(&self, obj: &ObjectOpts) -> bool {
+        if !obj.replica {
+            return true;
+        }
+
+        self.source_selection_criteria.as_ref().is_some_and(|s| {
+            s.replica_modifications
+                .clone()
+                .is_some_and(|r| r.status == ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED))
+        })
+    }
+}
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use std::collections::HashMap;
+
 use s3s::dto::Tag;
 use url::form_urlencoded;

@@ -34,6 +36,20 @@ pub fn decode_tags(tags: &str) -> Vec<Tag> {
    list
 }

+pub fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
+    let mut list = HashMap::new();
+
+    for (k, v) in form_urlencoded::parse(tags.as_bytes()) {
+        if k.is_empty() || v.is_empty() {
+            continue;
+        }
+
+        list.insert(k.to_string(), v.to_string());
+    }
+
+    list
+}
+
 pub fn encode_tags(tags: Vec<Tag>) -> String {
    let mut encoded = form_urlencoded::Serializer::new(String::new());

@@ -0,0 +1,66 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::BucketTargetType;
+use std::fmt::Display;
+use std::str::FromStr;
+
+pub struct ARN {
+    pub arn_type: BucketTargetType,
+    pub id: String,
+    pub region: String,
+    pub bucket: String,
+}
+
+impl ARN {
+    pub fn new(arn_type: BucketTargetType, id: String, region: String, bucket: String) -> Self {
+        Self {
+            arn_type,
+            id,
+            region,
+            bucket,
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.arn_type.is_valid()
+    }
+}
+
+impl Display for ARN {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
+    }
+}
+
+impl FromStr for ARN {
+    type Err = std::io::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if !s.starts_with("arn:rustfs:") {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
+        }
+
+        let parts: Vec<&str> = s.split(':').collect();
+        if parts.len() != 6 {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
+        }
+        Ok(ARN {
+            arn_type: BucketTargetType::from_str(parts[2]).unwrap_or_default(),
+            id: parts[3].to_string(),
+            region: parts[4].to_string(),
+            bucket: parts[5].to_string(),
+        })
+    }
+}
@@ -0,0 +1,800 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use rmp_serde::Serializer as rmpSerializer;
+use serde::{Deserialize, Serialize};
+use std::{
+    fmt::{self, Display},
+    str::FromStr,
+    time::Duration,
+};
+use time::OffsetDateTime;
+use url::Url;
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct Credentials {
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub session_token: Option<String>,
+    pub expiration: Option<chrono::DateTime<chrono::Utc>>,
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub enum ServiceType {
+    #[default]
+    Replication,
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct LatencyStat {
+    #[serde(with = "duration_milliseconds")]
+    pub curr: Duration, // Current latency
+    #[serde(with = "duration_milliseconds")]
+    pub avg: Duration, // Average latency
+    #[serde(with = "duration_milliseconds")]
+    pub max: Duration, // Maximum latency
+}
+
+mod duration_milliseconds {
+    use serde::{Deserialize, Deserializer, Serializer};
+    use std::time::Duration;
+
+    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_u64(duration.as_millis() as u64)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let millis = u64::deserialize(deserializer)?;
+        Ok(Duration::from_millis(millis))
+    }
+}
+
+mod duration_seconds {
+    use serde::{Deserialize, Deserializer, Serializer};
+    use std::time::Duration;
+
+    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_u64(duration.as_secs())
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let secs = u64::deserialize(deserializer)?;
+        Ok(Duration::from_secs(secs))
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
+pub enum BucketTargetType {
+    #[default]
+    None,
+    #[serde(rename = "replication")]
+    ReplicationService,
+    #[serde(rename = "ilm")]
+    IlmService,
+}
+
+impl BucketTargetType {
+    pub fn is_valid(&self) -> bool {
+        match self {
+            BucketTargetType::None => false,
+            BucketTargetType::ReplicationService | BucketTargetType::IlmService => true,
+        }
+    }
+}
+
+impl FromStr for BucketTargetType {
+    type Err = std::io::Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s {
+            "replication" => Ok(BucketTargetType::ReplicationService),
+            "ilm" => Ok(BucketTargetType::IlmService),
+            _ => Ok(BucketTargetType::None),
+        }
+    }
+}
+
+impl fmt::Display for BucketTargetType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BucketTargetType::None => write!(f, ""),
+            BucketTargetType::ReplicationService => write!(f, "replication"),
+            BucketTargetType::IlmService => write!(f, "ilm"),
+        }
+    }
+}
+
+// Define BucketTarget structure
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct BucketTarget {
+    #[serde(rename = "sourcebucket", default)]
+    pub source_bucket: String,
+
+    #[serde(default)]
+    pub endpoint: String,
+
+    #[serde(default)]
+    pub credentials: Option<Credentials>,
+    #[serde(rename = "targetbucket", default)]
+    pub target_bucket: String,
+
+    #[serde(default)]
+    pub secure: bool,
+    #[serde(default)]
+    pub path: String,
+    #[serde(default)]
+    pub api: String,
+    #[serde(default)]
+    pub arn: String,
+    #[serde(rename = "type", default)]
+    pub target_type: BucketTargetType,
+
+    #[serde(default)]
+    pub region: String,
+
+    #[serde(alias = "bandwidth", default)]
+    pub bandwidth_limit: i64,
+
+    #[serde(rename = "replicationSync", default)]
+    pub replication_sync: bool,
+    #[serde(default)]
+    pub storage_class: String,
+    #[serde(rename = "healthCheckDuration", with = "duration_seconds", default)]
+    pub health_check_duration: Duration,
+    #[serde(rename = "disableProxy", default)]
+    pub disable_proxy: bool,
+
+    #[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
+    pub reset_before_date: Option<OffsetDateTime>,
+    #[serde(default)]
+    pub reset_id: String,
+    #[serde(rename = "totalDowntime", with = "duration_seconds", default)]
+    pub total_downtime: Duration,
+
+    #[serde(rename = "lastOnline", with = "time::serde::rfc3339::option", default)]
+    pub last_online: Option<OffsetDateTime>,
+    #[serde(rename = "isOnline", default)]
+    pub online: bool,
+
+    #[serde(default)]
+    pub latency: LatencyStat,
+
+    #[serde(default)]
+    pub deployment_id: String,
+
+    #[serde(default)]
+    pub edge: bool,
+    #[serde(rename = "edgeSyncBeforeExpiry", default)]
+    pub edge_sync_before_expiry: bool,
+    #[serde(rename = "offlineCount", default)]
+    pub offline_count: u64,
+}
+
+impl BucketTarget {
+    pub fn is_empty(self) -> bool {
+        self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_empty()
+    }
+    pub fn url(&self) -> Result<Url> {
+        let scheme = if self.secure { "https" } else { "http" };
+        Url::parse(&format!("{}://{}", scheme, self.endpoint)).map_err(Error::other)
+    }
+}
+
+impl Display for BucketTarget {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} ", self.endpoint)?;
+        write!(f, "{}", self.target_bucket.clone())?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct BucketTargets {
+    pub targets: Vec<BucketTarget>,
+}
+
+impl BucketTargets {
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: BucketTargets = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        if self.targets.is_empty() {
+            return true;
+        }
+
+        for target in &self.targets {
+            if !target.clone().is_empty() {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json;
+    use std::time::Duration;
+    use time::OffsetDateTime;
+
+    #[test]
+    fn test_bucket_target_json_deserialize() {
+        let json = r#"
+        {
+            "sourcebucket": "source-bucket-name",
+            "endpoint": "s3.amazonaws.com",
+            "credentials": {
+                "accessKey": "test-access-key",
+                "secretKey": "test-secret-key",
+                "session_token": "test-session-token",
+                "expiration": "2024-12-31T23:59:59Z"
+            },
+            "targetbucket": "target-bucket-name",
+            "secure": true,
+            "path": "/api/v1",
+            "api": "s3v4",
+            "arn": "arn:aws:s3:::target-bucket-name",
+            "type": "replication",
+            "region": "us-east-1",
+            "bandwidth_limit": 1000000,
+            "replicationSync": true,
+            "storage_class": "STANDARD",
+            "healthCheckDuration": 30,
+            "disableProxy": false,
+            "resetBeforeDate": null,
+            "reset_id": "reset-123",
+            "totalDowntime": 3600,
+            "last_online": null,
+            "isOnline": true,
+            "latency": {
+                "curr": 100,
+                "avg": 150,
+                "max": 300
+            },
+            "deployment_id": "deployment-456",
+            "edge": false,
+            "edgeSyncBeforeExpiry": true,
+            "offlineCount": 5
+        }
+        "#;
+
+        let result: std::result::Result<BucketTarget, _> = serde_json::from_str(json);
+        assert!(result.is_ok(), "Failed to deserialize BucketTarget: {:?}", result.err());
+
+        let target = result.unwrap();
+
+        // Verify basic fields
+        assert_eq!(target.source_bucket, "source-bucket-name");
+        assert_eq!(target.endpoint, "s3.amazonaws.com");
+        assert_eq!(target.target_bucket, "target-bucket-name");
+        assert!(target.secure);
+        assert_eq!(target.path, "/api/v1");
+        assert_eq!(target.api, "s3v4");
+        assert_eq!(target.arn, "arn:aws:s3:::target-bucket-name");
+        assert_eq!(target.target_type, BucketTargetType::ReplicationService);
+        assert_eq!(target.region, "us-east-1");
+        assert_eq!(target.bandwidth_limit, 1000000);
+        assert!(target.replication_sync);
+        assert_eq!(target.storage_class, "STANDARD");
+        assert_eq!(target.health_check_duration, Duration::from_secs(30));
+        assert!(!target.disable_proxy);
+        assert_eq!(target.reset_id, "reset-123");
+        assert_eq!(target.total_downtime, Duration::from_secs(3600));
+        assert!(target.online);
+        assert_eq!(target.deployment_id, "deployment-456");
+        assert!(!target.edge);
+        assert!(target.edge_sync_before_expiry);
+        assert_eq!(target.offline_count, 5);
+
+        // Verify credentials
+        assert!(target.credentials.is_some());
+        let credentials = target.credentials.unwrap();
+        assert_eq!(credentials.access_key, "test-access-key");
+        assert_eq!(credentials.secret_key, "test-secret-key");
+        assert_eq!(credentials.session_token, Some("test-session-token".to_string()));
+        assert!(credentials.expiration.is_some());
+
+        // Verify latency statistics
+        assert_eq!(target.latency.curr, Duration::from_millis(100));
+        assert_eq!(target.latency.avg, Duration::from_millis(150));
+        assert_eq!(target.latency.max, Duration::from_millis(300));
+
+        // Verify time fields
+        assert!(target.reset_before_date.is_none());
+        assert!(target.last_online.is_none());
+    }
+
+    #[test]
+    fn test_bucket_target_json_serialize_deserialize_roundtrip() {
+        let original = BucketTarget {
+            source_bucket: "test-source".to_string(),
+            endpoint: "rustfs.example.com".to_string(),
+            credentials: Some(Credentials {
+                access_key: "rustfsaccess".to_string(),
+                secret_key: "rustfssecret".to_string(),
+                session_token: None,
+                expiration: None,
+            }),
+            target_bucket: "test-target".to_string(),
+            secure: false,
+            path: "/".to_string(),
+            api: "s3v4".to_string(),
+            arn: "arn:rustfs:s3:::test-target".to_string(),
+            target_type: BucketTargetType::ReplicationService,
+            region: "us-west-2".to_string(),
+            bandwidth_limit: 500000,
+            replication_sync: false,
+            storage_class: "REDUCED_REDUNDANCY".to_string(),
+            health_check_duration: Duration::from_secs(60),
+            disable_proxy: true,
+            reset_before_date: Some(OffsetDateTime::now_utc()),
+            reset_id: "reset-456".to_string(),
+            total_downtime: Duration::from_secs(1800),
+            last_online: Some(OffsetDateTime::now_utc()),
+            online: false,
+            latency: LatencyStat {
+                curr: Duration::from_millis(250),
+                avg: Duration::from_millis(200),
+                max: Duration::from_millis(500),
+            },
+            deployment_id: "deploy-789".to_string(),
+            edge: true,
+            edge_sync_before_expiry: false,
+            offline_count: 10,
+        };
+
+        // Serialize to JSON
+        let json = serde_json::to_string(&original).expect("Failed to serialize to JSON");
+
+        // Deserialize from JSON
+        let deserialized: BucketTarget = serde_json::from_str(&json).expect("Failed to deserialize from JSON");
+
+        // Verify key fields are equal
+        assert_eq!(original.source_bucket, deserialized.source_bucket);
+        assert_eq!(original.endpoint, deserialized.endpoint);
+        assert_eq!(original.target_bucket, deserialized.target_bucket);
+        assert_eq!(original.secure, deserialized.secure);
+        assert_eq!(original.target_type, deserialized.target_type);
+        assert_eq!(original.region, deserialized.region);
+        assert_eq!(original.bandwidth_limit, deserialized.bandwidth_limit);
+        assert_eq!(original.replication_sync, deserialized.replication_sync);
+        assert_eq!(original.health_check_duration, deserialized.health_check_duration);
+        assert_eq!(original.online, deserialized.online);
+        assert_eq!(original.edge, deserialized.edge);
+        assert_eq!(original.offline_count, deserialized.offline_count);
+    }
+
+    #[test]
+    fn test_bucket_target_type_json_deserialize() {
+        // Test BucketTargetType JSON deserialization
+        let replication_json = r#""replication""#;
+        let ilm_json = r#""ilm""#;
+
+        let replication_type: BucketTargetType =
+            serde_json::from_str(replication_json).expect("Failed to deserialize replication type");
+        let ilm_type: BucketTargetType = serde_json::from_str(ilm_json).expect("Failed to deserialize ilm type");
+
+        assert_eq!(replication_type, BucketTargetType::ReplicationService);
+        assert_eq!(ilm_type, BucketTargetType::IlmService);
+
+        // Verify type validity
+        assert!(replication_type.is_valid());
+        assert!(ilm_type.is_valid());
+        assert!(!BucketTargetType::None.is_valid());
+    }
+
+    #[test]
+    fn test_credentials_json_deserialize() {
+        let json = r#"
+        {
+            "accessKey": "AKIAIOSFODNN7EXAMPLE",
+            "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+            "session_token": "AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT",
+            "expiration": "2024-12-31T23:59:59Z"
+        }
+        "#;
+
+        let credentials: Credentials = serde_json::from_str(json).expect("Failed to deserialize credentials");
+
+        assert_eq!(credentials.access_key, "AKIAIOSFODNN7EXAMPLE");
+        assert_eq!(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+        assert_eq!(
+            credentials.session_token,
+            Some("AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT".to_string())
+        );
+        assert!(credentials.expiration.is_some());
+    }
+
+    #[test]
+    fn test_latency_stat_json_deserialize() {
+        let json = r#"
+        {
+            "curr": 50,
+            "avg": 75,
+            "max": 200
+        }
+        "#;
+
+        let latency: LatencyStat = serde_json::from_str(json).expect("Failed to deserialize latency stat");
+
+        assert_eq!(latency.curr, Duration::from_millis(50));
+        assert_eq!(latency.avg, Duration::from_millis(75));
+        assert_eq!(latency.max, Duration::from_millis(200));
+    }
+
+    #[test]
+    fn test_bucket_targets_json_deserialize() {
+        let json = r#"
+        {
+            "targets": [
+                {
+                    "sourcebucket": "bucket1",
+                    "endpoint": "s3.amazonaws.com",
+                    "targetbucket": "target1",
+                    "secure": true,
+                    "path": "/",
+                    "api": "s3v4",
+                    "arn": "arn:aws:s3:::target1",
+                    "type": "replication",
+                    "region": "us-east-1",
+                    "bandwidth_limit": 0,
+                    "replicationSync": false,
+                    "storage_class": "",
+                    "healthCheckDuration": 0,
+                    "disableProxy": false,
+                    "resetBeforeDate": null,
+                    "reset_id": "",
+                    "totalDowntime": 0,
+                    "lastOnline": null,
+                    "isOnline": false,
+                    "latency": {
+                        "curr": 0,
+                        "avg": 0,
+                        "max": 0
+                    },
+                    "deployment_id": "",
+                    "edge": false,
+                    "edgeSyncBeforeExpiry": false,
+                    "offlineCount": 0
+                }
+            ]
+        }
+        "#;
+
+        let targets: BucketTargets = serde_json::from_str(json).expect("Failed to deserialize bucket targets");
+
+        assert_eq!(targets.targets.len(), 1);
+        assert_eq!(targets.targets[0].source_bucket, "bucket1");
+        assert_eq!(targets.targets[0].endpoint, "s3.amazonaws.com");
+        assert_eq!(targets.targets[0].target_bucket, "target1");
+        assert!(!targets.is_empty());
+    }
+
+    #[test]
+    fn test_user_provided_json_deserialize() {
+        // Test the specific JSON provided by the user with missing required fields added
+        let json = r#"
+        {
+            "sourcebucket": "mc-test-bucket-22139",
+            "endpoint": "localhost:8000",
+            "credentials": {
+                "accessKey": "rustfsadmin",
+                "secretKey": "rustfsadmin",
+                "expiration": "0001-01-01T00:00:00Z"
+            },
+            "targetbucket": "test",
+            "secure": false,
+            "path": "auto",
+            "api": "s3v4",
+            "type": "replication",
+            "replicationSync": false,
+            "healthCheckDuration": 60,
+            "disableProxy": false,
+            "resetBeforeDate": "0001-01-01T00:00:00Z",
+            "totalDowntime": 0,
+            "lastOnline": "0001-01-01T00:00:00Z",
+            "isOnline": false,
+            "latency": {
+                "curr": 0,
+                "avg": 0,
+                "max": 0
+            },
+            "deployment_id": "",
+            "edge": false,
+            "edgeSyncBeforeExpiry": false,
+            "offlineCount": 0,
+            "bandwidth": 107374182400
+        }
+        "#;
+
+        let target: BucketTarget = serde_json::from_str(json).expect("Failed to deserialize user provided JSON to BucketTarget");
+
+        // Verify the deserialized values match the original JSON
+        assert_eq!(target.source_bucket, "mc-test-bucket-22139");
+        assert_eq!(target.endpoint, "localhost:8000");
+        assert_eq!(target.target_bucket, "test");
+        assert!(!target.secure);
+        assert_eq!(target.path, "auto");
+        assert_eq!(target.api, "s3v4");
+        assert_eq!(target.target_type, BucketTargetType::ReplicationService);
+        assert!(!target.replication_sync);
+        assert_eq!(target.health_check_duration, Duration::from_secs(60));
+        assert!(!target.disable_proxy);
+        assert!(!target.online);
+        assert!(!target.edge);
+        assert!(!target.edge_sync_before_expiry);
+        assert_eq!(target.bandwidth_limit, 107374182400); // bandwidth field mapped to bandwidth_limit
+
+        // Verify credentials
+        assert!(target.credentials.is_some());
+        let credentials = target.credentials.unwrap();
+        assert_eq!(credentials.access_key, "rustfsadmin");
+        assert_eq!(credentials.secret_key, "rustfsadmin");
+
+        // Verify latency statistics
+        assert_eq!(target.latency.curr, Duration::from_millis(0));
+        assert_eq!(target.latency.avg, Duration::from_millis(0));
+        assert_eq!(target.latency.max, Duration::from_millis(0));
+
+        // Verify time fields parsing (should handle "0001-01-01T00:00:00Z" as None due to being the zero time)
+        assert!(target.reset_before_date.is_some());
+        assert!(target.last_online.is_some());
+
+        println!("✅ User provided JSON successfully deserialized to BucketTarget");
+    }
+
+    #[test]
+    fn test_user_provided_json_as_bucket_targets() {
+        // Test wrapping the user JSON in BucketTargets structure
+        let json = r#"
+        {
+            "targets": [
+                {
+                    "sourcebucket": "mc-test-bucket-22139",
+                    "endpoint": "localhost:8000",
+                    "credentials": {
+                        "accessKey": "rustfsadmin",
+                        "secretKey": "rustfsadmin",
+                        "expiration": "0001-01-01T00:00:00Z"
+                    },
+                    "targetbucket": "test",
+                    "secure": false,
+                    "path": "auto",
+                    "api": "s3v4",
+                    "arn": "",
+                    "type": "replication",
+                    "region": "",
+                    "replicationSync": false,
+                    "storage_class": "",
+                    "healthCheckDuration": 60,
+                    "disableProxy": false,
+                    "resetBeforeDate": "0001-01-01T00:00:00Z",
+                    "reset_id": "",
+                    "totalDowntime": 0,
+                    "lastOnline": "0001-01-01T00:00:00Z",
+                    "isOnline": false,
+                    "latency": {
+                        "curr": 0,
+                        "avg": 0,
+                        "max": 0
+                    },
+                    "deployment_id": "",
+                    "edge": false,
+                    "edgeSyncBeforeExpiry": false,
+                    "offlineCount": 0,
+                    "bandwidth": 107374182400
+                }
+            ]
+        }
+        "#;
+
+        let bucket_targets: BucketTargets =
+            serde_json::from_str(json).expect("Failed to deserialize user provided JSON to BucketTargets");
+
+        assert_eq!(bucket_targets.targets.len(), 1);
+        assert!(!bucket_targets.is_empty());
+
+        let target = &bucket_targets.targets[0];
+        assert_eq!(target.source_bucket, "mc-test-bucket-22139");
+        assert_eq!(target.endpoint, "localhost:8000");
+        assert_eq!(target.target_bucket, "test");
+        assert_eq!(target.bandwidth_limit, 107374182400);
+
+        println!("✅ User provided JSON successfully deserialized to BucketTargets");
+    }
+
+    #[test]
+    fn test_bucket_target_minimal_json_with_defaults() {
+        // Test that BucketTarget can be deserialized with minimal JSON using defaults
+        let minimal_json = r#"
+        {
+            "sourcebucket": "test-source",
+            "endpoint": "localhost:9000",
+            "targetbucket": "test-target"
+        }
+        "#;
+
+        let target: BucketTarget =
+            serde_json::from_str(minimal_json).expect("Failed to deserialize minimal JSON to BucketTarget");
+
+        // Verify required fields
+        assert_eq!(target.source_bucket, "test-source");
+        assert_eq!(target.endpoint, "localhost:9000");
+        assert_eq!(target.target_bucket, "test-target");
+
+        // Verify default values
+        assert!(!target.secure); // bool default is false
+        assert_eq!(target.path, ""); // String default is empty
+        assert_eq!(target.api, ""); // String default is empty
+        assert_eq!(target.arn, ""); // String default is empty
+        assert_eq!(target.target_type, BucketTargetType::None); // enum default
+        assert_eq!(target.region, ""); // String default is empty
+        assert_eq!(target.bandwidth_limit, 0); // i64 default is 0
+        assert!(!target.replication_sync); // bool default is false
+        assert_eq!(target.storage_class, ""); // String default is empty
+        assert_eq!(target.health_check_duration, Duration::from_secs(0)); // Duration default
+        assert!(!target.disable_proxy); // bool default is false
+        assert!(target.reset_before_date.is_none()); // Option default is None
+        assert_eq!(target.reset_id, ""); // String default is empty
+        assert_eq!(target.total_downtime, Duration::from_secs(0)); // Duration default
+        assert!(target.last_online.is_none()); // Option default is None
+        assert!(!target.online); // bool default is false
+        assert_eq!(target.latency.curr, Duration::from_millis(0)); // LatencyStat default
+        assert_eq!(target.latency.avg, Duration::from_millis(0));
+        assert_eq!(target.latency.max, Duration::from_millis(0));
+        assert_eq!(target.deployment_id, ""); // String default is empty
+        assert!(!target.edge); // bool default is false
+        assert!(!target.edge_sync_before_expiry); // bool default is false
+        assert_eq!(target.offline_count, 0); // u64 default is 0
+        assert!(target.credentials.is_none()); // Option default is None
+
+        println!("✅ Minimal JSON with defaults successfully deserialized to BucketTarget");
+    }
+
+    #[test]
+    fn test_bucket_target_empty_json_with_defaults() {
+        // Test that BucketTarget can be deserialized with completely empty JSON using all defaults
+        let empty_json = r#"{}"#;
+
+        let target: BucketTarget = serde_json::from_str(empty_json).expect("Failed to deserialize empty JSON to BucketTarget");
+
+        // Verify all fields use default values
+        assert_eq!(target.source_bucket, "");
+        assert_eq!(target.endpoint, "");
+        assert_eq!(target.target_bucket, "");
+        assert!(!target.secure);
+        assert_eq!(target.path, "");
+        assert_eq!(target.api, "");
+        assert_eq!(target.arn, "");
+        assert_eq!(target.target_type, BucketTargetType::None);
+        assert_eq!(target.region, "");
+        assert_eq!(target.bandwidth_limit, 0);
+        assert!(!target.replication_sync);
+        assert_eq!(target.storage_class, "");
+        assert_eq!(target.health_check_duration, Duration::from_secs(0));
+        assert!(!target.disable_proxy);
+        assert!(target.reset_before_date.is_none());
+        assert_eq!(target.reset_id, "");
+        assert_eq!(target.total_downtime, Duration::from_secs(0));
+        assert!(target.last_online.is_none());
+        assert!(!target.online);
+        assert_eq!(target.latency.curr, Duration::from_millis(0));
+        assert_eq!(target.latency.avg, Duration::from_millis(0));
+        assert_eq!(target.latency.max, Duration::from_millis(0));
+        assert_eq!(target.deployment_id, "");
+        assert!(!target.edge);
+        assert!(!target.edge_sync_before_expiry);
+        assert_eq!(target.offline_count, 0);
+        assert!(target.credentials.is_none());
+
+        println!("✅ Empty JSON with all defaults successfully deserialized to BucketTarget");
+    }
+
+    #[test]
+    fn test_original_user_json_with_defaults() {
+        // Test the original user JSON without extra required fields
+        let json = r#"
+        {
+            "sourcebucket": "mc-test-bucket-22139",
+            "endpoint": "localhost:8000",
+            "credentials": {
+                "accessKey": "rustfsadmin",
+                "secretKey": "rustfsadmin",
+                "expiration": "0001-01-01T00:00:00Z"
+            },
+            "targetbucket": "test",
+            "secure": false,
+            "path": "auto",
+            "api": "s3v4",
+            "type": "replication",
+            "replicationSync": false,
+            "healthCheckDuration": 60,
+            "disableProxy": false,
+            "resetBeforeDate": "0001-01-01T00:00:00Z",
+            "totalDowntime": 0,
+            "lastOnline": "0001-01-01T00:00:00Z",
+            "isOnline": false,
+            "latency": {
+                "curr": 0,
+                "avg": 0,
+                "max": 0
+            },
+            "edge": false,
+            "edgeSyncBeforeExpiry": false,
+            "bandwidth": 107374182400
+        }
+        "#;
+
+        let target: BucketTarget = serde_json::from_str(json).expect("Failed to deserialize original user JSON to BucketTarget");
+
+        // Verify the deserialized values
+        assert_eq!(target.source_bucket, "mc-test-bucket-22139");
+        assert_eq!(target.endpoint, "localhost:8000");
+        assert_eq!(target.target_bucket, "test");
+        assert!(!target.secure);
+        assert_eq!(target.path, "auto");
+        assert_eq!(target.api, "s3v4");
+        assert_eq!(target.target_type, BucketTargetType::ReplicationService);
+        assert!(!target.replication_sync);
+        assert_eq!(target.health_check_duration, Duration::from_secs(60));
+        assert!(!target.disable_proxy);
+        assert!(!target.online);
+        assert!(!target.edge);
+        assert!(!target.edge_sync_before_expiry);
+        assert_eq!(target.bandwidth_limit, 107374182400);
+
+        // Fields not specified should use defaults
+        assert_eq!(target.arn, ""); // default empty string
+        assert_eq!(target.region, ""); // default empty string
+        assert_eq!(target.storage_class, ""); // default empty string
+        assert_eq!(target.reset_id, ""); // default empty string
+        assert_eq!(target.deployment_id, ""); // default empty string
+        assert_eq!(target.offline_count, 0); // default u64
+
+        // Verify credentials
+        assert!(target.credentials.is_some());
+        let credentials = target.credentials.unwrap();
+        assert_eq!(credentials.access_key, "rustfsadmin");
+        assert_eq!(credentials.secret_key, "rustfsadmin");
+
+        println!("✅ Original user JSON with defaults successfully deserialized to BucketTarget");
+    }
+}
@@ -12,124 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::error::Result;
-use rmp_serde::Serializer as rmpSerializer;
-use serde::{Deserialize, Serialize};
-use time::OffsetDateTime;
+mod arn;
+mod bucket_target;

-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct Credentials {
-    #[serde(rename = "accessKey")]
-    pub access_key: String,
-    #[serde(rename = "secretKey")]
-    pub secret_key: String,
-    pub session_token: Option<String>,
-    pub expiration: Option<chrono::DateTime<chrono::Utc>>,
-}
-
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub enum ServiceType {
-    #[default]
-    Replication,
-}
-
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct LatencyStat {
-    curr: u64, // current latency
-    avg: u64,  // average latency
-    max: u64,  // maximum latency
-}
-
-// Define BucketTarget struct
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct BucketTarget {
-    #[serde(rename = "sourcebucket")]
-    pub source_bucket: String,
-
-    pub endpoint: String,
-
-    pub credentials: Option<Credentials>,
-    #[serde(rename = "targetbucket")]
-    pub target_bucket: String,
-
-    secure: bool,
-    pub path: Option<String>,
-
-    api: Option<String>,
-
-    pub arn: Option<String>,
-    #[serde(rename = "type")]
-    pub type_: Option<String>,
-
-    pub region: Option<String>,
-
-    bandwidth_limit: Option<i64>,
-
-    #[serde(rename = "replicationSync")]
-    replication_sync: bool,
-
-    storage_class: Option<String>,
-    #[serde(rename = "healthCheckDuration")]
-    health_check_duration: u64,
-    #[serde(rename = "disableProxy")]
-    disable_proxy: bool,
-
-    #[serde(rename = "resetBeforeDate")]
-    reset_before_date: String,
-    reset_id: Option<String>,
-    #[serde(rename = "totalDowntime")]
-    total_downtime: u64,
-
-    last_online: Option<OffsetDateTime>,
-    #[serde(rename = "isOnline")]
-    online: bool,
-
-    latency: Option<LatencyStat>,
-
-    deployment_id: Option<String>,
-
-    edge: bool,
-    #[serde(rename = "edgeSyncBeforeExpiry")]
-    edge_sync_before_expiry: bool,
-}
-
-impl BucketTarget {
-    pub fn is_empty(self) -> bool {
-        //self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_empty()
-        self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_none()
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct BucketTargets {
-    pub targets: Vec<BucketTarget>,
-}
-
-impl BucketTargets {
-    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
-        let mut buf = Vec::new();
-
-        self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
-
-        Ok(buf)
-    }
-
-    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
-        let t: BucketTargets = rmp_serde::from_slice(buf)?;
-        Ok(t)
-    }
-
-    pub fn is_empty(&self) -> bool {
-        if self.targets.is_empty() {
-            return true;
-        }
-
-        for target in &self.targets {
-            if !target.clone().is_empty() {
-                return false;
-            }
-        }
-
-        true
-    }
-}
+pub use arn::*;
+pub use bucket_target::*;
@@ -17,7 +17,8 @@ use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
 use futures::future::join_all;
 use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
 use std::{future::Future, pin::Pin, sync::Arc};
-use tokio::{spawn, sync::broadcast::Receiver as B_Receiver};
+use tokio::spawn;
+use tokio_util::sync::CancellationToken;
 use tracing::{error, warn};

 pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
@@ -63,7 +64,7 @@ impl Clone for ListPathRawOptions {
    }
 }

-pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -> disk::error::Result<()> {
+pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
    if opts.disks.is_empty() {
        return Err(DiskError::other("list_path_raw: 0 drives provided"));
    }
@@ -72,13 +73,13 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
    let mut readers = Vec::with_capacity(opts.disks.len());
    let fds = Arc::new(opts.fallback_disks.clone());

-    let (cancel_tx, cancel_rx) = tokio::sync::broadcast::channel::<bool>(1);
+    let cancel_rx = CancellationToken::new();

    for disk in opts.disks.iter() {
        let opdisk = disk.clone();
        let opts_clone = opts.clone();
        let fds_clone = fds.clone();
-        let mut cancel_rx_clone = cancel_rx.resubscribe();
+        let cancel_rx_clone = cancel_rx.clone();
        let (rd, mut wr) = tokio::io::duplex(64);
        readers.push(MetacacheReader::new(rd));
        jobs.push(spawn(async move {
@@ -106,7 +107,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
                need_fallback = true;
            }

-            if cancel_rx_clone.try_recv().is_ok() {
+            if cancel_rx_clone.is_cancelled() {
                // warn!("list_path_raw: cancel_rx_clone.try_recv().await.is_ok()");
                return Ok(());
            }
@@ -173,7 +174,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
            //     opts.bucket, opts.path, &current.name
            // );

-            if rx.try_recv().is_ok() {
+            if rx.is_cancelled() {
                return Err(DiskError::other("canceled"));
            }

@@ -351,7 +352,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -

    if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
        error!("list_path_raw: revjob err {:?}", err);
-        let _ = cancel_tx.send(true);
+        cancel_rx.cancel();

        return Err(err);
    }
@@ -44,6 +44,8 @@ pub struct GetObjectOptions {
    pub internal: AdvancedGetOptions,
 }

+pub type StatObjectOptions = GetObjectOptions;
+
 impl Default for GetObjectOptions {
    fn default() -> Self {
        Self {
@@ -30,7 +30,8 @@ use s3s::header::{
    X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
 };
 //use crate::disk::{BufferReader, Reader};
-use crate::checksum::ChecksumMode;
+use crate::client::checksum::ChecksumMode;
+use crate::client::utils::base64_encode;
 use crate::client::{
    api_error_response::{err_entity_too_large, err_invalid_argument},
    api_put_object_common::optimal_part_info,
@@ -41,7 +42,6 @@ use crate::client::{
    transition_api::{ReaderImpl, TransitionClient, UploadInfo},
    utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
 };
-use rustfs_utils::crypto::base64_encode;

 #[derive(Debug, Clone)]
 pub struct AdvancedPutOptions {
@@ -25,7 +25,8 @@ use time::OffsetDateTime;
 use tracing::warn;
 use uuid::Uuid;

-use crate::checksum::ChecksumMode;
+use crate::client::checksum::ChecksumMode;
+use crate::client::utils::base64_encode;
 use crate::client::{
    api_error_response::{
        err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
@@ -38,7 +39,7 @@ use crate::client::{
    constants::{ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
 };
-use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+use rustfs_utils::path::trim_etag;
 use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};

 impl TransitionClient {
@@ -29,7 +29,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::warn;
 use uuid::Uuid;

-use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
+use crate::client::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
 use crate::client::{
    api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
    api_put_object::PutObjectOptions,
@@ -40,7 +40,8 @@ use crate::client::{
    transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
 };

-use rustfs_utils::{crypto::base64_encode, path::trim_etag};
+use crate::client::utils::base64_encode;
+use rustfs_utils::path::trim_etag;
 use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};

 pub struct UploadedPartRes {
@@ -20,7 +20,7 @@

 use bytes::Bytes;
 use http::{HeaderMap, HeaderValue, Method, StatusCode};
-use rustfs_utils::{HashAlgorithm, crypto::base64_encode};
+use rustfs_utils::HashAlgorithm;
 use s3s::S3ErrorCode;
 use s3s::dto::ReplicationStatus;
 use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
@@ -29,6 +29,7 @@ use std::{collections::HashMap, sync::Arc};
 use time::OffsetDateTime;
 use tokio::sync::mpsc::{self, Receiver, Sender};

+use crate::client::utils::base64_encode;
 use crate::client::{
    api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
@@ -46,11 +47,11 @@ pub struct RemoveBucketOptions {
 #[derive(Debug)]
 #[allow(dead_code)]
 pub struct AdvancedRemoveOptions {
-    replication_delete_marker: bool,
-    replication_status: ReplicationStatus,
-    replication_mtime: OffsetDateTime,
-    replication_request: bool,
-    replication_validity_check: bool,
+    pub replication_delete_marker: bool,
+    pub replication_status: ReplicationStatus,
+    pub replication_mtime: Option<OffsetDateTime>,
+    pub replication_request: bool,
+    pub replication_validity_check: bool,
 }

 impl Default for AdvancedRemoveOptions {
@@ -58,7 +59,7 @@ impl Default for AdvancedRemoveOptions {
        Self {
            replication_delete_marker: false,
            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
-            replication_mtime: OffsetDateTime::now_utc(),
+            replication_mtime: None,
            replication_request: false,
            replication_validity_check: false,
        }
@@ -140,8 +141,7 @@ impl TransitionClient {
    }

    pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
-        let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err");
-        res.err
+        self.remove_object_inner(bucket_name, object_name, opts).await.err()
    }

    pub async fn remove_object_inner(
@@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use time::OffsetDateTime;

-use crate::checksum::ChecksumMode;
+use crate::client::checksum::ChecksumMode;
 use crate::client::transition_api::ObjectMultipartInfo;
-use rustfs_utils::crypto::base64_decode;
+use crate::client::utils::base64_decode;

 use super::transition_api;

@@ -23,6 +23,7 @@ use http::{HeaderMap, HeaderValue};
 use rustfs_utils::EMPTY_STRING_SHA256_HASH;
 use std::{collections::HashMap, str::FromStr};
 use tokio::io::BufReader;
+use tracing::warn;
 use uuid::Uuid;

 use crate::client::{
@@ -30,7 +31,10 @@ use crate::client::{
    api_get_options::GetObjectOptions,
    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
 };
-use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
+use s3s::{
+    dto::VersioningConfiguration,
+    header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID},
+};

 impl TransitionClient {
    pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> {
@@ -58,8 +62,14 @@ impl TransitionClient {
            .await;

        if let Ok(resp) = resp {
+            if resp.status() != http::StatusCode::OK {
+                return Ok(false);
+            }
+
            let b = resp.body().bytes().expect("err").to_vec();
            let resperr = http_resp_to_error_response(&resp, b, bucket_name, "");
+
+            warn!("bucket exists, resp: {:?}, resperr: {:?}", resp, resperr);
            /*if to_error_response(resperr).code == "NoSuchBucket" {
                return Ok(false);
            }
@@ -70,6 +80,46 @@ impl TransitionClient {
        Ok(true)
    }

+    pub async fn get_bucket_versioning(&self, bucket_name: &str) -> Result<VersioningConfiguration, std::io::Error> {
+        let mut query_values = HashMap::new();
+        query_values.insert("versioning".to_string(), "".to_string());
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values,
+                    custom_header: HeaderMap::new(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_md5_base64: "".to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await;
+
+        match resp {
+            Ok(resp) => {
+                let b = resp.body().bytes().expect("get bucket versioning err").to_vec();
+                let resperr = http_resp_to_error_response(&resp, b, bucket_name, "");
+
+                warn!("get bucket versioning, resp: {:?}, resperr: {:?}", resp, resperr);
+
+                Ok(VersioningConfiguration::default())
+            }
+
+            Err(err) => Err(std::io::Error::other(err)),
+        }
+    }
+
    pub async fn stat_object(
        &self,
        bucket_name: &str,
@@ -131,24 +181,20 @@ impl TransitionClient {
                            ..Default::default()
                        };
                        return Ok(ObjectInfo {
-                            version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
-                                Ok(v) => v,
-                                Err(e) => {
-                                    return Err(std::io::Error::other(e));
-                                }
-                            },
+                            version_id: h
+                                .get(X_AMZ_VERSION_ID)
+                                .and_then(|v| v.to_str().ok())
+                                .and_then(|s| Uuid::from_str(s).ok()),
                            is_delete_marker: delete_marker,
                            ..Default::default()
                        });
                        //err_resp
                    }
                    return Ok(ObjectInfo {
-                        version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
-                            Ok(v) => v,
-                            Err(e) => {
-                                return Err(std::io::Error::other(e));
-                            }
-                        },
+                        version_id: h
+                            .get(X_AMZ_VERSION_ID)
+                            .and_then(|v| v.to_str().ok())
+                            .and_then(|s| Uuid::from_str(s).ok()),
                        is_delete_marker: delete_marker,
                        replication_ready: replication_ready,
                        ..Default::default()
@@ -36,6 +36,7 @@ use s3s::S3ErrorCode;
 use super::constants::UNSIGNED_PAYLOAD;
 use super::credentials::SignatureType;

+#[derive(Debug, Clone)]
 pub struct BucketLocationCache {
    items: HashMap<String, String>,
 }
@@ -0,0 +1,351 @@
+#![allow(clippy::map_entry)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+#![allow(unused_mut)]
+#![allow(unused_assignments)]
+#![allow(unused_must_use)]
+#![allow(clippy::all)]
+
+use lazy_static::lazy_static;
+use rustfs_checksums::ChecksumAlgorithm;
+use std::collections::HashMap;
+
+use crate::client::utils::base64_decode;
+use crate::client::utils::base64_encode;
+use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
+use crate::{disk::DiskAPI, store_api::GetObjectReader};
+use s3s::header::{
+    X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
+};
+
+use enumset::{EnumSet, EnumSetType, enum_set};
+
+#[derive(Debug, EnumSetType, Default)]
+#[enumset(repr = "u8")]
+pub enum ChecksumMode {
+    #[default]
+    ChecksumNone,
+    ChecksumSHA256,
+    ChecksumSHA1,
+    ChecksumCRC32,
+    ChecksumCRC32C,
+    ChecksumCRC64NVME,
+    ChecksumFullObject,
+}
+
+lazy_static! {
+    static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
+        let mut s = EnumSet::all();
+        s.remove(ChecksumMode::ChecksumFullObject);
+        s
+    };
+    static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
+    static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
+        enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
+}
+const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
+
+impl ChecksumMode {
+    //pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
+
+    pub fn base(&self) -> ChecksumMode {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            1_u8 => ChecksumMode::ChecksumNone,
+            2_u8 => ChecksumMode::ChecksumSHA256,
+            4_u8 => ChecksumMode::ChecksumSHA1,
+            8_u8 => ChecksumMode::ChecksumCRC32,
+            16_u8 => ChecksumMode::ChecksumCRC32C,
+            32_u8 => ChecksumMode::ChecksumCRC64NVME,
+            _ => panic!("enum err."),
+        }
+    }
+
+    pub fn is(&self, t: ChecksumMode) -> bool {
+        *self & t == t
+    }
+
+    pub fn key(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return X_AMZ_CHECKSUM_CRC32.to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return X_AMZ_CHECKSUM_CRC32C.to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return X_AMZ_CHECKSUM_SHA1.to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return X_AMZ_CHECKSUM_SHA256.to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return AMZ_CHECKSUM_CRC64NVME.to_string();
+            }
+            _ => {
+                return "".to_string();
+            }
+        }
+    }
+
+    pub fn can_composite(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            2_u8 => true,
+            4_u8 => true,
+            8_u8 => true,
+            16_u8 => true,
+            _ => false,
+        }
+    }
+
+    pub fn can_merge_crc(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            8_u8 => true,
+            16_u8 => true,
+            32_u8 => true,
+            _ => false,
+        }
+    }
+
+    pub fn full_object_requested(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        match s.as_u8() {
+            //C_ChecksumFullObjectCRC32 as u8 => true,
+            //C_ChecksumFullObjectCRC32C as u8 => true,
+            32_u8 => true,
+            _ => false,
+        }
+    }
+
+    pub fn key_capitalized(&self) -> String {
+        self.key()
+    }
+
+    pub fn raw_byte_len(&self) -> usize {
+        let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
+        if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
+            4
+        } else if u == ChecksumMode::ChecksumSHA1 as u8 {
+            use sha1::Digest;
+            sha1::Sha1::output_size() as usize
+        } else if u == ChecksumMode::ChecksumSHA256 as u8 {
+            use sha2::Digest;
+            sha2::Sha256::output_size() as usize
+        } else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
+            8
+        } else {
+            0
+        }
+    }
+
+    pub fn hasher(&self) -> Result<Box<dyn rustfs_checksums::http::HttpChecksum>, std::io::Error> {
+        match /*C_ChecksumMask & **/self {
+            ChecksumMode::ChecksumCRC32 => {
+                return Ok(ChecksumAlgorithm::Crc32.into_impl());
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return Ok(ChecksumAlgorithm::Crc32c.into_impl());
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return Ok(ChecksumAlgorithm::Sha1.into_impl());
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return Ok(ChecksumAlgorithm::Sha256.into_impl());
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return Ok(ChecksumAlgorithm::Crc64Nvme.into_impl());
+            }
+            _ => return Err(std::io::Error::other("unsupported checksum type")),
+        }
+    }
+
+    pub fn is_set(&self) -> bool {
+        let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
+        s.len() == 1
+    }
+
+    pub fn set_default(&mut self, t: ChecksumMode) {
+        if !self.is_set() {
+            *self = t;
+        }
+    }
+
+    pub fn encode_to_string(&self, b: &[u8]) -> Result<String, std::io::Error> {
+        if !self.is_set() {
+            return Ok("".to_string());
+        }
+        let mut h = self.hasher()?;
+        h.update(b);
+        let hash = h.finalize();
+        Ok(base64_encode(hash.as_ref()))
+    }
+
+    pub fn to_string(&self) -> String {
+        //match c & checksumMask {
+        match self {
+            ChecksumMode::ChecksumCRC32 => {
+                return "CRC32".to_string();
+            }
+            ChecksumMode::ChecksumCRC32C => {
+                return "CRC32C".to_string();
+            }
+            ChecksumMode::ChecksumSHA1 => {
+                return "SHA1".to_string();
+            }
+            ChecksumMode::ChecksumSHA256 => {
+                return "SHA256".to_string();
+            }
+            ChecksumMode::ChecksumNone => {
+                return "".to_string();
+            }
+            ChecksumMode::ChecksumCRC64NVME => {
+                return "CRC64NVME".to_string();
+            }
+            _ => {
+                return "<invalid>".to_string();
+            }
+        }
+    }
+
+    // pub fn check_sum_reader(&self, r: GetObjectReader) -> Result<Checksum, std::io::Error> {
+    //     let mut h = self.hasher()?;
+    //     Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    // }
+
+    // pub fn check_sum_bytes(&self, b: &[u8]) -> Result<Checksum, std::io::Error> {
+    //     let mut h = self.hasher()?;
+    //     Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
+    // }
+
+    pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        if !self.can_composite() {
+            return Err(std::io::Error::other("cannot do composite checksum"));
+        }
+        p.sort_by(|i, j| {
+            if i.part_num < j.part_num {
+                std::cmp::Ordering::Less
+            } else if i.part_num > j.part_num {
+                std::cmp::Ordering::Greater
+            } else {
+                std::cmp::Ordering::Equal
+            }
+        });
+        let c = self.base();
+        let crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
+        let mut h = self.hasher()?;
+        h.update(crc_bytes.as_ref());
+        let hash = h.finalize();
+        Ok(Checksum {
+            checksum_type: self.clone(),
+            r: hash.as_ref().to_vec(),
+            computed: false,
+        })
+    }
+
+    pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
+        todo!();
+    }
+}
+
+#[derive(Default)]
+pub struct Checksum {
+    checksum_type: ChecksumMode,
+    r: Vec<u8>,
+    computed: bool,
+}
+
+#[allow(dead_code)]
+impl Checksum {
+    fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Checksum {
+                checksum_type: t,
+                r: b.to_vec(),
+                computed: false,
+            };
+        }
+        Checksum::default()
+    }
+
+    #[allow(dead_code)]
+    fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
+        let b = match base64_decode(s.as_bytes()) {
+            Ok(b) => b,
+            Err(err) => return Err(std::io::Error::other(err.to_string())),
+        };
+        if t.is_set() && b.len() == t.raw_byte_len() {
+            return Ok(Checksum {
+                checksum_type: t,
+                r: b,
+                computed: false,
+            });
+        }
+        Ok(Checksum::default())
+    }
+
+    fn is_set(&self) -> bool {
+        self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len()
+    }
+
+    fn encoded(&self) -> String {
+        if !self.is_set() {
+            return "".to_string();
+        }
+        base64_encode(&self.r)
+    }
+
+    #[allow(dead_code)]
+    fn raw(&self) -> Option<Vec<u8>> {
+        if !self.is_set() {
+            return None;
+        }
+        Some(self.r.clone())
+    }
+}
+
+pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
+    opts.user_metadata
+        .insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
+    if opts.auto_checksum.full_object_requested() {
+        opts.user_metadata
+            .insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+    }
+}
+
+pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
+    if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
+        let crc = opts.auto_checksum.composite_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key(), crc.encoded());
+            hm
+        }
+    } else if opts.auto_checksum.can_merge_crc() {
+        let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
+        opts.user_metadata = {
+            let mut hm = HashMap::new();
+            hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
+            hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
+            hm
+        }
+    }
+
+    Ok(())
+}
@@ -30,6 +30,7 @@ pub mod api_restore;
 pub mod api_s3_datatypes;
 pub mod api_stat;
 pub mod bucket_cache;
+pub mod checksum;
 pub mod constants;
 pub mod credentials;
 pub mod object_api_utils;
@@ -20,8 +20,8 @@
 #![allow(clippy::all)]

 use http::HeaderMap;
-use std::io::Cursor;
-use std::{collections::HashMap, sync::Arc};
+use s3s::dto::ETag;
+use std::{collections::HashMap, io::Cursor, sync::Arc};
 use tokio::io::BufReader;

 use crate::error::ErrorResponse;
@@ -148,10 +148,78 @@ pub fn new_getobjectreader(
    Ok((get_fn, off as i64, length as i64))
 }

-pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
-    if let Some(etag) = metadata.get("etag") {
-        etag.clone()
-    } else {
-        metadata["md5Sum"].clone()
+/// Convert a raw stored ETag into the strongly-typed `s3s::dto::ETag`.
+///
+/// Supports already quoted (`"abc"`), weak (`W/"abc"`), or plain (`abc`) values.
+pub fn to_s3s_etag(etag: &str) -> ETag {
+    if let Some(rest) = etag.strip_prefix("W/\"") {
+        if let Some(body) = rest.strip_suffix('"') {
+            return ETag::Weak(body.to_string());
+        }
+        return ETag::Weak(rest.to_string());
+    }
+
+    if let Some(body) = etag.strip_prefix('"').and_then(|rest| rest.strip_suffix('"')) {
+        return ETag::Strong(body.to_string());
+    }
+
+    ETag::Strong(etag.to_string())
+}
+
+pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
+    metadata
+        .get("etag")
+        .cloned()
+        .or_else(|| metadata.get("md5Sum").cloned())
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_to_s3s_etag() {
+        // Test unquoted ETag - should become strong etag
+        assert_eq!(
+            to_s3s_etag("6af8d12c0c74b78094884349f3c8a079"),
+            ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
+        );
+
+        assert_eq!(
+            to_s3s_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
+            ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
+        );
+
+        assert_eq!(
+            to_s3s_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
+            ETag::Weak("6af8d12c0c74b78094884349f3c8a079".to_string())
+        );
+
+        assert_eq!(to_s3s_etag(""), ETag::Strong(String::new()));
+
+        assert_eq!(to_s3s_etag("\"incomplete"), ETag::Strong("\"incomplete".to_string()));
+
+        assert_eq!(to_s3s_etag("incomplete\""), ETag::Strong("incomplete\"".to_string()));
+    }
+
+    #[test]
+    fn test_extract_etag() {
+        let mut metadata = HashMap::new();
+
+        // Test with etag field
+        metadata.insert("etag".to_string(), "abc123".to_string());
+        assert_eq!(get_raw_etag(&metadata), "abc123");
+
+        metadata.insert("etag".to_string(), "\"def456\"".to_string());
+        assert_eq!(get_raw_etag(&metadata), "\"def456\"");
+
+        // Test fallback to md5Sum
+        metadata.remove("etag");
+        metadata.insert("md5Sum".to_string(), "xyz789".to_string());
+        assert_eq!(get_raw_etag(&metadata), "xyz789");
+
+        metadata.clear();
+        assert_eq!(get_raw_etag(&metadata), "");
    }
 }
@@ -61,7 +61,7 @@ use crate::client::{
    constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
    credentials::{CredContext, Credentials, SignatureType, Static},
 };
-use crate::{checksum::ChecksumMode, store_api::GetObjectReader};
+use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
 use rustfs_rio::HashReader;
 use rustfs_utils::{
    net::get_endpoint_url,
@@ -89,6 +89,7 @@ pub enum ReaderImpl {

 pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;

+#[derive(Debug)]
 pub struct TransitionClient {
    pub endpoint_url: Url,
    pub creds_provider: Arc<Mutex<Credentials<Static>>>,
@@ -809,6 +810,7 @@ impl TransitionCore {
    }
 }

+#[derive(Debug, Clone, Default)]
 pub struct PutObjectPartOptions {
    pub md5_base64: String,
    pub sha256_hex: String,
@@ -820,23 +822,23 @@ pub struct PutObjectPartOptions {

 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct ObjectInfo {
-    pub etag: String,
+    pub etag: Option<String>,
    pub name: String,
-    pub mod_time: OffsetDateTime,
-    pub size: usize,
+    pub mod_time: Option<OffsetDateTime>,
+    pub size: i64,
    pub content_type: Option<String>,
    #[serde(skip)]
    pub metadata: HeaderMap,
    pub user_metadata: HashMap<String, String>,
    pub user_tags: String,
-    pub user_tag_count: i64,
+    pub user_tag_count: usize,
    #[serde(skip)]
    pub owner: Owner,
    //pub grant: Vec<Grant>,
    pub storage_class: String,
    pub is_latest: bool,
    pub is_delete_marker: bool,
-    pub version_id: Uuid,
+    pub version_id: Option<Uuid>,

    #[serde(skip, default = "replication_status_default")]
    pub replication_status: ReplicationStatus,
@@ -862,9 +864,9 @@ fn replication_status_default() -> ReplicationStatus {
 impl Default for ObjectInfo {
    fn default() -> Self {
        Self {
-            etag: "".to_string(),
+            etag: None,
            name: "".to_string(),
-            mod_time: OffsetDateTime::now_utc(),
+            mod_time: None,
            size: 0,
            content_type: None,
            metadata: HeaderMap::new(),
@@ -875,7 +877,7 @@ impl Default for ObjectInfo {
            storage_class: "".to_string(),
            is_latest: false,
            is_delete_marker: false,
-            version_id: Uuid::nil(),
+            version_id: None,
            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
            replication_ready: false,
            expiration: OffsetDateTime::now_utc(),
@@ -90,3 +90,11 @@ pub fn is_rustfs_header(header_key: &str) -> bool {
 pub fn is_minio_header(header_key: &str) -> bool {
    header_key.to_lowercase().starts_with("x-minio-")
 }
+
+pub fn base64_encode(input: &[u8]) -> String {
+    base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
+}
+
+pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
+    base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
+}
@@ -1,69 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use chrono::{DateTime, Utc};
-
-// Representation of the replication status
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum StatusType {
-    Pending,
-    Completed,
-    CompletedLegacy,
-    Failed,
-    Replica,
-}
-
-// Representation of version purge status type (customize as needed)
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum VersionPurgeStatusType {
-    Pending,
-    Completed,
-    Failed,
-}
-
-// ReplicationState struct definition
-#[derive(Debug, Clone)]
-pub struct ReplicationState {
-    // Timestamp when the last replica update was received
-    pub replica_time_stamp: DateTime<Utc>,
-
-    // Replica status
-    pub replica_status: StatusType,
-
-    // Represents DeleteMarker replication state
-    pub delete_marker: bool,
-
-    // Timestamp when the last replication activity happened
-    pub replication_time_stamp: DateTime<Utc>,
-
-    // Stringified representation of all replication activity
-    pub replication_status_internal: String,
-
-    // Stringified representation of all version purge statuses
-    // Example format: "arn1=PENDING;arn2=COMPLETED;"
-    pub version_purge_status_internal: String,
-
-    // Stringified representation of replication decision for each target
-    pub replicate_decision_str: String,
-
-    // Map of ARN -> replication status for ongoing replication activity
-    pub targets: HashMap<String, StatusType>,
-
-    // Map of ARN -> VersionPurgeStatus for all the targets
-    pub purge_targets: HashMap<String, VersionPurgeStatusType>,
-
-    // Map of ARN -> stringified reset id and timestamp for all the targets
-    pub reset_statuses_map: HashMap<String, String>,
-}
@@ -1,890 +0,0 @@
-#![allow(unused_variables)]
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-#![allow(dead_code)]
-use crate::{
-    StorageAPI,
-    bucket::{metadata_sys, target::BucketTarget},
-    endpoints::Node,
-    rpc::{PeerS3Client, RemotePeerS3Client},
-};
-use crate::{
-    bucket::{self, target::BucketTargets},
-    new_object_layer_fn, store_api,
-};
-//use tokio::sync::RwLock;
-use aws_sdk_s3::Client as S3Client;
-use chrono::Utc;
-use lazy_static::lazy_static;
-use std::sync::Arc;
-use std::{
-    collections::HashMap,
-    time::{Duration, SystemTime},
-};
-use thiserror::Error;
-use tokio::sync::RwLock;
-
-pub struct TClient {
-    pub s3cli: S3Client,
-    pub remote_peer_client: RemotePeerS3Client,
-    pub arn: String,
-}
-impl TClient {
-    pub fn new(s3cli: S3Client, remote_peer_client: RemotePeerS3Client, arn: String) -> Self {
-        TClient {
-            s3cli,
-            remote_peer_client,
-            arn,
-        }
-    }
-}
-
-pub struct EpHealth {
-    pub endpoint: String,
-    pub scheme: String,
-    pub online: bool,
-    pub last_online: SystemTime,
-    pub last_hc_at: SystemTime,
-    pub offline_duration: Duration,
-    pub latency: LatencyStat, // Assuming LatencyStat is a custom struct
-}
-
-impl EpHealth {
-    pub fn new(
-        endpoint: String,
-        scheme: String,
-        online: bool,
-        last_online: SystemTime,
-        last_hc_at: SystemTime,
-        offline_duration: Duration,
-        latency: LatencyStat,
-    ) -> Self {
-        EpHealth {
-            endpoint,
-            scheme,
-            online,
-            last_online,
-            last_hc_at,
-            offline_duration,
-            latency,
-        }
-    }
-}
-
-pub struct LatencyStat {
-    // Define the fields of LatencyStat as per your requirements
-}
-
-pub struct ArnTarget {
-    client: TargetClient,
-    last_refresh: chrono::DateTime<Utc>,
-}
-impl ArnTarget {
-    pub fn new(bucket: String, endpoint: String, ak: String, sk: String) -> Self {
-        Self {
-            client: TargetClient {
-                bucket,
-                storage_class: "STANDARD".to_string(),
-                disable_proxy: false,
-                health_check_duration: Duration::from_secs(100),
-                endpoint,
-                reset_id: "0".to_string(),
-                replicate_sync: false,
-                secure: false,
-                arn: "".to_string(),
-                client: reqwest::Client::new(),
-                ak,
-                sk,
-            },
-            last_refresh: Utc::now(),
-        }
-    }
-}
-
-// pub fn get_s3client_from_para(
-//     ak: &str,
-//     sk: &str,
-//     url: &str,
-//     _region: &str,
-// ) -> Result<S3Client, Box<dyn Error>> {
-//     let credentials = Credentials::new(ak, sk, None, None, "");
-//     let region = Region::new("us-east-1".to_string());
-
-//     let config = Config::builder()
-//         .region(region)
-//         .endpoint_url(url.to_string())
-//         .credentials_provider(credentials)
-//         .behavior_version(BehaviorVersion::latest()) // Adjust as necessary
-//         .build();
-//     Ok(S3Client::from_conf(config))
-// }
-
-pub struct BucketTargetSys {
-    arn_remote_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
-    targets_map: Arc<RwLock<HashMap<String, Vec<bucket::target::BucketTarget>>>>,
-    hc: HashMap<String, EpHealth>,
-    //store:Option<Arc<ecstore::store::ECStore>>,
-}
-
-lazy_static! {
-    pub static ref GLOBAL_Bucket_Target_Sys: std::sync::OnceLock<BucketTargetSys> = BucketTargetSys::new().into();
-}
-
-//#[derive(Debug)]
-// pub enum SetTargetError {
-//   NotFound,
-// }
-
-pub async fn get_bucket_target_client(bucket: &str, arn: &str) -> Result<TargetClient, SetTargetError> {
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        sys.get_remote_target_client2(arn).await
-    } else {
-        Err(SetTargetError::TargetNotFound(bucket.to_string()))
-    }
-}
-
-#[derive(Debug)]
-pub struct BucketRemoteTargetNotFound {
-    pub bucket: String,
-}
-
-pub async fn init_bucket_targets(bucket: &str, meta: Arc<bucket::metadata::BucketMetadata>) {
-    println!("140 {bucket}");
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        if let Some(tgts) = meta.bucket_target_config.clone() {
-            for tgt in tgts.targets {
-                warn!("ak and sk is:{:?}", tgt.credentials);
-                let _ = sys.set_target(bucket, &tgt, false, true).await;
-                //sys.targets_map.
-            }
-        }
-    }
-}
-
-pub async fn remove_bucket_target(bucket: &str, arn_str: &str) {
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        let _ = sys.remove_target(bucket, arn_str).await;
-    }
-}
-
-pub async fn list_bucket_targets(bucket: &str) -> Result<BucketTargets, BucketRemoteTargetNotFound> {
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        sys.list_bucket_targets(bucket).await
-    } else {
-        Err(BucketRemoteTargetNotFound {
-            bucket: bucket.to_string(),
-        })
-    }
-}
-
-impl Default for BucketTargetSys {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl BucketTargetSys {
-    pub fn new() -> Self {
-        BucketTargetSys {
-            arn_remote_map: Arc::new(RwLock::new(HashMap::new())),
-            targets_map: Arc::new(RwLock::new(HashMap::new())),
-            hc: HashMap::new(),
-        }
-    }
-
-    pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketRemoteTargetNotFound> {
-        let targets_map = self.targets_map.read().await;
-        if let Some(targets) = targets_map.get(bucket) {
-            Ok(BucketTargets {
-                targets: targets.clone(),
-            })
-        } else {
-            Err(BucketRemoteTargetNotFound {
-                bucket: bucket.to_string(),
-            })
-        }
-    }
-
-    pub async fn list_targets(&self, bucket: Option<&str>, _arn_type: Option<&str>) -> Vec<BucketTarget> {
-        let _ = _arn_type;
-        //let health_stats = self.health_stats();
-
-        let mut targets = Vec::new();
-
-        if let Some(bucket_name) = bucket {
-            if let Ok(ts) = self.list_bucket_targets(bucket_name).await {
-                for t in ts.targets {
-                    //if arn_type.map_or(true, |arn| t.target_type == arn) {
-                    //if let Some(hs) = health_stats.get(&t.url().host) {
-                    // t.total_downtime = hs.offline_duration;
-                    // t.online = hs.online;
-                    // t.last_online = hs.last_online;
-                    // t.latency = LatencyStat {
-                    //     curr: hs.latency.curr,
-                    //     avg: hs.latency.avg,
-                    //     max: hs.latency.peak,
-                    // };
-                    //}
-                    targets.push(t.clone());
-                    //}
-                }
-            }
-            return targets;
-        }
-
-        // Locking and iterating over all targets in the system
-        let targets_map = self.targets_map.read().await;
-        for tgts in targets_map.values() {
-            for t in tgts {
-                //if arn_type.map_or(true, |arn| t.target_type == arn) {
-                // if let Some(hs) = health_stats.get(&t.url().host) {
-                //     t.total_downtime = hs.offline_duration;
-                //     t.online = hs.online;
-                //     t.last_online = hs.last_online;
-                //     t.latency = LatencyStat {
-                //         curr: hs.latency.curr,
-                //         avg: hs.latency.avg,
-                //         max: hs.latency.peak,
-                //     };
-                // }
-                targets.push(t.clone());
-                //}
-            }
-        }
-
-        targets
-    }
-
-    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<(), SetTargetError> {
-        //to do need lock;
-        let mut targets_map = self.targets_map.write().await;
-        let tgts = targets_map.get(bucket);
-        let mut arn_remotes_map = self.arn_remote_map.write().await;
-        if tgts.is_none() {
-            //Err(SetTargetError::TargetNotFound(bucket.to_string()));
-            return Ok(());
-        }
-
-        let tgts = tgts.unwrap(); // 安全解引用
-        let mut targets = Vec::with_capacity(tgts.len());
-        let mut found = false;
-
-        // 遍历 targets，找出不匹配的 ARN
-        for tgt in tgts {
-            if tgt.arn != Some(arn_str.to_string()) {
-                targets.push(tgt.clone()); // 克隆符合条件的项
-            } else {
-                found = true; // 找到匹配的 ARN
-            }
-        }
-
-        // 如果没有找到匹配的 ARN，则返回错误
-        if !found {
-            return Ok(());
-        }
-
-        // 更新 targets_map
-        targets_map.insert(bucket.to_string(), targets);
-        arn_remotes_map.remove(arn_str);
-
-        let targets = self.list_targets(Some(bucket), None).await;
-        println!("targets is {}", targets.len());
-        match serde_json::to_vec(&targets) {
-            Ok(json) => {
-                let _ = metadata_sys::update(bucket, "bucket-targets.json", json).await;
-            }
-            Err(e) => {
-                println!("序列化失败{e}");
-            }
-        }
-
-        Ok(())
-    }
-
-    pub async fn get_remote_arn(&self, bucket: &str, target: Option<&BucketTarget>, depl_id: &str) -> (Option<String>, bool) {
-        if target.is_none() {
-            return (None, false);
-        }
-
-        let target = target.unwrap();
-
-        let targets_map = self.targets_map.read().await;
-
-        // 获取锁以访问 arn_remote_map
-        let mut _arn_remotes_map = self.arn_remote_map.read().await;
-        if let Some(tgts) = targets_map.get(bucket) {
-            for tgt in tgts {
-                if tgt.type_ == target.type_
-                    && tgt.target_bucket == target.target_bucket
-                    && tgt.endpoint == target.endpoint
-                    && tgt.credentials.as_ref().unwrap().access_key == target.credentials.as_ref().unwrap().access_key
-                {
-                    return (tgt.arn.clone(), true);
-                }
-            }
-        }
-
-        // if !target.type_.is_valid() {
-        //     return (None, false);
-        // }
-
-        println!("generate_arn");
-
-        (Some(generate_arn(target.clone(), depl_id.to_string())), false)
-    }
-
-    pub async fn get_remote_target_client2(&self, arn: &str) -> Result<TargetClient, SetTargetError> {
-        let map = self.arn_remote_map.read().await;
-        info!("get remote target client and arn is: {}", arn);
-        if let Some(value) = map.get(arn) {
-            let mut x = value.client.clone();
-            x.arn = arn.to_string();
-            Ok(x)
-        } else {
-            error!("not find target");
-            Err(SetTargetError::TargetNotFound(arn.to_string()))
-        }
-    }
-
-    // pub async fn get_remote_target_client(&self, _tgt: &BucketTarget) -> Result<TargetClient, SetTargetError> {
-    //     // Mocked implementation for obtaining a remote client
-    //     let tcli = TargetClient {
-    //         bucket: _tgt.target_bucket.clone(),
-    //         storage_class: "STANDARD".to_string(),
-    //         disable_proxy: false,
-    //         health_check_duration: Duration::from_secs(100),
-    //         endpoint: _tgt.endpoint.clone(),
-    //         reset_id: "0".to_string(),
-    //         replicate_sync: false,
-    //         secure: false,
-    //         arn: "".to_string(),
-    //         client: reqwest::Client::new(),
-    //         ak: _tgt.
-
-    //     };
-    //     Ok(tcli)
-    // }
-    // pub async fn get_remote_target_client_with_bucket(&self, _bucket: String) -> Result<TargetClient, SetTargetError> {
-    //     // Mocked implementation for obtaining a remote client
-    //     let tcli = TargetClient {
-    //         bucket: _tgt.target_bucket.clone(),
-    //         storage_class: "STANDARD".to_string(),
-    //         disable_proxy: false,
-    //         health_check_duration: Duration::from_secs(100),
-    //         endpoint: _tgt.endpoint.clone(),
-    //         reset_id: "0".to_string(),
-    //         replicate_sync: false,
-    //         secure: false,
-    //         arn: "".to_string(),
-    //         client: reqwest::Client::new(),
-    //     };
-    //     Ok(tcli)
-    // }
-
-    async fn local_is_bucket_versioned(&self, _bucket: &str) -> bool {
-        let Some(store) = new_object_layer_fn() else {
-            return false;
-        };
-        //store.get_bucket_info(bucket, opts)
-
-        // let binfo:BucketInfo = store
-        // .get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default()).await;
-        match store.get_bucket_info(_bucket, &store_api::BucketOptions::default()).await {
-            Ok(info) => {
-                println!("Bucket Info: {info:?}");
-                info.versioning
-            }
-            Err(err) => {
-                eprintln!("Error: {err:?}");
-                false
-            }
-        }
-    }
-
-    async fn is_bucket_versioned(&self, _bucket: &str) -> bool {
-        true
-        // let url_str = "http://127.0.0.1:9001";
-
-        // // 转换为 Url 类型
-        // let parsed_url = url::Url::parse(url_str).unwrap();
-
-        // let node = Node {
-        //     url: parsed_url,
-        //     pools: vec![],
-        //     is_local: false,
-        //     grid_host: "".to_string(),
-        // };
-        // let cli = ecstore::peer::RemotePeerS3Client::new(Some(node), None);
-
-        // match cli.get_bucket_info(_bucket, &ecstore::store_api::BucketOptions::default()).await
-        // {
-        //     Ok(info) => {
-        //         println!("Bucket Info: {:?}", info);
-        //         info.versioning
-        //     }
-        //     Err(err) => {
-        //         eprintln!("Error: {:?}", err);
-        //         return false;
-        //     }
-        // }
-    }
-
-    pub async fn set_target(&self, bucket: &str, tgt: &BucketTarget, update: bool, fromdisk: bool) -> Result<(), SetTargetError> {
-        // if !tgt.type_.is_valid() && !update {
-        //     return Err(SetTargetError::InvalidTargetType(bucket.to_string()));
-        // }
-
-        //let client = self.get_remote_target_client(tgt).await?;
-        if tgt.type_ == Some("replication".to_string()) && !fromdisk {
-            let versioning_config = self.local_is_bucket_versioned(bucket).await;
-            if !versioning_config {
-                // println!("111111111");
-                return Err(SetTargetError::TargetNotVersioned(bucket.to_string()));
-            }
-        }
-
-        let url_str = format!("http://{}", tgt.endpoint.clone());
-
-        println!("url str is {url_str}");
-        // 转换为 Url 类型
-        let parsed_url = url::Url::parse(&url_str).unwrap();
-
-        let node = Node {
-            url: parsed_url,
-            pools: vec![],
-            is_local: false,
-            grid_host: "".to_string(),
-        };
-
-        let cli = RemotePeerS3Client::new(Some(node), None);
-
-        match cli
-            .get_bucket_info(&tgt.target_bucket, &store_api::BucketOptions::default())
-            .await
-        {
-            Ok(info) => {
-                println!("Bucket Info: {info:?}");
-                if !info.versioning {
-                    return Err(SetTargetError::TargetNotVersioned(tgt.target_bucket.to_string()));
-                }
-            }
-            Err(err) => {
-                println!("remote bucket 369 is:{}", tgt.target_bucket);
-                eprintln!("Error: {err:?}");
-                return Err(SetTargetError::SourceNotVersioned(tgt.target_bucket.to_string()));
-            }
-        }
-
-        //if tgt.target_type == BucketTargetType::ReplicationService {
-        // Check if target is a rustfs server and alive
-        // let hc_result = tokio::time::timeout(Duration::from_secs(3), client.health_check(&tgt.endpoint)).await;
-        // match hc_result {
-        //     Ok(Ok(true)) => {} // Server is alive
-        //     Ok(Ok(false)) | Ok(Err(_)) | Err(_) => {
-        //         return Err(SetTargetError::HealthCheckFailed(tgt.target_bucket.clone()));
-        //     }
-        // }
-
-        //Lock and update target maps
-        let mut targets_map = self.targets_map.write().await;
-        let mut arn_remotes_map = self.arn_remote_map.write().await;
-
-        let targets = targets_map.entry(bucket.to_string()).or_default();
-        let mut found = false;
-
-        for existing_target in targets.iter_mut() {
-            println!("418 exist:{}", existing_target.source_bucket.clone());
-            if existing_target.type_ == tgt.type_ {
-                if existing_target.arn == tgt.arn {
-                    if !update {
-                        return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-                    }
-                    *existing_target = tgt.clone();
-                    found = true;
-                    break;
-                }
-
-                if existing_target.endpoint == tgt.endpoint {
-                    println!("endpoint is same:{}", tgt.endpoint.clone());
-                    return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-                }
-            }
-        }
-
-        if !found && !update {
-            println!("437 exist:{}", tgt.arn.clone().unwrap());
-            targets.push(tgt.clone());
-        }
-        let arntgt: ArnTarget = ArnTarget::new(
-            tgt.target_bucket.clone(),
-            tgt.endpoint.clone(),
-            tgt.credentials.clone().unwrap().access_key.clone(),
-            tgt.credentials.clone().unwrap().secret_key,
-        );
-
-        arn_remotes_map.insert(tgt.arn.clone().unwrap().clone(), arntgt);
-        //self.update_bandwidth_limit(bucket, &tgt.arn, tgt.bandwidth_limit).await;
-
-        Ok(())
-    }
-}
-
-#[derive(Clone)]
-pub struct TargetClient {
-    pub client: reqwest::Client, // Using reqwest HTTP client
-    pub health_check_duration: Duration,
-    pub bucket: String, // Remote bucket target
-    pub replicate_sync: bool,
-    pub storage_class: String, // Storage class on remote
-    pub disable_proxy: bool,
-    pub arn: String, // ARN to uniquely identify remote target
-    pub reset_id: String,
-    pub endpoint: String,
-    pub secure: bool,
-    pub ak: String,
-    pub sk: String,
-}
-
-#[allow(clippy::too_many_arguments)]
-impl TargetClient {
-    #[allow(clippy::too_many_arguments)]
-    pub fn new(
-        client: reqwest::Client,
-        health_check_duration: Duration,
-        bucket: String,
-        replicate_sync: bool,
-        storage_class: String,
-        disable_proxy: bool,
-        arn: String,
-        reset_id: String,
-        endpoint: String,
-        secure: bool,
-        ak: String,
-        sk: String,
-    ) -> Self {
-        TargetClient {
-            client,
-            health_check_duration,
-            bucket,
-            replicate_sync,
-            storage_class,
-            disable_proxy,
-            arn,
-            reset_id,
-            endpoint,
-            secure,
-            ak,
-            sk,
-        }
-    }
-    pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
-        Ok(true) // Mocked implementation
-    }
-}
-use tracing::{error, info, warn};
-use uuid::Uuid;
-
-#[derive(Debug, Clone)]
-pub struct VersioningConfig {
-    pub enabled: bool,
-}
-
-impl VersioningConfig {
-    pub fn is_enabled(&self) -> bool {
-        self.enabled
-    }
-}
-
-#[derive(Debug)]
-pub struct Client;
-
-impl Client {
-    pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
-        Ok(true) // Mocked implementation
-    }
-
-    pub async fn get_bucket_versioning(&self, _bucket: &str) -> Result<VersioningConfig, SetTargetError> {
-        Ok(VersioningConfig { enabled: true })
-    }
-
-    pub async fn health_check(&self, _endpoint: &str) -> Result<bool, SetTargetError> {
-        Ok(true) // Mocked health check
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct ServiceType(String);
-
-impl ServiceType {
-    pub fn is_valid(&self) -> bool {
-        !self.0.is_empty() // 根据需求添加具体的验证逻辑
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct ARN {
-    pub arn_type: String,
-    pub id: String,
-    pub region: String,
-    pub bucket: String,
-}
-
-impl ARN {
-    /// 检查 ARN 是否为空
-    pub fn is_empty(&self) -> bool {
-        //!self.arn_type.is_valid()
-        false
-    }
-
-    // 从字符串解析 ARN
-    pub fn parse(s: &str) -> Result<Self, String> {
-        // ARN 必须是格式 arn:rustfs:<Type>:<REGION>:<ID>:<remote-bucket>
-        if !s.starts_with("arn:rustfs:") {
-            return Err(format!("Invalid ARN {s}"));
-        }
-
-        let tokens: Vec<&str> = s.split(':').collect();
-        if tokens.len() != 6 || tokens[4].is_empty() || tokens[5].is_empty() {
-            return Err(format!("Invalid ARN {s}"));
-        }
-
-        Ok(ARN {
-            arn_type: tokens[2].to_string(),
-            region: tokens[3].to_string(),
-            id: tokens[4].to_string(),
-            bucket: tokens[5].to_string(),
-        })
-    }
-}
-
-// 实现 `Display` trait，使得可以直接使用 `format!` 或 `{}` 输出 ARN
-impl std::fmt::Display for ARN {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
-    }
-}
-
-fn must_get_uuid() -> String {
-    Uuid::new_v4().to_string()
-    // match Uuid::new_v4() {
-    //     Ok(uuid) => uuid.to_string(),
-    //     Err(err) => {
-    //         error!("Critical error: {}", err);
-    //         panic!("Failed to generate UUID: {}", err); // Ensures similar behavior as Go's logger.CriticalIf
-    //     }
-    // }
-}
-fn generate_arn(target: BucketTarget, depl_id: String) -> String {
-    let mut uuid: String = depl_id;
-    if uuid.is_empty() {
-        uuid = must_get_uuid();
-    }
-
-    let arn: ARN = ARN {
-        arn_type: target.type_.unwrap(),
-        id: (uuid),
-        region: "us-east-1".to_string(),
-        bucket: (target.target_bucket),
-    };
-    arn.to_string()
-}
-
-// use std::collections::HashMap;
-// use std::sync::{Arc, Mutex, RwLock};
-// use std::time::Duration;
-// use tokio::time::timeout;
-// use tokio::sync::RwLock as AsyncRwLock;
-// use serde::Deserialize;
-// use thiserror::Error;
-
-// #[derive(Debug, Clone, PartialEq)]
-// pub enum BucketTargetType {
-//     ReplicationService,
-//     // Add other service types as needed
-// }
-
-// impl BucketTargetType {
-//     pub fn is_valid(&self) -> bool {
-//         matches!(self, BucketTargetType::ReplicationService)
-//     }
-// }
-
-// #[derive(Debug, Clone)]
-// pub struct BucketTarget {
-//     pub arn: String,
-//     pub target_bucket: String,
-//     pub endpoint: String,
-//     pub credentials: Credentials,
-//     pub secure: bool,
-//     pub bandwidth_limit: Option<u64>,
-//     pub target_type: BucketTargetType,
-// }
-
-// #[derive(Debug, Clone)]
-// pub struct Credentials {
-//     pub access_key: String,
-//     pub secret_key: String,
-// }
-
-// #[derive(Debug)]
-// pub struct BucketTargetSys {
-//     targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
-//     arn_remotes_map: Arc<Mutex<HashMap<String, ArnTarget>>>,
-// }
-
-// impl BucketTargetSys {
-//     pub fn new() -> Self {
-//         Self {
-//             targets_map: Arc::new(RwLock::new(HashMap::new())),
-//             arn_remotes_map: Arc::new(Mutex::new(HashMap::new())),
-//         }
-//     }
-
-//     pub async fn set_target(
-//         &self,
-//         bucket: &str,
-//         tgt: &BucketTarget,
-//         update: bool,
-//     ) -> Result<(), SetTargetError> {
-//         if !tgt.target_type.is_valid() && !update {
-//             return Err(SetTargetError::InvalidTargetType(bucket.to_string()));
-//         }
-
-//         let client = self.get_remote_target_client(tgt).await?;
-
-//         // Validate if target credentials are OK
-//         let exists = client.bucket_exists(&tgt.target_bucket).await?;
-//         if !exists {
-//             return Err(SetTargetError::TargetNotFound(tgt.target_bucket.clone()));
-//         }
-
-//         if tgt.target_type == BucketTargetType::ReplicationService {
-//             if !self.is_bucket_versioned(bucket).await {
-//                 return Err(SetTargetError::SourceNotVersioned(bucket.to_string()));
-//             }
-
-//             let versioning_config = client.get_bucket_versioning(&tgt.target_bucket).await?;
-//             if !versioning_config.is_enabled() {
-//                 return Err(SetTargetError::TargetNotVersioned(tgt.target_bucket.clone()));
-//             }
-//         }
-
-//         // Check if target is a rustfs server and alive
-//         let hc_result = timeout(Duration::from_secs(3), client.health_check(&tgt.endpoint)).await;
-//         match hc_result {
-//             Ok(Ok(true)) => {} // Server is alive
-//             Ok(Ok(false)) | Ok(Err(_)) | Err(_) => {
-//                 return Err(SetTargetError::HealthCheckFailed(tgt.target_bucket.clone()));
-//             }
-//         }
-
-//         // Lock and update target maps
-//         let mut targets_map = self.targets_map.write().await;
-//         let mut arn_remotes_map = self.arn_remotes_map.lock().unwrap();
-
-//         let targets = targets_map.entry(bucket.to_string()).or_default();
-//         let mut found = false;
-
-//         for existing_target in targets.iter_mut() {
-//             if existing_target.target_type == tgt.target_type {
-//                 if existing_target.arn == tgt.arn {
-//                     if !update {
-//                         return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-//                     }
-//                     *existing_target = tgt.clone();
-//                     found = true;
-//                     break;
-//                 }
-
-//                 if existing_target.endpoint == tgt.endpoint {
-//                     return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-//                 }
-//             }
-//         }
-
-//         if !found && !update {
-//             targets.push(tgt.clone());
-//         }
-
-//         arn_remotes_map.insert(tgt.arn.clone(), ArnTarget { client });
-//         self.update_bandwidth_limit(bucket, &tgt.arn, tgt.bandwidth_limit).await;
-
-//         Ok(())
-//     }
-
-//     async fn get_remote_target_client(&self, tgt: &BucketTarget) -> Result<Client, SetTargetError> {
-//         // Mocked implementation for obtaining a remote client
-//         Ok(Client {})
-//     }
-
-//     async fn is_bucket_versioned(&self, bucket: &str) -> bool {
-//         // Mocked implementation for checking if a bucket is versioned
-//         true
-//     }
-
-//     async fn update_bandwidth_limit(
-//         &self,
-//         bucket: &str,
-//         arn: &str,
-//         limit: Option<u64>,
-//     ) {
-//         // Mocked implementation for updating bandwidth limits
-//     }
-// }
-
-// #[derive(Debug)]
-// pub struct Client;
-
-// impl Client {
-//     pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
-//         Ok(true) // Mocked implementation
-//     }
-
-//     pub async fn get_bucket_versioning(
-//         &self,
-//         _bucket: &str,
-//     ) -> Result<VersioningConfig, SetTargetError> {
-//         Ok(VersioningConfig { enabled: true })
-//     }
-
-//     pub async fn health_check(&self, _endpoint: &str) -> Result<bool, SetTargetError> {
-//         Ok(true) // Mocked health check
-//     }
-// }
-
-// #[derive(Debug, Clone)]
-// pub struct ArnTarget {
-//     pub client: Client,
-// }
-
-#[derive(Debug, Error)]
-pub enum SetTargetError {
-    #[error("Invalid target type for bucket {0}")]
-    InvalidTargetType(String),
-
-    #[error("Target bucket {0} not found")]
-    TargetNotFound(String),
-
-    #[error("Source bucket {0} is not versioned")]
-    SourceNotVersioned(String),
-
-    #[error("Target bucket {0} is not versioned")]
-    TargetNotVersioned(String),
-
-    #[error("Health check failed for bucket {0}")]
-    HealthCheckFailed(String),
-
-    #[error("Target bucket {0} already exists")]
-    TargetAlreadyExists(String),
-}
@@ -88,7 +88,7 @@ impl LocalUsageSnapshot {

 /// Build the snapshot file name `<disk-id>.json`.
 pub fn snapshot_file_name(disk_id: &str) -> String {
-    format!("{}.json", disk_id)
+    format!("{disk_id}.json")
 }

 /// Build the object path relative to `RUSTFS_META_BUCKET`, e.g. `datausage/<disk-id>.json`.
@@ -2087,6 +2087,7 @@ impl DiskAPI for LocalDisk {
        for vol in volumes {
            if let Err(e) = self.make_volume(vol).await {
                if e != DiskError::VolumeExists {
+                    error!("local disk make volumes failed: {e}");
                    return Err(e);
                }
            }
@@ -2108,6 +2109,7 @@ impl DiskAPI for LocalDisk {
                os::make_dir_all(&volume_dir, self.root.as_path()).await?;
                return Ok(());
            }
+            error!("local disk make volume failed: {e}");
            return Err(to_volume_error(e).into());
        }

@@ -2349,12 +2351,7 @@ impl DiskAPI for LocalDisk {
        self.delete_file(&volume_dir, &xl_path, true, false).await
    }
    #[tracing::instrument(level = "debug", skip(self))]
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        _opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>> {
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, _opts: DeleteOptions) -> Vec<Option<Error>> {
        let mut errs = Vec::with_capacity(versions.len());
        for _ in 0..versions.len() {
            errs.push(None);
@@ -2368,7 +2365,7 @@ impl DiskAPI for LocalDisk {
            }
        }

-        Ok(errs)
+        errs
    }

    #[tracing::instrument(skip(self))]
@@ -201,12 +201,7 @@ impl DiskAPI for Disk {
    }

    #[tracing::instrument(skip(self))]
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>> {
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
        match self {
            Disk::Local(local_disk) => local_disk.delete_versions(volume, versions, opts).await,
            Disk::Remote(remote_disk) => remote_disk.delete_versions(volume, versions, opts).await,
@@ -448,12 +443,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
        force_del_marker: bool,
        opts: DeleteOptions,
    ) -> Result<()>;
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>>;
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>>;
    async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()>;
    async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
    async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
@@ -301,6 +301,10 @@ impl Erasure {
            written += n;
        }

+        if ret_err.is_some() {
+            return (written, ret_err);
+        }
+
        if written < length {
            ret_err = Some(Error::LessData.into());
        }
@@ -145,7 +145,9 @@ impl Erasure {
                            return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                        }
                    }
-                    Ok(_) => break,
+                    Ok(_) => {
+                        break;
+                    }
                    Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
                        break;
                    }
@@ -468,15 +468,21 @@ impl Erasure {
            let mut buf = vec![0u8; block_size];
            match rustfs_utils::read_full(&mut *reader, &mut buf).await {
                Ok(n) if n > 0 => {
+                    warn!("encode_stream_callback_async read n={}", n);
                    total += n;
                    let res = self.encode_data(&buf[..n]);
                    on_block(res).await?
                }
-                Ok(_) => break,
+                Ok(_) => {
+                    warn!("encode_stream_callback_async read unexpected ok");
+                    break;
+                }
                Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
+                    warn!("encode_stream_callback_async read unexpected eof");
                    break;
                }
                Err(e) => {
+                    warn!("encode_stream_callback_async read error={:?}", e);
                    on_block(Err(e)).await?;
                    break;
                }
@@ -38,7 +38,7 @@ pub const DISK_RESERVE_FRACTION: f64 = 0.15;

 lazy_static! {
    static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
-    static ref GLOBAL_RUSTFS_EXTERNAL_PORT: OnceLock<u16> = OnceLock::new();
+    static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
    pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
    pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
    pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
@@ -51,8 +51,6 @@ lazy_static! {
    pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
    pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
    pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
-    //pub static ref GLOBAL_RemoteTargetTransport
-    static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
    pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
    pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
    pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
@@ -60,12 +58,22 @@ lazy_static! {
    pub static ref GLOBAL_REGION: OnceLock<String> = OnceLock::new();
 }

-// Global cancellation token for background services (data scanner and auto heal)
+/// Global cancellation token for background services (data scanner and auto heal)
 static GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN: OnceLock<CancellationToken> = OnceLock::new();

+/// Global active credentials
 static GLOBAL_ACTIVE_CRED: OnceLock<Credentials> = OnceLock::new();

-pub fn init_global_action_cred(ak: Option<String>, sk: Option<String>) {
+/// Initialize the global action credentials
+///
+/// # Arguments
+/// * `ak` - Optional access key
+/// * `sk` - Optional secret key
+///
+/// # Returns
+/// * None
+///
+pub fn init_global_action_credentials(ak: Option<String>, sk: Option<String>) {
    let ak = {
        if let Some(k) = ak {
            k
@@ -91,11 +99,16 @@ pub fn init_global_action_cred(ak: Option<String>, sk: Option<String>) {
        .unwrap();
 }

+/// Get the global action credentials
 pub fn get_global_action_cred() -> Option<Credentials> {
    GLOBAL_ACTIVE_CRED.get().cloned()
 }

 /// Get the global rustfs port
+///
+/// # Returns
+/// * `u16` - The global rustfs port
+///
 pub fn global_rustfs_port() -> u16 {
    if let Some(p) = GLOBAL_RUSTFS_PORT.get() {
        *p
@@ -105,36 +118,44 @@ pub fn global_rustfs_port() -> u16 {
 }

 /// Set the global rustfs port
+///
+/// # Arguments
+/// * `value` - The port value to set globally
+///
+/// # Returns
+/// * None
 pub fn set_global_rustfs_port(value: u16) {
    GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail");
 }

-/// Get the global rustfs external port
-pub fn global_rustfs_external_port() -> u16 {
-    if let Some(p) = GLOBAL_RUSTFS_EXTERNAL_PORT.get() {
-        *p
-    } else {
-        rustfs_config::DEFAULT_PORT
-    }
-}
-
-/// Set the global rustfs external port
-pub fn set_global_rustfs_external_port(value: u16) {
-    GLOBAL_RUSTFS_EXTERNAL_PORT
-        .set(value)
-        .expect("set_global_rustfs_external_port fail");
-}
-
-/// Get the global rustfs port
+/// Set the global deployment id
+///
+/// # Arguments
+/// * `id` - The Uuid to set as the global deployment id
+///
+/// # Returns
+/// * None
+///
 pub fn set_global_deployment_id(id: Uuid) {
    globalDeploymentIDPtr.set(id).unwrap();
 }

 /// Get the global deployment id
+///
+/// # Returns
+/// * `Option<String>` - The global deployment id as a string, if set
+///
 pub fn get_global_deployment_id() -> Option<String> {
    globalDeploymentIDPtr.get().map(|v| v.to_string())
 }
-/// Get the global deployment id
+/// Set the global endpoints
+///
+/// # Arguments
+/// * `eps` - A vector of PoolEndpoints to set globally
+///
+/// # Returns
+/// * None
+///
 pub fn set_global_endpoints(eps: Vec<PoolEndpoints>) {
    GLOBAL_Endpoints
        .set(EndpointServerPools::from(eps))
@@ -142,6 +163,10 @@ pub fn set_global_endpoints(eps: Vec<PoolEndpoints>) {
 }

 /// Get the global endpoints
+///
+/// # Returns
+/// * `EndpointServerPools` - The global endpoints
+///
 pub fn get_global_endpoints() -> EndpointServerPools {
    if let Some(eps) = GLOBAL_Endpoints.get() {
        eps.clone()
@@ -150,29 +175,63 @@ pub fn get_global_endpoints() -> EndpointServerPools {
    }
 }

+/// Create a new object layer instance
+///
+/// # Returns
+/// * `Option<Arc<ECStore>>` - The global object layer instance, if set
+///
 pub fn new_object_layer_fn() -> Option<Arc<ECStore>> {
    GLOBAL_OBJECT_API.get().cloned()
 }

+/// Set the global object layer
+///
+/// # Arguments
+/// * `o` - The ECStore instance to set globally
+///
+/// # Returns
+/// * None
 pub async fn set_object_layer(o: Arc<ECStore>) {
    GLOBAL_OBJECT_API.set(o).expect("set_object_layer fail ")
 }

+/// Check if the setup type is distributed erasure coding
+///
+/// # Returns
+/// * `bool` - True if the setup type is distributed erasure coding, false otherwise
+///
 pub async fn is_dist_erasure() -> bool {
    let lock = GLOBAL_IsDistErasure.read().await;
    *lock
 }

+/// Check if the setup type is erasure coding with single data center
+///
+/// # Returns
+/// * `bool` - True if the setup type is erasure coding with single data center, false otherwise
+///
 pub async fn is_erasure_sd() -> bool {
    let lock = GLOBAL_IsErasureSD.read().await;
    *lock
 }

+/// Check if the setup type is erasure coding
+///
+/// # Returns
+/// * `bool` - True if the setup type is erasure coding, false otherwise
+///
 pub async fn is_erasure() -> bool {
    let lock = GLOBAL_IsErasure.read().await;
    *lock
 }

+/// Update the global erasure type based on the setup type
+///
+/// # Arguments
+/// * `setup_type` - The SetupType to update the global erasure type
+///
+/// # Returns
+/// * None
 pub async fn update_erasure_type(setup_type: SetupType) {
    let mut is_erasure = GLOBAL_IsErasure.write().await;
    *is_erasure = setup_type == SetupType::Erasure;
@@ -198,25 +257,53 @@ pub async fn update_erasure_type(setup_type: SetupType) {

 type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;

+/// Set the global region
+///
+/// # Arguments
+/// * `region` - The region string to set globally
+///
+/// # Returns
+/// * None
 pub fn set_global_region(region: String) {
    GLOBAL_REGION.set(region).unwrap();
 }

+/// Get the global region
+///
+/// # Returns
+/// * `Option<String>` - The global region string, if set
+///
 pub fn get_global_region() -> Option<String> {
    GLOBAL_REGION.get().cloned()
 }

 /// Initialize the global background services cancellation token
+///
+/// # Arguments
+/// * `cancel_token` - The CancellationToken instance to set globally
+///
+/// # Returns
+/// * `Ok(())` if successful
+/// * `Err(CancellationToken)` if setting fails
+///
 pub fn init_background_services_cancel_token(cancel_token: CancellationToken) -> Result<(), CancellationToken> {
    GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN.set(cancel_token)
 }

 /// Get the global background services cancellation token
+///
+/// # Returns
+/// * `Option<&'static CancellationToken>` - The global cancellation token, if set
+///
 pub fn get_background_services_cancel_token() -> Option<&'static CancellationToken> {
    GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN.get()
 }

 /// Create and initialize the global background services cancellation token
+///
+/// # Returns
+/// * `CancellationToken` - The newly created global cancellation token
+///
 pub fn create_background_services_cancel_token() -> CancellationToken {
    let cancel_token = CancellationToken::new();
    init_background_services_cancel_token(cancel_token.clone()).expect("Background services cancel token already initialized");
@@ -224,6 +311,9 @@ pub fn create_background_services_cancel_token() -> CancellationToken {
 }

 /// Shutdown all background services gracefully
+///
+/// # Returns
+/// * None
 pub fn shutdown_background_services() {
    if let Some(cancel_token) = GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN.get() {
        cancel_token.cancel();
@@ -21,7 +21,6 @@ pub mod bitrot;
 pub mod bucket;
 pub mod cache_value;
 mod chunk_stream;
-pub mod cmd;
 pub mod compress;
 pub mod config;
 pub mod data_usage;
@@ -32,7 +31,6 @@ pub mod erasure_coding;
 pub mod error;
 pub mod file_cache;
 pub mod global;
-pub mod lock_utils;
 pub mod metrics_realtime;
 pub mod notification_sys;
 pub mod pools;
@@ -46,7 +44,7 @@ mod store_init;
 pub mod store_list_objects;
 pub mod store_utils;

-pub mod checksum;
+// pub mod checksum;
 pub mod client;
 pub mod event;
 pub mod event_notification;
@@ -1,136 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::disk::endpoint::Endpoint;
-use crate::error::Result;
-use rustfs_lock::client::{LockClient, local::LocalClient, remote::RemoteClient};
-use std::collections::HashMap;
-use std::sync::Arc;
-
-/// Create unique lock clients from endpoints
-/// This function creates one client per unique host:port combination
-/// to avoid duplicate connections to the same server
-pub async fn create_unique_clients(endpoints: &[Endpoint]) -> Result<Vec<Arc<dyn LockClient>>> {
-    let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
-
-    // Collect unique endpoints based on host:port
-    for endpoint in endpoints {
-        if endpoint.is_local {
-            // For local endpoints, use "local" as the key
-            unique_endpoints.insert("local".to_string(), endpoint);
-        } else {
-            // For remote endpoints, use host:port as the key
-            let host_port = format!(
-                "{}:{}",
-                endpoint.url.host_str().unwrap_or("localhost"),
-                endpoint.url.port().unwrap_or(9000)
-            );
-            unique_endpoints.insert(host_port, endpoint);
-        }
-    }
-
-    let mut clients = Vec::new();
-
-    // Create clients for unique endpoints
-    for (_key, endpoint) in unique_endpoints {
-        if endpoint.is_local {
-            // For local endpoints, create a local lock client
-            let local_client = LocalClient::new();
-            clients.push(Arc::new(local_client) as Arc<dyn LockClient>);
-        } else {
-            // For remote endpoints, create a remote lock client
-            let remote_client = RemoteClient::new(endpoint.url.to_string());
-            clients.push(Arc::new(remote_client) as Arc<dyn LockClient>);
-        }
-    }
-
-    Ok(clients)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use url::Url;
-
-    #[tokio::test]
-    async fn test_create_unique_clients_local() {
-        let endpoints = vec![
-            Endpoint {
-                url: Url::parse("http://localhost:9000").unwrap(),
-                is_local: true,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 0,
-            },
-            Endpoint {
-                url: Url::parse("http://localhost:9000").unwrap(),
-                is_local: true,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 1,
-            },
-        ];
-
-        let clients = create_unique_clients(&endpoints).await.unwrap();
-        // Should only create one client for local endpoints
-        assert_eq!(clients.len(), 1);
-        assert!(clients[0].is_local().await);
-    }
-
-    #[tokio::test]
-    async fn test_create_unique_clients_mixed() {
-        let endpoints = vec![
-            Endpoint {
-                url: Url::parse("http://localhost:9000").unwrap(),
-                is_local: true,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 0,
-            },
-            Endpoint {
-                url: Url::parse("http://remote1:9000").unwrap(),
-                is_local: false,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 1,
-            },
-            Endpoint {
-                url: Url::parse("http://remote1:9000").unwrap(),
-                is_local: false,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 2,
-            },
-            Endpoint {
-                url: Url::parse("http://remote2:9000").unwrap(),
-                is_local: false,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 3,
-            },
-        ];
-
-        let clients = create_unique_clients(&endpoints).await.unwrap();
-        // Should create 3 clients: 1 local + 2 unique remote
-        assert_eq!(clients.len(), 3);
-
-        // Check that we have one local client
-        let local_count = clients.iter().filter(|c| futures::executor::block_on(c.is_local())).count();
-        assert_eq!(local_count, 1);
-
-        // Check that we have two remote clients
-        let remote_count = clients.iter().filter(|c| !futures::executor::block_on(c.is_local())).count();
-        assert_eq!(remote_count, 2);
-    }
-}
@@ -23,7 +23,7 @@ use rustfs_common::{
 use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics};
 use rustfs_utils::os::get_drive_stats;
 use serde::{Deserialize, Serialize};
-use tracing::info;
+use tracing::{debug, info};

 use crate::{
    admin_server_info::get_local_server_property,
@@ -44,7 +44,7 @@ pub struct CollectMetricsOpts {
 pub struct MetricType(u32);

 impl MetricType {
-    // 定义一些常量
+    // Define some constants
    pub const NONE: MetricType = MetricType(0);
    pub const SCANNER: MetricType = MetricType(1 << 0);
    pub const DISK: MetricType = MetricType(1 << 1);
@@ -70,8 +70,18 @@ impl MetricType {
    }
 }

+/// Collect local metrics based on the specified types and options.
+///
+/// # Arguments
+///
+/// * `types` - A `MetricType` specifying which types of metrics to collect.
+/// * `opts` - A reference to `CollectMetricsOpts` containing additional options for metric collection.
+///
+/// # Returns
+/// * A `RealtimeMetrics` struct containing the collected metrics.
+///
 pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) -> RealtimeMetrics {
-    info!("collect_local_metrics");
+    debug!("collect_local_metrics");
    let mut real_time_metrics = RealtimeMetrics::default();
    if types.0 == MetricType::NONE.0 {
        info!("types is None, return");
@@ -93,13 +103,13 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
    }

    if types.contains(&MetricType::DISK) {
-        info!("start get disk metrics");
+        debug!("start get disk metrics");
        let mut aggr = DiskMetric {
            collected_at: Utc::now(),
            ..Default::default()
        };
        for (name, disk) in collect_local_disks_metrics(&opts.disks).await.into_iter() {
-            info!("got disk metric, name: {name}, metric: {disk:?}");
+            debug!("got disk metric, name: {name}, metric: {disk:?}");
            real_time_metrics.by_disk.insert(name, disk.clone());
            aggr.merge(&disk);
        }
@@ -107,7 +117,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
    }

    if types.contains(&MetricType::SCANNER) {
-        info!("start get scanner metrics");
+        debug!("start get scanner metrics");
        let metrics = globalMetrics.report().await;
        real_time_metrics.aggregated.scanner = Some(metrics);
    }
@@ -16,11 +16,17 @@ use crate::StorageAPI;
 use crate::admin_server_info::get_commit_id;
 use crate::error::{Error, Result};
 use crate::global::{GLOBAL_BOOT_TIME, get_global_endpoints};
+use crate::metrics_realtime::{CollectMetricsOpts, MetricType};
 use crate::rpc::PeerRestClient;
 use crate::{endpoints::EndpointServerPools, new_object_layer_fn};
 use futures::future::join_all;
 use lazy_static::lazy_static;
+use rustfs_madmin::health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysService};
+use rustfs_madmin::metrics::RealtimeMetrics;
+use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties};
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
 use std::sync::OnceLock;
 use std::time::SystemTime;
 use tracing::{error, warn};
@@ -62,21 +68,122 @@ pub struct NotificationPeerErr {
 }

 impl NotificationSys {
-    pub fn rest_client_from_hash(&self, _s: &str) -> Option<PeerRestClient> {
-        None
-    }
-    pub async fn delete_policy(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
-    }
-    pub async fn load_policy(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
+    pub fn rest_client_from_hash(&self, s: &str) -> Option<PeerRestClient> {
+        if self.all_peer_clients.is_empty() {
+            return None;
+        }
+        let mut hasher = DefaultHasher::new();
+        s.hash(&mut hasher);
+        let idx = (hasher.finish() as usize) % self.all_peer_clients.len();
+        self.all_peer_clients[idx].clone()
    }

-    pub async fn load_policy_mapping(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
+    pub async fn delete_policy(&self, policy_name: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let policy = policy_name.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_policy(&policy).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
    }
-    pub async fn delete_user(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
+
+    pub async fn load_policy(&self, policy_name: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let policy = policy_name.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_policy(&policy).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_policy_mapping(&self, user_or_group: &str, user_type: u64, is_group: bool) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let uog = user_or_group.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_policy_mapping(&uog, user_type, is_group).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn delete_user(&self, access_key: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_user(&ak).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
    }

    pub async fn storage_info<S: StorageAPI>(&self, api: &S) -> rustfs_madmin::StorageInfo {
@@ -140,6 +247,114 @@ impl NotificationSys {
        join_all(futures).await
    }

+    pub async fn load_user(&self, access_key: &str, temp: bool) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_user(&ak, temp).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_group(&self, group: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let gname = group.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_group(&gname).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn delete_service_account(&self, access_key: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_service_account(&ak).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_service_account(&self, access_key: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_service_account(&ak).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
    pub async fn reload_pool_meta(&self) {
        let mut futures = Vec::with_capacity(self.peer_clients.len());
        for client in self.peer_clients.iter().flatten() {
@@ -202,6 +417,281 @@ impl NotificationSys {
        let _ = store.stop_rebalance().await;
        warn!("notification stop_rebalance stop_rebalance done");
    }
+
+    pub async fn load_bucket_metadata(&self, bucket: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let b = bucket.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_bucket_metadata(&b).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn delete_bucket_metadata(&self, bucket: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let b = bucket.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_bucket_metadata(&b).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn start_profiling(&self, profiler: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let pf = profiler.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.start_profiling(&pf).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_cpus(&self) -> Vec<Cpus> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_cpus().await.unwrap_or_default()
+                } else {
+                    Cpus::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_net_info(&self) -> Vec<NetInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_net_info().await.unwrap_or_default()
+                } else {
+                    NetInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_partitions(&self) -> Vec<Partitions> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_partitions().await.unwrap_or_default()
+                } else {
+                    Partitions::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_os_info(&self) -> Vec<OsInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_os_info().await.unwrap_or_default()
+                } else {
+                    OsInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_sys_services(&self) -> Vec<SysService> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_se_linux_info().await.unwrap_or_default()
+                } else {
+                    SysService::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_sys_config(&self) -> Vec<SysConfig> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_sys_config().await.unwrap_or_default()
+                } else {
+                    SysConfig::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_sys_errors(&self) -> Vec<SysErrors> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_sys_errors().await.unwrap_or_default()
+                } else {
+                    SysErrors::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_mem_info(&self) -> Vec<MemInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_mem_info().await.unwrap_or_default()
+                } else {
+                    MemInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_proc_info(&self) -> Vec<ProcInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_proc_info().await.unwrap_or_default()
+                } else {
+                    ProcInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_metrics(&self, t: MetricType, opts: &CollectMetricsOpts) -> Vec<RealtimeMetrics> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            let t_clone = t;
+            let opts_clone = opts;
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_metrics(t_clone, opts_clone).await.unwrap_or_default()
+                } else {
+                    RealtimeMetrics::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.reload_site_replication_config().await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_transition_tier_config(&self) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_transition_tier_config().await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
 }

 fn get_offline_disks(offline_host: &str, endpoints: &EndpointServerPools) -> Vec<rustfs_madmin::Disk> {
@@ -48,7 +48,7 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use time::{Duration, OffsetDateTime};
 use tokio::io::{AsyncReadExt, BufReader};
-use tokio::sync::broadcast::Receiver as B_Receiver;
+use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};

 pub const POOL_META_NAME: &str = "pool.bin";
@@ -651,7 +651,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    pub async fn decommission(&self, rx: B_Receiver<bool>, indices: Vec<usize>) -> Result<()> {
+    pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
        warn!("decommission: {:?}", indices);
        if indices.is_empty() {
            return Err(Error::other("InvalidArgument"));
@@ -663,13 +663,14 @@ impl ECStore {

        self.start_decommission(indices.clone()).await?;

+        let rx_clone = rx.clone();
        tokio::spawn(async move {
            let Some(store) = new_object_layer_fn() else {
                error!("store not init");
                return;
            };
            for idx in indices.iter() {
-                store.do_decommission_in_routine(rx.resubscribe(), *idx).await;
+                store.do_decommission_in_routine(rx_clone.clone(), *idx).await;
            }
        });

@@ -891,7 +892,7 @@ impl ECStore {
    #[tracing::instrument(skip(self, rx))]
    async fn decommission_pool(
        self: &Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        idx: usize,
        pool: Arc<Sets>,
        bi: DecomBucketInfo,
@@ -936,20 +937,20 @@ impl ECStore {
            });

            let set = set.clone();
-            let mut rx = rx.resubscribe();
+            let rx_clone = rx.clone();
            let bi = bi.clone();
            let set_id = set_idx;
            let wk_clone = wk.clone();
            tokio::spawn(async move {
                loop {
-                    if rx.try_recv().is_ok() {
+                    if rx_clone.is_cancelled() {
                        warn!("decommission_pool: cancel {}", set_id);
                        break;
                    }
                    warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name);

                    match set
-                        .list_objects_to_decommission(rx.resubscribe(), bi.clone(), decommission_entry.clone())
+                        .list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
                        .await
                    {
                        Ok(_) => {
@@ -982,7 +983,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    pub async fn do_decommission_in_routine(self: &Arc<Self>, rx: B_Receiver<bool>, idx: usize) {
+    pub async fn do_decommission_in_routine(self: &Arc<Self>, rx: CancellationToken, idx: usize) {
        if let Err(err) = self.decommission_in_background(rx, idx).await {
            error!("decom err {:?}", &err);
            if let Err(er) = self.decommission_failed(idx).await {
@@ -1060,7 +1061,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    async fn decommission_in_background(self: &Arc<Self>, rx: B_Receiver<bool>, idx: usize) -> Result<()> {
+    async fn decommission_in_background(self: &Arc<Self>, rx: CancellationToken, idx: usize) -> Result<()> {
        let pool = self.pools[idx].clone();

        let pending = {
@@ -1090,10 +1091,7 @@ impl ECStore {

            warn!("decommission: currently on bucket {}", &bucket.name);

-            if let Err(err) = self
-                .decommission_pool(rx.resubscribe(), idx, pool.clone(), bucket.clone())
-                .await
-            {
+            if let Err(err) = self.decommission_pool(rx.clone(), idx, pool.clone(), bucket.clone()).await {
                error!("decommission: decommission_pool err {:?}", &err);
                return Err(err);
            } else {
@@ -1142,6 +1140,7 @@ impl ECStore {
                .await
            {
                if !is_err_bucket_exists(&err) {
+                    error!("decommission: make bucket failed: {err}");
                    return Err(err);
                }
            }
@@ -1264,6 +1263,8 @@ impl ECStore {
                parts[i] = CompletePart {
                    part_num: pi.part_num,
                    etag: pi.etag,
+
+                    ..Default::default()
                };
            }

@@ -1291,7 +1292,7 @@ impl ECStore {
        }

        let reader = BufReader::new(rd.stream);
-        let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, false)?;
+        let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
        let mut data = PutObjReader::new(hrd);

        if let Err(err) = self
@@ -1329,7 +1330,7 @@ impl SetDisks {
    #[tracing::instrument(skip(self, rx, cb_func))]
    async fn list_objects_to_decommission(
        self: &Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        bucket_info: DecomBucketInfo,
        cb_func: ListCallback,
    ) -> Result<()> {
@@ -34,8 +34,8 @@ use std::io::Cursor;
 use std::sync::Arc;
 use time::OffsetDateTime;
 use tokio::io::{AsyncReadExt, BufReader};
-use tokio::sync::broadcast::{self, Receiver as B_Receiver};
 use tokio::time::{Duration, Instant};
+use tokio_util::sync::CancellationToken;
 use tracing::{error, info};
 use uuid::Uuid;

@@ -151,7 +151,7 @@ pub struct DiskStat {
 #[derive(Debug, Default, Serialize, Deserialize, Clone)]
 pub struct RebalanceMeta {
    #[serde(skip)]
-    pub cancel: Option<broadcast::Sender<bool>>, // To be invoked on rebalance-stop
+    pub cancel: Option<CancellationToken>, // To be invoked on rebalance-stop
    #[serde(skip)]
    pub last_refreshed_at: Option<OffsetDateTime>,
    #[serde(rename = "stopTs")]
@@ -493,8 +493,8 @@ impl ECStore {
    pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
        let rebalance_meta = self.rebalance_meta.read().await;
        if let Some(meta) = rebalance_meta.as_ref() {
-            if let Some(tx) = meta.cancel.as_ref() {
-                let _ = tx.send(true);
+            if let Some(cancel_tx) = meta.cancel.as_ref() {
+                cancel_tx.cancel();
            }
        }

@@ -506,13 +506,14 @@ impl ECStore {
        info!("start_rebalance: start rebalance");
        // let rebalance_meta = self.rebalance_meta.read().await;

-        let (tx, rx) = broadcast::channel::<bool>(1);
+        let cancel_tx = CancellationToken::new();
+        let rx = cancel_tx.clone();

        {
            let mut rebalance_meta = self.rebalance_meta.write().await;

            if let Some(meta) = rebalance_meta.as_mut() {
-                meta.cancel = Some(tx)
+                meta.cancel = Some(cancel_tx)
            } else {
                info!("start_rebalance: rebalance_meta is None exit");
                return;
@@ -565,9 +566,9 @@ impl ECStore {

            let pool_idx = idx;
            let store = self.clone();
-            let rx = rx.resubscribe();
+            let rx_clone = rx.clone();
            tokio::spawn(async move {
-                if let Err(err) = store.rebalance_buckets(rx, pool_idx).await {
+                if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
                    error!("Rebalance failed for pool {}: {}", pool_idx, err);
                } else {
                    info!("Rebalance completed for pool {}", pool_idx);
@@ -579,7 +580,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    async fn rebalance_buckets(self: &Arc<Self>, mut rx: B_Receiver<bool>, pool_index: usize) -> Result<()> {
+    async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize) -> Result<()> {
        let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);

        // Save rebalance metadata periodically
@@ -651,7 +652,7 @@ impl ECStore {
        info!("Pool {} rebalancing is started", pool_index);

        loop {
-            if let Ok(true) = rx.try_recv() {
+            if rx.is_cancelled() {
                info!("Pool {} rebalancing is stopped", pool_index);
                done_tx.send(Err(Error::other("rebalance stopped canceled"))).await.ok();
                break;
@@ -660,7 +661,7 @@ impl ECStore {
            if let Some(bucket) = self.next_rebal_bucket(pool_index).await? {
                info!("Rebalancing bucket: start {}", bucket);

-                if let Err(err) = self.rebalance_bucket(rx.resubscribe(), bucket.clone(), pool_index).await {
+                if let Err(err) = self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index).await {
                    if err.to_string().contains("not initialized") {
                        info!("rebalance_bucket: rebalance not initialized, continue");
                        continue;
@@ -978,6 +979,7 @@ impl ECStore {
                parts[i] = CompletePart {
                    part_num: pi.part_num,
                    etag: pi.etag,
+                    ..Default::default()
                };
            }

@@ -1004,7 +1006,7 @@ impl ECStore {
        }

        let reader = BufReader::new(rd.stream);
-        let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, false)?;
+        let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
        let mut data = PutObjReader::new(hrd);

        if let Err(err) = self
@@ -1033,7 +1035,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    async fn rebalance_bucket(self: &Arc<Self>, rx: B_Receiver<bool>, bucket: String, pool_index: usize) -> Result<()> {
+    async fn rebalance_bucket(self: &Arc<Self>, rx: CancellationToken, bucket: String, pool_index: usize) -> Result<()> {
        // Placeholder for actual bucket rebalance logic
        info!("Rebalancing bucket {} in pool {}", bucket, pool_index);

@@ -1072,7 +1074,7 @@ impl ECStore {
            });

            let set = set.clone();
-            let rx = rx.resubscribe();
+            let rx = rx.clone();
            let bucket = bucket.clone();
            // let wk = wk.clone();

@@ -1144,7 +1146,7 @@ impl SetDisks {
    #[tracing::instrument(skip(self, rx, cb))]
    pub async fn list_objects_to_rebalance(
        self: &Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        bucket: String,
        cb: ListCallback,
    ) -> Result<()> {
@@ -16,10 +16,8 @@ mod http_auth;
 mod peer_rest_client;
 mod peer_s3_client;
 mod remote_disk;
-mod tonic_service;

 pub use http_auth::{build_auth_headers, verify_rpc_signature};
 pub use peer_rest_client::PeerRestClient;
 pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, RemotePeerS3Client, S3PeerSys};
 pub use remote_disk::RemoteDisk;
-pub use tonic_service::{NodeService, make_server};
@@ -345,21 +345,43 @@ impl DiskAPI for RemoteDisk {
    }

    #[tracing::instrument(skip(self))]
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>> {
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
        info!("delete_versions");
-        let opts = serde_json::to_string(&opts)?;
+
+        let opts = match serde_json::to_string(&opts) {
+            Ok(opts) => opts,
+            Err(err) => {
+                let mut errors = Vec::with_capacity(versions.len());
+                for _ in 0..versions.len() {
+                    errors.push(Some(Error::other(err.to_string())));
+                }
+                return errors;
+            }
+        };
        let mut versions_str = Vec::with_capacity(versions.len());
        for file_info_versions in versions.iter() {
-            versions_str.push(serde_json::to_string(file_info_versions)?);
+            versions_str.push(match serde_json::to_string(file_info_versions) {
+                Ok(versions_str) => versions_str,
+                Err(err) => {
+                    let mut errors = Vec::with_capacity(versions.len());
+                    for _ in 0..versions.len() {
+                        errors.push(Some(Error::other(err.to_string())));
+                    }
+                    return errors;
+                }
+            });
        }
-        let mut client = node_service_time_out_client(&self.addr)
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+        let mut client = match node_service_time_out_client(&self.addr).await {
+            Ok(client) => client,
+            Err(err) => {
+                let mut errors = Vec::with_capacity(versions.len());
+                for _ in 0..versions.len() {
+                    errors.push(Some(Error::other(err.to_string())));
+                }
+                return errors;
+            }
+        };
+
        let request = Request::new(DeleteVersionsRequest {
            disk: self.endpoint.to_string(),
            volume: volume.to_string(),
@@ -368,11 +390,27 @@ impl DiskAPI for RemoteDisk {
        });

        // TODO: use Error not string
-        let response = client.delete_versions(request).await?.into_inner();
+
+        let response = match client.delete_versions(request).await {
+            Ok(response) => response,
+            Err(err) => {
+                let mut errors = Vec::with_capacity(versions.len());
+                for _ in 0..versions.len() {
+                    errors.push(Some(Error::other(err.to_string())));
+                }
+                return errors;
+            }
+        };
+
+        let response = response.into_inner();
        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
+            let mut errors = Vec::with_capacity(versions.len());
+            for _ in 0..versions.len() {
+                errors.push(Some(Error::other(response.error.clone().map(|e| e.error_info).unwrap_or_default())));
+            }
+            return errors;
        }
-        let errors = response
+        response
            .errors
            .iter()
            .map(|error| {
@@ -382,9 +420,7 @@ impl DiskAPI for RemoteDisk {
                    Some(Error::other(error.to_string()))
                }
            })
-            .collect();
-
-        Ok(errors)
+            .collect()
    }

    #[tracing::instrument(skip(self))]
@@ -17,7 +17,7 @@ use std::{collections::HashMap, sync::Arc};

 use crate::disk::error_reduce::count_errs;
 use crate::error::{Error, Result};
-use crate::store_api::ListPartsInfo;
+use crate::store_api::{ListPartsInfo, ObjectInfoOrErr, WalkOptions};
 use crate::{
    disk::{
        DiskAPI, DiskInfo, DiskOption, DiskStore,
@@ -48,6 +48,7 @@ use rustfs_filemeta::FileInfo;
 use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
 use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
 use tokio::sync::RwLock;
+use tokio_util::sync::CancellationToken;
 use uuid::Uuid;

 use tokio::sync::broadcast::{Receiver, Sender};
@@ -55,8 +56,6 @@ use tokio::time::Duration;
 use tracing::warn;
 use tracing::{error, info};

-use crate::lock_utils::create_unique_clients;
-
 #[derive(Debug, Clone)]
 pub struct Sets {
    pub id: Uuid,
@@ -163,8 +162,6 @@ impl Sets {
                }
            }

-            let _lock_clients = create_unique_clients(&set_endpoints).await?;
-
            // Note: write_quorum was used for the old lock system, no longer needed with FastLock
            let _write_quorum = set_drive_count - parity_count;
            // Create fast lock manager for high performance
@@ -459,6 +456,17 @@ impl StorageAPI for Sets {
        unimplemented!()
    }

+    async fn walk(
+        self: Arc<Self>,
+        _rx: CancellationToken,
+        _bucket: &str,
+        _prefix: &str,
+        _result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
+        _opts: WalkOptions,
+    ) -> Result<()> {
+        unimplemented!()
+    }
+
    #[tracing::instrument(skip(self))]
    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
        self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
@@ -543,7 +551,7 @@ impl StorageAPI for Sets {
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
-    ) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)> {
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
        // Default return value
        let mut del_objects = vec![DeletedObject::default(); objects.len()];

@@ -576,38 +584,11 @@ impl StorageAPI for Sets {
            }
        }

-        // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
-        // let mut jhs = Vec::with_capacity(semaphore.available_permits());
-
-        // for (k, v) in set_obj_map {
-        //     let disks = self.get_disks(k);
-        //     let semaphore = semaphore.clone();
-        //     let opts = opts.clone();
-        //     let bucket = bucket.to_string();
-
-        //     let jh = tokio::spawn(async move {
-        //         let _permit = semaphore.acquire().await.unwrap();
-        //         let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-        //         disks.delete_objects(&bucket, objs, opts).await
-        //     });
-        //     jhs.push(jh);
-        // }
-
-        // let mut results = Vec::with_capacity(jhs.len());
-        // for jh in jhs {
-        //     results.push(jh.await?.unwrap());
-        // }
-
-        // for (dobjects, errs) in results {
-        //     del_objects.extend(dobjects);
-        //     del_errs.extend(errs);
-        // }
-
-        // TODO: Implement concurrency
+        // TODO: concurrency
        for (k, v) in set_obj_map {
            let disks = self.get_disks(k);
            let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-            let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await?;
+            let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await;

            for (i, err) in errs.into_iter().enumerate() {
                let obj = v.get(i).unwrap();
@@ -618,7 +599,7 @@ impl StorageAPI for Sets {
            }
        }

-        Ok((del_objects, del_errs))
+        (del_objects, del_errs)
    }

    async fn list_object_parts(
@@ -34,7 +34,9 @@ use crate::global::{
 use crate::notification_sys::get_global_notification_sys;
 use crate::pools::PoolMeta;
 use crate::rebalance::RebalanceMeta;
-use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, ListPartsInfo, MultipartInfo, ObjectIO};
+use crate::store_api::{
+    ListMultipartsInfo, ListObjectVersionsInfo, ListPartsInfo, MultipartInfo, ObjectIO, ObjectInfoOrErr, WalkOptions,
+};
 use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
 use crate::{
    bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
@@ -57,7 +59,6 @@ use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_
 use rustfs_common::heal_channel::{HealItemType, HealOpts};
 use rustfs_filemeta::FileInfo;
 use rustfs_madmin::heal_commands::HealResultItem;
-use rustfs_utils::crypto::base64_decode;
 use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
 use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
 use std::cmp::Ordering;
@@ -68,8 +69,9 @@ use std::time::SystemTime;
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use time::OffsetDateTime;
 use tokio::select;
-use tokio::sync::{RwLock, broadcast};
+use tokio::sync::RwLock;
 use tokio::time::sleep;
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, info};
 use tracing::{error, warn};
 use uuid::Uuid;
@@ -109,7 +111,7 @@ pub struct ECStore {
 impl ECStore {
    #[allow(clippy::new_ret_no_self)]
    #[tracing::instrument(level = "debug", skip(endpoint_pools))]
-    pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools) -> Result<Arc<Self>> {
+    pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools, ctx: CancellationToken) -> Result<Arc<Self>> {
        // let layouts = DisksLayout::from_volumes(endpoints.as_slice())?;

        let mut deployment_id = None;
@@ -251,7 +253,7 @@ impl ECStore {
        let wait_sec = 5;
        let mut exit_count = 0;
        loop {
-            if let Err(err) = ec.init().await {
+            if let Err(err) = ec.init(ctx.clone()).await {
                error!("init err: {}", err);
                error!("retry after  {} second", wait_sec);
                sleep(Duration::from_secs(wait_sec)).await;
@@ -273,7 +275,7 @@ impl ECStore {
        Ok(ec)
    }

-    pub async fn init(self: &Arc<Self>) -> Result<()> {
+    pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
        GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;

        if self.load_rebalance_meta().await.is_ok() {
@@ -317,18 +319,16 @@ impl ECStore {
        if !pool_indices.is_empty() {
            let idx = pool_indices[0];
            if endpoints.as_ref()[idx].endpoints.as_ref()[0].is_local {
-                let (_tx, rx) = broadcast::channel(1);
-
                let store = self.clone();

                tokio::spawn(async move {
                    // wait  3 minutes for cluster init
                    tokio::time::sleep(Duration::from_secs(60 * 3)).await;

-                    if let Err(err) = store.decommission(rx.resubscribe(), pool_indices.clone()).await {
+                    if let Err(err) = store.decommission(rx.clone(), pool_indices.clone()).await {
                        if err == StorageError::DecommissionAlreadyRunning {
                            for i in pool_indices.iter() {
-                                store.do_decommission_in_routine(rx.resubscribe(), *i).await;
+                                store.do_decommission_in_routine(rx.clone(), *i).await;
                            }
                            return;
                        }
@@ -700,9 +700,13 @@ impl ECStore {
        opts: &ObjectOptions,
    ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
        let mut futures = Vec::new();
-
        for pool in self.pools.iter() {
-            futures.push(pool.get_object_info(bucket, object, opts));
+            let mut pool_opts = opts.clone();
+            if !pool_opts.metadata_chg {
+                pool_opts.version_id = None;
+            }
+
+            futures.push(async move { pool.get_object_info(bucket, object, &pool_opts).await });
        }

        let results = join_all(futures).await;
@@ -1226,6 +1230,7 @@ impl StorageAPI for ECStore {
        if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
            let err = to_object_err(err.into(), vec![bucket]);
            if !is_err_bucket_exists(&err) {
+                error!("make bucket failed: {err}");
                let _ = self
                    .delete_bucket(
                        bucket,
@@ -1351,6 +1356,17 @@ impl StorageAPI for ECStore {
            .await
    }

+    async fn walk(
+        self: Arc<Self>,
+        rx: CancellationToken,
+        bucket: &str,
+        prefix: &str,
+        result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
+        opts: WalkOptions,
+    ) -> Result<()> {
+        self.walk_internal(rx, bucket, prefix, result, opts).await
+    }
+
    #[tracing::instrument(skip(self))]
    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
        check_object_args(bucket, object)?;
@@ -1450,9 +1466,12 @@ impl StorageAPI for ECStore {
        let object = encode_dir_object(object);
        let object = object.as_str();

+        let mut gopts = opts.clone();
+        gopts.no_lock = true;
+
        // 查询在哪个 pool
        let (mut pinfo, errs) = self
-            .get_pool_info_existing_with_opts(bucket, object, &opts)
+            .get_pool_info_existing_with_opts(bucket, object, &gopts)
            .await
            .map_err(|e| {
                if is_err_read_quorum(&e) {
@@ -1513,7 +1532,7 @@ impl StorageAPI for ECStore {
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
-    ) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)> {
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
        // encode object name
        let objects: Vec<ObjectToDelete> = objects
            .iter()
@@ -1534,131 +1553,160 @@ impl StorageAPI for ECStore {

        // TODO: nslock

-        let mut futures = Vec::with_capacity(objects.len());
+        let mut futures = Vec::with_capacity(self.pools.len());

-        for obj in objects.iter() {
-            futures.push(async move {
-                self.internal_get_pool_info_existing_with_opts(
-                    bucket,
-                    &obj.object_name,
-                    &ObjectOptions {
-                        no_lock: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-            });
+        for pool in self.pools.iter() {
+            futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone()));
        }

        let results = join_all(futures).await;

-        // let mut jhs = Vec::new();
-        // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
-        // let pools = Arc::new(self.pools.clone());
+        for idx in 0..del_objects.len() {
+            for (dels, errs) in results.iter() {
+                if errs[idx].is_none() && dels[idx].found {
+                    del_errs[idx] = None;
+                    del_objects[idx] = dels[idx].clone();
+                    break;
+                }
+
+                if del_errs[idx].is_none() {
+                    del_errs[idx] = errs[idx].clone();
+                    del_objects[idx] = dels[idx].clone();
+                }
+            }
+        }
+
+        del_objects.iter_mut().for_each(|v| {
+            v.object_name = decode_dir_object(&v.object_name);
+        });
+
+        (del_objects, del_errs)
+
+        // let mut futures = Vec::with_capacity(objects.len());

        // for obj in objects.iter() {
-        //     let (semaphore, pools, bucket, object_name, opt) = (
-        //         semaphore.clone(),
-        //         pools.clone(),
-        //         bucket.to_string(),
-        //         obj.object_name.to_string(),
-        //         ObjectOptions::default(),
-        //     );
-
-        //     let jh = tokio::spawn(async move {
-        //         let _permit = semaphore.acquire().await.unwrap();
-        //         self.internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt)
-        //             .await
+        //     futures.push(async move {
+        //         self.internal_get_pool_info_existing_with_opts(
+        //             bucket,
+        //             &obj.object_name,
+        //             &ObjectOptions {
+        //                 no_lock: true,
+        //                 ..Default::default()
+        //             },
+        //         )
+        //         .await
        //     });
-        //     jhs.push(jh);
-        // }
-        // let mut results = Vec::new();
-        // for jh in jhs {
-        //     results.push(jh.await.unwrap());
        // }

-        // 记录 pool Index 对应的 objects pool_idx -> objects idx
-        let mut pool_obj_idx_map = HashMap::new();
-        let mut orig_index_map = HashMap::new();
+        // let results = join_all(futures).await;

-        for (i, res) in results.into_iter().enumerate() {
-            match res {
-                Ok((pinfo, _)) => {
-                    if let Some(obj) = objects.get(i) {
-                        if pinfo.object_info.delete_marker && obj.version_id.is_none() {
-                            del_objects[i] = DeletedObject {
-                                delete_marker: pinfo.object_info.delete_marker,
-                                delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
-                                object_name: decode_dir_object(&pinfo.object_info.name),
-                                delete_marker_mtime: pinfo.object_info.mod_time,
-                                ..Default::default()
-                            };
-                            continue;
-                        }
+        // // let mut jhs = Vec::new();
+        // // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
+        // // let pools = Arc::new(self.pools.clone());

-                        if !pool_obj_idx_map.contains_key(&pinfo.index) {
-                            pool_obj_idx_map.insert(pinfo.index, vec![obj.clone()]);
-                        } else if let Some(val) = pool_obj_idx_map.get_mut(&pinfo.index) {
-                            val.push(obj.clone());
-                        }
+        // // for obj in objects.iter() {
+        // //     let (semaphore, pools, bucket, object_name, opt) = (
+        // //         semaphore.clone(),
+        // //         pools.clone(),
+        // //         bucket.to_string(),
+        // //         obj.object_name.to_string(),
+        // //         ObjectOptions::default(),
+        // //     );

-                        if !orig_index_map.contains_key(&pinfo.index) {
-                            orig_index_map.insert(pinfo.index, vec![i]);
-                        } else if let Some(val) = orig_index_map.get_mut(&pinfo.index) {
-                            val.push(i);
-                        }
-                    }
-                }
-                Err(e) => {
-                    if !is_err_object_not_found(&e) && is_err_version_not_found(&e) {
-                        del_errs[i] = Some(e)
-                    }
+        // //     let jh = tokio::spawn(async move {
+        // //         let _permit = semaphore.acquire().await.unwrap();
+        // //         self.internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt)
+        // //             .await
+        // //     });
+        // //     jhs.push(jh);
+        // // }
+        // // let mut results = Vec::new();
+        // // for jh in jhs {
+        // //     results.push(jh.await.unwrap());
+        // // }

-                    if let Some(obj) = objects.get(i) {
-                        del_objects[i] = DeletedObject {
-                            object_name: decode_dir_object(&obj.object_name),
-                            version_id: obj.version_id.map(|v| v.to_string()),
-                            ..Default::default()
-                        }
-                    }
-                }
-            }
-        }
+        // // 记录 pool Index 对应的 objects pool_idx -> objects idx
+        // let mut pool_obj_idx_map = HashMap::new();
+        // let mut orig_index_map = HashMap::new();

-        if !pool_obj_idx_map.is_empty() {
-            for (i, sets) in self.pools.iter().enumerate() {
-                //  取 pool idx 对应的 objects index
-                if let Some(objs) = pool_obj_idx_map.get(&i) {
-                    //  取对应 obj，理论上不会 none
-                    // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();
+        // for (i, res) in results.into_iter().enumerate() {
+        //     match res {
+        //         Ok((pinfo, _)) => {
+        //             if let Some(obj) = objects.get(i) {
+        //                 if pinfo.object_info.delete_marker && obj.version_id.is_none() {
+        //                     del_objects[i] = DeletedObject {
+        //                         delete_marker: pinfo.object_info.delete_marker,
+        //                         delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
+        //                         object_name: decode_dir_object(&pinfo.object_info.name),
+        //                         delete_marker_mtime: pinfo.object_info.mod_time,
+        //                         ..Default::default()
+        //                     };
+        //                     continue;
+        //                 }

-                    if objs.is_empty() {
-                        continue;
-                    }
+        //                 if !pool_obj_idx_map.contains_key(&pinfo.index) {
+        //                     pool_obj_idx_map.insert(pinfo.index, vec![obj.clone()]);
+        //                 } else if let Some(val) = pool_obj_idx_map.get_mut(&pinfo.index) {
+        //                     val.push(obj.clone());
+        //                 }

-                    let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;
+        //                 if !orig_index_map.contains_key(&pinfo.index) {
+        //                     orig_index_map.insert(pinfo.index, vec![i]);
+        //                 } else if let Some(val) = orig_index_map.get_mut(&pinfo.index) {
+        //                     val.push(i);
+        //                 }
+        //             }
+        //         }
+        //         Err(e) => {
+        //             if !is_err_object_not_found(&e) && is_err_version_not_found(&e) {
+        //                 del_errs[i] = Some(e)
+        //             }

-                    // 同时存入不可能为 none
-                    let org_indexes = orig_index_map.get(&i).unwrap();
+        //             if let Some(obj) = objects.get(i) {
+        //                 del_objects[i] = DeletedObject {
+        //                     object_name: decode_dir_object(&obj.object_name),
+        //                     version_id: obj.version_id.map(|v| v.to_string()),
+        //                     ..Default::default()
+        //                 }
+        //             }
+        //         }
+        //     }
+        // }

-                    // perrs 的顺序理论上跟 obj_idxs 顺序一致
-                    for (i, err) in perrs.into_iter().enumerate() {
-                        let obj_idx = org_indexes[i];
+        // if !pool_obj_idx_map.is_empty() {
+        //     for (i, sets) in self.pools.iter().enumerate() {
+        //         //  取 pool idx 对应的 objects index
+        //         if let Some(objs) = pool_obj_idx_map.get(&i) {
+        //             //  取对应 obj，理论上不会 none
+        //             // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();

-                        if err.is_some() {
-                            del_errs[obj_idx] = err;
-                        }
+        //             if objs.is_empty() {
+        //                 continue;
+        //             }

-                        let mut dobj = pdel_objs.get(i).unwrap().clone();
-                        dobj.object_name = decode_dir_object(&dobj.object_name);
+        //             let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;

-                        del_objects[obj_idx] = dobj;
-                    }
-                }
-            }
-        }
+        //             // 同时存入不可能为 none
+        //             let org_indexes = orig_index_map.get(&i).unwrap();

-        Ok((del_objects, del_errs))
+        //             // perrs 的顺序理论上跟 obj_idxs 顺序一致
+        //             for (i, err) in perrs.into_iter().enumerate() {
+        //                 let obj_idx = org_indexes[i];
+
+        //                 if err.is_some() {
+        //                     del_errs[obj_idx] = err;
+        //                 }
+
+        //                 let mut dobj = pdel_objs.get(i).unwrap().clone();
+        //                 dobj.object_name = decode_dir_object(&dobj.object_name);
+
+        //                 del_objects[obj_idx] = dobj;
+        //             }
+        //         }
+        //     }
+        // }
+
+        // Ok((del_objects, del_errs))
    }

    #[tracing::instrument(skip(self))]
@@ -2373,7 +2421,7 @@ fn check_list_multipart_args(
            }
        }

-        if let Err(_e) = base64_decode(upload_id_marker.as_bytes()) {
+        if let Err(_e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id_marker.as_bytes()) {
            return Err(StorageError::MalformedUploadID(upload_id_marker.to_owned()));
        }
    }
@@ -2400,7 +2448,7 @@ fn check_new_multipart_args(bucket: &str, object: &str) -> Result<()> {
 }

 fn check_multipart_object_args(bucket: &str, object: &str, upload_id: &str) -> Result<()> {
-    if let Err(e) = base64_decode(upload_id.as_bytes()) {
+    if let Err(e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes()) {
        return Err(StorageError::MalformedUploadID(format!("{bucket}/{object}-{upload_id},err:{e}")));
    };
    check_object_args(bucket, object)
@@ -14,7 +14,6 @@

 use crate::bucket::metadata_sys::get_versioning_config;
 use crate::bucket::versioning::VersioningApi as _;
-use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType};
 use crate::disk::DiskStore;
 use crate::error::{Error, Result};
 use crate::store_utils::clean_metadata;
@@ -23,22 +22,30 @@ use crate::{
    bucket::lifecycle::lifecycle::ExpirationOptions,
    bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions},
 };
+use bytes::Bytes;
 use http::{HeaderMap, HeaderValue};
 use rustfs_common::heal_channel::HealOpts;
-use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER;
-use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING};
+use rustfs_filemeta::{
+    FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicationState,
+    ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
+};
 use rustfs_madmin::heal_commands::HealResultItem;
+use rustfs_rio::Checksum;
 use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader};
 use rustfs_utils::CompressionAlgorithm;
+use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER};
 use rustfs_utils::path::decode_dir_object;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::io::Cursor;
+use std::pin::Pin;
 use std::str::FromStr as _;
 use std::sync::Arc;
+use std::task::{Context, Poll};
 use time::OffsetDateTime;
-use tokio::io::{AsyncRead, AsyncReadExt};
+use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
+use tokio_util::sync::CancellationToken;
 use tracing::warn;
 use uuid::Uuid;

@@ -85,11 +92,28 @@ impl PutObjReader {
        PutObjReader { stream }
    }

+    pub fn as_hash_reader(&self) -> &HashReader {
+        &self.stream
+    }
+
    pub fn from_vec(data: Vec<u8>) -> Self {
+        use sha2::{Digest, Sha256};
        let content_length = data.len() as i64;
+        let sha256hex = if content_length > 0 {
+            Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower))
+        } else {
+            None
+        };
        PutObjReader {
-            stream: HashReader::new(Box::new(WarpReader::new(Cursor::new(data))), content_length, content_length, None, false)
-                .unwrap(),
+            stream: HashReader::new(
+                Box::new(WarpReader::new(Cursor::new(data))),
+                content_length,
+                content_length,
+                None,
+                sha256hex,
+                false,
+            )
+            .unwrap(),
        }
    }

@@ -221,6 +245,12 @@ impl GetObjectReader {
    }
 }

+impl AsyncRead for GetObjectReader {
+    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        Pin::new(&mut self.stream).poll_read(cx, buf)
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct HTTPRangeSpec {
    pub is_suffix_length: bool,
@@ -234,11 +264,16 @@ impl HTTPRangeSpec {
            return None;
        }

-        let mut start = 0i64;
-        let mut end = -1i64;
-        for i in 0..oi.parts.len().min(part_number) {
+        if part_number == 0 || part_number > oi.parts.len() {
+            return None;
+        }
+
+        let mut start = 0_i64;
+        let mut end = -1_i64;
+        for i in 0..part_number {
+            let part = &oi.parts[i];
            start = end + 1;
-            end = start + (oi.parts[i].size as i64) - 1
+            end = start + (part.size as i64) - 1;
        }

        Some(HTTPRangeSpec {
@@ -253,8 +288,14 @@ impl HTTPRangeSpec {

        let mut start = self.start;
        if self.is_suffix_length {
-            start = res_size + self.start;
-
+            let suffix_len = if self.start < 0 {
+                self.start
+                    .checked_neg()
+                    .ok_or_else(|| Error::other("range value invalid: suffix length overflow"))?
+            } else {
+                self.start
+            };
+            start = res_size - suffix_len;
            if start < 0 {
                start = 0;
            }
@@ -267,7 +308,13 @@ impl HTTPRangeSpec {
        }

        if self.is_suffix_length {
-            let specified_len = self.start; // 假设 h.start 是一个 i64 类型
+            let specified_len = if self.start < 0 {
+                self.start
+                    .checked_neg()
+                    .ok_or_else(|| Error::other("range value invalid: suffix length overflow"))?
+            } else {
+                self.start
+            };
            let mut range_length = specified_len;

            if specified_len > res_size {
@@ -326,6 +373,7 @@ pub struct ObjectOptions {

    pub skip_decommissioned: bool,
    pub skip_rebalancing: bool,
+    pub skip_free_version: bool,

    pub data_movement: bool,
    pub src_pool_idx: usize,
@@ -334,27 +382,79 @@ pub struct ObjectOptions {
    pub metadata_chg: bool,
    pub http_preconditions: Option<HTTPPreconditions>,

+    pub delete_replication: Option<ReplicationState>,
    pub replication_request: bool,
    pub delete_marker: bool,

-    pub skip_free_version: bool,
-
    pub transition: TransitionOptions,
    pub expiration: ExpirationOptions,
    pub lifecycle_audit_event: LcAuditEvent,

    pub eval_metadata: Option<HashMap<String, String>>,
+
+    pub want_checksum: Option<Checksum>,
 }

-// impl Default for ObjectOptions {
-//     fn default() -> Self {
-//         Self {
-//             max_parity: Default::default(),
-//             mod_time: OffsetDateTime::UNIX_EPOCH,
-//             part_number: Default::default(),
-//         }
-//     }
-// }
+impl ObjectOptions {
+    pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) {
+        let mut rs = ReplicationState {
+            replicate_decision_str: dsc.to_string(),
+            ..Default::default()
+        };
+        if self.version_id.is_none() {
+            rs.replication_status_internal = dsc.pending_status();
+            rs.targets = replication_statuses_map(rs.replication_status_internal.as_deref().unwrap_or_default());
+        } else {
+            rs.version_purge_status_internal = dsc.pending_status();
+            rs.purge_targets = version_purge_statuses_map(rs.version_purge_status_internal.as_deref().unwrap_or_default());
+        }
+
+        self.delete_replication = Some(rs)
+    }
+
+    pub fn set_replica_status(&mut self, status: ReplicationStatusType) {
+        if let Some(rs) = self.delete_replication.as_mut() {
+            rs.replica_status = status;
+            rs.replica_timestamp = Some(OffsetDateTime::now_utc());
+        } else {
+            self.delete_replication = Some(ReplicationState {
+                replica_status: status,
+                replica_timestamp: Some(OffsetDateTime::now_utc()),
+                ..Default::default()
+            });
+        }
+    }
+
+    pub fn version_purge_status(&self) -> VersionPurgeStatusType {
+        self.delete_replication
+            .as_ref()
+            .map(|v| v.composite_version_purge_status())
+            .unwrap_or(VersionPurgeStatusType::Empty)
+    }
+
+    pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
+        self.delete_replication
+            .as_ref()
+            .map(|v| v.composite_replication_status())
+            .unwrap_or(ReplicationStatusType::Empty)
+    }
+
+    pub fn put_replication_state(&self) -> ReplicationState {
+        let rs = match self
+            .user_defined
+            .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_STATUS}").as_str())
+        {
+            Some(v) => v.to_string(),
+            None => return ReplicationState::default(),
+        };
+
+        ReplicationState {
+            replication_status_internal: Some(rs.to_string()),
+            targets: replication_statuses_map(rs.as_str()),
+            ..Default::default()
+        }
+    }
+}

 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct BucketOptions {
@@ -375,6 +475,8 @@ pub struct BucketInfo {
 #[derive(Debug, Default, Clone)]
 pub struct MultipartUploadResult {
    pub upload_id: String,
+    pub checksum_algo: Option<String>,
+    pub checksum_type: Option<String>,
 }

 #[derive(Debug, Default, Clone)]
@@ -390,13 +492,24 @@ pub struct PartInfo {
 pub struct CompletePart {
    pub part_num: usize,
    pub etag: Option<String>,
+    // pub size: Option<usize>,
+    pub checksum_crc32: Option<String>,
+    pub checksum_crc32c: Option<String>,
+    pub checksum_sha1: Option<String>,
+    pub checksum_sha256: Option<String>,
+    pub checksum_crc64nvme: Option<String>,
 }

 impl From<s3s::dto::CompletedPart> for CompletePart {
    fn from(value: s3s::dto::CompletedPart) -> Self {
        Self {
            part_num: value.part_number.unwrap_or_default() as usize,
-            etag: value.e_tag,
+            etag: value.e_tag.map(|v| v.value().to_owned()),
+            checksum_crc32: value.checksum_crc32,
+            checksum_crc32c: value.checksum_crc32c,
+            checksum_sha1: value.checksum_sha1,
+            checksum_sha256: value.checksum_sha256,
+            checksum_crc64nvme: value.checksum_crc64nvme,
        }
    }
 }
@@ -423,6 +536,7 @@ pub struct ObjectInfo {
    pub is_latest: bool,
    pub content_type: Option<String>,
    pub content_encoding: Option<String>,
+    pub expires: Option<OffsetDateTime>,
    pub num_versions: usize,
    pub successor_mod_time: Option<OffsetDateTime>,
    pub put_object_reader: Option<PutObjReader>,
@@ -430,11 +544,12 @@ pub struct ObjectInfo {
    pub inlined: bool,
    pub metadata_only: bool,
    pub version_only: bool,
-    pub replication_status_internal: String,
+    pub replication_status_internal: Option<String>,
    pub replication_status: ReplicationStatusType,
-    pub version_purge_status_internal: String,
+    pub version_purge_status_internal: Option<String>,
    pub version_purge_status: VersionPurgeStatusType,
-    pub checksum: Vec<u8>,
+    pub replication_decision: String,
+    pub checksum: Option<Bytes>,
 }

 impl Clone for ObjectInfo {
@@ -470,7 +585,9 @@ impl Clone for ObjectInfo {
            replication_status: self.replication_status.clone(),
            version_purge_status_internal: self.version_purge_status_internal.clone(),
            version_purge_status: self.version_purge_status.clone(),
-            checksum: Default::default(),
+            replication_decision: self.replication_decision.clone(),
+            checksum: self.checksum.clone(),
+            expires: self.expires,
        }
    }
 }
@@ -609,6 +726,7 @@ impl ObjectInfo {
            inlined,
            user_defined: metadata,
            transitioned_object,
+            checksum: fi.checksum.clone(),
            ..Default::default()
        }
    }
@@ -665,7 +783,10 @@ impl ObjectInfo {
                };

                for fi in versions.iter() {
-                    // TODO:VersionPurgeStatus
+                    if !fi.version_purge_status().is_empty() {
+                        continue;
+                    }
+
                    let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
                    objects.push(ObjectInfo::from_file_info(fi, bucket, &entry.name, versioned));
                }
@@ -770,6 +891,49 @@ impl ObjectInfo {

        objects
    }
+
+    pub fn replication_state(&self) -> ReplicationState {
+        ReplicationState {
+            replication_status_internal: self.replication_status_internal.clone(),
+            version_purge_status_internal: self.version_purge_status_internal.clone(),
+            replicate_decision_str: self.replication_decision.clone(),
+            targets: replication_statuses_map(self.replication_status_internal.clone().unwrap_or_default().as_str()),
+            purge_targets: version_purge_statuses_map(self.version_purge_status_internal.clone().unwrap_or_default().as_str()),
+            reset_statuses_map: self
+                .user_defined
+                .iter()
+                .filter_map(|(k, v)| {
+                    if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}")) {
+                        Some((
+                            k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-"))
+                                .to_string(),
+                            v.clone(),
+                        ))
+                    } else {
+                        None
+                    }
+                })
+                .collect(),
+            ..Default::default()
+        }
+    }
+
+    pub fn decrypt_checksums(&self, part: usize, _headers: &HeaderMap) -> Result<(HashMap<String, String>, bool)> {
+        if part > 0 {
+            if let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone()) {
+                return Ok((checksums, true));
+            }
+        }
+
+        // TODO: decrypt checksums
+
+        if let Some(data) = &self.checksum {
+            let (checksums, is_multipart) = rustfs_rio::read_checksums(data.as_ref(), 0);
+            return Ok((checksums, is_multipart));
+        }
+
+        Ok((HashMap::new(), false))
+    }
 }

 #[derive(Debug, Default)]
@@ -927,17 +1091,52 @@ pub struct ListPartsInfo {
 pub struct ObjectToDelete {
    pub object_name: String,
    pub version_id: Option<Uuid>,
+    pub delete_marker_replication_status: Option<String>,
+    pub version_purge_status: Option<VersionPurgeStatusType>,
+    pub version_purge_statuses: Option<String>,
+    pub replicate_decision_str: Option<String>,
 }
+
+impl ObjectToDelete {
+    pub fn replication_state(&self) -> ReplicationState {
+        ReplicationState {
+            replication_status_internal: self.delete_marker_replication_status.clone(),
+            version_purge_status_internal: self.version_purge_statuses.clone(),
+            replicate_decision_str: self.replicate_decision_str.clone().unwrap_or_default(),
+            targets: replication_statuses_map(self.delete_marker_replication_status.as_deref().unwrap_or_default()),
+            purge_targets: version_purge_statuses_map(self.version_purge_statuses.as_deref().unwrap_or_default()),
+            ..Default::default()
+        }
+    }
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct DeletedObject {
    pub delete_marker: bool,
-    pub delete_marker_version_id: Option<String>,
+    pub delete_marker_version_id: Option<Uuid>,
    pub object_name: String,
-    pub version_id: Option<String>,
+    pub version_id: Option<Uuid>,
    // MTime of DeleteMarker on source that needs to be propagated to replica
    pub delete_marker_mtime: Option<OffsetDateTime>,
    // to support delete marker replication
-    // pub replication_state: ReplicationState,
+    pub replication_state: Option<ReplicationState>,
+    pub found: bool,
+}
+
+impl DeletedObject {
+    pub fn version_purge_status(&self) -> VersionPurgeStatusType {
+        self.replication_state
+            .as_ref()
+            .map(|v| v.composite_version_purge_status())
+            .unwrap_or(VersionPurgeStatusType::Empty)
+    }
+
+    pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
+        self.replication_state
+            .as_ref()
+            .map(|v| v.composite_replication_status())
+            .unwrap_or(ReplicationStatusType::Empty)
+    }
 }

 #[derive(Debug, Default, Clone)]
@@ -949,8 +1148,33 @@ pub struct ListObjectVersionsInfo {
    pub prefixes: Vec<String>,
 }

+type WalkFilter = fn(&FileInfo) -> bool;
+
+#[derive(Clone, Default)]
+pub struct WalkOptions {
+    pub filter: Option<WalkFilter>,           // return WalkFilter returns 'true/false'
+    pub marker: Option<String>,               // set to skip until this object
+    pub latest_only: bool,                    // returns only latest versions for all matching objects
+    pub ask_disks: String,                    // dictates how many disks are being listed
+    pub versions_sort: WalkVersionsSortOrder, // sort order for versions of the same object; default: Ascending order in ModTime
+    pub limit: usize,                         // maximum number of items, 0 means no limit
+}
+
+#[derive(Clone, Default, PartialEq, Eq)]
+pub enum WalkVersionsSortOrder {
+    #[default]
+    Ascending,
+    Descending,
+}
+
+#[derive(Debug)]
+pub struct ObjectInfoOrErr {
+    pub item: Option<ObjectInfo>,
+    pub err: Option<Error>,
+}
+
 #[async_trait::async_trait]
-pub trait ObjectIO: Send + Sync + 'static {
+pub trait ObjectIO: Send + Sync + Debug + 'static {
    // GetObjectNInfo FIXME:
    async fn get_object_reader(
        &self,
@@ -966,7 +1190,7 @@ pub trait ObjectIO: Send + Sync + 'static {

 #[async_trait::async_trait]
 #[allow(clippy::too_many_arguments)]
-pub trait StorageAPI: ObjectIO {
+pub trait StorageAPI: ObjectIO + Debug {
    // NewNSLock TODO:
    // Shutdown TODO:
    // NSScanner TODO:
@@ -1000,7 +1224,15 @@ pub trait StorageAPI: ObjectIO {
        delimiter: Option<String>,
        max_keys: i32,
    ) -> Result<ListObjectVersionsInfo>;
-    // Walk TODO:
+
+    async fn walk(
+        self: Arc<Self>,
+        rx: CancellationToken,
+        bucket: &str,
+        prefix: &str,
+        result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
+        opts: WalkOptions,
+    ) -> Result<()>;

    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
    async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
@@ -1021,7 +1253,7 @@ pub trait StorageAPI: ObjectIO {
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
-    ) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)>;
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>);

    // TransitionObject TODO:
    // RestoreTransitionedObject TODO:
@@ -1396,6 +1628,83 @@ mod tests {
        assert_eq!(length, 10); // end - start + 1 = 14 - 5 + 1 = 10
    }

+    #[test]
+    fn test_http_range_spec_suffix_positive_start() {
+        let range_spec = HTTPRangeSpec {
+            is_suffix_length: true,
+            start: 5,
+            end: -1,
+        };
+
+        let (offset, length) = range_spec.get_offset_length(20).unwrap();
+        assert_eq!(offset, 15);
+        assert_eq!(length, 5);
+    }
+
+    #[test]
+    fn test_http_range_spec_suffix_negative_start() {
+        let range_spec = HTTPRangeSpec {
+            is_suffix_length: true,
+            start: -5,
+            end: -1,
+        };
+
+        let (offset, length) = range_spec.get_offset_length(20).unwrap();
+        assert_eq!(offset, 15);
+        assert_eq!(length, 5);
+    }
+
+    #[test]
+    fn test_http_range_spec_suffix_exceeds_object() {
+        let range_spec = HTTPRangeSpec {
+            is_suffix_length: true,
+            start: 50,
+            end: -1,
+        };
+
+        let (offset, length) = range_spec.get_offset_length(20).unwrap();
+        assert_eq!(offset, 0);
+        assert_eq!(length, 20);
+    }
+
+    #[test]
+    fn test_http_range_spec_from_object_info_valid_and_invalid_parts() {
+        let object_info = ObjectInfo {
+            size: 300,
+            parts: vec![
+                ObjectPartInfo {
+                    etag: String::new(),
+                    number: 1,
+                    size: 100,
+                    actual_size: 100,
+                    ..Default::default()
+                },
+                ObjectPartInfo {
+                    etag: String::new(),
+                    number: 2,
+                    size: 100,
+                    actual_size: 100,
+                    ..Default::default()
+                },
+                ObjectPartInfo {
+                    etag: String::new(),
+                    number: 3,
+                    size: 100,
+                    actual_size: 100,
+                    ..Default::default()
+                },
+            ],
+            ..Default::default()
+        };
+
+        let spec = HTTPRangeSpec::from_object_info(&object_info, 2).unwrap();
+        assert_eq!(spec.start, 100);
+        assert_eq!(spec.end, 199);
+
+        assert!(HTTPRangeSpec::from_object_info(&object_info, 0).is_none());
+        assert!(HTTPRangeSpec::from_object_info(&object_info, 4).is_none());
+    }
+
    #[tokio::test]
    async fn test_ranged_decompress_reader_zero_length() {
        let original_data = b"Hello, World!";
@@ -23,20 +23,23 @@ use crate::error::{
 };
 use crate::set_disk::SetDisks;
 use crate::store::check_list_objs_args;
-use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectOptions};
+use crate::store_api::{
+    ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions, WalkVersionsSortOrder,
+};
 use crate::store_utils::is_reserved_or_invalid_bucket;
 use crate::{store::ECStore, store_api::ListObjectsV2Info};
 use futures::future::join_all;
 use rand::seq::SliceRandom;
 use rustfs_filemeta::{
-    FileInfo, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
+    MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
    merge_file_meta_versions,
 };
 use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
 use std::collections::HashMap;
 use std::sync::Arc;
-use tokio::sync::broadcast::{self, Receiver as B_Receiver};
+use tokio::sync::broadcast::{self};
 use tokio::sync::mpsc::{self, Receiver, Sender};
+use tokio_util::sync::CancellationToken;
 use tracing::{error, info};
 use uuid::Uuid;

@@ -529,14 +532,15 @@ impl ECStore {
        }

        // cancel channel
-        let (cancel_tx, cancel_rx) = broadcast::channel(1);
+        let cancel = CancellationToken::new();
+
        let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);

        let (sender, recv) = mpsc::channel(o.limit as usize);

        let store = self.clone();
        let opts = o.clone();
-        let cancel_rx1 = cancel_rx.resubscribe();
+        let cancel_rx1 = cancel.clone();
        let err_tx1 = err_tx.clone();
        let job1 = tokio::spawn(async move {
            let mut opts = opts;
@@ -547,7 +551,7 @@ impl ECStore {
            }
        });

-        let cancel_rx2 = cancel_rx.resubscribe();
+        let cancel_rx2 = cancel.clone();

        let (result_tx, mut result_rx) = mpsc::channel(1);
        let err_tx2 = err_tx.clone();
@@ -559,7 +563,7 @@ impl ECStore {
            }

            // cancel call exit spawns
-            let _ = cancel_tx.send(true);
+            cancel.cancel();
        });

        let mut result = {
@@ -615,7 +619,7 @@ impl ECStore {
    // Read all
    async fn list_merged(
        &self,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        opts: ListPathOptions,
        sender: Sender<MetaCacheEntry>,
    ) -> Result<Vec<ObjectInfo>> {
@@ -631,9 +635,8 @@ impl ECStore {

                inputs.push(recv);
                let opts = opts.clone();
-
-                let rx = rx.resubscribe();
-                futures.push(set.list_path(rx, opts, send));
+                let rx_clone = rx.clone();
+                futures.push(set.list_path(rx_clone, opts, send));
            }
        }

@@ -695,9 +698,9 @@ impl ECStore {
    }

    #[allow(unused_assignments)]
-    pub async fn walk(
+    pub async fn walk_internal(
        self: Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        bucket: &str,
        prefix: &str,
        result: Sender<ObjectInfoOrErr>,
@@ -711,11 +714,11 @@ impl ECStore {
        for eset in self.pools.iter() {
            for set in eset.disk_set.iter() {
                let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
-                let rx = rx.resubscribe();
                let opts = opts.clone();

                let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
                inputs.push(list_out_rx);
+                let rx_clone = rx.clone();
                futures.push(async move {
                    let mut ask_disks = get_list_quorum(&opts.ask_disks, set.set_drive_count as i32);
                    if ask_disks == -1 {
@@ -770,7 +773,7 @@ impl ECStore {
                    let tx2 = sender.clone();

                    list_path_raw(
-                        rx.resubscribe(),
+                        rx_clone,
                        ListPathRawOptions {
                            disks: disks.iter().cloned().map(Some).collect(),
                            fallback_disks: fallback_disks.iter().cloned().map(Some).collect(),
@@ -936,33 +939,8 @@ impl ECStore {
    }
 }

-type WalkFilter = fn(&FileInfo) -> bool;
-
-#[derive(Clone, Default)]
-pub struct WalkOptions {
-    pub filter: Option<WalkFilter>,           // return WalkFilter returns 'true/false'
-    pub marker: Option<String>,               // set to skip until this object
-    pub latest_only: bool,                    // returns only latest versions for all matching objects
-    pub ask_disks: String,                    // dictates how many disks are being listed
-    pub versions_sort: WalkVersionsSortOrder, // sort order for versions of the same object; default: Ascending order in ModTime
-    pub limit: usize,                         // maximum number of items, 0 means no limit
-}
-
-#[derive(Clone, Default, PartialEq, Eq)]
-pub enum WalkVersionsSortOrder {
-    #[default]
-    Ascending,
-    Descending,
-}
-
-#[derive(Debug)]
-pub struct ObjectInfoOrErr {
-    pub item: Option<ObjectInfo>,
-    pub err: Option<Error>,
-}
-
 async fn gather_results(
-    _rx: B_Receiver<bool>,
+    _rx: CancellationToken,
    opts: ListPathOptions,
    recv: Receiver<MetaCacheEntry>,
    results_tx: Sender<MetaCacheEntriesSortedResult>,
@@ -1067,12 +1045,11 @@ async fn select_from(

 // TODO: exit when cancel
 async fn merge_entry_channels(
-    rx: B_Receiver<bool>,
+    rx: CancellationToken,
    in_channels: Vec<Receiver<MetaCacheEntry>>,
    out_channel: Sender<MetaCacheEntry>,
    read_quorum: usize,
 ) -> Result<()> {
-    let mut rx = rx;
    let mut in_channels = in_channels;
    if in_channels.len() == 1 {
        loop {
@@ -1085,7 +1062,7 @@ async fn merge_entry_channels(
                        return Ok(())
                    }
                },
-                _ = rx.recv()=>{
+                _ = rx.cancelled()=>{
                    info!("merge_entry_channels rx.recv() cancel");
                    return Ok(())
                },
@@ -1228,7 +1205,7 @@ async fn merge_entry_channels(
 }

 impl SetDisks {
-    pub async fn list_path(&self, rx: B_Receiver<bool>, opts: ListPathOptions, sender: Sender<MetaCacheEntry>) -> Result<()> {
+    pub async fn list_path(&self, rx: CancellationToken, opts: ListPathOptions, sender: Sender<MetaCacheEntry>) -> Result<()> {
        let (mut disks, infos, _) = self.get_online_disks_with_healing_and_info(true).await;

        let mut ask_disks = get_list_quorum(&opts.ask_disks, self.set_drive_count as i32);
@@ -15,8 +15,8 @@
 use crate::config::storageclass::STANDARD;
 use crate::disk::RUSTFS_META_BUCKET;
 use regex::Regex;
-use rustfs_filemeta::headers::AMZ_OBJECT_TAGGING;
-use rustfs_filemeta::headers::AMZ_STORAGE_CLASS;
+use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
+use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
 use std::collections::HashMap;
 use std::io::{Error, Result};

@@ -35,11 +35,13 @@ uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
 tokio = { workspace = true, features = ["io-util", "macros", "sync"] }
 xxhash-rust = { workspace = true, features = ["xxh64"] }
 bytes.workspace = true
-rustfs-utils = { workspace = true, features = ["hash"] }
+rustfs-utils = { workspace = true, features = ["hash","http"] }
 byteorder = { workspace = true }
 tracing.workspace = true
 thiserror.workspace = true
 s3s.workspace = true
+lazy_static.workspace = true
+regex.workspace = true

 [dev-dependencies]
 criterion = { workspace = true }
@@ -13,11 +13,11 @@
 // limitations under the License.

 use crate::error::{Error, Result};
-use crate::headers::RESERVED_METADATA_PREFIX_LOWER;
-use crate::headers::RUSTFS_HEALING;
+use crate::{ReplicationState, ReplicationStatusType, VersionPurgeStatusType};
 use bytes::Bytes;
 use rmp_serde::Serializer;
 use rustfs_utils::HashAlgorithm;
+use rustfs_utils::http::headers::{RESERVED_METADATA_PREFIX_LOWER, RUSTFS_HEALING};
 use serde::Deserialize;
 use serde::Serialize;
 use std::collections::HashMap;
@@ -202,7 +202,7 @@ pub struct FileInfo {
    // MarkDeleted marks this version as deleted
    pub mark_deleted: bool,
    // ReplicationState - Internal replication state to be passed back in ObjectInfo
-    // pub replication_state: Option<ReplicationState>, // TODO: implement ReplicationState
+    pub replication_state_internal: Option<ReplicationState>,
    pub data: Option<Bytes>,
    pub num_versions: usize,
    pub successor_mod_time: Option<OffsetDateTime>,
@@ -284,6 +284,7 @@ impl FileInfo {
        Ok(t)
    }

+    #[allow(clippy::too_many_arguments)]
    pub fn add_object_part(
        &mut self,
        num: usize,
@@ -292,6 +293,7 @@ impl FileInfo {
        mod_time: Option<OffsetDateTime>,
        actual_size: i64,
        index: Option<Bytes>,
+        checksums: Option<HashMap<String, String>>,
    ) {
        let part = ObjectPartInfo {
            etag,
@@ -300,7 +302,7 @@ impl FileInfo {
            mod_time,
            actual_size,
            index,
-            checksums: None,
+            checksums,
            error: None,
        };

@@ -471,6 +473,29 @@ impl FileInfo {
        // TODO: Add replication_state comparison when implemented
        // && self.replication_state == other.replication_state
    }
+
+    pub fn version_purge_status(&self) -> VersionPurgeStatusType {
+        self.replication_state_internal
+            .as_ref()
+            .map(|v| v.composite_version_purge_status())
+            .unwrap_or(VersionPurgeStatusType::Empty)
+    }
+    pub fn replication_status(&self) -> ReplicationStatusType {
+        self.replication_state_internal
+            .as_ref()
+            .map(|v| v.composite_replication_status())
+            .unwrap_or(ReplicationStatusType::Empty)
+    }
+    pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
+        if self.deleted {
+            self.replication_state_internal
+                .as_ref()
+                .map(|v| v.composite_replication_status())
+                .unwrap_or(ReplicationStatusType::Empty)
+        } else {
+            ReplicationStatusType::Empty
+        }
+    }
 }

 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
houseme	416d3ad5b7	Refactor: Add observability enable flag, improve comments, remove unused config params, and enhance run function error logging. (#689 ) * improve code for dns log * fix * Improve comments, remove unused parameters in config.rs (opt), add observability enable flag, and enhance error logging in run function execution.	2025-10-23 13:59:57 +08:00
weisd	f30698ec7f	Refactor Console Server Architecture (#685 ) * todo * fix console server * fix console server * fix console server * fix console server * fix console server	2025-10-23 00:06:09 +08:00
houseme	7dcf01f127	feat: adjust metrics push interval to 3 seconds (#686 ) - Reduce metrics push frequency from default to 3s for better performance - Optimize resource utilization during metrics collection - Improve real-time monitoring responsiveness Related to admin metrics optimization on fix/admin-metrics branch	2025-10-22 23:47:11 +08:00
weisd	e524a106c5	add make bucket error logs (#683 ) * add make bucket error logs	2025-10-22 16:23:08 +08:00
weisd	d9e5f5d2e3	fix (#682 )	2025-10-22 10:35:40 +08:00
livelycode36	684e832530	fix: prevent duplicate data volumes in entrypoint.sh (#681 )	2025-10-22 09:04:04 +08:00
weisd	a65856bdf4	Fix CRC32C Checksum Implementation and Enhance Authentication System (#678 ) * fix: get_condition_values * fix checksum crc32c * fix clippy	2025-10-21 21:28:00 +08:00
weisd	2edb2929b2	fix: DataUsageInfo add list bucket permission (#674 )	2025-10-21 10:05:54 +08:00
majinghe	14bc55479b	fix docker healthcheck unhealthy issue (#672 )	2025-10-21 09:39:15 +08:00
weisd	cd1e244c68	Refactor: Introduce content checksums and improve multipart/object metadata handling (#671 ) * feat: adapt to s3s typed etag support * refactor: move replication struct to rustfs_filemeta, fix filemeta transition bug * add head_object checksum, filter object metadata output * fix multipart checksum * fix multipart checksum * add content md5,sha256 check * fix test * fix cargo --------- Co-authored-by: overtrue <anzhengchao@gmail.com>	2025-10-20 23:46:13 +08:00
songhahaha66	46797dc815	fix(export): fix the policy and service account export (#665 ) * fix(export): fix the policy export mechanism * fix: correct service account check logic in IamSys	2025-10-20 19:40:54 +08:00
Nugine	7f24dbda19	build(deps): upgrade s3s (#667 ) Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>	2025-10-19 18:32:01 +08:00
loverustfs	ef11d3a2eb	fix words error	2025-10-19 18:13:58 +08:00
loverustfs	d1398cb3ab	fix error	2025-10-19 18:10:45 +08:00
majinghe	95019c4cb5	add ansible installation with mnmd (#664 ) * add ansible installation with mnmd * change script install dir name	2025-10-18 22:20:17 +08:00
houseme	4168e6c180	chore(docs): move root `examples` to `docs/examples/docker` and update README (#663 ) * chore(docs): move root `examples` to `docs/examples/docker` and update README - Move root `examples/` contents into `docs/examples/docker/`. - Update `docs/examples/README.md` to add migration note, new `docker/` entry and usage examples. - Replace references from `examples/` to `docs/examples/docker/` where applicable. - Reminder: verify CI and external links still point to the correct paths. * fix	2025-10-17 17:17:36 +08:00
houseme	42d3645d6f	fix(targets): make target removal and reload transactional; prevent reappearing entries (#662 ) * feat: improve code for notify * upgrade starshard version * upgrade version * Fix ETag format to comply with HTTP standards by wrapping with quotes (#592) * Initial plan * Fix ETag format to comply with HTTP standards by wrapping with quotes Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> * bufigx --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> Co-authored-by: overtrue <anzhengchao@gmail.com> * Improve lock (#596) * improve lock Signed-off-by: Mu junxiang <1948535941@qq.com> * feat(tests): add wait_for_object_absence helper and improve lifecycle test reliability Signed-off-by: Mu junxiang <1948535941@qq.com> * chore: remove dirty docs Signed-off-by: Mu junxiang <1948535941@qq.com> --------- Signed-off-by: Mu junxiang <1948535941@qq.com> * feat(append): implement object append operations with state tracking (#599) * feat(append): implement object append operations with state tracking Signed-off-by: junxiang Mu <1948535941@qq.com> * chore: rebase Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: junxiang Mu <1948535941@qq.com> * build(deps): upgrade s3s (#595) Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com> * fix: validate mqtt broker * improve code for `import` * fix * improve * remove logger from `rustfs-obs` crate * remove code for config Observability * fix * improve code * fix comment * up * up * upgrade version * fix * fmt * upgrade tokio version to 1.48.0 * upgrade `datafusion` and `reed-solomon-simd` version * fix * fmt * improve code for notify webhook example * improve code * fix * fix * fmt --------- Signed-off-by: Mu junxiang <1948535941@qq.com> Signed-off-by: junxiang Mu <1948535941@qq.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> Co-authored-by: overtrue <anzhengchao@gmail.com> Co-authored-by: guojidan <63799833+guojidan@users.noreply.github.com> Co-authored-by: Nugine <nugine@foxmail.com> Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>	2025-10-17 15:34:53 +08:00
安正超	30e7f00b02	fix: update ahm integration test fixture (#659 )	2025-10-17 09:13:56 +08:00
overtrue	58f8a8f46b	fix: correct HTTP range suffix handling	2025-10-16 21:39:21 +08:00
gatewayJ	aae768f446	feat: Simple OPA support (#644 ) * opa-feature * Update crates/policy/src/policy/opa.rs * add the content related to 'Copyright' --------- Co-authored-by: root <root@debian.localdomain> Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>	2025-10-16 10:35:26 +08:00
安正超	d447b3e426	feat: adapt to s3s typed etag support (#653 ) * feat: adapt to s3s typed etag support * refactor: streamline etag handling	2025-10-15 21:27:20 +08:00
安正超	8f310cd4a8	test: allow mocking dns resolver (#656 )	2025-10-15 21:24:03 +08:00
majinghe	8ed01a3e06	Refactor mnmd docker compose for extendence (#652 )	2025-10-15 03:48:05 +08:00
loverustfs	9e1739ed8d	chore(docs): update README and README_ZH (#649 )	2025-10-13 18:49:34 +08:00
loverustfs	7abbfc9c2c	RustFS trending images RustFS trending	2025-10-13 17:45:54 +08:00
安正超	639bf0c233	Revert "feat(append): implement object append operations with state tracking (#599 )" (#646 ) This reverts commit `4f73760a45`.	2025-10-12 23:47:51 +08:00
Copilot	ad99019749	Add complete MNMD Docker deployment example with startup coordination and VolumeNotFound fix (#642 ) * Initial plan * Add MNMD Docker deployment example with 4 nodes x 4 drives - Create docs/examples/mnmd/ directory structure - Add docker-compose.yml with proper disk indexing (1..4) - Add wait-and-start.sh for startup coordination - Add README.md with usage instructions and alternatives - Add CHECKLIST.md with step-by-step verification - Fixes VolumeNotFound issue by using correct volume paths - Implements health checks and startup ordering - Uses service names for stable inter-node addressing Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add docs/examples README as index for deployment examples Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add automated test script for MNMD deployment - Add test-deployment.sh with comprehensive validation - Test container status, health, endpoints, connectivity - Update README to reference test script - Make script executable Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * improve code * improve code * improve dep crates `cargo shear --fix` * upgrade aws-sdk-s3 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>	2025-10-12 13:15:14 +08:00
houseme	aac9b1edb7	chore: improve event and docker-compose ,Improve the permissions of the `endpoint` health interface, upgrade otel from 0.30.0 to 0.31.0 (#620 ) * feat: improve code for notify * upgrade starshard version * upgrade version * Fix ETag format to comply with HTTP standards by wrapping with quotes (#592) * Initial plan * Fix ETag format to comply with HTTP standards by wrapping with quotes Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> * bufigx --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> Co-authored-by: overtrue <anzhengchao@gmail.com> * Improve lock (#596) * improve lock Signed-off-by: Mu junxiang <1948535941@qq.com> * feat(tests): add wait_for_object_absence helper and improve lifecycle test reliability Signed-off-by: Mu junxiang <1948535941@qq.com> * chore: remove dirty docs Signed-off-by: Mu junxiang <1948535941@qq.com> --------- Signed-off-by: Mu junxiang <1948535941@qq.com> * feat(append): implement object append operations with state tracking (#599) * feat(append): implement object append operations with state tracking Signed-off-by: junxiang Mu <1948535941@qq.com> * chore: rebase Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: junxiang Mu <1948535941@qq.com> * build(deps): upgrade s3s (#595) Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com> * fix: validate mqtt broker * improve code for `import` * upgrade otel relation crates version * fix：dep("jsonwebtoken") feature = 'rust_crypto' * fix * fix * fix * upgrade version * improve code for ecfs * chore: improve event and docker-compose ,Improve the permissions of the `endpoint` health interface * fix * fix * fix * fix * improve code * fix --------- Signed-off-by: Mu junxiang <1948535941@qq.com> Signed-off-by: junxiang Mu <1948535941@qq.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> Co-authored-by: overtrue <anzhengchao@gmail.com> Co-authored-by: guojidan <63799833+guojidan@users.noreply.github.com> Co-authored-by: Nugine <nugine@foxmail.com> Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>	2025-10-11 09:08:25 +08:00
weisd	5689311cff	fix:#630 (#633 )	2025-10-10 15:16:28 +08:00
安正超	007d9c0b21	fix: normalize ETag comparison in multipart upload and replication (#627 ) - Normalize ETags by removing quotes before comparison in complete_multipart_upload - Fix ETag comparison in replication logic to handle quoted ETags from API responses - Fix ETag comparison in transition object logic - Add unit tests for trim_etag function This fixes the ETag mismatch error when uploading large files (5GB+) via multipart upload, which was caused by PR #592 adding quotes to ETag responses while internal storage remains unquoted. Fixes #625	2025-10-08 21:19:57 +08:00
Nugine	626c7ed34a	fix: CompleteMultipartUpload encryption (#626 )	2025-10-08 20:27:40 +08:00
houseme	0e680eae31	fix typos and bump the dependencies group with 9 updates (#614 ) * fix typos * build(deps): bump the dependencies group with 9 updates (#613) Bumps the dependencies group with 9 updates: \| Package \| From \| To \| \| --- \| --- \| --- \| \| [axum](https://github.com/tokio-rs/axum) \| `0.8.4` \| `0.8.6` \| \| [axum-extra](https://github.com/tokio-rs/axum) \| `0.10.1` \| `0.10.3` \| \| [regex](https://github.com/rust-lang/regex) \| `1.11.2` \| `1.11.3` \| \| [serde](https://github.com/serde-rs/serde) \| `1.0.226` \| `1.0.228` \| \| [shadow-rs](https://github.com/baoyachi/shadow-rs) \| `1.3.0` \| `1.4.0` \| \| [sysinfo](https://github.com/GuillaumeGomez/sysinfo) \| `0.37.0` \| `0.37.1` \| \| [thiserror](https://github.com/dtolnay/thiserror) \| `2.0.16` \| `2.0.17` \| \| [tokio-rustls](https://github.com/rustls/tokio-rustls) \| `0.26.3` \| `0.26.4` \| \| [zeroize](https://github.com/RustCrypto/utils) \| `1.8.1` \| `1.8.2` \| Updates `axum` from 0.8.4 to 0.8.6 - [Release notes](https://github.com/tokio-rs/axum/releases) - [Changelog](https://github.com/tokio-rs/axum/blob/main/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/axum/compare/axum-v0.8.4...axum-v0.8.6) Updates `axum-extra` from 0.10.1 to 0.10.3 - [Release notes](https://github.com/tokio-rs/axum/releases) - [Changelog](https://github.com/tokio-rs/axum/blob/main/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/axum/compare/axum-extra-v0.10.1...axum-extra-v0.10.3) Updates `regex` from 1.11.2 to 1.11.3 - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.11.2...1.11.3) Updates `serde` from 1.0.226 to 1.0.228 - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.226...v1.0.228) Updates `shadow-rs` from 1.3.0 to 1.4.0 - [Release notes](https://github.com/baoyachi/shadow-rs/releases) - [Commits](https://github.com/baoyachi/shadow-rs/compare/1.3.0...v1.4.0) Updates `sysinfo` from 0.37.0 to 0.37.1 - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/compare/v0.37.0...v0.37.1) Updates `thiserror` from 2.0.16 to 2.0.17 - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/2.0.16...2.0.17) Updates `tokio-rustls` from 0.26.3 to 0.26.4 - [Release notes](https://github.com/rustls/tokio-rustls/releases) - [Commits](https://github.com/rustls/tokio-rustls/compare/v/0.26.3...v/0.26.4) Updates `zeroize` from 1.8.1 to 1.8.2 - [Commits](https://github.com/RustCrypto/utils/compare/zeroize-v1.8.1...zeroize-v1.8.2) --- updated-dependencies: - dependency-name: axum dependency-version: 0.8.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: axum-extra dependency-version: 0.10.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: regex dependency-version: 1.11.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: serde dependency-version: 1.0.228 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: shadow-rs dependency-version: 1.4.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dependencies - dependency-name: sysinfo dependency-version: 0.37.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: thiserror dependency-version: 2.0.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: tokio-rustls dependency-version: 0.26.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: zeroize dependency-version: 1.8.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2025-10-02 23:29:18 +08:00
weisd	7622b37f7b	add iam notification (#604 ) move tonic service to rustfs	2025-09-30 17:32:23 +08:00
Nugine	f1dd3a982e	build(deps): upgrade s3s (#595 ) Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>	2025-09-28 21:10:42 +08:00
guojidan	4f73760a45	feat(append): implement object append operations with state tracking (#599 ) * feat(append): implement object append operations with state tracking Signed-off-by: junxiang Mu <1948535941@qq.com> * chore: rebase Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: junxiang Mu <1948535941@qq.com>	2025-09-27 20:06:26 -07:00
guojidan	be66cf8bd3	Improve lock (#596 ) * improve lock Signed-off-by: Mu junxiang <1948535941@qq.com> * feat(tests): add wait_for_object_absence helper and improve lifecycle test reliability Signed-off-by: Mu junxiang <1948535941@qq.com> * chore: remove dirty docs Signed-off-by: Mu junxiang <1948535941@qq.com> --------- Signed-off-by: Mu junxiang <1948535941@qq.com>	2025-09-27 17:57:56 -07:00
Copilot	23b40d398f	Fix ETag format to comply with HTTP standards by wrapping with quotes (#592 ) * Initial plan * Fix ETag format to comply with HTTP standards by wrapping with quotes Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> * bufigx --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> Co-authored-by: overtrue <anzhengchao@gmail.com>	2025-09-27 10:03:05 +08:00
weisd	90f21a9102	refactor: Reimplement bucket replication system with enhanced architecture (#590 ) * feat:refactor replication * use aws sdk for replication client * refactor/replication * merge main * fix lifecycle test	2025-09-26 14:27:53 +08:00
guojidan	9b029d18b2	feat(lock): enhance lock management with timeout and ownership tracking (#589 ) - Add lock timeout support and track acquisition time in lock state - Improve lock conflict handling with detailed error messages - Optimize lock reuse when already held by same owner - Refactor lock state to store owner info and timeout duration - Update all lock operations to handle new state structure Signed-off-by: junxiang Mu <1948535941@qq.com>	2025-09-25 20:21:53 -07:00