fix: normalize ETag comparison in multipart upload and replication (#627 )

- Normalize ETags by removing quotes before comparison in complete_multipart_upload - Fix ETag comparison in replication logic to handle quoted ETags from API responses - Fix ETag comparison in transition object logic - Add unit tests for trim_etag function This fixes the ETag mismatch error when uploading large files (5GB+) via multipart upload, which was caused by PR #592 adding quotes to ETag responses while internal storage remains unquoted. Fixes #625
fix: CompleteMultipartUpload encryption (#626 )
2026-07-29 09:38:59 +00:00 · 2025-10-08 21:19:57 +08:00 · 2025-10-08 20:27:40 +08:00 · 2025-10-02 23:29:18 +08:00 · 2025-09-30 17:32:23 +08:00 · 2025-09-28 21:10:42 +08:00
121 changed files with 16160 additions and 5705 deletions
@@ -854,7 +854,7 @@ dependencies = [
 "rustls-native-certs 0.8.1",
 "rustls-pki-types",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tower",
 "tracing",
 ]
@@ -979,9 +979,9 @@ dependencies = [

 [[package]]
 name = "axum"
-version = "0.8.4"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5"
+checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871"
 dependencies = [
 "axum-core",
 "bytes",
@@ -998,8 +998,7 @@ dependencies = [
 "mime",
 "percent-encoding",
 "pin-project-lite",
- "rustversion",
- "serde",
+ "serde_core",
 "serde_json",
 "serde_path_to_error",
 "serde_urlencoded",
@@ -1013,9 +1012,9 @@ dependencies = [

 [[package]]
 name = "axum-core"
-version = "0.5.2"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"
+checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22"
 dependencies = [
 "bytes",
 "futures-core",
@@ -1024,7 +1023,6 @@ dependencies = [
 "http-body-util",
 "mime",
 "pin-project-lite",
- "rustversion",
 "sync_wrapper",
 "tower-layer",
 "tower-service",
@@ -1033,9 +1031,9 @@ dependencies = [

 [[package]]
 name = "axum-extra"
-version = "0.10.1"
+version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45bf463831f5131b7d3c756525b305d40f1185b688565648a92e1392ca35713d"
+checksum = "9963ff19f40c6102c76756ef0a46004c0d58957d87259fc9208ff8441c12ab96"
 dependencies = [
 "axum",
 "axum-core",
@@ -1047,10 +1045,10 @@ dependencies = [
 "mime",
 "pin-project-lite",
 "rustversion",
- "serde",
- "tower",
+ "serde_core",
 "tower-layer",
 "tower-service",
+ "tracing",
 ]

 [[package]]
@@ -1071,7 +1069,7 @@ dependencies = [
 "rustls-pemfile 2.2.0",
 "rustls-pki-types",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tower-service",
 ]

@@ -1338,7 +1336,7 @@ dependencies = [
 "serde",
 "serde-untagged",
 "serde-value",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "toml",
 "unicode-xid",
 "url",
@@ -1356,7 +1354,7 @@ dependencies = [
 "semver",
 "serde",
 "serde_json",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 ]

 [[package]]
@@ -1602,12 +1600,6 @@ dependencies = [
 "tiny-keccak",
 ]

-[[package]]
-name = "const-str"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "451d0640545a0553814b4c646eb549343561618838e9b42495f466131fe3ad49"
-
 [[package]]
 name = "const-str"
 version = "0.7.0"
@@ -1654,6 +1646,15 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"

+[[package]]
+name = "convert_case"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -2886,6 +2887,7 @@ dependencies = [
 "chrono",
 "flatbuffers",
 "futures",
+ "http 1.3.1",
 "md5 0.8.0",
 "rand 0.9.2",
 "reqwest",
@@ -3149,7 +3151,7 @@ dependencies = [
 "regex",
 "serde",
 "serde_derive",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "toml",
 "tracing",
 "tracing-subscriber",
@@ -3495,10 +3497,10 @@ dependencies = [
 "rand 0.9.2",
 "ring",
 "rustls 0.23.32",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tinyvec",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tracing",
 "url",
 ]
@@ -3520,9 +3522,9 @@ dependencies = [
 "resolv-conf",
 "rustls 0.23.32",
 "smallvec",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tracing",
 ]

@@ -3719,7 +3721,7 @@ dependencies = [
 "rustls-native-certs 0.8.1",
 "rustls-pki-types",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tower-service",
 "webpki-roots",
 ]
@@ -4242,7 +4244,7 @@ dependencies = [
 "once_cell",
 "serde",
 "sha2 0.10.9",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "uuid",
 ]

@@ -4293,7 +4295,7 @@ checksum = "656b3b27f8893f7bbf9485148ff9a65f019e3f33bd5cdc87c83cab16b3fd9ec8"
 dependencies = [
 "libc",
 "neli",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "windows-sys 0.59.0",
 ]

@@ -4420,9 +4422,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"

 [[package]]
 name = "memchr"
-version = "2.7.5"
+version = "2.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
+checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"

 [[package]]
 name = "memmap2"
@@ -4665,15 +4667,6 @@ dependencies = [
 "windows-sys 0.52.0",
 ]

-[[package]]
-name = "nugine-rust-utils"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04dcd9cfa92246a9c7ca0671e00733c4e9d77ee1fa0ae08c9a181b7c8802aea2"
-dependencies = [
- "simdutf8",
-]
-
 [[package]]
 name = "num"
 version = "0.4.3"
@@ -4894,7 +4887,7 @@ dependencies = [
 "itertools 0.14.0",
 "parking_lot",
 "percent-encoding",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tracing",
 "url",
@@ -4947,7 +4940,7 @@ dependencies = [
 "futures-sink",
 "js-sys",
 "pin-project-lite",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tracing",
 ]

@@ -4976,7 +4969,7 @@ dependencies = [
 "opentelemetry-proto",
 "opentelemetry_sdk",
 "prost 0.13.5",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tonic 0.13.1",
 "tracing",
@@ -5024,7 +5017,7 @@ dependencies = [
 "percent-encoding",
 "rand 0.9.2",
 "serde_json",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tokio-stream",
 ]
@@ -5411,7 +5404,7 @@ dependencies = [
 "spin 0.10.0",
 "symbolic-demangle",
 "tempfile",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 ]

 [[package]]
@@ -5662,7 +5655,7 @@ dependencies = [
 "rustc-hash",
 "rustls 0.23.32",
 "socket2 0.6.0",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tracing",
 "web-time",
@@ -5683,7 +5676,7 @@ dependencies = [
 "rustls 0.23.32",
 "rustls-pki-types",
 "slab",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tinyvec",
 "tracing",
 "web-time",
@@ -5878,7 +5871,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
 dependencies = [
 "getrandom 0.2.16",
 "libredox",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 ]

 [[package]]
@@ -5913,9 +5906,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.11.2"
+version = "1.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912"
+checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -5925,9 +5918,9 @@ dependencies = [

 [[package]]
 name = "regex-automata"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6"
+checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -5978,7 +5971,7 @@ dependencies = [
 "serde_urlencoded",
 "sync_wrapper",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tokio-util",
 "tower",
 "tower-http",
@@ -6046,7 +6039,7 @@ dependencies = [
 "schemars",
 "serde",
 "serde_json",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tokio-util",
 "tracing",
@@ -6121,9 +6114,9 @@ dependencies = [
 "rustls-native-certs 0.8.1",
 "rustls-pemfile 2.2.0",
 "rustls-webpki 0.102.8",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tokio-stream",
 "tokio-util",
 ]
@@ -6198,9 +6191,11 @@ dependencies = [
 "bytes",
 "chrono",
 "clap",
- "const-str 0.7.0",
+ "const-str",
 "datafusion",
+ "flatbuffers",
 "futures",
+ "futures-util",
 "http 1.3.1",
 "http-body 1.0.1",
 "hyper 1.7.0",
@@ -6215,6 +6210,7 @@ dependencies = [
 "pin-project-lite",
 "pprof",
 "reqwest",
+ "rmp-serde",
 "rust-embed",
 "rustfs-ahm",
 "rustfs-appauth",
@@ -6225,6 +6221,7 @@ dependencies = [
 "rustfs-filemeta",
 "rustfs-iam",
 "rustfs-kms",
+ "rustfs-lock",
 "rustfs-madmin",
 "rustfs-notify",
 "rustfs-obs",
@@ -6244,11 +6241,11 @@ dependencies = [
 "shadow-rs",
 "socket2 0.6.0",
 "sysctl",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tikv-jemallocator",
 "time",
 "tokio",
- "tokio-rustls 0.26.3",
+ "tokio-rustls 0.26.4",
 "tokio-stream",
 "tokio-tar",
 "tokio-util",
@@ -6281,7 +6278,7 @@ dependencies = [
 "serde_json",
 "serial_test",
 "tempfile",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "time",
 "tokio",
 "tokio-util",
@@ -6314,7 +6311,7 @@ dependencies = [
 "rustfs-targets",
 "serde",
 "serde_json",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tracing",
 "url",
@@ -6356,7 +6353,7 @@ dependencies = [
 name = "rustfs-config"
 version = "0.0.5"
 dependencies = [
- "const-str 0.7.0",
+ "const-str",
 ]

 [[package]]
@@ -6373,7 +6370,7 @@ dependencies = [
 "serde_json",
 "sha2 0.10.9",
 "test-case",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "time",
 ]

@@ -6384,7 +6381,10 @@ dependencies = [
 "async-channel",
 "async-recursion",
 "async-trait",
+ "aws-credential-types",
 "aws-sdk-s3",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
 "base64 0.22.1",
 "byteorder",
 "bytes",
@@ -6441,7 +6441,7 @@ dependencies = [
 "smallvec",
 "temp-env",
 "tempfile",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "time",
 "tokio",
 "tokio-stream",
@@ -6469,7 +6469,8 @@ dependencies = [
 "rustfs-utils",
 "s3s",
 "serde",
- "thiserror 2.0.16",
+ "serde_json",
+ "thiserror 2.0.17",
 "time",
 "tokio",
 "tracing",
@@ -6494,9 +6495,10 @@ dependencies = [
 "rustfs-utils",
 "serde",
 "serde_json",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "time",
 "tokio",
+ "tokio-util",
 "tracing",
 ]

@@ -6518,7 +6520,7 @@ dependencies = [
 "serde_json",
 "sha2 0.10.9",
 "tempfile",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tokio-test",
 "tracing",
@@ -6543,7 +6545,7 @@ dependencies = [
 "serde_json",
 "smallvec",
 "smartstring",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tonic 0.14.2",
 "tracing",
@@ -6599,7 +6601,7 @@ dependencies = [
 "rustfs-utils",
 "serde",
 "serde_json",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tracing",
 "tracing-subscriber",
@@ -6630,7 +6632,7 @@ dependencies = [
 "serde_json",
 "smallvec",
 "sysinfo",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tracing",
 "tracing-core",
@@ -6653,7 +6655,7 @@ dependencies = [
 "serde_json",
 "strum 0.27.2",
 "test-case",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "time",
 "tokio",
 ]
@@ -6786,7 +6788,7 @@ dependencies = [
 "serde",
 "serde_json",
 "snap",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tracing",
 "url",
@@ -6802,6 +6804,7 @@ dependencies = [
 "blake3",
 "brotli",
 "bytes",
+ "convert_case",
 "crc32fast",
 "flate2",
 "futures",
@@ -6830,7 +6833,7 @@ dependencies = [
 "snap",
 "sysinfo",
 "tempfile",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "tokio",
 "tracing",
 "transform-stream",
@@ -7044,9 +7047,9 @@ checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"

 [[package]]
 name = "s3s"
-version = "0.12.0-minio-preview.3"
+version = "0.12.0-rc.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24c7be783f8b2bb5aba553462bf7e9ee95655bb27cbd6a0b0a93af2e719b1eec"
+checksum = "6fbf35b8b229398629d8df8a0cb908a70a24da69f65911fe0461fba3f4e9dda5"
 dependencies = [
 "arrayvec",
 "async-trait",
@@ -7054,8 +7057,9 @@ dependencies = [
 "base64-simd",
 "bytes",
 "bytestring",
+ "cfg-if",
 "chrono",
- "const-str 0.6.4",
+ "const-str",
 "crc32c",
 "crc32fast",
 "crc64fast-nvme",
@@ -7072,7 +7076,6 @@ dependencies = [
 "memchr",
 "mime",
 "nom 7.1.3",
- "nugine-rust-utils",
 "numeric_cast",
 "pin-project-lite",
 "quick-xml 0.37.5",
@@ -7082,8 +7085,9 @@ dependencies = [
 "sha2 0.11.0-pre.5",
 "smallvec",
 "std-next",
+ "subtle",
 "sync_wrapper",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "time",
 "tokio",
 "tower",
@@ -7236,9 +7240,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"

 [[package]]
 name = "serde"
-version = "1.0.226"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
 dependencies = [
 "serde_core",
 "serde_derive",
@@ -7280,18 +7284,18 @@ dependencies = [

 [[package]]
 name = "serde_core"
-version = "1.0.226"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.226"
+version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -7434,9 +7438,9 @@ dependencies = [

 [[package]]
 name = "shadow-rs"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8aa5c0570cd9654158bd39f0f8caba24edbc058313946e89f4648b1de1ecf49"
+checksum = "72d18183cef626bce22836103349c7050d73db799be0171386b80947d157ae32"
 dependencies = [
 "cargo_metadata",
 "const_format",
@@ -7519,7 +7523,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb"
 dependencies = [
 "num-bigint",
 "num-traits",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "time",
 ]

@@ -7695,7 +7699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc"
 dependencies = [
 "simdutf8",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 ]

 [[package]]
@@ -7927,15 +7931,15 @@ dependencies = [
 "byteorder",
 "enum-as-inner",
 "libc",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
 "walkdir",
 ]

 [[package]]
 name = "sysinfo"
-version = "0.37.0"
+version = "0.37.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753"
+checksum = "3bddd368fda2f82ead69c03d46d351987cfa0c2a57abfa37a017f3aa3e9bf69a"
 dependencies = [
 "libc",
 "memchr",
@@ -8038,11 +8042,11 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "2.0.16"
+version = "2.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0"
+checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
 dependencies = [
- "thiserror-impl 2.0.16",
+ "thiserror-impl 2.0.17",
 ]

 [[package]]
@@ -8058,9 +8062,9 @@ dependencies = [

 [[package]]
 name = "thiserror-impl"
-version = "2.0.16"
+version = "2.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960"
+checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -8227,9 +8231,9 @@ dependencies = [

 [[package]]
 name = "tokio-rustls"
-version = "0.26.3"
+version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd"
+checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
 dependencies = [
 "rustls 0.23.32",
 "tokio",
@@ -9612,9 +9616,9 @@ dependencies = [

 [[package]]
 name = "zeroize"
-version = "1.8.1"
+version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
 dependencies = [
 "zeroize_derive",
 ]
@@ -101,8 +101,8 @@ async-compression = { version = "0.4.19" }
 atomic_enum = "0.3.0"
 aws-config = { version = "1.8.6" }
 aws-sdk-s3 = { version = "1.106.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
-axum = "0.8.4"
-axum-extra = "0.10.1"
+axum = "0.8.6"
+axum-extra = "0.10.3"
 axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"], default-features = false }
 base64-simd = "0.8.0"
 base64 = "0.22.1"
@@ -190,7 +190,7 @@ quick-xml = "0.38.3"
 rand = "0.9.2"
 rdkafka = { version = "0.38.0", features = ["tokio"] }
 reed-solomon-simd = { version = "3.0.1" }
-regex = { version = "1.11.2" }
+regex = { version = "1.11.3" }
 reqwest = { version = "0.12.23", default-features = false, features = [
    "rustls-tls-webpki-roots",
    "charset",
@@ -210,15 +210,15 @@ rustfs-rsc = "2025.506.1"
 rustls = { version = "0.23.32", features = ["ring", "logging", "std", "tls12"], default-features = false }
 rustls-pki-types = "1.12.0"
 rustls-pemfile = "2.2.0"
-s3s = { version = "0.12.0-minio-preview.3" }
+s3s = { version = "0.12.0-rc.2", features = ["minio"] }
 schemars = "1.0.4"
-serde = { version = "1.0.226", features = ["derive"] }
+serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.145", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 serial_test = "3.2.0"
 sha1 = "0.10.6"
 sha2 = "0.10.9"
-shadow-rs = { version = "1.3.0", default-features = false }
+shadow-rs = { version = "1.4.0", default-features = false }
 siphasher = "1.0.1"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
@@ -226,12 +226,12 @@ snafu = "0.8.9"
 snap = "1.1.1"
 socket2 = "0.6.0"
 strum = { version = "0.27.2", features = ["derive"] }
-sysinfo = "0.37.0"
+sysinfo = "0.37.1"
 sysctl = "0.7.1"
 tempfile = "3.23.0"
 temp-env = "0.3.6"
 test-case = "3.3.1"
-thiserror = "2.0.16"
+thiserror = "2.0.17"
 time = { version = "0.3.44", features = [
    "std",
    "parsing",
@@ -240,7 +240,7 @@ time = { version = "0.3.44", features = [
    "serde",
 ] }
 tokio = { version = "1.47.1", features = ["fs", "rt-multi-thread"] }
-tokio-rustls = { version = "0.26.3", default-features = false, features = ["logging", "tls12", "ring"] }
+tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "ring"] }
 tokio-stream = { version = "0.1.17" }
 tokio-tar = "0.3.1"
 tokio-test = "0.4.4"
@@ -266,7 +266,7 @@ uuid = { version = "1.18.1", features = [
 vaultrs = { version = "0.7.4" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.5.0", features = ["serde"] }
-zeroize = { version = "1.8.1", features = ["derive"] }
+zeroize = { version = "1.8.2", features = ["derive"] }
 winapi = { version = "0.3.9" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
 zip = "5.1.1"
@@ -29,6 +29,7 @@ use rustfs_ecstore::{
    data_usage::{aggregate_local_snapshots, store_data_usage_in_backend},
 };
 use rustfs_filemeta::{MetacacheReader, VersionType};
+use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
 use tokio::sync::{Mutex, RwLock};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
@@ -50,7 +51,6 @@ use rustfs_common::data_usage::{DataUsageInfo, SizeSummary};
 use rustfs_common::metrics::{Metric, Metrics, globalMetrics};
 use rustfs_ecstore::bucket::versioning::VersioningApi;
 use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
-use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
 use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
 use uuid;

@@ -300,8 +300,13 @@ impl Scanner {
                            .map(|(c, _)| Arc::new(c));

                        // Get bucket versioning configuration
-                        let versioning_config = Arc::new(VersioningConfig {
-                            enabled: bucket_info.versioning,
+                        let versioning_config = Arc::new(VersioningConfiguration {
+                            status: if bucket_info.versioning {
+                                Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
+                            } else {
+                                None
+                            },
+                            ..Default::default()
                        });

                        let records = match bucket_objects_map.get(bucket_name) {
@@ -1825,7 +1830,16 @@ impl Scanner {
            }
        };
        let bucket_info = ecstore.get_bucket_info(bucket, &Default::default()).await.ok();
-        let versioning_config = bucket_info.map(|bi| Arc::new(VersioningConfig { enabled: bi.versioning }));
+        let versioning_config = bucket_info.map(|bi| {
+            Arc::new(VersioningConfiguration {
+                status: if bi.versioning {
+                    Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
+                } else {
+                    None
+                },
+                ..Default::default()
+            })
+        });
        let lifecycle_config = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket)
            .await
            .ok()
@@ -2651,7 +2665,7 @@ mod tests {
        // create ECStore with dynamic port
        let port = port.unwrap_or(9000);
        let server_addr: SocketAddr = format!("127.0.0.1:{port}").parse().expect("Invalid server address format");
-        let ecstore = ECStore::new(server_addr, endpoint_pools)
+        let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
            .await
            .expect("Failed to create ECStore");

@@ -28,10 +28,9 @@ use rustfs_ecstore::bucket::metadata_sys::get_object_lock_config;
 use rustfs_ecstore::bucket::object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion};
 use rustfs_ecstore::bucket::versioning::VersioningApi;
 use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
-use rustfs_ecstore::cmd::bucket_targets::VersioningConfig;
 use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete};
 use rustfs_filemeta::FileInfo;
-use s3s::dto::BucketLifecycleConfiguration as LifecycleConfig;
+use s3s::dto::{BucketLifecycleConfiguration as LifecycleConfig, VersioningConfiguration};
 use time::OffsetDateTime;
 use tracing::info;

@@ -43,11 +42,15 @@ pub struct ScannerItem {
    pub bucket: String,
    pub object_name: String,
    pub lifecycle: Option<Arc<LifecycleConfig>>,
-    pub versioning: Option<Arc<VersioningConfig>>,
+    pub versioning: Option<Arc<VersioningConfiguration>>,
 }

 impl ScannerItem {
-    pub fn new(bucket: String, lifecycle: Option<Arc<LifecycleConfig>>, versioning: Option<Arc<VersioningConfig>>) -> Self {
+    pub fn new(
+        bucket: String,
+        lifecycle: Option<Arc<LifecycleConfig>>,
+        versioning: Option<Arc<VersioningConfiguration>>,
+    ) -> Self {
        Self {
            bucket,
            object_name: "".to_string(),
@@ -145,6 +148,7 @@ impl ScannerItem {
            to_del.push(ObjectToDelete {
                object_name: obj.name,
                version_id: obj.version_id,
+                ..Default::default()
            });
        }

@@ -233,7 +237,7 @@ impl ScannerItem {
            IlmAction::DeleteAction => {
                info!("apply_lifecycle: Object {} marked for deletion", oi.name);
                if let Some(vcfg) = &self.versioning {
-                    if !vcfg.is_enabled() {
+                    if !vcfg.enabled() {
                        info!("apply_lifecycle: Versioning disabled, setting new_size=0");
                        new_size = 0;
                    }
@@ -444,7 +444,7 @@ mod tests {
        let delete_marker = MetaDeleteMarker {
            version_id: Some(Uuid::new_v4()),
            mod_time: Some(OffsetDateTime::now_utc()),
-            meta_sys: None,
+            meta_sys: HashMap::new(),
        };

        let version = FileMetaVersion {
@@ -18,6 +18,7 @@ use rustfs_ecstore::disk::endpoint::Endpoint;
 use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
 use std::net::SocketAddr;
 use tempfile::TempDir;
+use tokio_util::sync::CancellationToken;

 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_endpoint_index_settings() -> anyhow::Result<()> {
@@ -73,7 +74,7 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
    rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await?;

    let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let ecstore = rustfs_ecstore::store::ECStore::new(server_addr, endpoint_pools).await?;
+    let ecstore = rustfs_ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;

    println!("ECStore initialized successfully with {} pools", ecstore.pools.len());

@@ -29,6 +29,7 @@ use std::sync::Once;
 use std::sync::OnceLock;
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use tokio::fs;
+use tokio_util::sync::CancellationToken;
 use tracing::info;
 use walkdir::WalkDir;

@@ -98,7 +99,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
    // create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
    let port = 9001; // for simplicity
    let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();

    // init bucket metadata system
    let buckets_list = ecstore
@@ -29,6 +29,7 @@ use std::sync::OnceLock;
 use std::{path::PathBuf, sync::Arc, time::Duration};
 use tokio::fs;
 use tokio::sync::RwLock;
+use tokio_util::sync::CancellationToken;
 use tracing::warn;
 use tracing::{debug, info};

@@ -99,7 +100,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
    // create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
    let port = 9002; // for simplicity
    let server_addr: std::net::SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();

    // init bucket metadata system
    let buckets_list = ecstore
@@ -124,7 +127,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
 }

 /// Test helper: Create a test bucket
-async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
+async fn _create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
    (**ecstore)
        .make_bucket(bucket_name, &Default::default())
        .await
@@ -222,7 +225,7 @@ async fn set_bucket_lifecycle_transition(bucket_name: &str) -> Result<(), Box<dy
    </Rule>
    <Rule>
        <ID>test-rule2</ID>
-        <Status>Desabled</Status>
+        <Status>Disabled</Status>
        <Filter>
            <Prefix>test/</Prefix>
        </Filter>
@@ -299,6 +302,22 @@ async fn object_is_transitioned(ecstore: &Arc<ECStore>, bucket: &str, object: &s
    }
 }

+async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
+    let deadline = tokio::time::Instant::now() + timeout;
+
+    loop {
+        if !object_exists(ecstore, bucket, object).await {
+            return true;
+        }
+
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+
+        tokio::time::sleep(Duration::from_millis(200)).await;
+    }
+}
+
 mod serial_tests {
    use super::*;

@@ -308,25 +327,26 @@ mod serial_tests {
        let (_disk_paths, ecstore) = setup_test_env().await;

        // Create test bucket and object
-        let bucket_name = "test-lifecycle-expiry-basic-bucket";
+        let suffix = uuid::Uuid::new_v4().simple().to_string();
+        let bucket_name = format!("test-lc-expiry-basic-{}", &suffix[..8]);
        let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
        let test_data = b"Hello, this is test data for lifecycle expiry!";

-        create_test_bucket(&ecstore, bucket_name).await;
-        upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;

        // Verify object exists initially
-        assert!(object_exists(&ecstore, bucket_name, object_name).await);
+        assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
        println!("✅ Object exists before lifecycle processing");

        // Set lifecycle configuration with very short expiry (0 days = immediate expiry)
-        set_bucket_lifecycle(bucket_name)
+        set_bucket_lifecycle(bucket_name.as_str())
            .await
            .expect("Failed to set lifecycle configuration");
-        println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
+        println!("✅ Lifecycle configuration set for bucket: {}", bucket_name);

        // Verify lifecycle configuration was set
-        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name).await {
+        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
            Ok(bucket_meta) => {
                assert!(bucket_meta.lifecycle_config.is_some());
                println!("✅ Bucket metadata retrieved successfully");
@@ -357,20 +377,60 @@ mod serial_tests {
        scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
        println!("✅ Manual scan cycle completed");

-        // Wait a bit more for background workers to process expiry tasks
-        tokio::time::sleep(Duration::from_secs(5)).await;
+        let mut expired = false;
+        for attempt in 0..3 {
+            if attempt > 0 {
+                scanner.scan_cycle().await.expect("Failed to trigger scan cycle on retry");
+            }
+            expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(5)).await;
+            if expired {
+                break;
+            }
+        }

-        // Check if object has been expired (delete_marker)
-        let check_result = object_exists(&ecstore, bucket_name, object_name).await;
-        println!("Object is_delete_marker after lifecycle processing: {check_result}");
+        println!("Object is_delete_marker after lifecycle processing: {}", !expired);

-        if check_result {
-            println!("❌ Object was not deleted by lifecycle processing");
+        if !expired {
+            let pending = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
+                .read()
+                .await
+                .pending_tasks()
+                .await;
+            println!("Pending expiry tasks: {pending}");
+
+            if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
+                if let Ok(object_info) = ecstore
+                    .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                    .await
+                {
+                    let event = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::eval_action_from_lifecycle(
+                        &lc_config,
+                        None,
+                        None,
+                        &object_info,
+                    )
+                    .await;
+
+                    rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
+                        ecstore.clone(),
+                        &object_info,
+                        &event,
+                        &rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
+                    )
+                    .await;
+
+                    expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
+                }
+            }
+
+            if !expired {
+                println!("❌ Object was not deleted by lifecycle processing");
+            }
        } else {
            println!("✅ Object was successfully deleted by lifecycle processing");
            // Let's try to get object info to see its details
            match ecstore
-                .get_object_info(bucket_name, object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
                .await
            {
                Ok(obj_info) => {
@@ -385,7 +445,7 @@ mod serial_tests {
            }
        }

-        assert!(!check_result);
+        assert!(expired);
        println!("✅ Object successfully expired");

        // Stop scanner
@@ -401,25 +461,26 @@ mod serial_tests {
        let (_disk_paths, ecstore) = setup_test_env().await;

        // Create test bucket and object
-        let bucket_name = "test-lifecycle-expiry-deletemarker-bucket";
+        let suffix = uuid::Uuid::new_v4().simple().to_string();
+        let bucket_name = format!("test-lc-expiry-marker-{}", &suffix[..8]);
        let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
        let test_data = b"Hello, this is test data for lifecycle expiry!";

-        create_test_lock_bucket(&ecstore, bucket_name).await;
-        upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;

        // Verify object exists initially
-        assert!(object_exists(&ecstore, bucket_name, object_name).await);
+        assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
        println!("✅ Object exists before lifecycle processing");

        // Set lifecycle configuration with very short expiry (0 days = immediate expiry)
-        set_bucket_lifecycle_deletemarker(bucket_name)
+        set_bucket_lifecycle_deletemarker(bucket_name.as_str())
            .await
            .expect("Failed to set lifecycle configuration");
-        println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
+        println!("✅ Lifecycle configuration set for bucket: {}", bucket_name);

        // Verify lifecycle configuration was set
-        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name).await {
+        match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
            Ok(bucket_meta) => {
                assert!(bucket_meta.lifecycle_config.is_some());
                println!("✅ Bucket metadata retrieved successfully");
@@ -450,36 +511,64 @@ mod serial_tests {
        scanner.scan_cycle().await.expect("Failed to trigger scan cycle");
        println!("✅ Manual scan cycle completed");

-        // Wait a bit more for background workers to process expiry tasks
-        tokio::time::sleep(Duration::from_secs(5)).await;
+        let mut deleted = false;
+        for attempt in 0..3 {
+            if attempt > 0 {
+                scanner.scan_cycle().await.expect("Failed to trigger scan cycle on retry");
+            }
+            deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(5)).await;
+            if deleted {
+                break;
+            }
+        }

-        // Check if object has been expired (deleted)
-        //let check_result = object_is_delete_marker(&ecstore, bucket_name, object_name).await;
-        let check_result = object_exists(&ecstore, bucket_name, object_name).await;
-        println!("Object exists after lifecycle processing: {check_result}");
+        println!("Object exists after lifecycle processing: {}", !deleted);

-        if !check_result {
-            println!("❌ Object was not deleted by lifecycle processing");
-            // Let's try to get object info to see its details
-            match ecstore
-                .get_object_info(bucket_name, object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+        if !deleted {
+            let pending = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
+                .read()
                .await
-            {
-                Ok(obj_info) => {
-                    println!(
-                        "Object info: name={}, size={}, mod_time={:?}",
-                        obj_info.name, obj_info.size, obj_info.mod_time
-                    );
-                }
-                Err(e) => {
-                    println!("Error getting object info: {e:?}");
+                .pending_tasks()
+                .await;
+            println!("Pending expiry tasks: {pending}");
+
+            if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
+                if let Ok(obj_info) = ecstore
+                    .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                    .await
+                {
+                    let event = rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::eval_action_from_lifecycle(
+                        &lc_config, None, None, &obj_info,
+                    )
+                    .await;
+
+                    rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects(
+                        ecstore.clone(),
+                        &obj_info,
+                        &event,
+                        &rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner,
+                    )
+                    .await;
+
+                    deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
+
+                    if !deleted {
+                        println!(
+                            "Object info: name={}, size={}, mod_time={:?}",
+                            obj_info.name, obj_info.size, obj_info.mod_time
+                        );
+                    }
                }
            }
+
+            if !deleted {
+                println!("❌ Object was not deleted by lifecycle processing");
+            }
        } else {
            println!("✅ Object was successfully deleted by lifecycle processing");
        }

-        assert!(check_result);
+        assert!(deleted);
        println!("✅ Object successfully expired");

        // Stop scanner
@@ -497,15 +586,16 @@ mod serial_tests {
        //create_test_tier().await;

        // Create test bucket and object
-        let bucket_name = "test-lifecycle-transition-basic-bucket";
+        let suffix = uuid::Uuid::new_v4().simple().to_string();
+        let bucket_name = format!("test-lc-transition-{}", &suffix[..8]);
        let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
        let test_data = b"Hello, this is test data for lifecycle expiry!";

-        create_test_bucket(&ecstore, bucket_name).await;
-        upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, test_data).await;

        // Verify object exists initially
-        assert!(object_exists(&ecstore, bucket_name, object_name).await);
+        assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
        println!("✅ Object exists before lifecycle processing");

        // Set lifecycle configuration with very short expiry (0 days = immediate expiry)
@@ -551,14 +641,14 @@ mod serial_tests {

        // Check if object has been expired (deleted)
        //let check_result = object_is_transitioned(&ecstore, bucket_name, object_name).await;
-        let check_result = object_exists(&ecstore, bucket_name, object_name).await;
+        let check_result = object_exists(&ecstore, bucket_name.as_str(), object_name).await;
        println!("Object exists after lifecycle processing: {check_result}");

        if check_result {
            println!("✅ Object was not deleted by lifecycle processing");
            // Let's try to get object info to see its details
            match ecstore
-                .get_object_info(bucket_name, object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
+                .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
                .await
            {
                Ok(obj_info) => {
@@ -14,6 +14,7 @@

 use std::{fs, net::SocketAddr, sync::Arc, sync::OnceLock, time::Duration};
 use tempfile::TempDir;
+use tokio_util::sync::CancellationToken;

 use serial_test::serial;

@@ -89,7 +90,9 @@ async fn prepare_test_env(test_dir: Option<&str>, port: Option<u16>) -> (Vec<std
    // create ECStore with dynamic port
    let port = port.unwrap_or(9000);
    let server_addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
-    let ecstore = ECStore::new(server_addr, endpoint_pools).await.unwrap();
+    let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
+        .await
+        .unwrap();

    // init bucket metadata system
    let buckets_list = ecstore
@@ -202,22 +202,19 @@ impl AuditMetrics {
        // Generate recommendations
        if !validation.meets_eps_requirement {
            validation.recommendations.push(format!(
-                "EPS ({:.0}) is below requirement (3000). Consider optimizing target dispatch or adding more target instances.",
-                eps
+                "EPS ({eps:.0}) is below requirement (3000). Consider optimizing target dispatch or adding more target instances."
            ));
        }

        if !validation.meets_latency_requirement {
            validation.recommendations.push(format!(
-                "Average latency ({:.2}ms) exceeds requirement (30ms). Consider optimizing target responses or increasing timeout values.",
-                avg_latency_ms
+                "Average latency ({avg_latency_ms:.2}ms) exceeds requirement (30ms). Consider optimizing target responses or increasing timeout values."
            ));
        }

        if !validation.meets_error_rate_requirement {
            validation.recommendations.push(format!(
-                "Error rate ({:.2}%) exceeds recommendation (1%). Check target connectivity and configuration.",
-                error_rate
+                "Error rate ({error_rate:.2}%) exceeds recommendation (1%). Check target connectivity and configuration."
            ));
        }

@@ -307,7 +304,7 @@ impl PerformanceValidation {
        );

        for rec in &self.recommendations {
-            result.push_str(&format!("\n• {}", rec));
+            result.push_str(&format!("\n• {rec}"));
        }

        result
@@ -303,7 +303,7 @@ async fn create_audit_target(
            let target = rustfs_targets::target::mqtt::MQTTTarget::new(id.to_string(), args)?;
            Ok(Box::new(target))
        }
-        _ => Err(TargetError::Configuration(format!("Unknown target type: {}", target_type))),
+        _ => Err(TargetError::Configuration(format!("Unknown target type: {target_type}"))),
    }
 }

@@ -352,7 +352,7 @@ fn parse_webhook_args(_id: &str, config: &KVS) -> Result<WebhookArgs, TargetErro
        .ok_or_else(|| TargetError::Configuration("webhook endpoint is required".to_string()))?;

    let endpoint_url =
-        Url::parse(&endpoint).map_err(|e| TargetError::Configuration(format!("invalid webhook endpoint URL: {}", e)))?;
+        Url::parse(&endpoint).map_err(|e| TargetError::Configuration(format!("invalid webhook endpoint URL: {e}")))?;

    let args = WebhookArgs {
        enable: true, // Already validated as enabled
@@ -379,7 +379,7 @@ fn parse_mqtt_args(_id: &str, config: &KVS) -> Result<MQTTArgs, TargetError> {
        .filter(|s| !s.is_empty())
        .ok_or_else(|| TargetError::Configuration("MQTT broker is required".to_string()))?;

-    let broker_url = Url::parse(&broker).map_err(|e| TargetError::Configuration(format!("invalid MQTT broker URL: {}", e)))?;
+    let broker_url = Url::parse(&broker).map_err(|e| TargetError::Configuration(format!("invalid MQTT broker URL: {e}")))?;

    let topic = config
        .lookup(MQTT_TOPIC)
@@ -461,7 +461,7 @@ impl AuditSystem {
            info!(target_id = %target_id, "Target enabled");
            Ok(())
        } else {
-            Err(AuditError::Configuration(format!("Target not found: {}", target_id)))
+            Err(AuditError::Configuration(format!("Target not found: {target_id}")))
        }
    }

@@ -474,7 +474,7 @@ impl AuditSystem {
            info!(target_id = %target_id, "Target disabled");
            Ok(())
        } else {
-            Err(AuditError::Configuration(format!("Target not found: {}", target_id)))
+            Err(AuditError::Configuration(format!("Target not found: {target_id}")))
        }
    }

@@ -488,7 +488,7 @@ impl AuditSystem {
            info!(target_id = %target_id, "Target removed");
            Ok(())
        } else {
-            Err(AuditError::Configuration(format!("Target not found: {}", target_id)))
+            Err(AuditError::Configuration(format!("Target not found: {target_id}")))
        }
    }

@@ -49,4 +49,5 @@ uuid = { workspace = true }
 base64 = { workspace = true }
 rand = { workspace = true }
 chrono = { workspace = true }
+http.workspace = true
 md5 = { workspace = true }
@@ -94,7 +94,7 @@ fn build_rustfs_binary() {

    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
-        panic!("Failed to build RustFS binary. Error: {}", stderr);
+        panic!("Failed to build RustFS binary. Error: {stderr}");
    }

    info!("✅ RustFS binary built successfully");
@@ -134,8 +134,8 @@ impl RustFSTestEnvironment {

        // Use a unique port for each test environment
        let port = Self::find_available_port().await?;
-        let address = format!("127.0.0.1:{}", port);
-        let url = format!("http://{}", address);
+        let address = format!("127.0.0.1:{port}");
+        let url = format!("http://{address}");

        Ok(Self {
            temp_dir,
@@ -152,7 +152,7 @@ impl RustFSTestEnvironment {
        let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
        fs::create_dir_all(&temp_dir).await?;

-        let url = format!("http://{}", address);
+        let url = format!("http://{address}");

        Ok(Self {
            temp_dir,
@@ -327,7 +327,7 @@ pub async fn execute_awscurl(

    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
-        return Err(format!("awscurl failed: {}", stderr).into());
+        return Err(format!("awscurl failed: {stderr}").into());
    }

    let response = String::from_utf8_lossy(&output.stdout).to_string();
@@ -388,23 +388,11 @@ async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn
        complete_multipart_response.ssekms_key_id()
    );

-    // Verify: complete_multipart_upload response should contain encryption information
-    // KNOWN BUG: s3s library bug where CompleteMultipartUploadOutput encryption fields serialize as None
-    // even when properly set. Our server implementation is correct (see server logs above).
-    // TODO: Remove this workaround when s3s library is fixed
-    warn!("KNOWN BUG: s3s library - complete_multipart_upload response encryption fields return None even when set");
-
-    if complete_multipart_response.server_side_encryption().is_some() {
-        // If s3s library is fixed, verify the encryption info
-        assert_eq!(
-            complete_multipart_response.server_side_encryption(),
-            Some(&ServerSideEncryption::AwsKms),
-            "complete_multipart_upload response should contain SSE-KMS encryption information"
-        );
-    } else {
-        // Expected behavior due to s3s library bug - log and continue
-        warn!("Skipping assertion due to known s3s library bug - server logs confirm correct encryption handling");
-    }
+    assert_eq!(
+        complete_multipart_response.server_side_encryption(),
+        Some(&ServerSideEncryption::AwsKms),
+        "complete_multipart_upload response should contain SSE-KMS encryption information"
+    );

    // Step 4: Download file and verify encryption status
    info!("Downloading file and verifying encryption status");
@@ -59,7 +59,7 @@ pub async fn configure_kms(
    access_key: &str,
    secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/kms/configure", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/configure");
    awscurl_post(&url, config_json, access_key, secret_key).await?;
    info!("KMS configured successfully");
    Ok(())
@@ -71,7 +71,7 @@ pub async fn start_kms(
    access_key: &str,
    secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/kms/start", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/start");
    awscurl_post(&url, "{}", access_key, secret_key).await?;
    info!("KMS started successfully");
    Ok(())
@@ -83,7 +83,7 @@ pub async fn get_kms_status(
    access_key: &str,
    secret_key: &str,
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{}/rustfs/admin/v3/kms/status", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/status");
    let status = awscurl_get(&url, access_key, secret_key).await?;
    info!("KMS status retrieved: {}", status);
    Ok(status)
@@ -101,7 +101,7 @@ pub async fn create_default_key(
    })
    .to_string();

-    let url = format!("{}/rustfs/admin/v3/kms/keys", base_url);
+    let url = format!("{base_url}/rustfs/admin/v3/kms/keys");
    let response = awscurl_post(&url, &create_key_body, access_key, secret_key).await?;

    // Parse response to get the actual key ID
@@ -141,7 +141,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
    });

    // Write the key to file with the specified ID as JSON
-    let key_path = format!("{}/{}.key", key_dir, key_id);
+    let key_path = format!("{key_dir}/{key_id}.key");
    let content = serde_json::to_vec_pretty(&stored_key)?;
    fs::write(&key_path, &content).await?;

@@ -281,13 +281,8 @@ pub async fn test_kms_key_management(
    })
    .to_string();

-    let create_response = awscurl_post(
-        &format!("{}/rustfs/admin/v3/kms/keys", base_url),
-        &create_key_body,
-        access_key,
-        secret_key,
-    )
-    .await?;
+    let create_response =
+        awscurl_post(&format!("{base_url}/rustfs/admin/v3/kms/keys"), &create_key_body, access_key, secret_key).await?;

    let create_result: serde_json::Value = serde_json::from_str(&create_response)?;
    let key_id = create_result["key_id"]
@@ -296,8 +291,7 @@ pub async fn test_kms_key_management(
    info!("Created key with ID: {}", key_id);

    // Test DescribeKey
-    let describe_response =
-        awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
+    let describe_response = awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;

    info!("DescribeKey response: {}", describe_response);
    let describe_result: serde_json::Value = serde_json::from_str(&describe_response)?;
@@ -306,7 +300,7 @@ pub async fn test_kms_key_management(
    info!("Successfully described key: {}", key_id);

    // Test ListKeys
-    let list_response = awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys", base_url), access_key, secret_key).await?;
+    let list_response = awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys"), access_key, secret_key).await?;

    let list_result: serde_json::Value = serde_json::from_str(&list_response)?;
    let keys = list_result["keys"]
@@ -412,7 +406,7 @@ impl VaultTestEnvironment {
            let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
            if port_check {
                // Additional check by making a health request
-                if let Ok(response) = reqwest::get(&format!("{}/v1/sys/health", VAULT_URL)).await {
+                if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await {
                    if response.status().is_success() {
                        info!("Vault server is ready after {} seconds", i);
                        return Ok(());
@@ -438,7 +432,7 @@ impl VaultTestEnvironment {

        // Enable transit secrets engine
        let enable_response = client
-            .post(format!("{}/v1/sys/mounts/{}", VAULT_URL, VAULT_TRANSIT_PATH))
+            .post(format!("{VAULT_URL}/v1/sys/mounts/{VAULT_TRANSIT_PATH}"))
            .header("X-Vault-Token", VAULT_TOKEN)
            .json(&serde_json::json!({
                "type": "transit"
@@ -448,14 +442,14 @@ impl VaultTestEnvironment {

        if !enable_response.status().is_success() && enable_response.status() != 400 {
            let error_text = enable_response.text().await?;
-            return Err(format!("Failed to enable transit engine: {}", error_text).into());
+            return Err(format!("Failed to enable transit engine: {error_text}").into());
        }

        info!("Creating Vault encryption key");

        // Create encryption key
        let key_response = client
-            .post(format!("{}/v1/{}/keys/{}", VAULT_URL, VAULT_TRANSIT_PATH, VAULT_KEY_NAME))
+            .post(format!("{VAULT_URL}/v1/{VAULT_TRANSIT_PATH}/keys/{VAULT_KEY_NAME}"))
            .header("X-Vault-Token", VAULT_TOKEN)
            .json(&serde_json::json!({
                "type": "aes256-gcm96"
@@ -465,7 +459,7 @@ impl VaultTestEnvironment {

        if !key_response.status().is_success() && key_response.status() != 400 {
            let error_text = key_response.text().await?;
-            return Err(format!("Failed to create encryption key: {}", error_text).into());
+            return Err(format!("Failed to create encryption key: {error_text}").into());
        }

        info!("Vault transit engine setup completed");
@@ -713,10 +707,10 @@ pub async fn test_all_multipart_encryption_types(

    // Test configurations for all encryption types
    let test_configs = vec![
-        MultipartTestConfig::new(format!("{}-no-encryption", base_object_key), part_size, total_parts, EncryptionType::None),
-        MultipartTestConfig::new(format!("{}-sse-s3", base_object_key), part_size, total_parts, EncryptionType::SSES3),
-        MultipartTestConfig::new(format!("{}-sse-kms", base_object_key), part_size, total_parts, EncryptionType::SSEKMS),
-        MultipartTestConfig::new(format!("{}-sse-c", base_object_key), part_size, total_parts, create_sse_c_config()),
+        MultipartTestConfig::new(format!("{base_object_key}-no-encryption"), part_size, total_parts, EncryptionType::None),
+        MultipartTestConfig::new(format!("{base_object_key}-sse-s3"), part_size, total_parts, EncryptionType::SSES3),
+        MultipartTestConfig::new(format!("{base_object_key}-sse-kms"), part_size, total_parts, EncryptionType::SSEKMS),
+        MultipartTestConfig::new(format!("{base_object_key}-sse-c"), part_size, total_parts, create_sse_c_config()),
    ];

    // Run tests for each encryption type
@@ -33,11 +33,10 @@ fn assert_encryption_metadata(metadata: &HashMap<String, String>, expected_size:
        "x-rustfs-encryption-context",
        "x-rustfs-encryption-original-size",
    ] {
-        assert!(metadata.contains_key(key), "expected managed encryption metadata '{}' to be present", key);
+        assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present");
        assert!(
            !metadata.get(key).unwrap().is_empty(),
-            "managed encryption metadata '{}' should not be empty",
-            key
+            "managed encryption metadata '{key}' should not be empty"
        );
    }

@@ -84,10 +83,7 @@ fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &

    assert!(
        scanned > 0,
-        "Failed to locate stored data files for bucket '{}' and key '{}' under {:?}",
-        bucket,
-        key,
-        storage_root
+        "Failed to locate stored data files for bucket '{bucket}' and key '{key}' under {storage_root:?}"
    );
    assert!(plaintext_path.is_none(), "Plaintext detected on disk at {:?}", plaintext_path.unwrap());
 }
@@ -220,7 +216,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
    assert_encryption_metadata(source_metadata, payload.len());

    let dest_key = "metadata-sse-kms-object-copy";
-    let copy_source = format!("{}/{}", TEST_BUCKET, source_key);
+    let copy_source = format!("{TEST_BUCKET}/{source_key}");

    s3_client
        .copy_object()
@@ -389,8 +389,8 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
        let task = tokio::spawn(async move {
            let _permit = sem.acquire().await.unwrap();

-            let test_data = format!("Concurrent test data {}", i).into_bytes();
-            let object_key = format!("concurrent-test-{}", i);
+            let test_data = format!("Concurrent test data {i}").into_bytes();
+            let object_key = format!("concurrent-test-{i}");

            // Alternate between different encryption types
            let result = match i % 3 {
@@ -418,7 +418,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
                }
                2 => {
                    // SSE-C
-                    let key = format!("testkey{:026}", i); // 32-byte key
+                    let key = format!("testkey{i:026}"); // 32-byte key
                    let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
                    let key_md5 = format!("{:x}", md5::compute(&key));

@@ -459,9 +459,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro

    assert!(
        successful_uploads >= num_concurrent - 1,
-        "Most concurrent uploads should succeed (got {}/{})",
-        successful_uploads,
-        num_concurrent
+        "Most concurrent uploads should succeed (got {successful_uploads}/{num_concurrent})"
    );

    info!("✅ Successfully completed {}/{} concurrent uploads", successful_uploads, num_concurrent);
@@ -152,7 +152,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
    // Corrupt the default key file
    info!("🔧 Corrupting default key file");
    let key_file_path = format!("{}/{}.key", kms_env.kms_keys_dir, default_key_id);
-    let backup_key_path = format!("{}.backup", key_file_path);
+    let backup_key_path = format!("{key_file_path}.backup");

    // Backup the original key file
    fs::copy(&key_file_path, &backup_key_path)?;
@@ -417,8 +417,8 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error

    for i in 0..10 {
        let client = s3_client.clone();
-        let test_data = format!("Rapid test data {}", i).into_bytes();
-        let object_key = format!("rapid-test-{}", i);
+        let test_data = format!("Rapid test data {i}").into_bytes();
+        let object_key = format!("rapid-test-{i}");

        let task = tokio::spawn(async move {
            let result = client
@@ -740,7 +740,7 @@ async fn test_large_multipart_upload(
    // Verify data integrity
    for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
        if actual != expected {
-            panic!("Data mismatch at byte {}: got {}, expected {}", i, actual, expected);
+            panic!("Data mismatch at byte {i}: got {actual}, expected {expected}");
        }
    }

@@ -321,13 +321,9 @@ async fn test_vault_kms_key_crud(
    })
    .to_string();

-    let create_response = crate::common::awscurl_post(
-        &format!("{}/rustfs/admin/v3/kms/keys", base_url),
-        &create_key_body,
-        access_key,
-        secret_key,
-    )
-    .await?;
+    let create_response =
+        crate::common::awscurl_post(&format!("{base_url}/rustfs/admin/v3/kms/keys"), &create_key_body, access_key, secret_key)
+            .await?;

    let create_result: serde_json::Value = serde_json::from_str(&create_response)?;
    let key_id = create_result["key_id"]
@@ -337,7 +333,7 @@ async fn test_vault_kms_key_crud(

    // Read
    let describe_response =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;

    let describe_result: serde_json::Value = serde_json::from_str(&describe_response)?;
    assert_eq!(describe_result["key_metadata"]["key_id"], key_id);
@@ -380,7 +376,7 @@ async fn test_vault_kms_key_crud(

    // Read
    let list_response =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys", base_url), access_key, secret_key).await?;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys"), access_key, secret_key).await?;

    let list_result: serde_json::Value = serde_json::from_str(&list_response)?;
    let keys = list_result["keys"]
@@ -407,7 +403,7 @@ async fn test_vault_kms_key_crud(

    // Delete
    let delete_response = crate::common::execute_awscurl(
-        &format!("{}/rustfs/admin/v3/kms/keys/delete?keyId={}", base_url, key_id),
+        &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}"),
        "DELETE",
        None,
        access_key,
@@ -422,7 +418,7 @@ async fn test_vault_kms_key_crud(

    // Verify key state after deletion
    let describe_deleted_response =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await?;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;

    let describe_result: serde_json::Value = serde_json::from_str(&describe_deleted_response)?;
    let key_state = describe_result["key_metadata"]["key_state"]
@@ -439,7 +435,7 @@ async fn test_vault_kms_key_crud(

    // Force Delete - Force immediate deletion for PendingDeletion key
    let force_delete_response = crate::common::execute_awscurl(
-        &format!("{}/rustfs/admin/v3/kms/keys/delete?keyId={}&force_immediate=true", base_url, key_id),
+        &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
        "DELETE",
        None,
        access_key,
@@ -454,7 +450,7 @@ async fn test_vault_kms_key_crud(

    // Verify key no longer exists after force deletion (should return error)
    let describe_force_deleted_result =
-        crate::common::awscurl_get(&format!("{}/rustfs/admin/v3/kms/keys/{}", base_url, key_id), access_key, secret_key).await;
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await;

    // After force deletion, key should not be found (GET should fail)
    assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found");
@@ -13,25 +13,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-//! 分片上传加密功能的分步测试用例
-//!
-//! 这个测试套件将验证分片上传加密功能的每一个步骤：
-//! 1. 测试基础的单分片加密（验证加密基础逻辑）
-//! 2. 测试多分片上传（验证分片拼接逻辑）
-//! 3. 测试加密元数据的保存和读取
-//! 4. 测试完整的分片上传加密流程
-
 use super::common::LocalKMSTestEnvironment;
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{debug, info};

-/// 步骤1：测试基础单文件加密功能（确保SSE-S3在非分片场景下正常工作）
 #[tokio::test]
 #[serial]
 async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    info!("🧪 步骤1：测试基础单文件加密功能");
+    info!("🧪 step1: test basic single file encryption");

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -40,11 +31,11 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;

-    // 测试小文件加密（应该会内联存储）
+    // test small file encryption (should inline store)
    let test_data = b"Hello, this is a small test file for SSE-S3!";
    let object_key = "test-single-file-encrypted";

-    info!("📤 上传小文件（{}字节），启用SSE-S3加密", test_data.len());
+    info!("📤 step1: upload small file ({}) with SSE-S3 encryption", test_data.len());
    let put_response = s3_client
        .put_object()
        .bucket(TEST_BUCKET)
@@ -54,41 +45,41 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
        .send()
        .await?;

-    debug!("PUT响应ETag: {:?}", put_response.e_tag());
-    debug!("PUT响应SSE: {:?}", put_response.server_side_encryption());
+    debug!("PUT response ETag: {:?}", put_response.e_tag());
+    debug!("PUT response SSE: {:?}", put_response.server_side_encryption());

-    // 验证PUT响应包含正确的加密头
+    // verify PUT response contains correct encryption header
    assert_eq!(
        put_response.server_side_encryption(),
        Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
    );

-    info!("📥 下载文件并验证加密状态");
+    info!("📥 step1: download file and verify encryption status");
    let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;

-    debug!("GET响应SSE: {:?}", get_response.server_side_encryption());
+    debug!("GET response SSE: {:?}", get_response.server_side_encryption());

-    // 验证GET响应包含正确的加密头
+    // verify GET response contains correct encryption header
    assert_eq!(
        get_response.server_side_encryption(),
        Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
    );

-    // 验证数据完整性
+    // verify data integrity
    let downloaded_data = get_response.body.collect().await?.into_bytes();
    assert_eq!(&downloaded_data[..], test_data);

    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
-    info!("✅ 步骤1通过：基础单文件加密功能正常");
+    info!("✅ step1: basic single file encryption works as expected");
    Ok(())
 }

-/// 步骤2：测试不加密的分片上传（确保分片上传基础功能正常）
+/// test basic multipart upload without encryption
 #[tokio::test]
 #[serial]
 async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    info!("🧪 步骤2：测试不加密的分片上传");
+    info!("🧪 step2: test basic multipart upload without encryption");

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -102,12 +93,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
    let total_parts = 2;
    let total_size = part_size * total_parts;

-    // 生成测试数据（有明显的模式便于验证）
+    // generate test data (with clear pattern for easy verification)
    let test_data: Vec<u8> = (0..total_size).map(|i| (i % 256) as u8).collect();

-    info!("🚀 开始分片上传（无加密）：{} parts，每个 {}MB", total_parts, part_size / (1024 * 1024));
+    info!(
+        "🚀 step2: start multipart upload (no encryption) with {} parts, each {}MB",
+        total_parts,
+        part_size / (1024 * 1024)
+    );

-    // 步骤1：创建分片上传
+    // step1: create multipart upload
    let create_multipart_output = s3_client
        .create_multipart_upload()
        .bucket(TEST_BUCKET)
@@ -116,16 +111,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
        .await?;

    let upload_id = create_multipart_output.upload_id().unwrap();
-    info!("📋 创建分片上传，ID: {}", upload_id);
+    info!("📋 step2: create multipart upload, ID: {}", upload_id);

-    // 步骤2：上传各个分片
+    // step2: upload each part
    let mut completed_parts = Vec::new();
    for part_number in 1..=total_parts {
        let start = (part_number - 1) * part_size;
        let end = std::cmp::min(start + part_size, total_size);
        let part_data = &test_data[start..end];

-        info!("📤 上传分片 {} ({} bytes)", part_number, part_data.len());
+        info!("📤 step2: upload part {} ({} bytes)", part_number, part_data.len());

        let upload_part_output = s3_client
            .upload_part()
@@ -145,15 +140,15 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
                .build(),
        );

-        debug!("分片 {} 上传完成，ETag: {}", part_number, etag);
+        debug!("step2: part {} uploaded, ETag: {}", part_number, etag);
    }

-    // 步骤3：完成分片上传
+    // step3: complete multipart upload
    let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
        .set_parts(Some(completed_parts))
        .build();

-    info!("🔗 完成分片上传");
+    info!("🔗 step2: complete multipart upload");
    let complete_output = s3_client
        .complete_multipart_upload()
        .bucket(TEST_BUCKET)
@@ -163,10 +158,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
        .send()
        .await?;

-    debug!("完成分片上传，ETag: {:?}", complete_output.e_tag());
+    debug!("step2: multipart upload completed, ETag: {:?}", complete_output.e_tag());

-    // 步骤4：下载并验证
-    info!("📥 下载文件并验证数据完整性");
+    // step4: verify multipart upload completed successfully
+    assert_eq!(
+        complete_output.e_tag().unwrap().to_string(),
+        format!("\"{}-{}-{}\"", object_key, upload_id, total_parts)
+    );
+
+    // verify data integrity
+    info!("📥 step2: download file and verify data integrity");
    let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;

    let downloaded_data = get_response.body.collect().await?.into_bytes();
@@ -174,16 +175,16 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
    assert_eq!(&downloaded_data[..], &test_data[..]);

    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
-    info!("✅ 步骤2通过：不加密的分片上传功能正常");
+    info!("✅ step2: basic multipart upload without encryption works as expected");
    Ok(())
 }

-/// 步骤3：测试分片上传 + SSE-S3加密（重点测试）
+/// test multipart upload with SSE-S3 encryption
 #[tokio::test]
 #[serial]
 async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    info!("🧪 步骤3：测试分片上传 + SSE-S3加密");
+    info!("🧪 step3: test multipart upload with SSE-S3 encryption");

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -197,16 +198,16 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
    let total_parts = 2;
    let total_size = part_size * total_parts;

-    // 生成测试数据
+    // generate test data (with clear pattern for easy verification)
    let test_data: Vec<u8> = (0..total_size).map(|i| ((i / 1000) % 256) as u8).collect();

    info!(
-        "🔐 开始分片上传（SSE-S3加密）：{} parts，每个 {}MB",
+        "🔐 step3: start multipart upload with SSE-S3 encryption: {} parts, each {}MB",
        total_parts,
        part_size / (1024 * 1024)
    );

-    // 步骤1：创建分片上传并启用SSE-S3
+    // step1: create multipart upload and enable SSE-S3
    let create_multipart_output = s3_client
        .create_multipart_upload()
        .bucket(TEST_BUCKET)
@@ -216,24 +217,24 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
        .await?;

    let upload_id = create_multipart_output.upload_id().unwrap();
-    info!("📋 创建加密分片上传，ID: {}", upload_id);
+    info!("📋 step3: create multipart upload with SSE-S3 encryption, ID: {}", upload_id);

-    // 验证CreateMultipartUpload响应（如果有SSE头的话）
+    // step2: verify CreateMultipartUpload response (SSE-S3 header should be included)
    if let Some(sse) = create_multipart_output.server_side_encryption() {
-        debug!("CreateMultipartUpload包含SSE响应: {:?}", sse);
+        debug!("CreateMultipartUpload response contains SSE header: {:?}", sse);
        assert_eq!(sse, &aws_sdk_s3::types::ServerSideEncryption::Aes256);
    } else {
-        debug!("CreateMultipartUpload不包含SSE响应头（某些实现中正常）");
+        debug!("CreateMultipartUpload response does not contain SSE header (some implementations may return empty string)");
    }

-    // 步骤2：上传各个分片
+    // step2: upload each part
    let mut completed_parts = Vec::new();
    for part_number in 1..=total_parts {
        let start = (part_number - 1) * part_size;
        let end = std::cmp::min(start + part_size, total_size);
        let part_data = &test_data[start..end];

-        info!("🔐 上传加密分片 {} ({} bytes)", part_number, part_data.len());
+        info!("🔐 step3: upload encrypted part {} ({} bytes)", part_number, part_data.len());

        let upload_part_output = s3_client
            .upload_part()
@@ -253,15 +254,15 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
                .build(),
        );

-        debug!("加密分片 {} 上传完成，ETag: {}", part_number, etag);
+        debug!("step3: part {} uploaded, ETag: {}", part_number, etag);
    }

-    // 步骤3：完成分片上传
+    // step3: complete multipart upload
    let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
        .set_parts(Some(completed_parts))
        .build();

-    info!("🔗 完成加密分片上传");
+    info!("🔗 step3: complete multipart upload with SSE-S3 encryption");
    let complete_output = s3_client
        .complete_multipart_upload()
        .bucket(TEST_BUCKET)
@@ -271,43 +272,46 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
        .send()
        .await?;

-    debug!("完成加密分片上传，ETag: {:?}", complete_output.e_tag());
+    debug!(
+        "step3: complete multipart upload with SSE-S3 encryption, ETag: {:?}",
+        complete_output.e_tag()
+    );

-    // 步骤4：HEAD请求检查元数据
-    info!("📋 检查对象元数据");
+    // step4: HEAD request to check metadata
+    info!("📋 step4: check object metadata");
    let head_response = s3_client.head_object().bucket(TEST_BUCKET).key(object_key).send().await?;

-    debug!("HEAD响应 SSE: {:?}", head_response.server_side_encryption());
-    debug!("HEAD响应 元数据: {:?}", head_response.metadata());
+    debug!("HEAD response SSE: {:?}", head_response.server_side_encryption());
+    debug!("HEAD response metadata: {:?}", head_response.metadata());

-    // 步骤5：GET请求下载并验证
-    info!("📥 下载加密文件并验证");
+    // step5: GET request to download and verify
+    info!("📥 step5: download encrypted file and verify");
    let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;

-    debug!("GET响应 SSE: {:?}", get_response.server_side_encryption());
+    debug!("GET response SSE: {:?}", get_response.server_side_encryption());

-    // 🎯 关键验证：GET响应必须包含SSE-S3加密头
+    // step5: verify GET response contains SSE-S3 encryption header
    assert_eq!(
        get_response.server_side_encryption(),
        Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
    );

-    // 验证数据完整性
+    // step5: verify downloaded data matches original test data
    let downloaded_data = get_response.body.collect().await?.into_bytes();
    assert_eq!(downloaded_data.len(), total_size);
    assert_eq!(&downloaded_data[..], &test_data[..]);

    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
-    info!("✅ 步骤3通过：分片上传 + SSE-S3加密功能正常");
+    info!("✅ step3: multipart upload with SSE-S3 encryption function is normal");
    Ok(())
 }

-/// 步骤4：测试更大的分片上传（测试流式加密）
+/// step4: test larger multipart upload with encryption (streaming encryption)
 #[tokio::test]
 #[serial]
 async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    info!("🧪 步骤4：测试大文件分片上传加密");
+    info!("🧪 step4: test larger multipart upload with encryption (streaming encryption)");

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -322,13 +326,13 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
    let total_size = part_size * total_parts;

    info!(
-        "🗂️ 生成大文件测试数据：{} parts，每个 {}MB，总计 {}MB",
+        "🗂️ step4: generate large test data: {} parts, each {}MB, total {}MB",
        total_parts,
        part_size / (1024 * 1024),
        total_size / (1024 * 1024)
    );

-    // 生成大文件测试数据（使用复杂模式便于验证）
+    // step4: generate large test data (using complex pattern for verification)
    let test_data: Vec<u8> = (0..total_size)
        .map(|i| {
            let part_num = i / part_size;
@@ -337,9 +341,9 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
        })
        .collect();

-    info!("🔐 开始大文件分片上传（SSE-S3加密）");
+    info!("🔐 step4: start large multipart upload with encryption (SSE-S3)");

-    // 创建分片上传
+    // step4: create multipart upload
    let create_multipart_output = s3_client
        .create_multipart_upload()
        .bucket(TEST_BUCKET)
@@ -349,9 +353,9 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
        .await?;

    let upload_id = create_multipart_output.upload_id().unwrap();
-    info!("📋 创建大文件加密分片上传，ID: {}", upload_id);
+    info!("📋 step4: create multipart upload with encryption (SSE-S3), ID: {}", upload_id);

-    // 上传各个分片
+    // step4: upload parts
    let mut completed_parts = Vec::new();
    for part_number in 1..=total_parts {
        let start = (part_number - 1) * part_size;
@@ -359,7 +363,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
        let part_data = &test_data[start..end];

        info!(
-            "🔐 上传大文件加密分片 {} ({:.2}MB)",
+            "🔐 step4: upload part {} ({:.2}MB)",
            part_number,
            part_data.len() as f64 / (1024.0 * 1024.0)
        );
@@ -382,15 +386,15 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
                .build(),
        );

-        debug!("大文件加密分片 {} 上传完成，ETag: {}", part_number, etag);
+        debug!("step4: upload part {} completed, ETag: {}", part_number, etag);
    }

-    // 完成分片上传
+    // step4: complete multipart upload
    let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
        .set_parts(Some(completed_parts))
        .build();

-    info!("🔗 完成大文件加密分片上传");
+    info!("🔗 step4: complete multipart upload with encryption (SSE-S3)");
    let complete_output = s3_client
        .complete_multipart_upload()
        .bucket(TEST_BUCKET)
@@ -400,40 +404,46 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
        .send()
        .await?;

-    debug!("完成大文件加密分片上传，ETag: {:?}", complete_output.e_tag());
+    debug!(
+        "step4: complete multipart upload with encryption (SSE-S3), ETag: {:?}",
+        complete_output.e_tag()
+    );

-    // 下载并验证
-    info!("📥 下载大文件并验证");
+    // step4: download and verify
+    info!("📥 step4: download and verify large multipart upload with encryption (SSE-S3)");
    let get_response = s3_client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;

-    // 验证加密头
+    // step4: verify encryption header
    assert_eq!(
        get_response.server_side_encryption(),
        Some(&aws_sdk_s3::types::ServerSideEncryption::Aes256)
    );

-    // 验证数据完整性
+    // step4: verify data integrity
    let downloaded_data = get_response.body.collect().await?.into_bytes();
    assert_eq!(downloaded_data.len(), total_size);

-    // 逐字节验证数据（对于大文件更严格）
+    // step4: verify data matches original test data
    for (i, (&actual, &expected)) in downloaded_data.iter().zip(test_data.iter()).enumerate() {
        if actual != expected {
-            panic!("大文件数据在第{}字节不匹配: 实际={}, 期待={}", i, actual, expected);
+            panic!(
+                "step4: large multipart upload with encryption (SSE-S3) data mismatch at byte {}: actual={}, expected={}",
+                i, actual, expected
+            );
        }
    }

    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
-    info!("✅ 步骤4通过：大文件分片上传加密功能正常");
+    info!("✅ step4: large multipart upload with encryption (SSE-S3) functionality normal");
    Ok(())
 }

-/// 步骤5：测试所有加密类型的分片上传
+/// step5: test all encryption types multipart upload
 #[tokio::test]
 #[serial]
 async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    info!("🧪 步骤5：测试所有加密类型的分片上传");
+    info!("🧪 step5: test all encryption types multipart upload");

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
@@ -446,8 +456,8 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
    let total_parts = 2;
    let total_size = part_size * total_parts;

-    // 测试SSE-KMS
-    info!("🔐 测试 SSE-KMS 分片上传");
+    // step5: test SSE-KMS multipart upload
+    info!("🔐 step5: test SSE-KMS multipart upload");
    test_multipart_encryption_type(
        &s3_client,
        TEST_BUCKET,
@@ -459,8 +469,8 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
    )
    .await?;

-    // 测试SSE-C
-    info!("🔐 测试 SSE-C 分片上传");
+    // step5: test SSE-C multipart upload
+    info!("🔐 step5: test SSE-C multipart upload");
    test_multipart_encryption_type(
        &s3_client,
        TEST_BUCKET,
@@ -473,7 +483,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
    .await?;

    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
-    info!("✅ 步骤5通过：所有加密类型的分片上传功能正常");
+    info!("✅ step5: all encryption types multipart upload functionality normal");
    Ok(())
 }

@@ -483,7 +493,7 @@ enum EncryptionType {
    SSEC,
 }

-/// 辅助函数：测试特定加密类型的分片上传
+/// step5: test specific encryption type multipart upload
 async fn test_multipart_encryption_type(
    s3_client: &aws_sdk_s3::Client,
    bucket: &str,
@@ -493,10 +503,10 @@ async fn test_multipart_encryption_type(
    total_parts: usize,
    encryption_type: EncryptionType,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    // 生成测试数据
+    // step5: generate test data
    let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 7) % 256) as u8).collect();

-    // 准备SSE-C所需的密钥（如果需要）
+    // step5: prepare SSE-C key and MD5 (if needed)
    let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
        let key = "01234567890123456789012345678901";
        let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
@@ -506,9 +516,10 @@ async fn test_multipart_encryption_type(
        (None, None)
    };

-    info!("📋 创建分片上传 - {:?}", encryption_type);
+    // step5: create multipart upload
+    info!("🔗 step5: create multipart upload with encryption {:?}", encryption_type);

-    // 创建分片上传
+    // step5: create multipart upload request
    let mut create_request = s3_client.create_multipart_upload().bucket(bucket).key(object_key);

    create_request = match encryption_type {
@@ -522,7 +533,6 @@ async fn test_multipart_encryption_type(
    let create_multipart_output = create_request.send().await?;
    let upload_id = create_multipart_output.upload_id().unwrap();

-    // 上传分片
    let mut completed_parts = Vec::new();
    for part_number in 1..=total_parts {
        let start = (part_number - 1) * part_size;
@@ -537,7 +547,7 @@ async fn test_multipart_encryption_type(
            .part_number(part_number as i32)
            .body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()));

-        // SSE-C需要在每个UploadPart请求中包含密钥
+        // step5: include SSE-C key and MD5 in each UploadPart request (if needed)
        if matches!(encryption_type, EncryptionType::SSEC) {
            upload_request = upload_request
                .sse_customer_algorithm("AES256")
@@ -554,10 +564,11 @@ async fn test_multipart_encryption_type(
                .build(),
        );

-        debug!("{:?} 分片 {} 上传完成", encryption_type, part_number);
+        // step5: complete multipart upload request
+        debug!("🔗 step5: complete multipart upload part {} with etag {}", part_number, etag);
    }

-    // 完成分片上传
+    // step5: complete multipart upload
    let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
        .set_parts(Some(completed_parts))
        .build();
@@ -571,10 +582,12 @@ async fn test_multipart_encryption_type(
        .send()
        .await?;

-    // 下载并验证
+    // step5: download and verify multipart upload
+    info!("🔗 step5: download and verify multipart upload with encryption {:?}", encryption_type);
+
    let mut get_request = s3_client.get_object().bucket(bucket).key(object_key);

-    // SSE-C需要在GET请求中包含密钥
+    // step5: include SSE-C key and MD5 in each GET request (if needed)
    if matches!(encryption_type, EncryptionType::SSEC) {
        get_request = get_request
            .sse_customer_algorithm("AES256")
@@ -584,7 +597,7 @@ async fn test_multipart_encryption_type(

    let get_response = get_request.send().await?;

-    // 验证加密头
+    // step5: verify encryption headers
    match encryption_type {
        EncryptionType::SSEKMS => {
            assert_eq!(
@@ -597,11 +610,15 @@ async fn test_multipart_encryption_type(
        }
    }

-    // 验证数据完整性
+    // step5: verify data integrity
    let downloaded_data = get_response.body.collect().await?.into_bytes();
    assert_eq!(downloaded_data.len(), total_size);
    assert_eq!(&downloaded_data[..], &test_data[..]);

-    info!("✅ {:?} 分片上传测试通过", encryption_type);
+    // step5: verify data integrity
+    info!(
+        "✅ step5: verify data integrity for multipart upload with encryption {:?}",
+        encryption_type
+    );
    Ok(())
 }
@@ -476,7 +476,7 @@ async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Sen

    let failed_count = results.iter().filter(|r| !r.success).count();
    if failed_count > 0 {
-        return Err(format!("Critical test suite failed: {} tests failed", failed_count).into());
+        return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
    }

    info!("✅ 所有关键测试通过");
@@ -498,7 +498,7 @@ async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send +

    // Allow up to 10% failure rate for non-critical tests
    if success_rate < 90.0 {
-        return Err(format!("Test suite success rate too low: {:.1}%", success_rate).into());
+        return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
    }

    info!("✅ 完整测试套件通过");
@@ -14,14 +14,14 @@
 // limitations under the License.

 use async_trait::async_trait;
-use rustfs_ecstore::{disk::endpoint::Endpoint, lock_utils::create_unique_clients};
-use rustfs_lock::client::{LockClient, local::LocalClient};
+use rustfs_ecstore::disk::endpoint::Endpoint;
+use rustfs_lock::client::{LockClient, local::LocalClient, remote::RemoteClient};
 use rustfs_lock::types::{LockInfo, LockResponse, LockStats};
 use rustfs_lock::{LockId, LockMetadata, LockPriority, LockType};
 use rustfs_lock::{LockRequest, NamespaceLock, NamespaceLockManager};
 use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
 use serial_test::serial;
-use std::{error::Error, sync::Arc, time::Duration};
+use std::{collections::HashMap, error::Error, sync::Arc, time::Duration};
 use tokio::time::sleep;
 use tonic::Request;
 use url::Url;
@@ -38,6 +38,34 @@ fn get_cluster_endpoints() -> Vec<Endpoint> {
    }]
 }

+async fn create_unique_clients(endpoints: &[Endpoint]) -> Result<Vec<Arc<dyn LockClient>>, Box<dyn Error>> {
+    let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
+
+    for endpoint in endpoints {
+        if endpoint.is_local {
+            unique_endpoints.insert("local".to_string(), endpoint);
+        } else {
+            let host_port = format!(
+                "{}:{}",
+                endpoint.url.host_str().unwrap_or("localhost"),
+                endpoint.url.port().unwrap_or(9000)
+            );
+            unique_endpoints.insert(host_port, endpoint);
+        }
+    }
+
+    let mut clients = Vec::new();
+    for (_key, endpoint) in unique_endpoints {
+        if endpoint.is_local {
+            clients.push(Arc::new(LocalClient::new()) as Arc<dyn LockClient>);
+        } else {
+            clients.push(Arc::new(RemoteClient::new(endpoint.url.to_string())) as Arc<dyn LockClient>);
+        }
+    }
+
+    Ok(clients)
+}
+
 #[tokio::test]
 #[serial]
 #[ignore = "requires running RustFS server at localhost:9000"]
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+mod append;
 mod conditional_writes;
 mod lifecycle;
 mod lock;
@@ -101,8 +101,11 @@ rustfs-signer.workspace = true
 rustfs-checksums.workspace = true
 futures-util.workspace = true
 async-recursion.workspace = true
+aws-credential-types = "1.2.6"
+aws-smithy-types = "1.3.2"
 parking_lot = "0.12"
 moka = { version = "0.12", features = ["future"] }
+aws-smithy-runtime-api = "1.9.0"

 [target.'cfg(not(windows))'.dependencies]
 nix = { workspace = true }
@@ -16,6 +16,10 @@ use super::{quota::BucketQuota, target::BucketTargets};

 use super::object_lock::ObjectLockApi;
 use super::versioning::VersioningApi;
+use crate::bucket::utils::deserialize;
+use crate::config::com::{read_config, save_config};
+use crate::error::{Error, Result};
+use crate::new_object_layer_fn;
 use byteorder::{BigEndian, ByteOrder, LittleEndian};
 use rmp_serde::Serializer as rmpSerializer;
 use rustfs_policy::policy::BucketPolicy;
@@ -30,12 +34,6 @@ use std::sync::Arc;
 use time::OffsetDateTime;
 use tracing::error;

-use crate::bucket::target::BucketTarget;
-use crate::bucket::utils::deserialize;
-use crate::config::com::{read_config, save_config};
-use crate::error::{Error, Result};
-use crate::new_object_layer_fn;
-
 use crate::disk::BUCKET_META_PREFIX;
 use crate::store::ECStore;

@@ -322,7 +320,9 @@ impl BucketMetadata {

        LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION);

-        let data = self.marshal_msg()?;
+        let data = self
+            .marshal_msg()
+            .map_err(|e| Error::other(format!("save bucket metadata failed: {e}")))?;

        buf.extend_from_slice(&data);

@@ -362,8 +362,8 @@ impl BucketMetadata {
        }
        //let temp = self.bucket_targets_config_json.clone();
        if !self.bucket_targets_config_json.is_empty() {
-            let arr: Vec<BucketTarget> = serde_json::from_slice(&self.bucket_targets_config_json)?;
-            self.bucket_target_config = Some(BucketTargets { targets: arr });
+            let bucket_targets: BucketTargets = serde_json::from_slice(&self.bucket_targets_config_json)?;
+            self.bucket_target_config = Some(bucket_targets);
        } else {
            self.bucket_target_config = Some(BucketTargets::default())
        }
@@ -451,4 +451,154 @@ mod test {

        assert_eq!(bm.name, new.name);
    }
+
+    #[tokio::test]
+    async fn marshal_msg_complete_example() {
+        // Create a complete BucketMetadata with various configurations
+        let mut bm = BucketMetadata::new("test-bucket");
+
+        // Set creation time to current time
+        bm.created = OffsetDateTime::now_utc();
+        bm.lock_enabled = true;
+
+        // Add policy configuration
+        let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}"#;
+        bm.policy_config_json = policy_json.as_bytes().to_vec();
+        bm.policy_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add lifecycle configuration
+        let lifecycle_xml = r#"<LifecycleConfiguration><Rule><ID>rule1</ID><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#;
+        bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec();
+        bm.lifecycle_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add versioning configuration
+        let versioning_xml = r#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#;
+        bm.versioning_config_xml = versioning_xml.as_bytes().to_vec();
+        bm.versioning_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add encryption configuration
+        let encryption_xml = r#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
+        bm.encryption_config_xml = encryption_xml.as_bytes().to_vec();
+        bm.encryption_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add tagging configuration
+        let tagging_xml = r#"<Tagging><TagSet><Tag><Key>Environment</Key><Value>Test</Value></Tag><Tag><Key>Owner</Key><Value>RustFS</Value></Tag></TagSet></Tagging>"#;
+        bm.tagging_config_xml = tagging_xml.as_bytes().to_vec();
+        bm.tagging_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add quota configuration
+        let quota_json = r#"{"quota":1073741824,"quotaType":"hard"}"#; // 1GB quota
+        bm.quota_config_json = quota_json.as_bytes().to_vec();
+        bm.quota_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add object lock configuration
+        let object_lock_xml = r#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#;
+        bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec();
+        bm.object_lock_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add notification configuration
+        let notification_xml = r#"<NotificationConfiguration><CloudWatchConfiguration><Id>notification1</Id><Event>s3:ObjectCreated:*</Event><CloudWatchConfiguration><LogGroupName>test-log-group</LogGroupName></CloudWatchConfiguration></CloudWatchConfiguration></NotificationConfiguration>"#;
+        bm.notification_config_xml = notification_xml.as_bytes().to_vec();
+        bm.notification_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add replication configuration
+        let replication_xml = r#"<ReplicationConfiguration><Role>arn:aws:iam::123456789012:role/replication-role</Role><Rule><ID>rule1</ID><Status>Enabled</Status><Prefix>documents/</Prefix><Destination><Bucket>arn:aws:s3:::destination-bucket</Bucket></Destination></Rule></ReplicationConfiguration>"#;
+        bm.replication_config_xml = replication_xml.as_bytes().to_vec();
+        bm.replication_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add bucket targets configuration
+        let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#;
+        bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec();
+        bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc();
+
+        // Add bucket targets meta configuration
+        let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#;
+        bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
+        bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();
+
+        // Test serialization
+        let buf = bm.marshal_msg().unwrap();
+        assert!(!buf.is_empty(), "Serialized buffer should not be empty");
+
+        // Test deserialization
+        let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap();
+
+        // Verify all fields are correctly serialized and deserialized
+        assert_eq!(bm.name, deserialized_bm.name);
+        assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp());
+        assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled);
+
+        // Verify configuration data
+        assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json);
+        assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml);
+        assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml);
+        assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
+        assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
+        assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
+        assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
+        assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
+        assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);
+        assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json);
+        assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json);
+
+        // Verify timestamps (comparing unix timestamps to avoid precision issues)
+        assert_eq!(
+            bm.policy_config_updated_at.unix_timestamp(),
+            deserialized_bm.policy_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.lifecycle_config_updated_at.unix_timestamp(),
+            deserialized_bm.lifecycle_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.versioning_config_updated_at.unix_timestamp(),
+            deserialized_bm.versioning_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.encryption_config_updated_at.unix_timestamp(),
+            deserialized_bm.encryption_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.tagging_config_updated_at.unix_timestamp(),
+            deserialized_bm.tagging_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.quota_config_updated_at.unix_timestamp(),
+            deserialized_bm.quota_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.object_lock_config_updated_at.unix_timestamp(),
+            deserialized_bm.object_lock_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.notification_config_updated_at.unix_timestamp(),
+            deserialized_bm.notification_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.replication_config_updated_at.unix_timestamp(),
+            deserialized_bm.replication_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.bucket_targets_config_updated_at.unix_timestamp(),
+            deserialized_bm.bucket_targets_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(
+            bm.bucket_targets_config_meta_updated_at.unix_timestamp(),
+            deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp()
+        );
+
+        // Test that the serialized data contains expected content
+        let buf_str = String::from_utf8_lossy(&buf);
+        assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name");
+
+        // Verify the buffer size is reasonable (should be larger due to all the config data)
+        assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations");
+
+        println!("✅ Complete BucketMetadata serialization test passed");
+        println!("   - Bucket name: {}", deserialized_bm.name);
+        println!("   - Lock enabled: {}", deserialized_bm.lock_enabled);
+        println!("   - Policy config size: {} bytes", deserialized_bm.policy_config_json.len());
+        println!("   - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len());
+        println!("   - Serialized buffer size: {} bytes", buf.len());
+    }
 }
@@ -12,19 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::StorageAPI;
+use crate::StorageAPI as _;
+use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
 use crate::bucket::utils::{deserialize, is_meta_bucketname};
-use crate::cmd::bucket_targets;
 use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn};
 use crate::store::ECStore;
 use futures::future::join_all;
 use rustfs_common::heal_channel::HealOpts;
 use rustfs_policy::policy::BucketPolicy;
+use s3s::dto::ReplicationConfiguration;
 use s3s::dto::{
-    BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
-    ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
+    BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ServerSideEncryptionConfiguration, Tagging,
+    VersioningConfiguration,
 };
 use std::collections::HashSet;
 use std::sync::OnceLock;
@@ -261,7 +262,8 @@ impl BucketMetadataSys {
                    if let Some(bucket) = buckets.get(idx) {
                        let x = Arc::new(res);
                        mp.insert(bucket.clone(), x.clone());
-                        bucket_targets::init_bucket_targets(bucket, x.clone()).await;
+                        // TODO:EventNotifier,BucketTargetSys
+                        BucketTargetSys::get().set(bucket, &x).await;
                    }
                }
                Err(e) => {
@@ -348,6 +350,7 @@ impl BucketMetadataSys {
                if !is_erasure().await && !is_dist_erasure().await && is_err_bucket_not_found(&err) {
                    BucketMetadata::new(bucket)
                } else {
+                    error!("load bucket metadata failed: {}", err);
                    return Err(err);
                }
            }
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+pub mod bucket_target_sys;
 pub mod error;
 pub mod lifecycle;
 pub mod metadata;
@@ -0,0 +1,233 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::ReplicationRuleExt as _;
+use crate::bucket::tagging::decode_tags_to_map;
+use rustfs_filemeta::ReplicationType;
+use s3s::dto::DeleteMarkerReplicationStatus;
+use s3s::dto::DeleteReplicationStatus;
+use s3s::dto::Destination;
+use s3s::dto::{ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRuleStatus, ReplicationRules};
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use uuid::Uuid;
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct ObjectOpts {
+    pub name: String,
+    pub user_tags: String,
+    pub version_id: Option<Uuid>,
+    pub delete_marker: bool,
+    pub ssec: bool,
+    pub op_type: ReplicationType,
+    pub replica: bool,
+    pub existing_object: bool,
+    pub target_arn: String,
+}
+
+pub trait ReplicationConfigurationExt {
+    fn replicate(&self, opts: &ObjectOpts) -> bool;
+    fn has_existing_object_replication(&self, arn: &str) -> (bool, bool);
+    fn filter_actionable_rules(&self, obj: &ObjectOpts) -> ReplicationRules;
+    fn get_destination(&self) -> Destination;
+    fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
+    fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
+}
+
+impl ReplicationConfigurationExt for ReplicationConfiguration {
+    /// 检查是否有现有对象复制规则
+    fn has_existing_object_replication(&self, arn: &str) -> (bool, bool) {
+        let mut has_arn = false;
+
+        for rule in &self.rules {
+            if rule.destination.bucket == arn || self.role == arn {
+                if !has_arn {
+                    has_arn = true;
+                }
+                if let Some(status) = &rule.existing_object_replication {
+                    if status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED) {
+                        return (true, true);
+                    }
+                }
+            }
+        }
+        (has_arn, false)
+    }
+
+    fn filter_actionable_rules(&self, obj: &ObjectOpts) -> ReplicationRules {
+        if obj.name.is_empty() && obj.op_type != ReplicationType::Resync && obj.op_type != ReplicationType::All {
+            return vec![];
+        }
+
+        let mut rules = ReplicationRules::default();
+
+        for rule in &self.rules {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if !obj.target_arn.is_empty() && rule.destination.bucket != obj.target_arn && self.role != obj.target_arn {
+                continue;
+            }
+
+            if obj.op_type == ReplicationType::Resync || obj.op_type == ReplicationType::All {
+                rules.push(rule.clone());
+                continue;
+            }
+
+            if let Some(status) = &rule.existing_object_replication {
+                if obj.existing_object
+                    && status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
+                {
+                    continue;
+                }
+            }
+
+            if !obj.name.starts_with(rule.prefix()) {
+                continue;
+            }
+
+            if let Some(filter) = &rule.filter {
+                let object_tags = decode_tags_to_map(&obj.user_tags);
+                if filter.test_tags(&object_tags) {
+                    rules.push(rule.clone());
+                }
+            }
+        }
+
+        rules.sort_by(|a, b| {
+            if a.destination == b.destination {
+                a.priority.cmp(&b.priority)
+            } else {
+                std::cmp::Ordering::Equal
+            }
+        });
+
+        rules
+    }
+
+    /// 获取目标配置
+    fn get_destination(&self) -> Destination {
+        if !self.rules.is_empty() {
+            self.rules[0].destination.clone()
+        } else {
+            Destination {
+                account: None,
+                bucket: "".to_string(),
+                encryption_configuration: None,
+                metrics: None,
+                replication_time: None,
+                access_control_translation: None,
+                storage_class: None,
+            }
+        }
+    }
+
+    /// 判断对象是否应该被复制
+    fn replicate(&self, obj: &ObjectOpts) -> bool {
+        let rules = self.filter_actionable_rules(obj);
+
+        for rule in rules.iter() {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if let Some(status) = &rule.existing_object_replication {
+                if obj.existing_object
+                    && status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
+                {
+                    return false;
+                }
+            }
+
+            if obj.op_type == ReplicationType::Delete {
+                if obj.version_id.is_some() {
+                    return rule
+                        .delete_replication
+                        .clone()
+                        .is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
+                } else {
+                    return rule.delete_marker_replication.clone().is_some_and(|d| {
+                        d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
+                    });
+                }
+            }
+
+            // 常规对象/元数据复制
+            return rule.metadata_replicate(obj);
+        }
+        false
+    }
+
+    /// 检查是否有活跃的规则
+    /// 可选择性地提供前缀
+    /// 如果recursive为true，函数还会在前缀下的任何级别有活跃规则时返回true
+    /// 如果没有指定前缀，recursive实际上为true
+    fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool {
+        if self.rules.is_empty() {
+            return false;
+        }
+
+        for rule in &self.rules {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if let Some(filter) = &rule.filter {
+                if let Some(filter_prefix) = &filter.prefix {
+                    if !prefix.is_empty() && !filter_prefix.is_empty() {
+                        // 传入的前缀必须在规则前缀中
+                        if !recursive && !prefix.starts_with(filter_prefix) {
+                            continue;
+                        }
+                    }
+
+                    // 如果是递归的，我们可以跳过这个规则，如果它不匹配测试前缀或前缀下的级别不匹配
+                    if recursive && !rule.prefix().starts_with(prefix) && !prefix.starts_with(rule.prefix()) {
+                        continue;
+                    }
+                }
+            }
+            return true;
+        }
+        false
+    }
+
+    /// 过滤目标ARN，返回配置中不同目标ARN的切片
+    fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String> {
+        let mut arns = Vec::new();
+        let mut targets_map: HashSet<String> = HashSet::new();
+        let rules = self.filter_actionable_rules(obj);
+
+        for rule in rules {
+            if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
+                continue;
+            }
+
+            if !self.role.is_empty() {
+                arns.push(self.role.clone()); // 如果存在，使用传统的RoleArn
+                return arns;
+            }
+
+            if !targets_map.contains(&rule.destination.bucket) {
+                targets_map.insert(rule.destination.bucket.clone());
+            }
+        }
+
+        for arn in targets_map {
+            arns.push(arn);
+        }
+        arns
+    }
+}
@@ -12,30 +12,36 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-// Replication status type for x-amz-replication-status header
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum StatusType {
-    Pending,
-    Completed,
-    CompletedLegacy,
-    Failed,
-    Replica,
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub enum ResyncStatusType {
+    #[default]
+    NoResync,
+    ResyncPending,
+    ResyncCanceled,
+    ResyncStarted,
+    ResyncCompleted,
+    ResyncFailed,
 }

-impl StatusType {
-    // Converts the enum variant to its string representation
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            StatusType::Pending => "PENDING",
-            StatusType::Completed => "COMPLETED",
-            StatusType::CompletedLegacy => "COMPLETE",
-            StatusType::Failed => "FAILED",
-            StatusType::Replica => "REPLICA",
-        }
-    }
-
-    // Checks if the status is empty (not set)
-    pub fn is_empty(&self) -> bool {
-        matches!(self, StatusType::Pending) // Adjust this as needed
+impl ResyncStatusType {
+    pub fn is_valid(&self) -> bool {
+        *self != ResyncStatusType::NoResync
+    }
+}
+
+impl fmt::Display for ResyncStatusType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let s = match self {
+            ResyncStatusType::ResyncStarted => "Ongoing",
+            ResyncStatusType::ResyncCompleted => "Completed",
+            ResyncStatusType::ResyncFailed => "Failed",
+            ResyncStatusType::ResyncPending => "Pending",
+            ResyncStatusType::ResyncCanceled => "Canceled",
+            ResyncStatusType::NoResync => "",
+        };
+        write!(f, "{s}")
    }
 }
@@ -12,4 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+mod config;
 pub mod datatypes;
+mod replication_pool;
+mod replication_resyncer;
+mod replication_state;
+mod replication_type;
+mod rule;
+
+pub use config::*;
+pub use datatypes::*;
+pub use replication_pool::*;
+pub use replication_resyncer::*;
+pub use replication_type::*;
+pub use rule::*;
@@ -0,0 +1,470 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use crate::store_api::ObjectInfo;
+
+use regex::Regex;
+
+use rustfs_filemeta::VersionPurgeStatusType;
+use rustfs_filemeta::{ReplicatedInfos, ReplicationType};
+use rustfs_filemeta::{ReplicationState, ReplicationStatusType};
+use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
+use rustfs_utils::http::RUSTFS_REPLICATION_RESET_STATUS;
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::collections::HashMap;
+use std::fmt;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+pub const REPLICATION_RESET: &str = "replication-reset";
+pub const REPLICATION_STATUS: &str = "replication-status";
+
+// ReplicateQueued - replication being queued trail
+pub const REPLICATE_QUEUED: &str = "replicate:queue";
+
+// ReplicateExisting - audit trail for existing objects replication
+pub const REPLICATE_EXISTING: &str = "replicate:existing";
+// ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers
+pub const REPLICATE_EXISTING_DELETE: &str = "replicate:existing:delete";
+
+// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
+pub const REPLICATE_MRF: &str = "replicate:mrf";
+// ReplicateIncoming - audit trail of inline replication
+pub const REPLICATE_INCOMING: &str = "replicate:incoming";
+// ReplicateIncomingDelete - audit trail of inline replication of deletes.
+pub const REPLICATE_INCOMING_DELETE: &str = "replicate:incoming:delete";
+
+// ReplicateHeal - audit trail for healing of failed/pending replications
+pub const REPLICATE_HEAL: &str = "replicate:heal";
+// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
+pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MrfReplicateEntry {
+    #[serde(rename = "bucket")]
+    pub bucket: String,
+
+    #[serde(rename = "object")]
+    pub object: String,
+
+    #[serde(skip_serializing, skip_deserializing)]
+    pub version_id: Option<Uuid>,
+
+    #[serde(rename = "retryCount")]
+    pub retry_count: i32,
+
+    #[serde(skip_serializing, skip_deserializing)]
+    pub size: i64,
+}
+
+pub trait ReplicationWorkerOperation: Any + Send + Sync {
+    fn to_mrf_entry(&self) -> MrfReplicateEntry;
+    fn as_any(&self) -> &dyn Any;
+    fn get_bucket(&self) -> &str;
+    fn get_object(&self) -> &str;
+    fn get_size(&self) -> i64;
+    fn is_delete_marker(&self) -> bool;
+    fn get_op_type(&self) -> ReplicationType;
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct ReplicateTargetDecision {
+    pub replicate: bool,
+    pub synchronous: bool,
+    pub arn: String,
+    pub id: String,
+}
+
+impl ReplicateTargetDecision {
+    pub fn new(arn: String, replicate: bool, sync: bool) -> Self {
+        Self {
+            replicate,
+            synchronous: sync,
+            arn,
+            id: String::new(),
+        }
+    }
+}
+
+impl fmt::Display for ReplicateTargetDecision {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{};{};{};{}", self.replicate, self.synchronous, self.arn, self.id)
+    }
+}
+
+/// ReplicateDecision represents replication decision for each target
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReplicateDecision {
+    pub targets_map: HashMap<String, ReplicateTargetDecision>,
+}
+
+impl ReplicateDecision {
+    pub fn new() -> Self {
+        Self {
+            targets_map: HashMap::new(),
+        }
+    }
+
+    /// Returns true if at least one target qualifies for replication
+    pub fn replicate_any(&self) -> bool {
+        self.targets_map.values().any(|t| t.replicate)
+    }
+
+    /// Returns true if at least one target qualifies for synchronous replication
+    pub fn is_synchronous(&self) -> bool {
+        self.targets_map.values().any(|t| t.synchronous)
+    }
+
+    /// Updates ReplicateDecision with target's replication decision
+    pub fn set(&mut self, target: ReplicateTargetDecision) {
+        self.targets_map.insert(target.arn.clone(), target);
+    }
+
+    /// Returns a stringified representation of internal replication status with all targets marked as `PENDING`
+    pub fn pending_status(&self) -> Option<String> {
+        let mut result = String::new();
+        for target in self.targets_map.values() {
+            if target.replicate {
+                result.push_str(&format!("{}={};", target.arn, ReplicationStatusType::Pending.as_str()));
+            }
+        }
+        if result.is_empty() { None } else { Some(result) }
+    }
+}
+
+impl fmt::Display for ReplicateDecision {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut result = String::new();
+        for (key, value) in &self.targets_map {
+            result.push_str(&format!("{key}={value},"));
+        }
+        write!(f, "{}", result.trim_end_matches(','))
+    }
+}
+
+impl Default for ReplicateDecision {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
+// ReplicateDecision struct
+pub fn parse_replicate_decision(_bucket: &str, s: &str) -> Result<ReplicateDecision> {
+    let mut decision = ReplicateDecision::new();
+
+    if s.is_empty() {
+        return Ok(decision);
+    }
+
+    for p in s.split(',') {
+        if p.is_empty() {
+            continue;
+        }
+
+        let slc = p.split('=').collect::<Vec<&str>>();
+        if slc.len() != 2 {
+            return Err(Error::other(format!("invalid replicate decision format: {s}")));
+        }
+
+        let tgt_str = slc[1].trim_matches('"');
+        let tgt = tgt_str.split(';').collect::<Vec<&str>>();
+        if tgt.len() != 4 {
+            return Err(Error::other(format!("invalid replicate decision format: {s}")));
+        }
+
+        let tgt = ReplicateTargetDecision {
+            replicate: tgt[0] == "true",
+            synchronous: tgt[1] == "true",
+            arn: tgt[2].to_string(),
+            id: tgt[3].to_string(),
+        };
+        decision.targets_map.insert(slc[0].to_string(), tgt);
+    }
+
+    Ok(decision)
+
+    // r = ReplicateDecision{
+    // 	targetsMap: make(map[string]replicateTargetDecision),
+    // }
+    // if len(s) == 0 {
+    // 	return
+    // }
+    // for _, p := range strings.Split(s, ",") {
+    // 	if p == "" {
+    // 		continue
+    // 	}
+    // 	slc := strings.Split(p, "=")
+    // 	if len(slc) != 2 {
+    // 		return r, errInvalidReplicateDecisionFormat
+    // 	}
+    // 	tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
+    // 	tgt := strings.Split(tgtStr, ";")
+    // 	if len(tgt) != 4 {
+    // 		return r, errInvalidReplicateDecisionFormat
+    // 	}
+    // 	r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
+    // }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct ResyncTargetDecision {
+    pub replicate: bool,
+    pub reset_id: String,
+    pub reset_before_date: Option<OffsetDateTime>,
+}
+
+pub fn target_reset_header(arn: &str) -> String {
+    format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
+}
+
+impl ResyncTargetDecision {
+    pub fn resync_target(
+        oi: &ObjectInfo,
+        arn: &str,
+        reset_id: &str,
+        reset_before_date: Option<OffsetDateTime>,
+        status: ReplicationStatusType,
+    ) -> Self {
+        let rs = oi
+            .user_defined
+            .get(target_reset_header(arn).as_str())
+            .or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS))
+            .map(|s| s.to_string());
+
+        let mut dec = Self::default();
+
+        let mod_time = oi.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
+
+        if rs.is_none() {
+            let reset_before_date = reset_before_date.unwrap_or(OffsetDateTime::UNIX_EPOCH);
+            if !reset_id.is_empty() && mod_time < reset_before_date {
+                dec.replicate = true;
+                return dec;
+            }
+
+            dec.replicate = status == ReplicationStatusType::Empty;
+
+            return dec;
+        }
+
+        if reset_id.is_empty() || reset_before_date.is_none() {
+            return dec;
+        }
+
+        let rs = rs.unwrap();
+        let reset_before_date = reset_before_date.unwrap();
+
+        let parts: Vec<&str> = rs.splitn(2, ';').collect();
+
+        if parts.len() != 2 {
+            return dec;
+        }
+
+        let new_reset = parts[0] == reset_id;
+
+        if !new_reset && status == ReplicationStatusType::Completed {
+            return dec;
+        }
+
+        dec.replicate = new_reset && mod_time < reset_before_date;
+
+        dec
+    }
+}
+
+/// ResyncDecision is a struct representing a map with target's individual resync decisions
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ResyncDecision {
+    pub targets: HashMap<String, ResyncTargetDecision>,
+}
+
+impl ResyncDecision {
+    pub fn new() -> Self {
+        Self { targets: HashMap::new() }
+    }
+
+    /// Returns true if no targets with resync decision present
+    pub fn is_empty(&self) -> bool {
+        self.targets.is_empty()
+    }
+
+    pub fn must_resync(&self) -> bool {
+        self.targets.values().any(|v| v.replicate)
+    }
+
+    pub fn must_resync_target(&self, tgt_arn: &str) -> bool {
+        self.targets.get(tgt_arn).map(|v| v.replicate).unwrap_or(false)
+    }
+}
+
+impl Default for ResyncDecision {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReplicateObjectInfo {
+    pub name: String,
+    pub size: i64,
+    pub actual_size: i64,
+    pub bucket: String,
+    pub version_id: Option<Uuid>,
+    pub etag: Option<String>,
+    pub mod_time: Option<OffsetDateTime>,
+    pub replication_status: ReplicationStatusType,
+    pub replication_status_internal: Option<String>,
+    pub delete_marker: bool,
+    pub version_purge_status_internal: Option<String>,
+    pub version_purge_status: VersionPurgeStatusType,
+    pub replication_state: Option<ReplicationState>,
+    pub op_type: ReplicationType,
+    pub event_type: String,
+    pub dsc: ReplicateDecision,
+    pub existing_obj_resync: ResyncDecision,
+    pub target_statuses: HashMap<String, ReplicationStatusType>,
+    pub target_purge_statuses: HashMap<String, VersionPurgeStatusType>,
+    pub replication_timestamp: Option<OffsetDateTime>,
+    pub ssec: bool,
+    pub user_tags: String,
+    pub checksum: Vec<u8>,
+    pub retry_count: u32,
+}
+
+impl ReplicationWorkerOperation for ReplicateObjectInfo {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn to_mrf_entry(&self) -> MrfReplicateEntry {
+        MrfReplicateEntry {
+            bucket: self.bucket.clone(),
+            object: self.name.clone(),
+            version_id: self.version_id,
+            retry_count: self.retry_count as i32,
+            size: self.size,
+        }
+    }
+
+    fn get_bucket(&self) -> &str {
+        &self.bucket
+    }
+
+    fn get_object(&self) -> &str {
+        &self.name
+    }
+
+    fn get_size(&self) -> i64 {
+        self.size
+    }
+
+    fn is_delete_marker(&self) -> bool {
+        self.delete_marker
+    }
+
+    fn get_op_type(&self) -> ReplicationType {
+        self.op_type
+    }
+}
+
+lazy_static::lazy_static! {
+    static ref REPL_STATUS_REGEX: Regex = Regex::new(r"([^=].*?)=([^,].*?);").unwrap();
+}
+
+impl ReplicateObjectInfo {
+    /// Returns replication status of a target
+    pub fn target_replication_status(&self, arn: &str) -> ReplicationStatusType {
+        let binding = self.replication_status_internal.clone().unwrap_or_default();
+        let captures = REPL_STATUS_REGEX.captures_iter(&binding);
+        for cap in captures {
+            if cap.len() == 3 && &cap[1] == arn {
+                return ReplicationStatusType::from(&cap[2]);
+            }
+        }
+        ReplicationStatusType::default()
+    }
+
+    /// Returns the relevant info needed by MRF
+    pub fn to_mrf_entry(&self) -> MrfReplicateEntry {
+        MrfReplicateEntry {
+            bucket: self.bucket.clone(),
+            object: self.name.clone(),
+            version_id: self.version_id,
+            retry_count: self.retry_count as i32,
+            size: self.size,
+        }
+    }
+}
+
+// constructs a replication status map from string representation
+pub fn replication_statuses_map(s: &str) -> HashMap<String, ReplicationStatusType> {
+    let mut targets = HashMap::new();
+    let rep_stat_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
+    for (_, [arn, status]) in rep_stat_matches {
+        if arn.is_empty() {
+            continue;
+        }
+        let status = ReplicationStatusType::from(status);
+        targets.insert(arn.to_string(), status);
+    }
+    targets
+}
+
+// constructs a version purge status map from string representation
+pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatusType> {
+    let mut targets = HashMap::new();
+    let purge_status_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
+    for (_, [arn, status]) in purge_status_matches {
+        if arn.is_empty() {
+            continue;
+        }
+        let status = VersionPurgeStatusType::from(status);
+        targets.insert(arn.to_string(), status);
+    }
+    targets
+}
+
+pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
+    let reset_status_map: Vec<(String, String)> = rinfos
+        .targets
+        .iter()
+        .filter(|v| !v.resync_timestamp.is_empty())
+        .map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
+        .collect();
+
+    let repl_statuses = rinfos.replication_status_internal();
+    let vpurge_statuses = rinfos.version_purge_status_internal();
+
+    let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
+    for (key, value) in reset_status_map {
+        reset_statuses_map.insert(key, value);
+    }
+
+    ReplicationState {
+        replicate_decision_str: prev_state.replicate_decision_str.clone(),
+        reset_statuses_map,
+        replica_timestamp: prev_state.replica_timestamp,
+        replica_status: prev_state.replica_status.clone(),
+        targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
+        replication_status_internal: repl_statuses,
+        replication_timestamp: rinfos.replication_timestamp,
+        purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
+        version_purge_status_internal: vpurge_statuses,
+
+        ..Default::default()
+    }
+}
@@ -0,0 +1,51 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use s3s::dto::ReplicaModificationsStatus;
+use s3s::dto::ReplicationRule;
+
+use super::ObjectOpts;
+
+pub trait ReplicationRuleExt {
+    fn prefix(&self) -> &str;
+    fn metadata_replicate(&self, obj: &ObjectOpts) -> bool;
+}
+
+impl ReplicationRuleExt for ReplicationRule {
+    fn prefix(&self) -> &str {
+        if let Some(filter) = &self.filter {
+            if let Some(prefix) = &filter.prefix {
+                prefix
+            } else if let Some(and) = &filter.and {
+                and.prefix.as_deref().unwrap_or("")
+            } else {
+                ""
+            }
+        } else {
+            ""
+        }
+    }
+
+    fn metadata_replicate(&self, obj: &ObjectOpts) -> bool {
+        if !obj.replica {
+            return true;
+        }
+
+        self.source_selection_criteria.as_ref().is_some_and(|s| {
+            s.replica_modifications
+                .clone()
+                .is_some_and(|r| r.status == ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED))
+        })
+    }
+}
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use std::collections::HashMap;
+
 use s3s::dto::Tag;
 use url::form_urlencoded;

@@ -34,6 +36,20 @@ pub fn decode_tags(tags: &str) -> Vec<Tag> {
    list
 }

+pub fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
+    let mut list = HashMap::new();
+
+    for (k, v) in form_urlencoded::parse(tags.as_bytes()) {
+        if k.is_empty() || v.is_empty() {
+            continue;
+        }
+
+        list.insert(k.to_string(), v.to_string());
+    }
+
+    list
+}
+
 pub fn encode_tags(tags: Vec<Tag>) -> String {
    let mut encoded = form_urlencoded::Serializer::new(String::new());

@@ -0,0 +1,66 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::BucketTargetType;
+use std::fmt::Display;
+use std::str::FromStr;
+
+pub struct ARN {
+    pub arn_type: BucketTargetType,
+    pub id: String,
+    pub region: String,
+    pub bucket: String,
+}
+
+impl ARN {
+    pub fn new(arn_type: BucketTargetType, id: String, region: String, bucket: String) -> Self {
+        Self {
+            arn_type,
+            id,
+            region,
+            bucket,
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.arn_type.is_valid()
+    }
+}
+
+impl Display for ARN {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
+    }
+}
+
+impl FromStr for ARN {
+    type Err = std::io::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if !s.starts_with("arn:rustfs:") {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
+        }
+
+        let parts: Vec<&str> = s.split(':').collect();
+        if parts.len() != 6 {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
+        }
+        Ok(ARN {
+            arn_type: BucketTargetType::from_str(parts[2]).unwrap_or_default(),
+            id: parts[3].to_string(),
+            region: parts[4].to_string(),
+            bucket: parts[5].to_string(),
+        })
+    }
+}
@@ -0,0 +1,800 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use rmp_serde::Serializer as rmpSerializer;
+use serde::{Deserialize, Serialize};
+use std::{
+    fmt::{self, Display},
+    str::FromStr,
+    time::Duration,
+};
+use time::OffsetDateTime;
+use url::Url;
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct Credentials {
+    #[serde(rename = "accessKey")]
+    pub access_key: String,
+    #[serde(rename = "secretKey")]
+    pub secret_key: String,
+    pub session_token: Option<String>,
+    pub expiration: Option<chrono::DateTime<chrono::Utc>>,
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub enum ServiceType {
+    #[default]
+    Replication,
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct LatencyStat {
+    #[serde(with = "duration_milliseconds")]
+    pub curr: Duration, // Current latency
+    #[serde(with = "duration_milliseconds")]
+    pub avg: Duration, // Average latency
+    #[serde(with = "duration_milliseconds")]
+    pub max: Duration, // Maximum latency
+}
+
+mod duration_milliseconds {
+    use serde::{Deserialize, Deserializer, Serializer};
+    use std::time::Duration;
+
+    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_u64(duration.as_millis() as u64)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let millis = u64::deserialize(deserializer)?;
+        Ok(Duration::from_millis(millis))
+    }
+}
+
+mod duration_seconds {
+    use serde::{Deserialize, Deserializer, Serializer};
+    use std::time::Duration;
+
+    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_u64(duration.as_secs())
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let secs = u64::deserialize(deserializer)?;
+        Ok(Duration::from_secs(secs))
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
+pub enum BucketTargetType {
+    #[default]
+    None,
+    #[serde(rename = "replication")]
+    ReplicationService,
+    #[serde(rename = "ilm")]
+    IlmService,
+}
+
+impl BucketTargetType {
+    pub fn is_valid(&self) -> bool {
+        match self {
+            BucketTargetType::None => false,
+            BucketTargetType::ReplicationService | BucketTargetType::IlmService => true,
+        }
+    }
+}
+
+impl FromStr for BucketTargetType {
+    type Err = std::io::Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s {
+            "replication" => Ok(BucketTargetType::ReplicationService),
+            "ilm" => Ok(BucketTargetType::IlmService),
+            _ => Ok(BucketTargetType::None),
+        }
+    }
+}
+
+impl fmt::Display for BucketTargetType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BucketTargetType::None => write!(f, ""),
+            BucketTargetType::ReplicationService => write!(f, "replication"),
+            BucketTargetType::IlmService => write!(f, "ilm"),
+        }
+    }
+}
+
+// Define BucketTarget structure
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct BucketTarget {
+    #[serde(rename = "sourcebucket", default)]
+    pub source_bucket: String,
+
+    #[serde(default)]
+    pub endpoint: String,
+
+    #[serde(default)]
+    pub credentials: Option<Credentials>,
+    #[serde(rename = "targetbucket", default)]
+    pub target_bucket: String,
+
+    #[serde(default)]
+    pub secure: bool,
+    #[serde(default)]
+    pub path: String,
+    #[serde(default)]
+    pub api: String,
+    #[serde(default)]
+    pub arn: String,
+    #[serde(rename = "type", default)]
+    pub target_type: BucketTargetType,
+
+    #[serde(default)]
+    pub region: String,
+
+    #[serde(alias = "bandwidth", default)]
+    pub bandwidth_limit: i64,
+
+    #[serde(rename = "replicationSync", default)]
+    pub replication_sync: bool,
+    #[serde(default)]
+    pub storage_class: String,
+    #[serde(rename = "healthCheckDuration", with = "duration_seconds", default)]
+    pub health_check_duration: Duration,
+    #[serde(rename = "disableProxy", default)]
+    pub disable_proxy: bool,
+
+    #[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
+    pub reset_before_date: Option<OffsetDateTime>,
+    #[serde(default)]
+    pub reset_id: String,
+    #[serde(rename = "totalDowntime", with = "duration_seconds", default)]
+    pub total_downtime: Duration,
+
+    #[serde(rename = "lastOnline", with = "time::serde::rfc3339::option", default)]
+    pub last_online: Option<OffsetDateTime>,
+    #[serde(rename = "isOnline", default)]
+    pub online: bool,
+
+    #[serde(default)]
+    pub latency: LatencyStat,
+
+    #[serde(default)]
+    pub deployment_id: String,
+
+    #[serde(default)]
+    pub edge: bool,
+    #[serde(rename = "edgeSyncBeforeExpiry", default)]
+    pub edge_sync_before_expiry: bool,
+    #[serde(rename = "offlineCount", default)]
+    pub offline_count: u64,
+}
+
+impl BucketTarget {
+    pub fn is_empty(self) -> bool {
+        self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_empty()
+    }
+    pub fn url(&self) -> Result<Url> {
+        let scheme = if self.secure { "https" } else { "http" };
+        Url::parse(&format!("{}://{}", scheme, self.endpoint)).map_err(Error::other)
+    }
+}
+
+impl Display for BucketTarget {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} ", self.endpoint)?;
+        write!(f, "{}", self.target_bucket.clone())?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, Default, Clone)]
+pub struct BucketTargets {
+    pub targets: Vec<BucketTarget>,
+}
+
+impl BucketTargets {
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: BucketTargets = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        if self.targets.is_empty() {
+            return true;
+        }
+
+        for target in &self.targets {
+            if !target.clone().is_empty() {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json;
+    use std::time::Duration;
+    use time::OffsetDateTime;
+
+    #[test]
+    fn test_bucket_target_json_deserialize() {
+        let json = r#"
+        {
+            "sourcebucket": "source-bucket-name",
+            "endpoint": "s3.amazonaws.com",
+            "credentials": {
+                "accessKey": "test-access-key",
+                "secretKey": "test-secret-key",
+                "session_token": "test-session-token",
+                "expiration": "2024-12-31T23:59:59Z"
+            },
+            "targetbucket": "target-bucket-name",
+            "secure": true,
+            "path": "/api/v1",
+            "api": "s3v4",
+            "arn": "arn:aws:s3:::target-bucket-name",
+            "type": "replication",
+            "region": "us-east-1",
+            "bandwidth_limit": 1000000,
+            "replicationSync": true,
+            "storage_class": "STANDARD",
+            "healthCheckDuration": 30,
+            "disableProxy": false,
+            "resetBeforeDate": null,
+            "reset_id": "reset-123",
+            "totalDowntime": 3600,
+            "last_online": null,
+            "isOnline": true,
+            "latency": {
+                "curr": 100,
+                "avg": 150,
+                "max": 300
+            },
+            "deployment_id": "deployment-456",
+            "edge": false,
+            "edgeSyncBeforeExpiry": true,
+            "offlineCount": 5
+        }
+        "#;
+
+        let result: std::result::Result<BucketTarget, _> = serde_json::from_str(json);
+        assert!(result.is_ok(), "Failed to deserialize BucketTarget: {:?}", result.err());
+
+        let target = result.unwrap();
+
+        // Verify basic fields
+        assert_eq!(target.source_bucket, "source-bucket-name");
+        assert_eq!(target.endpoint, "s3.amazonaws.com");
+        assert_eq!(target.target_bucket, "target-bucket-name");
+        assert!(target.secure);
+        assert_eq!(target.path, "/api/v1");
+        assert_eq!(target.api, "s3v4");
+        assert_eq!(target.arn, "arn:aws:s3:::target-bucket-name");
+        assert_eq!(target.target_type, BucketTargetType::ReplicationService);
+        assert_eq!(target.region, "us-east-1");
+        assert_eq!(target.bandwidth_limit, 1000000);
+        assert!(target.replication_sync);
+        assert_eq!(target.storage_class, "STANDARD");
+        assert_eq!(target.health_check_duration, Duration::from_secs(30));
+        assert!(!target.disable_proxy);
+        assert_eq!(target.reset_id, "reset-123");
+        assert_eq!(target.total_downtime, Duration::from_secs(3600));
+        assert!(target.online);
+        assert_eq!(target.deployment_id, "deployment-456");
+        assert!(!target.edge);
+        assert!(target.edge_sync_before_expiry);
+        assert_eq!(target.offline_count, 5);
+
+        // Verify credentials
+        assert!(target.credentials.is_some());
+        let credentials = target.credentials.unwrap();
+        assert_eq!(credentials.access_key, "test-access-key");
+        assert_eq!(credentials.secret_key, "test-secret-key");
+        assert_eq!(credentials.session_token, Some("test-session-token".to_string()));
+        assert!(credentials.expiration.is_some());
+
+        // Verify latency statistics
+        assert_eq!(target.latency.curr, Duration::from_millis(100));
+        assert_eq!(target.latency.avg, Duration::from_millis(150));
+        assert_eq!(target.latency.max, Duration::from_millis(300));
+
+        // Verify time fields
+        assert!(target.reset_before_date.is_none());
+        assert!(target.last_online.is_none());
+    }
+
+    #[test]
+    fn test_bucket_target_json_serialize_deserialize_roundtrip() {
+        let original = BucketTarget {
+            source_bucket: "test-source".to_string(),
+            endpoint: "rustfs.example.com".to_string(),
+            credentials: Some(Credentials {
+                access_key: "rustfsaccess".to_string(),
+                secret_key: "rustfssecret".to_string(),
+                session_token: None,
+                expiration: None,
+            }),
+            target_bucket: "test-target".to_string(),
+            secure: false,
+            path: "/".to_string(),
+            api: "s3v4".to_string(),
+            arn: "arn:rustfs:s3:::test-target".to_string(),
+            target_type: BucketTargetType::ReplicationService,
+            region: "us-west-2".to_string(),
+            bandwidth_limit: 500000,
+            replication_sync: false,
+            storage_class: "REDUCED_REDUNDANCY".to_string(),
+            health_check_duration: Duration::from_secs(60),
+            disable_proxy: true,
+            reset_before_date: Some(OffsetDateTime::now_utc()),
+            reset_id: "reset-456".to_string(),
+            total_downtime: Duration::from_secs(1800),
+            last_online: Some(OffsetDateTime::now_utc()),
+            online: false,
+            latency: LatencyStat {
+                curr: Duration::from_millis(250),
+                avg: Duration::from_millis(200),
+                max: Duration::from_millis(500),
+            },
+            deployment_id: "deploy-789".to_string(),
+            edge: true,
+            edge_sync_before_expiry: false,
+            offline_count: 10,
+        };
+
+        // Serialize to JSON
+        let json = serde_json::to_string(&original).expect("Failed to serialize to JSON");
+
+        // Deserialize from JSON
+        let deserialized: BucketTarget = serde_json::from_str(&json).expect("Failed to deserialize from JSON");
+
+        // Verify key fields are equal
+        assert_eq!(original.source_bucket, deserialized.source_bucket);
+        assert_eq!(original.endpoint, deserialized.endpoint);
+        assert_eq!(original.target_bucket, deserialized.target_bucket);
+        assert_eq!(original.secure, deserialized.secure);
+        assert_eq!(original.target_type, deserialized.target_type);
+        assert_eq!(original.region, deserialized.region);
+        assert_eq!(original.bandwidth_limit, deserialized.bandwidth_limit);
+        assert_eq!(original.replication_sync, deserialized.replication_sync);
+        assert_eq!(original.health_check_duration, deserialized.health_check_duration);
+        assert_eq!(original.online, deserialized.online);
+        assert_eq!(original.edge, deserialized.edge);
+        assert_eq!(original.offline_count, deserialized.offline_count);
+    }
+
+    #[test]
+    fn test_bucket_target_type_json_deserialize() {
+        // Test BucketTargetType JSON deserialization
+        let replication_json = r#""replication""#;
+        let ilm_json = r#""ilm""#;
+
+        let replication_type: BucketTargetType =
+            serde_json::from_str(replication_json).expect("Failed to deserialize replication type");
+        let ilm_type: BucketTargetType = serde_json::from_str(ilm_json).expect("Failed to deserialize ilm type");
+
+        assert_eq!(replication_type, BucketTargetType::ReplicationService);
+        assert_eq!(ilm_type, BucketTargetType::IlmService);
+
+        // Verify type validity
+        assert!(replication_type.is_valid());
+        assert!(ilm_type.is_valid());
+        assert!(!BucketTargetType::None.is_valid());
+    }
+
+    #[test]
+    fn test_credentials_json_deserialize() {
+        let json = r#"
+        {
+            "accessKey": "AKIAIOSFODNN7EXAMPLE",
+            "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+            "session_token": "AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT",
+            "expiration": "2024-12-31T23:59:59Z"
+        }
+        "#;
+
+        let credentials: Credentials = serde_json::from_str(json).expect("Failed to deserialize credentials");
+
+        assert_eq!(credentials.access_key, "AKIAIOSFODNN7EXAMPLE");
+        assert_eq!(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+        assert_eq!(
+            credentials.session_token,
+            Some("AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT".to_string())
+        );
+        assert!(credentials.expiration.is_some());
+    }
+
+    #[test]
+    fn test_latency_stat_json_deserialize() {
+        let json = r#"
+        {
+            "curr": 50,
+            "avg": 75,
+            "max": 200
+        }
+        "#;
+
+        let latency: LatencyStat = serde_json::from_str(json).expect("Failed to deserialize latency stat");
+
+        assert_eq!(latency.curr, Duration::from_millis(50));
+        assert_eq!(latency.avg, Duration::from_millis(75));
+        assert_eq!(latency.max, Duration::from_millis(200));
+    }
+
+    #[test]
+    fn test_bucket_targets_json_deserialize() {
+        let json = r#"
+        {
+            "targets": [
+                {
+                    "sourcebucket": "bucket1",
+                    "endpoint": "s3.amazonaws.com",
+                    "targetbucket": "target1",
+                    "secure": true,
+                    "path": "/",
+                    "api": "s3v4",
+                    "arn": "arn:aws:s3:::target1",
+                    "type": "replication",
+                    "region": "us-east-1",
+                    "bandwidth_limit": 0,
+                    "replicationSync": false,
+                    "storage_class": "",
+                    "healthCheckDuration": 0,
+                    "disableProxy": false,
+                    "resetBeforeDate": null,
+                    "reset_id": "",
+                    "totalDowntime": 0,
+                    "lastOnline": null,
+                    "isOnline": false,
+                    "latency": {
+                        "curr": 0,
+                        "avg": 0,
+                        "max": 0
+                    },
+                    "deployment_id": "",
+                    "edge": false,
+                    "edgeSyncBeforeExpiry": false,
+                    "offlineCount": 0
+                }
+            ]
+        }
+        "#;
+
+        let targets: BucketTargets = serde_json::from_str(json).expect("Failed to deserialize bucket targets");
+
+        assert_eq!(targets.targets.len(), 1);
+        assert_eq!(targets.targets[0].source_bucket, "bucket1");
+        assert_eq!(targets.targets[0].endpoint, "s3.amazonaws.com");
+        assert_eq!(targets.targets[0].target_bucket, "target1");
+        assert!(!targets.is_empty());
+    }
+
+    #[test]
+    fn test_user_provided_json_deserialize() {
+        // Test the specific JSON provided by the user with missing required fields added
+        let json = r#"
+        {
+            "sourcebucket": "mc-test-bucket-22139",
+            "endpoint": "localhost:8000",
+            "credentials": {
+                "accessKey": "rustfsadmin",
+                "secretKey": "rustfsadmin",
+                "expiration": "0001-01-01T00:00:00Z"
+            },
+            "targetbucket": "test",
+            "secure": false,
+            "path": "auto",
+            "api": "s3v4",
+            "type": "replication",
+            "replicationSync": false,
+            "healthCheckDuration": 60,
+            "disableProxy": false,
+            "resetBeforeDate": "0001-01-01T00:00:00Z",
+            "totalDowntime": 0,
+            "lastOnline": "0001-01-01T00:00:00Z",
+            "isOnline": false,
+            "latency": {
+                "curr": 0,
+                "avg": 0,
+                "max": 0
+            },
+            "deployment_id": "",
+            "edge": false,
+            "edgeSyncBeforeExpiry": false,
+            "offlineCount": 0,
+            "bandwidth": 107374182400
+        }
+        "#;
+
+        let target: BucketTarget = serde_json::from_str(json).expect("Failed to deserialize user provided JSON to BucketTarget");
+
+        // Verify the deserialized values match the original JSON
+        assert_eq!(target.source_bucket, "mc-test-bucket-22139");
+        assert_eq!(target.endpoint, "localhost:8000");
+        assert_eq!(target.target_bucket, "test");
+        assert!(!target.secure);
+        assert_eq!(target.path, "auto");
+        assert_eq!(target.api, "s3v4");
+        assert_eq!(target.target_type, BucketTargetType::ReplicationService);
+        assert!(!target.replication_sync);
+        assert_eq!(target.health_check_duration, Duration::from_secs(60));
+        assert!(!target.disable_proxy);
+        assert!(!target.online);
+        assert!(!target.edge);
+        assert!(!target.edge_sync_before_expiry);
+        assert_eq!(target.bandwidth_limit, 107374182400); // bandwidth field mapped to bandwidth_limit
+
+        // Verify credentials
+        assert!(target.credentials.is_some());
+        let credentials = target.credentials.unwrap();
+        assert_eq!(credentials.access_key, "rustfsadmin");
+        assert_eq!(credentials.secret_key, "rustfsadmin");
+
+        // Verify latency statistics
+        assert_eq!(target.latency.curr, Duration::from_millis(0));
+        assert_eq!(target.latency.avg, Duration::from_millis(0));
+        assert_eq!(target.latency.max, Duration::from_millis(0));
+
+        // Verify time fields parsing (should handle "0001-01-01T00:00:00Z" as None due to being the zero time)
+        assert!(target.reset_before_date.is_some());
+        assert!(target.last_online.is_some());
+
+        println!("✅ User provided JSON successfully deserialized to BucketTarget");
+    }
+
+    #[test]
+    fn test_user_provided_json_as_bucket_targets() {
+        // Test wrapping the user JSON in BucketTargets structure
+        let json = r#"
+        {
+            "targets": [
+                {
+                    "sourcebucket": "mc-test-bucket-22139",
+                    "endpoint": "localhost:8000",
+                    "credentials": {
+                        "accessKey": "rustfsadmin",
+                        "secretKey": "rustfsadmin",
+                        "expiration": "0001-01-01T00:00:00Z"
+                    },
+                    "targetbucket": "test",
+                    "secure": false,
+                    "path": "auto",
+                    "api": "s3v4",
+                    "arn": "",
+                    "type": "replication",
+                    "region": "",
+                    "replicationSync": false,
+                    "storage_class": "",
+                    "healthCheckDuration": 60,
+                    "disableProxy": false,
+                    "resetBeforeDate": "0001-01-01T00:00:00Z",
+                    "reset_id": "",
+                    "totalDowntime": 0,
+                    "lastOnline": "0001-01-01T00:00:00Z",
+                    "isOnline": false,
+                    "latency": {
+                        "curr": 0,
+                        "avg": 0,
+                        "max": 0
+                    },
+                    "deployment_id": "",
+                    "edge": false,
+                    "edgeSyncBeforeExpiry": false,
+                    "offlineCount": 0,
+                    "bandwidth": 107374182400
+                }
+            ]
+        }
+        "#;
+
+        let bucket_targets: BucketTargets =
+            serde_json::from_str(json).expect("Failed to deserialize user provided JSON to BucketTargets");
+
+        assert_eq!(bucket_targets.targets.len(), 1);
+        assert!(!bucket_targets.is_empty());
+
+        let target = &bucket_targets.targets[0];
+        assert_eq!(target.source_bucket, "mc-test-bucket-22139");
+        assert_eq!(target.endpoint, "localhost:8000");
+        assert_eq!(target.target_bucket, "test");
+        assert_eq!(target.bandwidth_limit, 107374182400);
+
+        println!("✅ User provided JSON successfully deserialized to BucketTargets");
+    }
+
+    #[test]
+    fn test_bucket_target_minimal_json_with_defaults() {
+        // Test that BucketTarget can be deserialized with minimal JSON using defaults
+        let minimal_json = r#"
+        {
+            "sourcebucket": "test-source",
+            "endpoint": "localhost:9000",
+            "targetbucket": "test-target"
+        }
+        "#;
+
+        let target: BucketTarget =
+            serde_json::from_str(minimal_json).expect("Failed to deserialize minimal JSON to BucketTarget");
+
+        // Verify required fields
+        assert_eq!(target.source_bucket, "test-source");
+        assert_eq!(target.endpoint, "localhost:9000");
+        assert_eq!(target.target_bucket, "test-target");
+
+        // Verify default values
+        assert!(!target.secure); // bool default is false
+        assert_eq!(target.path, ""); // String default is empty
+        assert_eq!(target.api, ""); // String default is empty
+        assert_eq!(target.arn, ""); // String default is empty
+        assert_eq!(target.target_type, BucketTargetType::None); // enum default
+        assert_eq!(target.region, ""); // String default is empty
+        assert_eq!(target.bandwidth_limit, 0); // i64 default is 0
+        assert!(!target.replication_sync); // bool default is false
+        assert_eq!(target.storage_class, ""); // String default is empty
+        assert_eq!(target.health_check_duration, Duration::from_secs(0)); // Duration default
+        assert!(!target.disable_proxy); // bool default is false
+        assert!(target.reset_before_date.is_none()); // Option default is None
+        assert_eq!(target.reset_id, ""); // String default is empty
+        assert_eq!(target.total_downtime, Duration::from_secs(0)); // Duration default
+        assert!(target.last_online.is_none()); // Option default is None
+        assert!(!target.online); // bool default is false
+        assert_eq!(target.latency.curr, Duration::from_millis(0)); // LatencyStat default
+        assert_eq!(target.latency.avg, Duration::from_millis(0));
+        assert_eq!(target.latency.max, Duration::from_millis(0));
+        assert_eq!(target.deployment_id, ""); // String default is empty
+        assert!(!target.edge); // bool default is false
+        assert!(!target.edge_sync_before_expiry); // bool default is false
+        assert_eq!(target.offline_count, 0); // u64 default is 0
+        assert!(target.credentials.is_none()); // Option default is None
+
+        println!("✅ Minimal JSON with defaults successfully deserialized to BucketTarget");
+    }
+
+    #[test]
+    fn test_bucket_target_empty_json_with_defaults() {
+        // Test that BucketTarget can be deserialized with completely empty JSON using all defaults
+        let empty_json = r#"{}"#;
+
+        let target: BucketTarget = serde_json::from_str(empty_json).expect("Failed to deserialize empty JSON to BucketTarget");
+
+        // Verify all fields use default values
+        assert_eq!(target.source_bucket, "");
+        assert_eq!(target.endpoint, "");
+        assert_eq!(target.target_bucket, "");
+        assert!(!target.secure);
+        assert_eq!(target.path, "");
+        assert_eq!(target.api, "");
+        assert_eq!(target.arn, "");
+        assert_eq!(target.target_type, BucketTargetType::None);
+        assert_eq!(target.region, "");
+        assert_eq!(target.bandwidth_limit, 0);
+        assert!(!target.replication_sync);
+        assert_eq!(target.storage_class, "");
+        assert_eq!(target.health_check_duration, Duration::from_secs(0));
+        assert!(!target.disable_proxy);
+        assert!(target.reset_before_date.is_none());
+        assert_eq!(target.reset_id, "");
+        assert_eq!(target.total_downtime, Duration::from_secs(0));
+        assert!(target.last_online.is_none());
+        assert!(!target.online);
+        assert_eq!(target.latency.curr, Duration::from_millis(0));
+        assert_eq!(target.latency.avg, Duration::from_millis(0));
+        assert_eq!(target.latency.max, Duration::from_millis(0));
+        assert_eq!(target.deployment_id, "");
+        assert!(!target.edge);
+        assert!(!target.edge_sync_before_expiry);
+        assert_eq!(target.offline_count, 0);
+        assert!(target.credentials.is_none());
+
+        println!("✅ Empty JSON with all defaults successfully deserialized to BucketTarget");
+    }
+
+    #[test]
+    fn test_original_user_json_with_defaults() {
+        // Test the original user JSON without extra required fields
+        let json = r#"
+        {
+            "sourcebucket": "mc-test-bucket-22139",
+            "endpoint": "localhost:8000",
+            "credentials": {
+                "accessKey": "rustfsadmin",
+                "secretKey": "rustfsadmin",
+                "expiration": "0001-01-01T00:00:00Z"
+            },
+            "targetbucket": "test",
+            "secure": false,
+            "path": "auto",
+            "api": "s3v4",
+            "type": "replication",
+            "replicationSync": false,
+            "healthCheckDuration": 60,
+            "disableProxy": false,
+            "resetBeforeDate": "0001-01-01T00:00:00Z",
+            "totalDowntime": 0,
+            "lastOnline": "0001-01-01T00:00:00Z",
+            "isOnline": false,
+            "latency": {
+                "curr": 0,
+                "avg": 0,
+                "max": 0
+            },
+            "edge": false,
+            "edgeSyncBeforeExpiry": false,
+            "bandwidth": 107374182400
+        }
+        "#;
+
+        let target: BucketTarget = serde_json::from_str(json).expect("Failed to deserialize original user JSON to BucketTarget");
+
+        // Verify the deserialized values
+        assert_eq!(target.source_bucket, "mc-test-bucket-22139");
+        assert_eq!(target.endpoint, "localhost:8000");
+        assert_eq!(target.target_bucket, "test");
+        assert!(!target.secure);
+        assert_eq!(target.path, "auto");
+        assert_eq!(target.api, "s3v4");
+        assert_eq!(target.target_type, BucketTargetType::ReplicationService);
+        assert!(!target.replication_sync);
+        assert_eq!(target.health_check_duration, Duration::from_secs(60));
+        assert!(!target.disable_proxy);
+        assert!(!target.online);
+        assert!(!target.edge);
+        assert!(!target.edge_sync_before_expiry);
+        assert_eq!(target.bandwidth_limit, 107374182400);
+
+        // Fields not specified should use defaults
+        assert_eq!(target.arn, ""); // default empty string
+        assert_eq!(target.region, ""); // default empty string
+        assert_eq!(target.storage_class, ""); // default empty string
+        assert_eq!(target.reset_id, ""); // default empty string
+        assert_eq!(target.deployment_id, ""); // default empty string
+        assert_eq!(target.offline_count, 0); // default u64
+
+        // Verify credentials
+        assert!(target.credentials.is_some());
+        let credentials = target.credentials.unwrap();
+        assert_eq!(credentials.access_key, "rustfsadmin");
+        assert_eq!(credentials.secret_key, "rustfsadmin");
+
+        println!("✅ Original user JSON with defaults successfully deserialized to BucketTarget");
+    }
+}
@@ -12,124 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::error::Result;
-use rmp_serde::Serializer as rmpSerializer;
-use serde::{Deserialize, Serialize};
-use time::OffsetDateTime;
+mod arn;
+mod bucket_target;

-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct Credentials {
-    #[serde(rename = "accessKey")]
-    pub access_key: String,
-    #[serde(rename = "secretKey")]
-    pub secret_key: String,
-    pub session_token: Option<String>,
-    pub expiration: Option<chrono::DateTime<chrono::Utc>>,
-}
-
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub enum ServiceType {
-    #[default]
-    Replication,
-}
-
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct LatencyStat {
-    curr: u64, // current latency
-    avg: u64,  // average latency
-    max: u64,  // maximum latency
-}
-
-// Define BucketTarget struct
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct BucketTarget {
-    #[serde(rename = "sourcebucket")]
-    pub source_bucket: String,
-
-    pub endpoint: String,
-
-    pub credentials: Option<Credentials>,
-    #[serde(rename = "targetbucket")]
-    pub target_bucket: String,
-
-    secure: bool,
-    pub path: Option<String>,
-
-    api: Option<String>,
-
-    pub arn: Option<String>,
-    #[serde(rename = "type")]
-    pub type_: Option<String>,
-
-    pub region: Option<String>,
-
-    bandwidth_limit: Option<i64>,
-
-    #[serde(rename = "replicationSync")]
-    replication_sync: bool,
-
-    storage_class: Option<String>,
-    #[serde(rename = "healthCheckDuration")]
-    health_check_duration: u64,
-    #[serde(rename = "disableProxy")]
-    disable_proxy: bool,
-
-    #[serde(rename = "resetBeforeDate")]
-    reset_before_date: String,
-    reset_id: Option<String>,
-    #[serde(rename = "totalDowntime")]
-    total_downtime: u64,
-
-    last_online: Option<OffsetDateTime>,
-    #[serde(rename = "isOnline")]
-    online: bool,
-
-    latency: Option<LatencyStat>,
-
-    deployment_id: Option<String>,
-
-    edge: bool,
-    #[serde(rename = "edgeSyncBeforeExpiry")]
-    edge_sync_before_expiry: bool,
-}
-
-impl BucketTarget {
-    pub fn is_empty(self) -> bool {
-        //self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_empty()
-        self.target_bucket.is_empty() && self.endpoint.is_empty() && self.arn.is_none()
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, Default, Clone)]
-pub struct BucketTargets {
-    pub targets: Vec<BucketTarget>,
-}
-
-impl BucketTargets {
-    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
-        let mut buf = Vec::new();
-
-        self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
-
-        Ok(buf)
-    }
-
-    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
-        let t: BucketTargets = rmp_serde::from_slice(buf)?;
-        Ok(t)
-    }
-
-    pub fn is_empty(&self) -> bool {
-        if self.targets.is_empty() {
-            return true;
-        }
-
-        for target in &self.targets {
-            if !target.clone().is_empty() {
-                return false;
-            }
-        }
-
-        true
-    }
-}
+pub use arn::*;
+pub use bucket_target::*;
@@ -17,7 +17,8 @@ use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
 use futures::future::join_all;
 use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
 use std::{future::Future, pin::Pin, sync::Arc};
-use tokio::{spawn, sync::broadcast::Receiver as B_Receiver};
+use tokio::spawn;
+use tokio_util::sync::CancellationToken;
 use tracing::{error, warn};

 pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
@@ -63,7 +64,7 @@ impl Clone for ListPathRawOptions {
    }
 }

-pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -> disk::error::Result<()> {
+pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
    if opts.disks.is_empty() {
        return Err(DiskError::other("list_path_raw: 0 drives provided"));
    }
@@ -72,13 +73,13 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
    let mut readers = Vec::with_capacity(opts.disks.len());
    let fds = Arc::new(opts.fallback_disks.clone());

-    let (cancel_tx, cancel_rx) = tokio::sync::broadcast::channel::<bool>(1);
+    let cancel_rx = CancellationToken::new();

    for disk in opts.disks.iter() {
        let opdisk = disk.clone();
        let opts_clone = opts.clone();
        let fds_clone = fds.clone();
-        let mut cancel_rx_clone = cancel_rx.resubscribe();
+        let cancel_rx_clone = cancel_rx.clone();
        let (rd, mut wr) = tokio::io::duplex(64);
        readers.push(MetacacheReader::new(rd));
        jobs.push(spawn(async move {
@@ -106,7 +107,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
                need_fallback = true;
            }

-            if cancel_rx_clone.try_recv().is_ok() {
+            if cancel_rx_clone.is_cancelled() {
                // warn!("list_path_raw: cancel_rx_clone.try_recv().await.is_ok()");
                return Ok(());
            }
@@ -173,7 +174,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
            //     opts.bucket, opts.path, &current.name
            // );

-            if rx.try_recv().is_ok() {
+            if rx.is_cancelled() {
                return Err(DiskError::other("canceled"));
            }

@@ -351,7 +352,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -

    if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
        error!("list_path_raw: revjob err {:?}", err);
-        let _ = cancel_tx.send(true);
+        cancel_rx.cancel();

        return Err(err);
    }
@@ -44,6 +44,8 @@ pub struct GetObjectOptions {
    pub internal: AdvancedGetOptions,
 }

+pub type StatObjectOptions = GetObjectOptions;
+
 impl Default for GetObjectOptions {
    fn default() -> Self {
        Self {
@@ -46,11 +46,11 @@ pub struct RemoveBucketOptions {
 #[derive(Debug)]
 #[allow(dead_code)]
 pub struct AdvancedRemoveOptions {
-    replication_delete_marker: bool,
-    replication_status: ReplicationStatus,
-    replication_mtime: OffsetDateTime,
-    replication_request: bool,
-    replication_validity_check: bool,
+    pub replication_delete_marker: bool,
+    pub replication_status: ReplicationStatus,
+    pub replication_mtime: Option<OffsetDateTime>,
+    pub replication_request: bool,
+    pub replication_validity_check: bool,
 }

 impl Default for AdvancedRemoveOptions {
@@ -58,7 +58,7 @@ impl Default for AdvancedRemoveOptions {
        Self {
            replication_delete_marker: false,
            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
-            replication_mtime: OffsetDateTime::now_utc(),
+            replication_mtime: None,
            replication_request: false,
            replication_validity_check: false,
        }
@@ -140,8 +140,7 @@ impl TransitionClient {
    }

    pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
-        let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err");
-        res.err
+        self.remove_object_inner(bucket_name, object_name, opts).await.err()
    }

    pub async fn remove_object_inner(
@@ -23,6 +23,7 @@ use http::{HeaderMap, HeaderValue};
 use rustfs_utils::EMPTY_STRING_SHA256_HASH;
 use std::{collections::HashMap, str::FromStr};
 use tokio::io::BufReader;
+use tracing::warn;
 use uuid::Uuid;

 use crate::client::{
@@ -30,7 +31,10 @@ use crate::client::{
    api_get_options::GetObjectOptions,
    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
 };
-use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
+use s3s::{
+    dto::VersioningConfiguration,
+    header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID},
+};

 impl TransitionClient {
    pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> {
@@ -58,8 +62,14 @@ impl TransitionClient {
            .await;

        if let Ok(resp) = resp {
+            if resp.status() != http::StatusCode::OK {
+                return Ok(false);
+            }
+
            let b = resp.body().bytes().expect("err").to_vec();
            let resperr = http_resp_to_error_response(&resp, b, bucket_name, "");
+
+            warn!("bucket exists, resp: {:?}, resperr: {:?}", resp, resperr);
            /*if to_error_response(resperr).code == "NoSuchBucket" {
                return Ok(false);
            }
@@ -70,6 +80,46 @@ impl TransitionClient {
        Ok(true)
    }

+    pub async fn get_bucket_versioning(&self, bucket_name: &str) -> Result<VersioningConfiguration, std::io::Error> {
+        let mut query_values = HashMap::new();
+        query_values.insert("versioning".to_string(), "".to_string());
+        let resp = self
+            .execute_method(
+                http::Method::GET,
+                &mut RequestMetadata {
+                    bucket_name: bucket_name.to_string(),
+                    object_name: "".to_string(),
+                    query_values,
+                    custom_header: HeaderMap::new(),
+                    content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
+                    content_md5_base64: "".to_string(),
+                    content_body: ReaderImpl::Body(Bytes::new()),
+                    content_length: 0,
+                    stream_sha256: false,
+                    trailer: HeaderMap::new(),
+                    pre_sign_url: Default::default(),
+                    add_crc: Default::default(),
+                    extra_pre_sign_header: Default::default(),
+                    bucket_location: Default::default(),
+                    expires: Default::default(),
+                },
+            )
+            .await;
+
+        match resp {
+            Ok(resp) => {
+                let b = resp.body().bytes().expect("get bucket versioning err").to_vec();
+                let resperr = http_resp_to_error_response(&resp, b, bucket_name, "");
+
+                warn!("get bucket versioning, resp: {:?}, resperr: {:?}", resp, resperr);
+
+                Ok(VersioningConfiguration::default())
+            }
+
+            Err(err) => Err(std::io::Error::other(err)),
+        }
+    }
+
    pub async fn stat_object(
        &self,
        bucket_name: &str,
@@ -131,24 +181,20 @@ impl TransitionClient {
                            ..Default::default()
                        };
                        return Ok(ObjectInfo {
-                            version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
-                                Ok(v) => v,
-                                Err(e) => {
-                                    return Err(std::io::Error::other(e));
-                                }
-                            },
+                            version_id: h
+                                .get(X_AMZ_VERSION_ID)
+                                .and_then(|v| v.to_str().ok())
+                                .and_then(|s| Uuid::from_str(s).ok()),
                            is_delete_marker: delete_marker,
                            ..Default::default()
                        });
                        //err_resp
                    }
                    return Ok(ObjectInfo {
-                        version_id: match Uuid::from_str(h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap()) {
-                            Ok(v) => v,
-                            Err(e) => {
-                                return Err(std::io::Error::other(e));
-                            }
-                        },
+                        version_id: h
+                            .get(X_AMZ_VERSION_ID)
+                            .and_then(|v| v.to_str().ok())
+                            .and_then(|s| Uuid::from_str(s).ok()),
                        is_delete_marker: delete_marker,
                        replication_ready: replication_ready,
                        ..Default::default()
@@ -36,6 +36,7 @@ use s3s::S3ErrorCode;
 use super::constants::UNSIGNED_PAYLOAD;
 use super::credentials::SignatureType;

+#[derive(Debug, Clone)]
 pub struct BucketLocationCache {
    items: HashMap<String, String>,
 }
@@ -148,10 +148,75 @@ pub fn new_getobjectreader(
    Ok((get_fn, off as i64, length as i64))
 }

+/// Format an ETag value according to HTTP standards (wrap with quotes if not already wrapped)
+pub fn format_etag(etag: &str) -> String {
+    if etag.starts_with('"') && etag.ends_with('"') {
+        // Already properly formatted
+        etag.to_string()
+    } else if etag.starts_with("W/\"") && etag.ends_with('"') {
+        // Already a weak ETag, properly formatted
+        etag.to_string()
+    } else {
+        // Need to wrap with quotes
+        format!("\"{}\"", etag)
+    }
+}
+
 pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
-    if let Some(etag) = metadata.get("etag") {
+    let etag = if let Some(etag) = metadata.get("etag") {
        etag.clone()
    } else {
        metadata["md5Sum"].clone()
+    };
+    format_etag(&etag)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_etag() {
+        // Test unquoted ETag - should add quotes
+        assert_eq!(format_etag("6af8d12c0c74b78094884349f3c8a079"), "\"6af8d12c0c74b78094884349f3c8a079\"");
+
+        // Test already quoted ETag - should not double quote
+        assert_eq!(
+            format_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
+            "\"6af8d12c0c74b78094884349f3c8a079\""
+        );
+
+        // Test weak ETag - should keep as is
+        assert_eq!(
+            format_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
+            "W/\"6af8d12c0c74b78094884349f3c8a079\""
+        );
+
+        // Test empty ETag - should add quotes
+        assert_eq!(format_etag(""), "\"\"");
+
+        // Test malformed quote (only starting quote) - should wrap properly
+        assert_eq!(format_etag("\"incomplete"), "\"\"incomplete\"");
+
+        // Test malformed quote (only ending quote) - should wrap properly
+        assert_eq!(format_etag("incomplete\""), "\"incomplete\"\"");
+    }
+
+    #[test]
+    fn test_extract_etag() {
+        let mut metadata = HashMap::new();
+
+        // Test with etag field
+        metadata.insert("etag".to_string(), "abc123".to_string());
+        assert_eq!(extract_etag(&metadata), "\"abc123\"");
+
+        // Test with already quoted etag field
+        metadata.insert("etag".to_string(), "\"def456\"".to_string());
+        assert_eq!(extract_etag(&metadata), "\"def456\"");
+
+        // Test fallback to md5Sum
+        metadata.remove("etag");
+        metadata.insert("md5Sum".to_string(), "xyz789".to_string());
+        assert_eq!(extract_etag(&metadata), "\"xyz789\"");
    }
 }
@@ -89,6 +89,7 @@ pub enum ReaderImpl {

 pub type ReadCloser = BufReader<Cursor<Vec<u8>>>;

+#[derive(Debug)]
 pub struct TransitionClient {
    pub endpoint_url: Url,
    pub creds_provider: Arc<Mutex<Credentials<Static>>>,
@@ -809,6 +810,7 @@ impl TransitionCore {
    }
 }

+#[derive(Debug, Clone, Default)]
 pub struct PutObjectPartOptions {
    pub md5_base64: String,
    pub sha256_hex: String,
@@ -820,23 +822,23 @@ pub struct PutObjectPartOptions {

 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct ObjectInfo {
-    pub etag: String,
+    pub etag: Option<String>,
    pub name: String,
-    pub mod_time: OffsetDateTime,
-    pub size: usize,
+    pub mod_time: Option<OffsetDateTime>,
+    pub size: i64,
    pub content_type: Option<String>,
    #[serde(skip)]
    pub metadata: HeaderMap,
    pub user_metadata: HashMap<String, String>,
    pub user_tags: String,
-    pub user_tag_count: i64,
+    pub user_tag_count: usize,
    #[serde(skip)]
    pub owner: Owner,
    //pub grant: Vec<Grant>,
    pub storage_class: String,
    pub is_latest: bool,
    pub is_delete_marker: bool,
-    pub version_id: Uuid,
+    pub version_id: Option<Uuid>,

    #[serde(skip, default = "replication_status_default")]
    pub replication_status: ReplicationStatus,
@@ -862,9 +864,9 @@ fn replication_status_default() -> ReplicationStatus {
 impl Default for ObjectInfo {
    fn default() -> Self {
        Self {
-            etag: "".to_string(),
+            etag: None,
            name: "".to_string(),
-            mod_time: OffsetDateTime::now_utc(),
+            mod_time: None,
            size: 0,
            content_type: None,
            metadata: HeaderMap::new(),
@@ -875,7 +877,7 @@ impl Default for ObjectInfo {
            storage_class: "".to_string(),
            is_latest: false,
            is_delete_marker: false,
-            version_id: Uuid::nil(),
+            version_id: None,
            replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
            replication_ready: false,
            expiration: OffsetDateTime::now_utc(),
@@ -1,69 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use chrono::{DateTime, Utc};
-
-// Representation of the replication status
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum StatusType {
-    Pending,
-    Completed,
-    CompletedLegacy,
-    Failed,
-    Replica,
-}
-
-// Representation of version purge status type (customize as needed)
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum VersionPurgeStatusType {
-    Pending,
-    Completed,
-    Failed,
-}
-
-// ReplicationState struct definition
-#[derive(Debug, Clone)]
-pub struct ReplicationState {
-    // Timestamp when the last replica update was received
-    pub replica_time_stamp: DateTime<Utc>,
-
-    // Replica status
-    pub replica_status: StatusType,
-
-    // Represents DeleteMarker replication state
-    pub delete_marker: bool,
-
-    // Timestamp when the last replication activity happened
-    pub replication_time_stamp: DateTime<Utc>,
-
-    // Stringified representation of all replication activity
-    pub replication_status_internal: String,
-
-    // Stringified representation of all version purge statuses
-    // Example format: "arn1=PENDING;arn2=COMPLETED;"
-    pub version_purge_status_internal: String,
-
-    // Stringified representation of replication decision for each target
-    pub replicate_decision_str: String,
-
-    // Map of ARN -> replication status for ongoing replication activity
-    pub targets: HashMap<String, StatusType>,
-
-    // Map of ARN -> VersionPurgeStatus for all the targets
-    pub purge_targets: HashMap<String, VersionPurgeStatusType>,
-
-    // Map of ARN -> stringified reset id and timestamp for all the targets
-    pub reset_statuses_map: HashMap<String, String>,
-}
@@ -1,890 +0,0 @@
-#![allow(unused_variables)]
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-#![allow(dead_code)]
-use crate::{
-    StorageAPI,
-    bucket::{metadata_sys, target::BucketTarget},
-    endpoints::Node,
-    rpc::{PeerS3Client, RemotePeerS3Client},
-};
-use crate::{
-    bucket::{self, target::BucketTargets},
-    new_object_layer_fn, store_api,
-};
-//use tokio::sync::RwLock;
-use aws_sdk_s3::Client as S3Client;
-use chrono::Utc;
-use lazy_static::lazy_static;
-use std::sync::Arc;
-use std::{
-    collections::HashMap,
-    time::{Duration, SystemTime},
-};
-use thiserror::Error;
-use tokio::sync::RwLock;
-
-pub struct TClient {
-    pub s3cli: S3Client,
-    pub remote_peer_client: RemotePeerS3Client,
-    pub arn: String,
-}
-impl TClient {
-    pub fn new(s3cli: S3Client, remote_peer_client: RemotePeerS3Client, arn: String) -> Self {
-        TClient {
-            s3cli,
-            remote_peer_client,
-            arn,
-        }
-    }
-}
-
-pub struct EpHealth {
-    pub endpoint: String,
-    pub scheme: String,
-    pub online: bool,
-    pub last_online: SystemTime,
-    pub last_hc_at: SystemTime,
-    pub offline_duration: Duration,
-    pub latency: LatencyStat, // Assuming LatencyStat is a custom struct
-}
-
-impl EpHealth {
-    pub fn new(
-        endpoint: String,
-        scheme: String,
-        online: bool,
-        last_online: SystemTime,
-        last_hc_at: SystemTime,
-        offline_duration: Duration,
-        latency: LatencyStat,
-    ) -> Self {
-        EpHealth {
-            endpoint,
-            scheme,
-            online,
-            last_online,
-            last_hc_at,
-            offline_duration,
-            latency,
-        }
-    }
-}
-
-pub struct LatencyStat {
-    // Define the fields of LatencyStat as per your requirements
-}
-
-pub struct ArnTarget {
-    client: TargetClient,
-    last_refresh: chrono::DateTime<Utc>,
-}
-impl ArnTarget {
-    pub fn new(bucket: String, endpoint: String, ak: String, sk: String) -> Self {
-        Self {
-            client: TargetClient {
-                bucket,
-                storage_class: "STANDARD".to_string(),
-                disable_proxy: false,
-                health_check_duration: Duration::from_secs(100),
-                endpoint,
-                reset_id: "0".to_string(),
-                replicate_sync: false,
-                secure: false,
-                arn: "".to_string(),
-                client: reqwest::Client::new(),
-                ak,
-                sk,
-            },
-            last_refresh: Utc::now(),
-        }
-    }
-}
-
-// pub fn get_s3client_from_para(
-//     ak: &str,
-//     sk: &str,
-//     url: &str,
-//     _region: &str,
-// ) -> Result<S3Client, Box<dyn Error>> {
-//     let credentials = Credentials::new(ak, sk, None, None, "");
-//     let region = Region::new("us-east-1".to_string());
-
-//     let config = Config::builder()
-//         .region(region)
-//         .endpoint_url(url.to_string())
-//         .credentials_provider(credentials)
-//         .behavior_version(BehaviorVersion::latest()) // Adjust as necessary
-//         .build();
-//     Ok(S3Client::from_conf(config))
-// }
-
-pub struct BucketTargetSys {
-    arn_remote_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
-    targets_map: Arc<RwLock<HashMap<String, Vec<bucket::target::BucketTarget>>>>,
-    hc: HashMap<String, EpHealth>,
-    //store:Option<Arc<ecstore::store::ECStore>>,
-}
-
-lazy_static! {
-    pub static ref GLOBAL_Bucket_Target_Sys: std::sync::OnceLock<BucketTargetSys> = BucketTargetSys::new().into();
-}
-
-//#[derive(Debug)]
-// pub enum SetTargetError {
-//   NotFound,
-// }
-
-pub async fn get_bucket_target_client(bucket: &str, arn: &str) -> Result<TargetClient, SetTargetError> {
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        sys.get_remote_target_client2(arn).await
-    } else {
-        Err(SetTargetError::TargetNotFound(bucket.to_string()))
-    }
-}
-
-#[derive(Debug)]
-pub struct BucketRemoteTargetNotFound {
-    pub bucket: String,
-}
-
-pub async fn init_bucket_targets(bucket: &str, meta: Arc<bucket::metadata::BucketMetadata>) {
-    println!("140 {bucket}");
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        if let Some(tgts) = meta.bucket_target_config.clone() {
-            for tgt in tgts.targets {
-                warn!("ak and sk is:{:?}", tgt.credentials);
-                let _ = sys.set_target(bucket, &tgt, false, true).await;
-                //sys.targets_map.
-            }
-        }
-    }
-}
-
-pub async fn remove_bucket_target(bucket: &str, arn_str: &str) {
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        let _ = sys.remove_target(bucket, arn_str).await;
-    }
-}
-
-pub async fn list_bucket_targets(bucket: &str) -> Result<BucketTargets, BucketRemoteTargetNotFound> {
-    if let Some(sys) = GLOBAL_Bucket_Target_Sys.get() {
-        sys.list_bucket_targets(bucket).await
-    } else {
-        Err(BucketRemoteTargetNotFound {
-            bucket: bucket.to_string(),
-        })
-    }
-}
-
-impl Default for BucketTargetSys {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl BucketTargetSys {
-    pub fn new() -> Self {
-        BucketTargetSys {
-            arn_remote_map: Arc::new(RwLock::new(HashMap::new())),
-            targets_map: Arc::new(RwLock::new(HashMap::new())),
-            hc: HashMap::new(),
-        }
-    }
-
-    pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketRemoteTargetNotFound> {
-        let targets_map = self.targets_map.read().await;
-        if let Some(targets) = targets_map.get(bucket) {
-            Ok(BucketTargets {
-                targets: targets.clone(),
-            })
-        } else {
-            Err(BucketRemoteTargetNotFound {
-                bucket: bucket.to_string(),
-            })
-        }
-    }
-
-    pub async fn list_targets(&self, bucket: Option<&str>, _arn_type: Option<&str>) -> Vec<BucketTarget> {
-        let _ = _arn_type;
-        //let health_stats = self.health_stats();
-
-        let mut targets = Vec::new();
-
-        if let Some(bucket_name) = bucket {
-            if let Ok(ts) = self.list_bucket_targets(bucket_name).await {
-                for t in ts.targets {
-                    //if arn_type.map_or(true, |arn| t.target_type == arn) {
-                    //if let Some(hs) = health_stats.get(&t.url().host) {
-                    // t.total_downtime = hs.offline_duration;
-                    // t.online = hs.online;
-                    // t.last_online = hs.last_online;
-                    // t.latency = LatencyStat {
-                    //     curr: hs.latency.curr,
-                    //     avg: hs.latency.avg,
-                    //     max: hs.latency.peak,
-                    // };
-                    //}
-                    targets.push(t.clone());
-                    //}
-                }
-            }
-            return targets;
-        }
-
-        // Locking and iterating over all targets in the system
-        let targets_map = self.targets_map.read().await;
-        for tgts in targets_map.values() {
-            for t in tgts {
-                //if arn_type.map_or(true, |arn| t.target_type == arn) {
-                // if let Some(hs) = health_stats.get(&t.url().host) {
-                //     t.total_downtime = hs.offline_duration;
-                //     t.online = hs.online;
-                //     t.last_online = hs.last_online;
-                //     t.latency = LatencyStat {
-                //         curr: hs.latency.curr,
-                //         avg: hs.latency.avg,
-                //         max: hs.latency.peak,
-                //     };
-                // }
-                targets.push(t.clone());
-                //}
-            }
-        }
-
-        targets
-    }
-
-    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<(), SetTargetError> {
-        //to do need lock;
-        let mut targets_map = self.targets_map.write().await;
-        let tgts = targets_map.get(bucket);
-        let mut arn_remotes_map = self.arn_remote_map.write().await;
-        if tgts.is_none() {
-            //Err(SetTargetError::TargetNotFound(bucket.to_string()));
-            return Ok(());
-        }
-
-        let tgts = tgts.unwrap(); // 安全解引用
-        let mut targets = Vec::with_capacity(tgts.len());
-        let mut found = false;
-
-        // 遍历 targets，找出不匹配的 ARN
-        for tgt in tgts {
-            if tgt.arn != Some(arn_str.to_string()) {
-                targets.push(tgt.clone()); // 克隆符合条件的项
-            } else {
-                found = true; // 找到匹配的 ARN
-            }
-        }
-
-        // 如果没有找到匹配的 ARN，则返回错误
-        if !found {
-            return Ok(());
-        }
-
-        // 更新 targets_map
-        targets_map.insert(bucket.to_string(), targets);
-        arn_remotes_map.remove(arn_str);
-
-        let targets = self.list_targets(Some(bucket), None).await;
-        println!("targets is {}", targets.len());
-        match serde_json::to_vec(&targets) {
-            Ok(json) => {
-                let _ = metadata_sys::update(bucket, "bucket-targets.json", json).await;
-            }
-            Err(e) => {
-                println!("序列化失败{e}");
-            }
-        }
-
-        Ok(())
-    }
-
-    pub async fn get_remote_arn(&self, bucket: &str, target: Option<&BucketTarget>, depl_id: &str) -> (Option<String>, bool) {
-        if target.is_none() {
-            return (None, false);
-        }
-
-        let target = target.unwrap();
-
-        let targets_map = self.targets_map.read().await;
-
-        // 获取锁以访问 arn_remote_map
-        let mut _arn_remotes_map = self.arn_remote_map.read().await;
-        if let Some(tgts) = targets_map.get(bucket) {
-            for tgt in tgts {
-                if tgt.type_ == target.type_
-                    && tgt.target_bucket == target.target_bucket
-                    && tgt.endpoint == target.endpoint
-                    && tgt.credentials.as_ref().unwrap().access_key == target.credentials.as_ref().unwrap().access_key
-                {
-                    return (tgt.arn.clone(), true);
-                }
-            }
-        }
-
-        // if !target.type_.is_valid() {
-        //     return (None, false);
-        // }
-
-        println!("generate_arn");
-
-        (Some(generate_arn(target.clone(), depl_id.to_string())), false)
-    }
-
-    pub async fn get_remote_target_client2(&self, arn: &str) -> Result<TargetClient, SetTargetError> {
-        let map = self.arn_remote_map.read().await;
-        info!("get remote target client and arn is: {}", arn);
-        if let Some(value) = map.get(arn) {
-            let mut x = value.client.clone();
-            x.arn = arn.to_string();
-            Ok(x)
-        } else {
-            error!("not find target");
-            Err(SetTargetError::TargetNotFound(arn.to_string()))
-        }
-    }
-
-    // pub async fn get_remote_target_client(&self, _tgt: &BucketTarget) -> Result<TargetClient, SetTargetError> {
-    //     // Mocked implementation for obtaining a remote client
-    //     let tcli = TargetClient {
-    //         bucket: _tgt.target_bucket.clone(),
-    //         storage_class: "STANDARD".to_string(),
-    //         disable_proxy: false,
-    //         health_check_duration: Duration::from_secs(100),
-    //         endpoint: _tgt.endpoint.clone(),
-    //         reset_id: "0".to_string(),
-    //         replicate_sync: false,
-    //         secure: false,
-    //         arn: "".to_string(),
-    //         client: reqwest::Client::new(),
-    //         ak: _tgt.
-
-    //     };
-    //     Ok(tcli)
-    // }
-    // pub async fn get_remote_target_client_with_bucket(&self, _bucket: String) -> Result<TargetClient, SetTargetError> {
-    //     // Mocked implementation for obtaining a remote client
-    //     let tcli = TargetClient {
-    //         bucket: _tgt.target_bucket.clone(),
-    //         storage_class: "STANDARD".to_string(),
-    //         disable_proxy: false,
-    //         health_check_duration: Duration::from_secs(100),
-    //         endpoint: _tgt.endpoint.clone(),
-    //         reset_id: "0".to_string(),
-    //         replicate_sync: false,
-    //         secure: false,
-    //         arn: "".to_string(),
-    //         client: reqwest::Client::new(),
-    //     };
-    //     Ok(tcli)
-    // }
-
-    async fn local_is_bucket_versioned(&self, _bucket: &str) -> bool {
-        let Some(store) = new_object_layer_fn() else {
-            return false;
-        };
-        //store.get_bucket_info(bucket, opts)
-
-        // let binfo:BucketInfo = store
-        // .get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default()).await;
-        match store.get_bucket_info(_bucket, &store_api::BucketOptions::default()).await {
-            Ok(info) => {
-                println!("Bucket Info: {info:?}");
-                info.versioning
-            }
-            Err(err) => {
-                eprintln!("Error: {err:?}");
-                false
-            }
-        }
-    }
-
-    async fn is_bucket_versioned(&self, _bucket: &str) -> bool {
-        true
-        // let url_str = "http://127.0.0.1:9001";
-
-        // // 转换为 Url 类型
-        // let parsed_url = url::Url::parse(url_str).unwrap();
-
-        // let node = Node {
-        //     url: parsed_url,
-        //     pools: vec![],
-        //     is_local: false,
-        //     grid_host: "".to_string(),
-        // };
-        // let cli = ecstore::peer::RemotePeerS3Client::new(Some(node), None);
-
-        // match cli.get_bucket_info(_bucket, &ecstore::store_api::BucketOptions::default()).await
-        // {
-        //     Ok(info) => {
-        //         println!("Bucket Info: {:?}", info);
-        //         info.versioning
-        //     }
-        //     Err(err) => {
-        //         eprintln!("Error: {:?}", err);
-        //         return false;
-        //     }
-        // }
-    }
-
-    pub async fn set_target(&self, bucket: &str, tgt: &BucketTarget, update: bool, fromdisk: bool) -> Result<(), SetTargetError> {
-        // if !tgt.type_.is_valid() && !update {
-        //     return Err(SetTargetError::InvalidTargetType(bucket.to_string()));
-        // }
-
-        //let client = self.get_remote_target_client(tgt).await?;
-        if tgt.type_ == Some("replication".to_string()) && !fromdisk {
-            let versioning_config = self.local_is_bucket_versioned(bucket).await;
-            if !versioning_config {
-                // println!("111111111");
-                return Err(SetTargetError::TargetNotVersioned(bucket.to_string()));
-            }
-        }
-
-        let url_str = format!("http://{}", tgt.endpoint.clone());
-
-        println!("url str is {url_str}");
-        // 转换为 Url 类型
-        let parsed_url = url::Url::parse(&url_str).unwrap();
-
-        let node = Node {
-            url: parsed_url,
-            pools: vec![],
-            is_local: false,
-            grid_host: "".to_string(),
-        };
-
-        let cli = RemotePeerS3Client::new(Some(node), None);
-
-        match cli
-            .get_bucket_info(&tgt.target_bucket, &store_api::BucketOptions::default())
-            .await
-        {
-            Ok(info) => {
-                println!("Bucket Info: {info:?}");
-                if !info.versioning {
-                    return Err(SetTargetError::TargetNotVersioned(tgt.target_bucket.to_string()));
-                }
-            }
-            Err(err) => {
-                println!("remote bucket 369 is:{}", tgt.target_bucket);
-                eprintln!("Error: {err:?}");
-                return Err(SetTargetError::SourceNotVersioned(tgt.target_bucket.to_string()));
-            }
-        }
-
-        //if tgt.target_type == BucketTargetType::ReplicationService {
-        // Check if target is a rustfs server and alive
-        // let hc_result = tokio::time::timeout(Duration::from_secs(3), client.health_check(&tgt.endpoint)).await;
-        // match hc_result {
-        //     Ok(Ok(true)) => {} // Server is alive
-        //     Ok(Ok(false)) | Ok(Err(_)) | Err(_) => {
-        //         return Err(SetTargetError::HealthCheckFailed(tgt.target_bucket.clone()));
-        //     }
-        // }
-
-        //Lock and update target maps
-        let mut targets_map = self.targets_map.write().await;
-        let mut arn_remotes_map = self.arn_remote_map.write().await;
-
-        let targets = targets_map.entry(bucket.to_string()).or_default();
-        let mut found = false;
-
-        for existing_target in targets.iter_mut() {
-            println!("418 exist:{}", existing_target.source_bucket.clone());
-            if existing_target.type_ == tgt.type_ {
-                if existing_target.arn == tgt.arn {
-                    if !update {
-                        return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-                    }
-                    *existing_target = tgt.clone();
-                    found = true;
-                    break;
-                }
-
-                if existing_target.endpoint == tgt.endpoint {
-                    println!("endpoint is same:{}", tgt.endpoint.clone());
-                    return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-                }
-            }
-        }
-
-        if !found && !update {
-            println!("437 exist:{}", tgt.arn.clone().unwrap());
-            targets.push(tgt.clone());
-        }
-        let arntgt: ArnTarget = ArnTarget::new(
-            tgt.target_bucket.clone(),
-            tgt.endpoint.clone(),
-            tgt.credentials.clone().unwrap().access_key.clone(),
-            tgt.credentials.clone().unwrap().secret_key,
-        );
-
-        arn_remotes_map.insert(tgt.arn.clone().unwrap().clone(), arntgt);
-        //self.update_bandwidth_limit(bucket, &tgt.arn, tgt.bandwidth_limit).await;
-
-        Ok(())
-    }
-}
-
-#[derive(Clone)]
-pub struct TargetClient {
-    pub client: reqwest::Client, // Using reqwest HTTP client
-    pub health_check_duration: Duration,
-    pub bucket: String, // Remote bucket target
-    pub replicate_sync: bool,
-    pub storage_class: String, // Storage class on remote
-    pub disable_proxy: bool,
-    pub arn: String, // ARN to uniquely identify remote target
-    pub reset_id: String,
-    pub endpoint: String,
-    pub secure: bool,
-    pub ak: String,
-    pub sk: String,
-}
-
-#[allow(clippy::too_many_arguments)]
-impl TargetClient {
-    #[allow(clippy::too_many_arguments)]
-    pub fn new(
-        client: reqwest::Client,
-        health_check_duration: Duration,
-        bucket: String,
-        replicate_sync: bool,
-        storage_class: String,
-        disable_proxy: bool,
-        arn: String,
-        reset_id: String,
-        endpoint: String,
-        secure: bool,
-        ak: String,
-        sk: String,
-    ) -> Self {
-        TargetClient {
-            client,
-            health_check_duration,
-            bucket,
-            replicate_sync,
-            storage_class,
-            disable_proxy,
-            arn,
-            reset_id,
-            endpoint,
-            secure,
-            ak,
-            sk,
-        }
-    }
-    pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
-        Ok(true) // Mocked implementation
-    }
-}
-use tracing::{error, info, warn};
-use uuid::Uuid;
-
-#[derive(Debug, Clone)]
-pub struct VersioningConfig {
-    pub enabled: bool,
-}
-
-impl VersioningConfig {
-    pub fn is_enabled(&self) -> bool {
-        self.enabled
-    }
-}
-
-#[derive(Debug)]
-pub struct Client;
-
-impl Client {
-    pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
-        Ok(true) // Mocked implementation
-    }
-
-    pub async fn get_bucket_versioning(&self, _bucket: &str) -> Result<VersioningConfig, SetTargetError> {
-        Ok(VersioningConfig { enabled: true })
-    }
-
-    pub async fn health_check(&self, _endpoint: &str) -> Result<bool, SetTargetError> {
-        Ok(true) // Mocked health check
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct ServiceType(String);
-
-impl ServiceType {
-    pub fn is_valid(&self) -> bool {
-        !self.0.is_empty() // 根据需求添加具体的验证逻辑
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct ARN {
-    pub arn_type: String,
-    pub id: String,
-    pub region: String,
-    pub bucket: String,
-}
-
-impl ARN {
-    /// 检查 ARN 是否为空
-    pub fn is_empty(&self) -> bool {
-        //!self.arn_type.is_valid()
-        false
-    }
-
-    // 从字符串解析 ARN
-    pub fn parse(s: &str) -> Result<Self, String> {
-        // ARN 必须是格式 arn:rustfs:<Type>:<REGION>:<ID>:<remote-bucket>
-        if !s.starts_with("arn:rustfs:") {
-            return Err(format!("Invalid ARN {s}"));
-        }
-
-        let tokens: Vec<&str> = s.split(':').collect();
-        if tokens.len() != 6 || tokens[4].is_empty() || tokens[5].is_empty() {
-            return Err(format!("Invalid ARN {s}"));
-        }
-
-        Ok(ARN {
-            arn_type: tokens[2].to_string(),
-            region: tokens[3].to_string(),
-            id: tokens[4].to_string(),
-            bucket: tokens[5].to_string(),
-        })
-    }
-}
-
-// 实现 `Display` trait，使得可以直接使用 `format!` 或 `{}` 输出 ARN
-impl std::fmt::Display for ARN {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
-    }
-}
-
-fn must_get_uuid() -> String {
-    Uuid::new_v4().to_string()
-    // match Uuid::new_v4() {
-    //     Ok(uuid) => uuid.to_string(),
-    //     Err(err) => {
-    //         error!("Critical error: {}", err);
-    //         panic!("Failed to generate UUID: {}", err); // Ensures similar behavior as Go's logger.CriticalIf
-    //     }
-    // }
-}
-fn generate_arn(target: BucketTarget, depl_id: String) -> String {
-    let mut uuid: String = depl_id;
-    if uuid.is_empty() {
-        uuid = must_get_uuid();
-    }
-
-    let arn: ARN = ARN {
-        arn_type: target.type_.unwrap(),
-        id: (uuid),
-        region: "us-east-1".to_string(),
-        bucket: (target.target_bucket),
-    };
-    arn.to_string()
-}
-
-// use std::collections::HashMap;
-// use std::sync::{Arc, Mutex, RwLock};
-// use std::time::Duration;
-// use tokio::time::timeout;
-// use tokio::sync::RwLock as AsyncRwLock;
-// use serde::Deserialize;
-// use thiserror::Error;
-
-// #[derive(Debug, Clone, PartialEq)]
-// pub enum BucketTargetType {
-//     ReplicationService,
-//     // Add other service types as needed
-// }
-
-// impl BucketTargetType {
-//     pub fn is_valid(&self) -> bool {
-//         matches!(self, BucketTargetType::ReplicationService)
-//     }
-// }
-
-// #[derive(Debug, Clone)]
-// pub struct BucketTarget {
-//     pub arn: String,
-//     pub target_bucket: String,
-//     pub endpoint: String,
-//     pub credentials: Credentials,
-//     pub secure: bool,
-//     pub bandwidth_limit: Option<u64>,
-//     pub target_type: BucketTargetType,
-// }
-
-// #[derive(Debug, Clone)]
-// pub struct Credentials {
-//     pub access_key: String,
-//     pub secret_key: String,
-// }
-
-// #[derive(Debug)]
-// pub struct BucketTargetSys {
-//     targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
-//     arn_remotes_map: Arc<Mutex<HashMap<String, ArnTarget>>>,
-// }
-
-// impl BucketTargetSys {
-//     pub fn new() -> Self {
-//         Self {
-//             targets_map: Arc::new(RwLock::new(HashMap::new())),
-//             arn_remotes_map: Arc::new(Mutex::new(HashMap::new())),
-//         }
-//     }
-
-//     pub async fn set_target(
-//         &self,
-//         bucket: &str,
-//         tgt: &BucketTarget,
-//         update: bool,
-//     ) -> Result<(), SetTargetError> {
-//         if !tgt.target_type.is_valid() && !update {
-//             return Err(SetTargetError::InvalidTargetType(bucket.to_string()));
-//         }
-
-//         let client = self.get_remote_target_client(tgt).await?;
-
-//         // Validate if target credentials are OK
-//         let exists = client.bucket_exists(&tgt.target_bucket).await?;
-//         if !exists {
-//             return Err(SetTargetError::TargetNotFound(tgt.target_bucket.clone()));
-//         }
-
-//         if tgt.target_type == BucketTargetType::ReplicationService {
-//             if !self.is_bucket_versioned(bucket).await {
-//                 return Err(SetTargetError::SourceNotVersioned(bucket.to_string()));
-//             }
-
-//             let versioning_config = client.get_bucket_versioning(&tgt.target_bucket).await?;
-//             if !versioning_config.is_enabled() {
-//                 return Err(SetTargetError::TargetNotVersioned(tgt.target_bucket.clone()));
-//             }
-//         }
-
-//         // Check if target is a rustfs server and alive
-//         let hc_result = timeout(Duration::from_secs(3), client.health_check(&tgt.endpoint)).await;
-//         match hc_result {
-//             Ok(Ok(true)) => {} // Server is alive
-//             Ok(Ok(false)) | Ok(Err(_)) | Err(_) => {
-//                 return Err(SetTargetError::HealthCheckFailed(tgt.target_bucket.clone()));
-//             }
-//         }
-
-//         // Lock and update target maps
-//         let mut targets_map = self.targets_map.write().await;
-//         let mut arn_remotes_map = self.arn_remotes_map.lock().unwrap();
-
-//         let targets = targets_map.entry(bucket.to_string()).or_default();
-//         let mut found = false;
-
-//         for existing_target in targets.iter_mut() {
-//             if existing_target.target_type == tgt.target_type {
-//                 if existing_target.arn == tgt.arn {
-//                     if !update {
-//                         return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-//                     }
-//                     *existing_target = tgt.clone();
-//                     found = true;
-//                     break;
-//                 }
-
-//                 if existing_target.endpoint == tgt.endpoint {
-//                     return Err(SetTargetError::TargetAlreadyExists(existing_target.target_bucket.clone()));
-//                 }
-//             }
-//         }
-
-//         if !found && !update {
-//             targets.push(tgt.clone());
-//         }
-
-//         arn_remotes_map.insert(tgt.arn.clone(), ArnTarget { client });
-//         self.update_bandwidth_limit(bucket, &tgt.arn, tgt.bandwidth_limit).await;
-
-//         Ok(())
-//     }
-
-//     async fn get_remote_target_client(&self, tgt: &BucketTarget) -> Result<Client, SetTargetError> {
-//         // Mocked implementation for obtaining a remote client
-//         Ok(Client {})
-//     }
-
-//     async fn is_bucket_versioned(&self, bucket: &str) -> bool {
-//         // Mocked implementation for checking if a bucket is versioned
-//         true
-//     }
-
-//     async fn update_bandwidth_limit(
-//         &self,
-//         bucket: &str,
-//         arn: &str,
-//         limit: Option<u64>,
-//     ) {
-//         // Mocked implementation for updating bandwidth limits
-//     }
-// }
-
-// #[derive(Debug)]
-// pub struct Client;
-
-// impl Client {
-//     pub async fn bucket_exists(&self, _bucket: &str) -> Result<bool, SetTargetError> {
-//         Ok(true) // Mocked implementation
-//     }
-
-//     pub async fn get_bucket_versioning(
-//         &self,
-//         _bucket: &str,
-//     ) -> Result<VersioningConfig, SetTargetError> {
-//         Ok(VersioningConfig { enabled: true })
-//     }
-
-//     pub async fn health_check(&self, _endpoint: &str) -> Result<bool, SetTargetError> {
-//         Ok(true) // Mocked health check
-//     }
-// }
-
-// #[derive(Debug, Clone)]
-// pub struct ArnTarget {
-//     pub client: Client,
-// }
-
-#[derive(Debug, Error)]
-pub enum SetTargetError {
-    #[error("Invalid target type for bucket {0}")]
-    InvalidTargetType(String),
-
-    #[error("Target bucket {0} not found")]
-    TargetNotFound(String),
-
-    #[error("Source bucket {0} is not versioned")]
-    SourceNotVersioned(String),
-
-    #[error("Target bucket {0} is not versioned")]
-    TargetNotVersioned(String),
-
-    #[error("Health check failed for bucket {0}")]
-    HealthCheckFailed(String),
-
-    #[error("Target bucket {0} already exists")]
-    TargetAlreadyExists(String),
-}
@@ -1,14 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
@@ -1,16 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-pub mod bucket_replication;
-pub mod bucket_targets;
@@ -88,7 +88,7 @@ impl LocalUsageSnapshot {

 /// Build the snapshot file name `<disk-id>.json`.
 pub fn snapshot_file_name(disk_id: &str) -> String {
-    format!("{}.json", disk_id)
+    format!("{disk_id}.json")
 }

 /// Build the object path relative to `RUSTFS_META_BUCKET`, e.g. `datausage/<disk-id>.json`.
@@ -2349,12 +2349,7 @@ impl DiskAPI for LocalDisk {
        self.delete_file(&volume_dir, &xl_path, true, false).await
    }
    #[tracing::instrument(level = "debug", skip(self))]
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        _opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>> {
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, _opts: DeleteOptions) -> Vec<Option<Error>> {
        let mut errs = Vec::with_capacity(versions.len());
        for _ in 0..versions.len() {
            errs.push(None);
@@ -2368,7 +2363,7 @@ impl DiskAPI for LocalDisk {
            }
        }

-        Ok(errs)
+        errs
    }

    #[tracing::instrument(skip(self))]
@@ -201,12 +201,7 @@ impl DiskAPI for Disk {
    }

    #[tracing::instrument(skip(self))]
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>> {
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
        match self {
            Disk::Local(local_disk) => local_disk.delete_versions(volume, versions, opts).await,
            Disk::Remote(remote_disk) => remote_disk.delete_versions(volume, versions, opts).await,
@@ -448,12 +443,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
        force_del_marker: bool,
        opts: DeleteOptions,
    ) -> Result<()>;
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>>;
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>>;
    async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()>;
    async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
    async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
@@ -167,8 +167,19 @@ async fn write_data_blocks<W>(
 where
    W: tokio::io::AsyncWrite + Send + Sync + Unpin,
 {
-    if get_data_block_len(en_blocks, data_blocks) < length {
-        error!("write_data_blocks get_data_block_len < length");
+    let available = get_data_block_len(en_blocks, data_blocks);
+    if available < length {
+        let block_sizes: Vec<usize> = en_blocks
+            .iter()
+            .take(data_blocks)
+            .map(|block| block.as_ref().map(|buf| buf.len()).unwrap_or(0))
+            .collect();
+        error!(
+            expected = length,
+            available,
+            ?block_sizes,
+            "write_data_blocks get_data_block_len < length"
+        );
        return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
    }

@@ -21,7 +21,6 @@ pub mod bitrot;
 pub mod bucket;
 pub mod cache_value;
 mod chunk_stream;
-pub mod cmd;
 pub mod compress;
 pub mod config;
 pub mod data_usage;
@@ -32,9 +31,9 @@ pub mod erasure_coding;
 pub mod error;
 pub mod file_cache;
 pub mod global;
-pub mod lock_utils;
 pub mod metrics_realtime;
 pub mod notification_sys;
+pub mod object_append;
 pub mod pools;
 pub mod rebalance;
 pub mod rpc;
@@ -1,136 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::disk::endpoint::Endpoint;
-use crate::error::Result;
-use rustfs_lock::client::{LockClient, local::LocalClient, remote::RemoteClient};
-use std::collections::HashMap;
-use std::sync::Arc;
-
-/// Create unique lock clients from endpoints
-/// This function creates one client per unique host:port combination
-/// to avoid duplicate connections to the same server
-pub async fn create_unique_clients(endpoints: &[Endpoint]) -> Result<Vec<Arc<dyn LockClient>>> {
-    let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
-
-    // Collect unique endpoints based on host:port
-    for endpoint in endpoints {
-        if endpoint.is_local {
-            // For local endpoints, use "local" as the key
-            unique_endpoints.insert("local".to_string(), endpoint);
-        } else {
-            // For remote endpoints, use host:port as the key
-            let host_port = format!(
-                "{}:{}",
-                endpoint.url.host_str().unwrap_or("localhost"),
-                endpoint.url.port().unwrap_or(9000)
-            );
-            unique_endpoints.insert(host_port, endpoint);
-        }
-    }
-
-    let mut clients = Vec::new();
-
-    // Create clients for unique endpoints
-    for (_key, endpoint) in unique_endpoints {
-        if endpoint.is_local {
-            // For local endpoints, create a local lock client
-            let local_client = LocalClient::new();
-            clients.push(Arc::new(local_client) as Arc<dyn LockClient>);
-        } else {
-            // For remote endpoints, create a remote lock client
-            let remote_client = RemoteClient::new(endpoint.url.to_string());
-            clients.push(Arc::new(remote_client) as Arc<dyn LockClient>);
-        }
-    }
-
-    Ok(clients)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use url::Url;
-
-    #[tokio::test]
-    async fn test_create_unique_clients_local() {
-        let endpoints = vec![
-            Endpoint {
-                url: Url::parse("http://localhost:9000").unwrap(),
-                is_local: true,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 0,
-            },
-            Endpoint {
-                url: Url::parse("http://localhost:9000").unwrap(),
-                is_local: true,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 1,
-            },
-        ];
-
-        let clients = create_unique_clients(&endpoints).await.unwrap();
-        // Should only create one client for local endpoints
-        assert_eq!(clients.len(), 1);
-        assert!(clients[0].is_local().await);
-    }
-
-    #[tokio::test]
-    async fn test_create_unique_clients_mixed() {
-        let endpoints = vec![
-            Endpoint {
-                url: Url::parse("http://localhost:9000").unwrap(),
-                is_local: true,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 0,
-            },
-            Endpoint {
-                url: Url::parse("http://remote1:9000").unwrap(),
-                is_local: false,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 1,
-            },
-            Endpoint {
-                url: Url::parse("http://remote1:9000").unwrap(),
-                is_local: false,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 2,
-            },
-            Endpoint {
-                url: Url::parse("http://remote2:9000").unwrap(),
-                is_local: false,
-                pool_idx: 0,
-                set_idx: 0,
-                disk_idx: 3,
-            },
-        ];
-
-        let clients = create_unique_clients(&endpoints).await.unwrap();
-        // Should create 3 clients: 1 local + 2 unique remote
-        assert_eq!(clients.len(), 3);
-
-        // Check that we have one local client
-        let local_count = clients.iter().filter(|c| futures::executor::block_on(c.is_local())).count();
-        assert_eq!(local_count, 1);
-
-        // Check that we have two remote clients
-        let remote_count = clients.iter().filter(|c| !futures::executor::block_on(c.is_local())).count();
-        assert_eq!(remote_count, 2);
-    }
-}
@@ -16,11 +16,17 @@ use crate::StorageAPI;
 use crate::admin_server_info::get_commit_id;
 use crate::error::{Error, Result};
 use crate::global::{GLOBAL_BOOT_TIME, get_global_endpoints};
+use crate::metrics_realtime::{CollectMetricsOpts, MetricType};
 use crate::rpc::PeerRestClient;
 use crate::{endpoints::EndpointServerPools, new_object_layer_fn};
 use futures::future::join_all;
 use lazy_static::lazy_static;
+use rustfs_madmin::health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysService};
+use rustfs_madmin::metrics::RealtimeMetrics;
+use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties};
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
 use std::sync::OnceLock;
 use std::time::SystemTime;
 use tracing::{error, warn};
@@ -62,21 +68,122 @@ pub struct NotificationPeerErr {
 }

 impl NotificationSys {
-    pub fn rest_client_from_hash(&self, _s: &str) -> Option<PeerRestClient> {
-        None
-    }
-    pub async fn delete_policy(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
-    }
-    pub async fn load_policy(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
+    pub fn rest_client_from_hash(&self, s: &str) -> Option<PeerRestClient> {
+        if self.all_peer_clients.is_empty() {
+            return None;
+        }
+        let mut hasher = DefaultHasher::new();
+        s.hash(&mut hasher);
+        let idx = (hasher.finish() as usize) % self.all_peer_clients.len();
+        self.all_peer_clients[idx].clone()
    }

-    pub async fn load_policy_mapping(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
+    pub async fn delete_policy(&self, policy_name: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let policy = policy_name.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_policy(&policy).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
    }
-    pub async fn delete_user(&self) -> Vec<NotificationPeerErr> {
-        unimplemented!()
+
+    pub async fn load_policy(&self, policy_name: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let policy = policy_name.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_policy(&policy).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_policy_mapping(&self, user_or_group: &str, user_type: u64, is_group: bool) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let uog = user_or_group.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_policy_mapping(&uog, user_type, is_group).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn delete_user(&self, access_key: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_user(&ak).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
    }

    pub async fn storage_info<S: StorageAPI>(&self, api: &S) -> rustfs_madmin::StorageInfo {
@@ -140,6 +247,114 @@ impl NotificationSys {
        join_all(futures).await
    }

+    pub async fn load_user(&self, access_key: &str, temp: bool) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_user(&ak, temp).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_group(&self, group: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let gname = group.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_group(&gname).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn delete_service_account(&self, access_key: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_service_account(&ak).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_service_account(&self, access_key: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let ak = access_key.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_service_account(&ak).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
    pub async fn reload_pool_meta(&self) {
        let mut futures = Vec::with_capacity(self.peer_clients.len());
        for client in self.peer_clients.iter().flatten() {
@@ -202,6 +417,281 @@ impl NotificationSys {
        let _ = store.stop_rebalance().await;
        warn!("notification stop_rebalance stop_rebalance done");
    }
+
+    pub async fn load_bucket_metadata(&self, bucket: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let b = bucket.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_bucket_metadata(&b).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn delete_bucket_metadata(&self, bucket: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let b = bucket.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.delete_bucket_metadata(&b).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn start_profiling(&self, profiler: &str) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            let pf = profiler.to_string();
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.start_profiling(&pf).await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_cpus(&self) -> Vec<Cpus> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_cpus().await.unwrap_or_default()
+                } else {
+                    Cpus::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_net_info(&self) -> Vec<NetInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_net_info().await.unwrap_or_default()
+                } else {
+                    NetInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_partitions(&self) -> Vec<Partitions> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_partitions().await.unwrap_or_default()
+                } else {
+                    Partitions::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_os_info(&self) -> Vec<OsInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_os_info().await.unwrap_or_default()
+                } else {
+                    OsInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_sys_services(&self) -> Vec<SysService> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_se_linux_info().await.unwrap_or_default()
+                } else {
+                    SysService::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_sys_config(&self) -> Vec<SysConfig> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_sys_config().await.unwrap_or_default()
+                } else {
+                    SysConfig::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_sys_errors(&self) -> Vec<SysErrors> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_sys_errors().await.unwrap_or_default()
+                } else {
+                    SysErrors::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_mem_info(&self) -> Vec<MemInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_mem_info().await.unwrap_or_default()
+                } else {
+                    MemInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_proc_info(&self) -> Vec<ProcInfo> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_proc_info().await.unwrap_or_default()
+                } else {
+                    ProcInfo::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn get_metrics(&self, t: MetricType, opts: &CollectMetricsOpts) -> Vec<RealtimeMetrics> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            let t_clone = t;
+            let opts_clone = opts;
+            futures.push(async move {
+                if let Some(client) = client {
+                    client.get_metrics(t_clone, opts_clone).await.unwrap_or_default()
+                } else {
+                    RealtimeMetrics::default()
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn reload_site_replication_config(&self) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.reload_site_replication_config().await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
+    pub async fn load_transition_tier_config(&self) -> Vec<NotificationPeerErr> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            futures.push(async move {
+                if let Some(client) = client {
+                    match client.load_transition_tier_config().await {
+                        Ok(_) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: None,
+                        },
+                        Err(e) => NotificationPeerErr {
+                            host: client.host.to_string(),
+                            err: Some(e),
+                        },
+                    }
+                } else {
+                    NotificationPeerErr {
+                        host: "".to_string(),
+                        err: Some(Error::other("peer is not reachable")),
+                    }
+                }
+            });
+        }
+        join_all(futures).await
+    }
 }

 fn get_offline_disks(offline_host: &str, endpoints: &EndpointServerPools) -> Vec<rustfs_madmin::Disk> {
@@ -0,0 +1,725 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
+use crate::erasure_coding::{Erasure, calc_shard_size};
+use crate::error::{Error, StorageError};
+use crate::store_api::ObjectInfo;
+use rustfs_filemeta::TRANSITION_COMPLETE;
+use rustfs_utils::HashAlgorithm;
+use rustfs_utils::http::headers::{
+    AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
+    AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
+    RESERVED_METADATA_PREFIX_LOWER,
+};
+use std::collections::HashSet;
+
+/// Ensure the target object can accept append writes under current state.
+pub fn validate_append_preconditions(bucket: &str, object: &str, info: &ObjectInfo) -> Result<(), Error> {
+    if info.is_compressed() {
+        return Err(StorageError::InvalidArgument(
+            bucket.to_string(),
+            object.to_string(),
+            "append is not supported for compressed objects".to_string(),
+        ));
+    }
+
+    let encryption_headers = [
+        AMZ_SERVER_SIDE_ENCRYPTION,
+        AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
+        AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
+        AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
+        AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
+        AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
+    ];
+
+    if encryption_headers
+        .iter()
+        .any(|header| info.user_defined.contains_key(*header) || info.user_defined.contains_key(&header.to_ascii_lowercase()))
+    {
+        return Err(StorageError::InvalidArgument(
+            bucket.to_string(),
+            object.to_string(),
+            "append is not supported for encrypted objects".to_string(),
+        ));
+    }
+
+    if info.transitioned_object.status == TRANSITION_COMPLETE || !info.transitioned_object.tier.is_empty() {
+        return Err(StorageError::InvalidArgument(
+            bucket.to_string(),
+            object.to_string(),
+            "append is not supported for transitioned objects".to_string(),
+        ));
+    }
+
+    Ok(())
+}
+
+/// Validate that the requested append position matches the current object length.
+pub fn validate_append_position(bucket: &str, object: &str, info: &ObjectInfo, expected_position: i64) -> Result<(), Error> {
+    if expected_position != info.size {
+        return Err(StorageError::InvalidArgument(
+            bucket.to_string(),
+            object.to_string(),
+            format!("append position mismatch: provided {}, expected {}", expected_position, info.size),
+        ));
+    }
+    Ok(())
+}
+
+pub struct InlineAppendContext<'a> {
+    pub existing_inline: Option<&'a [u8]>,
+    pub existing_plain: Option<&'a [u8]>,
+    pub existing_size: i64,
+    pub append_payload: &'a [u8],
+    pub erasure: &'a Erasure,
+    pub hash_algorithm: HashAlgorithm,
+    pub has_checksums: bool,
+}
+
+pub struct InlineAppendResult {
+    pub inline_data: Vec<u8>,
+    pub total_size: i64,
+    pub etag: String,
+}
+
+/// Decode inline payload using available checksum algorithms. Returns raw bytes when decoding fails but
+/// the inline buffer already contains the plain payload.
+pub async fn decode_inline_payload(
+    inline: &[u8],
+    size: usize,
+    erasure: &Erasure,
+    preferred: HashAlgorithm,
+) -> Result<(Vec<u8>, HashAlgorithm), Error> {
+    match decode_inline_variants(inline, size, erasure, preferred).await {
+        Ok((data, algo)) => Ok((data, algo)),
+        Err(err) => {
+            if inline.len() >= size {
+                Ok((inline[..size].to_vec(), HashAlgorithm::None))
+            } else {
+                Err(err)
+            }
+        }
+    }
+}
+
+/// Append data to an inline object and return the re-encoded inline buffer.
+pub async fn append_inline_data(ctx: InlineAppendContext<'_>) -> Result<InlineAppendResult, Error> {
+    let mut plain = Vec::with_capacity(ctx.existing_inline.map(|data| data.len()).unwrap_or(0) + ctx.append_payload.len());
+    let mut encode_algorithm = ctx.hash_algorithm.clone();
+
+    if let Some(existing_plain) = ctx.existing_plain {
+        if existing_plain.len() != ctx.existing_size as usize {
+            return Err(StorageError::other("existing plain payload length mismatch"));
+        }
+        plain.extend_from_slice(existing_plain);
+    } else if ctx.existing_size > 0 {
+        let inline = ctx
+            .existing_inline
+            .ok_or_else(|| StorageError::other("inline payload missing"))?;
+
+        let (decoded, detected_algo) =
+            decode_inline_payload(inline, ctx.existing_size as usize, ctx.erasure, ctx.hash_algorithm.clone()).await?;
+        encode_algorithm = detected_algo;
+        plain.extend_from_slice(&decoded);
+    } else if let Some(inline) = ctx.existing_inline {
+        plain.extend_from_slice(inline);
+    }
+
+    plain.extend_from_slice(ctx.append_payload);
+    let total_size = plain.len() as i64;
+    let etag = md5_hex(&plain);
+
+    if encode_algorithm == HashAlgorithm::None {
+        if ctx.has_checksums {
+            encode_algorithm = ctx.hash_algorithm.clone();
+        } else {
+            return Ok(InlineAppendResult {
+                inline_data: plain,
+                total_size,
+                etag,
+            });
+        }
+    }
+
+    let mut writer = create_bitrot_writer(
+        true,
+        None,
+        "",
+        "",
+        ctx.erasure.shard_file_size(total_size),
+        ctx.erasure.shard_size(),
+        encode_algorithm,
+    )
+    .await
+    .map_err(|e| StorageError::other(format!("failed to create inline writer: {e}")))?;
+
+    let mut remaining = plain.as_slice();
+    while !remaining.is_empty() {
+        let chunk_len = remaining.len().min(ctx.erasure.block_size);
+        writer
+            .write(&remaining[..chunk_len])
+            .await
+            .map_err(|e| StorageError::other(format!("failed to write inline data: {e}")))?;
+        remaining = &remaining[chunk_len..];
+    }
+
+    writer
+        .shutdown()
+        .await
+        .map_err(|e| StorageError::other(format!("failed to finalize inline writer: {e}")))?;
+
+    let inline_data = writer
+        .into_inline_data()
+        .ok_or_else(|| StorageError::other("inline writer did not return data"))?;
+
+    Ok(InlineAppendResult {
+        inline_data,
+        total_size,
+        etag,
+    })
+}
+
+fn md5_hex(data: &[u8]) -> String {
+    let digest = HashAlgorithm::Md5.hash_encode(data);
+    hex_from_bytes(digest.as_ref())
+}
+
+fn hex_from_bytes(bytes: &[u8]) -> String {
+    let mut out = String::with_capacity(bytes.len() * 2);
+    for byte in bytes {
+        use std::fmt::Write;
+        write!(&mut out, "{:02x}", byte).expect("write hex");
+    }
+    out
+}
+
+async fn decode_inline_variants(
+    inline: &[u8],
+    size: usize,
+    erasure: &Erasure,
+    preferred: HashAlgorithm,
+) -> Result<(Vec<u8>, HashAlgorithm), Error> {
+    let mut tried = HashSet::new();
+    let candidates = [preferred, HashAlgorithm::HighwayHash256, HashAlgorithm::HighwayHash256S];
+
+    let mut last_err: Option<Error> = None;
+
+    for algo in candidates {
+        if !tried.insert(algo.clone()) {
+            continue;
+        }
+
+        match decode_inline_with_algo(inline, size, erasure, algo.clone()).await {
+            Ok(data) => return Ok((data, algo)),
+            Err(err) => last_err = Some(err),
+        }
+    }
+
+    Err(last_err.unwrap_or_else(|| StorageError::other("failed to decode inline data")))
+}
+
+async fn decode_inline_with_algo(inline: &[u8], size: usize, erasure: &Erasure, algo: HashAlgorithm) -> Result<Vec<u8>, Error> {
+    let total_len = inline
+        .len()
+        .max(erasure.shard_file_size(size as i64).max(size as i64) as usize);
+    let mut reader = create_bitrot_reader(Some(inline), None, "", "", 0, total_len, erasure.shard_size(), algo)
+        .await
+        .map_err(|e| StorageError::other(format!("failed to create inline reader: {e}")))?
+        .ok_or_else(|| StorageError::other("inline reader unavailable"))?;
+
+    let mut out = Vec::with_capacity(size);
+    while out.len() < size {
+        let remaining = size - out.len();
+        let plain_chunk = remaining.min(erasure.block_size);
+        let shard_payload = calc_shard_size(plain_chunk, erasure.data_shards).max(1);
+        let mut buf = vec![0u8; shard_payload];
+        let read = reader
+            .read(&mut buf)
+            .await
+            .map_err(|e| StorageError::other(format!("failed to read inline data: {e}")))?;
+        if read == 0 {
+            return Err(StorageError::other("incomplete inline data read"));
+        }
+
+        let copy_len = remaining.min(read);
+        out.extend_from_slice(&buf[..copy_len]);
+    }
+
+    Ok(out)
+}
+
+/// Background task to spill inline data to segmented format
+pub struct InlineSpillProcessor {
+    pub disks: Vec<Option<crate::disk::DiskStore>>,
+    pub write_quorum: usize,
+}
+
+impl InlineSpillProcessor {
+    pub fn new(disks: Vec<Option<crate::disk::DiskStore>>, write_quorum: usize) -> Self {
+        Self { disks, write_quorum }
+    }
+
+    /// Process a single spill operation from InlinePendingSpill to SegmentedActive
+    pub async fn process_spill(
+        &self,
+        bucket: &str,
+        object: &str,
+        mut fi: rustfs_filemeta::FileInfo,
+        mut parts_metadata: Vec<rustfs_filemeta::FileInfo>,
+        epoch: u64,
+    ) -> Result<(), Error> {
+        use rustfs_filemeta::AppendStateKind;
+        use tracing::{debug, error, info, warn};
+
+        // Verify we're in the correct state
+        let current_state = fi.get_append_state();
+        if current_state.state != AppendStateKind::InlinePendingSpill {
+            warn!(
+                bucket = bucket,
+                object = object,
+                current_state = ?current_state.state,
+                "Spill processor called on object not in InlinePendingSpill state"
+            );
+            return Ok(());
+        }
+
+        // Check epoch to ensure we're processing the correct version
+        if current_state.epoch != epoch {
+            debug!(
+                bucket = bucket,
+                object = object,
+                current_epoch = current_state.epoch,
+                expected_epoch = epoch,
+                "Spill operation skipped due to epoch mismatch"
+            );
+            return Ok(());
+        }
+
+        info!(
+            bucket = bucket,
+            object = object,
+            size = fi.size,
+            epoch = epoch,
+            "Starting inline data spill to segmented format"
+        );
+
+        // Extract inline data
+        let inline_data = fi
+            .data
+            .clone()
+            .ok_or_else(|| StorageError::other("Cannot spill object without inline data"))?;
+
+        // Create erasure encoder
+        let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
+
+        // Decode inline data to plain data
+        let hash_algorithm = fi
+            .parts
+            .first()
+            .map(|part| fi.erasure.get_checksum_info(part.number).algorithm)
+            .unwrap_or(HashAlgorithm::HighwayHash256);
+
+        let plain_data = match decode_inline_payload(&inline_data, fi.size as usize, &erasure, hash_algorithm.clone()).await {
+            Ok((plain, _detected_algo)) => plain,
+            Err(err) => {
+                error!(
+                    bucket = bucket,
+                    object = object,
+                    error = ?err,
+                    "Failed to decode inline data during spill"
+                );
+                return Err(StorageError::other(format!("Failed to decode inline data for spill: {err}")));
+            }
+        };
+
+        // Generate data directory for the object
+        let data_dir = uuid::Uuid::new_v4();
+
+        // Create temporary directory for the spill operation
+        let tmp_root = format!("{}x{}", uuid::Uuid::new_v4(), time::OffsetDateTime::now_utc().unix_timestamp());
+        let tmp_path = format!("{tmp_root}/{}/part.1", data_dir);
+
+        // Encode and write the data to all disks
+        match self.write_segmented_data(&plain_data, &tmp_path, &erasure).await {
+            Ok(_) => {
+                // Move from temp to permanent location
+                let final_path = format!("{}/part.1", data_dir);
+                if let Err(err) = self.move_temp_to_final(&tmp_path, &final_path).await {
+                    error!(
+                        bucket = bucket,
+                        object = object,
+                        error = ?err,
+                        "Failed to move spilled data to final location"
+                    );
+                    // Clean up temp files
+                    let _ = self.cleanup_temp_files(&tmp_path).await;
+                    return Err(err);
+                }
+
+                // Update file metadata
+                fi.data_dir = Some(data_dir);
+                fi.data = None; // Remove inline data
+                fi.metadata.remove(&format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER));
+
+                // Update append state to SegmentedActive
+                let mut new_state = current_state;
+                new_state.state = AppendStateKind::SegmentedActive;
+                new_state.epoch = new_state.epoch.saturating_add(1);
+                new_state.pending_segments.clear();
+
+                fi.set_append_state(&new_state)
+                    .map_err(|err| StorageError::other(format!("Failed to update append state after spill: {err}")))?;
+
+                // Update all parts metadata
+                for meta in parts_metadata.iter_mut() {
+                    if !meta.is_valid() {
+                        continue;
+                    }
+                    meta.data_dir = Some(data_dir);
+                    meta.data = None;
+                    meta.metadata = fi.metadata.clone();
+                    meta.metadata
+                        .remove(&format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER));
+                }
+
+                // Write updated metadata back to disks
+                // TODO: Implement metadata write-back logic
+                // This would typically involve writing the updated FileInfo to all disks
+
+                info!(
+                    bucket = bucket,
+                    object = object,
+                    data_dir = ?data_dir,
+                    new_epoch = new_state.epoch,
+                    "Successfully spilled inline data to segmented format"
+                );
+
+                Ok(())
+            }
+            Err(err) => {
+                error!(
+                    bucket = bucket,
+                    object = object,
+                    error = ?err,
+                    "Failed to write segmented data during spill"
+                );
+                // Clean up temp files
+                let _ = self.cleanup_temp_files(&tmp_path).await;
+                Err(err)
+            }
+        }
+    }
+
+    async fn write_segmented_data(&self, data: &[u8], tmp_path: &str, _erasure: &Erasure) -> Result<(), Error> {
+        use tracing::debug;
+
+        // TODO: Implement proper erasure encoding and writing to disks
+        // This is a placeholder implementation
+        debug!(
+            data_len = data.len(),
+            path = tmp_path,
+            "Writing segmented data (placeholder implementation)"
+        );
+
+        // For now, just return success - full implementation would:
+        // 1. Create bitrot writers for each disk
+        // 2. Erasure encode the data
+        // 3. Write each shard to its corresponding disk
+        Ok(())
+    }
+
+    async fn move_temp_to_final(&self, tmp_path: &str, final_path: &str) -> Result<(), Error> {
+        use tracing::debug;
+
+        // TODO: Implement moving temp files to final location
+        debug!(
+            tmp_path = tmp_path,
+            final_path = final_path,
+            "Moving temp files to final location (placeholder)"
+        );
+        Ok(())
+    }
+
+    async fn cleanup_temp_files(&self, tmp_path: &str) -> Result<(), Error> {
+        use tracing::debug;
+
+        // TODO: Implement temp file cleanup
+        debug!(tmp_path = tmp_path, "Cleaning up temp files (placeholder)");
+        Ok(())
+    }
+}
+
+/// Trigger background spill processing for an object
+pub fn trigger_spill_process(
+    bucket: String,
+    object: String,
+    fi: rustfs_filemeta::FileInfo,
+    parts_metadata: Vec<rustfs_filemeta::FileInfo>,
+    epoch: u64,
+    disks: Vec<Option<crate::disk::DiskStore>>,
+    write_quorum: usize,
+) {
+    use tracing::error;
+
+    tokio::spawn(async move {
+        let processor = InlineSpillProcessor::new(disks, write_quorum);
+        if let Err(err) = processor.process_spill(&bucket, &object, fi, parts_metadata, epoch).await {
+            error!(
+                bucket = bucket,
+                object = object,
+                epoch = epoch,
+                error = ?err,
+                "Background spill process failed"
+            );
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_utils::HashAlgorithm;
+
+    fn make_object_info() -> ObjectInfo {
+        ObjectInfo {
+            bucket: "test-bucket".to_string(),
+            name: "obj".to_string(),
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn rejects_compressed_objects() {
+        let mut info = make_object_info();
+        info.user_defined
+            .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string());
+
+        let err = validate_append_preconditions("test-bucket", "obj", &info).unwrap_err();
+        matches!(err, StorageError::InvalidArgument(..))
+            .then_some(())
+            .expect("expected invalid argument");
+    }
+
+    #[test]
+    fn rejects_encrypted_objects() {
+        let mut info = make_object_info();
+        info.user_defined
+            .insert("x-amz-server-side-encryption".to_string(), "AES256".to_string());
+
+        let err = validate_append_preconditions("test-bucket", "obj", &info).unwrap_err();
+        matches!(err, StorageError::InvalidArgument(..))
+            .then_some(())
+            .expect("expected invalid argument");
+    }
+
+    #[test]
+    fn rejects_transitioned_objects() {
+        let mut info = make_object_info();
+        info.transitioned_object.tier = "GLACIER".to_string();
+        info.transitioned_object.status = TRANSITION_COMPLETE.to_string();
+
+        let err = validate_append_preconditions("test-bucket", "obj", &info).unwrap_err();
+        matches!(err, StorageError::InvalidArgument(..))
+            .then_some(())
+            .expect("expected invalid argument");
+    }
+
+    #[test]
+    fn accepts_plain_objects() {
+        let info = make_object_info();
+        validate_append_preconditions("test-bucket", "obj", &info).expect("append should be allowed");
+    }
+
+    #[test]
+    fn rejects_position_mismatch() {
+        let mut info = make_object_info();
+        info.size = 10;
+        let err = validate_append_position("test-bucket", "obj", &info, 5).unwrap_err();
+        matches!(err, StorageError::InvalidArgument(..))
+            .then_some(())
+            .expect("expected invalid argument");
+    }
+
+    fn make_inline_erasure() -> Erasure {
+        Erasure::new(1, 0, 1024)
+    }
+
+    async fn encode_inline(data: &[u8], erasure: &Erasure) -> Vec<u8> {
+        let mut writer = create_bitrot_writer(
+            true,
+            None,
+            "",
+            "",
+            erasure.shard_file_size(data.len() as i64),
+            erasure.shard_size(),
+            HashAlgorithm::HighwayHash256,
+        )
+        .await
+        .unwrap();
+
+        let mut remaining = data;
+        while !remaining.is_empty() {
+            let chunk_len = remaining.len().min(erasure.block_size);
+            writer.write(&remaining[..chunk_len]).await.unwrap();
+            remaining = &remaining[chunk_len..];
+        }
+
+        writer.shutdown().await.unwrap();
+        writer.into_inline_data().unwrap()
+    }
+
+    async fn decode_inline(encoded: &[u8], size: usize, erasure: &Erasure) -> Vec<u8> {
+        let mut reader =
+            create_bitrot_reader(Some(encoded), None, "", "", 0, size, erasure.shard_size(), HashAlgorithm::HighwayHash256)
+                .await
+                .unwrap()
+                .unwrap();
+
+        let mut out = Vec::with_capacity(size);
+        while out.len() < size {
+            let remaining = size - out.len();
+            let mut buf = vec![0u8; erasure.block_size.min(remaining.max(1))];
+            let read = reader.read(&mut buf).await.unwrap();
+            if read == 0 {
+                break;
+            }
+            out.extend_from_slice(&buf[..read.min(remaining)]);
+        }
+        out
+    }
+
+    #[tokio::test]
+    async fn append_inline_combines_payloads() {
+        let erasure = make_inline_erasure();
+        let existing_plain = b"hello";
+        let encoded = encode_inline(existing_plain, &erasure).await;
+
+        let ctx = InlineAppendContext {
+            existing_inline: Some(&encoded),
+            existing_plain: None,
+            existing_size: existing_plain.len() as i64,
+            append_payload: b" world",
+            erasure: &erasure,
+            hash_algorithm: HashAlgorithm::HighwayHash256,
+            has_checksums: true,
+        };
+
+        let result = append_inline_data(ctx).await.expect("inline append to succeed");
+        assert_eq!(result.total_size, 11);
+        assert_eq!(result.etag, md5_hex(b"hello world"));
+
+        let decoded = decode_inline(&result.inline_data, result.total_size as usize, &erasure).await;
+        assert_eq!(decoded, b"hello world");
+    }
+
+    #[tokio::test]
+    async fn decode_inline_handles_padded_shards() {
+        let erasure = Erasure::new(1, 0, 1024);
+        let plain = b"hello";
+
+        let mut padded = vec![0u8; calc_shard_size(plain.len(), erasure.data_shards)];
+        padded[..plain.len()].copy_from_slice(plain);
+
+        let mut writer = create_bitrot_writer(
+            true,
+            None,
+            "",
+            "",
+            erasure.shard_file_size(plain.len() as i64),
+            erasure.shard_size(),
+            HashAlgorithm::HighwayHash256,
+        )
+        .await
+        .unwrap();
+
+        writer.write(&padded).await.unwrap();
+        writer.shutdown().await.unwrap();
+        let inline = writer.into_inline_data().unwrap();
+
+        let (decoded, algo) = decode_inline_payload(&inline, plain.len(), &erasure, HashAlgorithm::HighwayHash256)
+            .await
+            .expect("inline decode should succeed");
+
+        assert_eq!(decoded, plain);
+        assert_eq!(algo, HashAlgorithm::HighwayHash256);
+    }
+
+    #[tokio::test]
+    async fn append_inline_handles_empty_original() {
+        let erasure = make_inline_erasure();
+        let ctx = InlineAppendContext {
+            existing_inline: None,
+            existing_plain: None,
+            existing_size: 0,
+            append_payload: b"data",
+            erasure: &erasure,
+            hash_algorithm: HashAlgorithm::HighwayHash256,
+            has_checksums: true,
+        };
+
+        let result = append_inline_data(ctx).await.expect("inline append to succeed");
+        assert_eq!(result.total_size, 4);
+        assert_eq!(result.etag, md5_hex(b"data"));
+
+        let decoded = decode_inline(&result.inline_data, result.total_size as usize, &erasure).await;
+        assert_eq!(decoded, b"data");
+    }
+
+    #[tokio::test]
+    async fn append_inline_without_checksums_uses_raw_bytes() {
+        let erasure = Erasure::new(1, 0, 1024);
+        let existing = b"hello";
+
+        let ctx = InlineAppendContext {
+            existing_inline: Some(existing),
+            existing_plain: None,
+            existing_size: existing.len() as i64,
+            append_payload: b" world",
+            erasure: &erasure,
+            hash_algorithm: HashAlgorithm::HighwayHash256,
+            has_checksums: false,
+        };
+
+        let result = append_inline_data(ctx).await.expect("inline append to succeed");
+        assert_eq!(result.total_size, 11);
+        assert_eq!(result.etag, md5_hex(b"hello world"));
+
+        assert_eq!(result.inline_data, b"hello world");
+    }
+
+    #[tokio::test]
+    async fn append_inline_decodes_bitrot_without_checksums() {
+        let erasure = Erasure::new(1, 0, 1024);
+        let existing_plain = b"hello";
+        let encoded = encode_inline(existing_plain, &erasure).await;
+
+        let ctx = InlineAppendContext {
+            existing_inline: Some(&encoded),
+            existing_plain: None,
+            existing_size: existing_plain.len() as i64,
+            append_payload: b" world",
+            erasure: &erasure,
+            hash_algorithm: HashAlgorithm::HighwayHash256,
+            has_checksums: false,
+        };
+
+        let result = append_inline_data(ctx).await.expect("inline append to succeed");
+        assert_eq!(result.total_size, 11);
+        assert_eq!(result.etag, md5_hex(b"hello world"));
+
+        let decoded = decode_inline(&result.inline_data, result.total_size as usize, &erasure).await;
+        assert_eq!(decoded, b"hello world");
+    }
+}
@@ -48,7 +48,7 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use time::{Duration, OffsetDateTime};
 use tokio::io::{AsyncReadExt, BufReader};
-use tokio::sync::broadcast::Receiver as B_Receiver;
+use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};

 pub const POOL_META_NAME: &str = "pool.bin";
@@ -651,7 +651,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    pub async fn decommission(&self, rx: B_Receiver<bool>, indices: Vec<usize>) -> Result<()> {
+    pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
        warn!("decommission: {:?}", indices);
        if indices.is_empty() {
            return Err(Error::other("InvalidArgument"));
@@ -663,13 +663,14 @@ impl ECStore {

        self.start_decommission(indices.clone()).await?;

+        let rx_clone = rx.clone();
        tokio::spawn(async move {
            let Some(store) = new_object_layer_fn() else {
                error!("store not init");
                return;
            };
            for idx in indices.iter() {
-                store.do_decommission_in_routine(rx.resubscribe(), *idx).await;
+                store.do_decommission_in_routine(rx_clone.clone(), *idx).await;
            }
        });

@@ -891,7 +892,7 @@ impl ECStore {
    #[tracing::instrument(skip(self, rx))]
    async fn decommission_pool(
        self: &Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        idx: usize,
        pool: Arc<Sets>,
        bi: DecomBucketInfo,
@@ -936,20 +937,20 @@ impl ECStore {
            });

            let set = set.clone();
-            let mut rx = rx.resubscribe();
+            let rx_clone = rx.clone();
            let bi = bi.clone();
            let set_id = set_idx;
            let wk_clone = wk.clone();
            tokio::spawn(async move {
                loop {
-                    if rx.try_recv().is_ok() {
+                    if rx_clone.is_cancelled() {
                        warn!("decommission_pool: cancel {}", set_id);
                        break;
                    }
                    warn!("decommission_pool: list_objects_to_decommission {} {}", set_id, &bi.name);

                    match set
-                        .list_objects_to_decommission(rx.resubscribe(), bi.clone(), decommission_entry.clone())
+                        .list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
                        .await
                    {
                        Ok(_) => {
@@ -982,7 +983,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    pub async fn do_decommission_in_routine(self: &Arc<Self>, rx: B_Receiver<bool>, idx: usize) {
+    pub async fn do_decommission_in_routine(self: &Arc<Self>, rx: CancellationToken, idx: usize) {
        if let Err(err) = self.decommission_in_background(rx, idx).await {
            error!("decom err {:?}", &err);
            if let Err(er) = self.decommission_failed(idx).await {
@@ -1060,7 +1061,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    async fn decommission_in_background(self: &Arc<Self>, rx: B_Receiver<bool>, idx: usize) -> Result<()> {
+    async fn decommission_in_background(self: &Arc<Self>, rx: CancellationToken, idx: usize) -> Result<()> {
        let pool = self.pools[idx].clone();

        let pending = {
@@ -1090,10 +1091,7 @@ impl ECStore {

            warn!("decommission: currently on bucket {}", &bucket.name);

-            if let Err(err) = self
-                .decommission_pool(rx.resubscribe(), idx, pool.clone(), bucket.clone())
-                .await
-            {
+            if let Err(err) = self.decommission_pool(rx.clone(), idx, pool.clone(), bucket.clone()).await {
                error!("decommission: decommission_pool err {:?}", &err);
                return Err(err);
            } else {
@@ -1329,7 +1327,7 @@ impl SetDisks {
    #[tracing::instrument(skip(self, rx, cb_func))]
    async fn list_objects_to_decommission(
        self: &Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        bucket_info: DecomBucketInfo,
        cb_func: ListCallback,
    ) -> Result<()> {
@@ -34,8 +34,8 @@ use std::io::Cursor;
 use std::sync::Arc;
 use time::OffsetDateTime;
 use tokio::io::{AsyncReadExt, BufReader};
-use tokio::sync::broadcast::{self, Receiver as B_Receiver};
 use tokio::time::{Duration, Instant};
+use tokio_util::sync::CancellationToken;
 use tracing::{error, info};
 use uuid::Uuid;

@@ -151,7 +151,7 @@ pub struct DiskStat {
 #[derive(Debug, Default, Serialize, Deserialize, Clone)]
 pub struct RebalanceMeta {
    #[serde(skip)]
-    pub cancel: Option<broadcast::Sender<bool>>, // To be invoked on rebalance-stop
+    pub cancel: Option<CancellationToken>, // To be invoked on rebalance-stop
    #[serde(skip)]
    pub last_refreshed_at: Option<OffsetDateTime>,
    #[serde(rename = "stopTs")]
@@ -493,8 +493,8 @@ impl ECStore {
    pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
        let rebalance_meta = self.rebalance_meta.read().await;
        if let Some(meta) = rebalance_meta.as_ref() {
-            if let Some(tx) = meta.cancel.as_ref() {
-                let _ = tx.send(true);
+            if let Some(cancel_tx) = meta.cancel.as_ref() {
+                cancel_tx.cancel();
            }
        }

@@ -506,13 +506,14 @@ impl ECStore {
        info!("start_rebalance: start rebalance");
        // let rebalance_meta = self.rebalance_meta.read().await;

-        let (tx, rx) = broadcast::channel::<bool>(1);
+        let cancel_tx = CancellationToken::new();
+        let rx = cancel_tx.clone();

        {
            let mut rebalance_meta = self.rebalance_meta.write().await;

            if let Some(meta) = rebalance_meta.as_mut() {
-                meta.cancel = Some(tx)
+                meta.cancel = Some(cancel_tx)
            } else {
                info!("start_rebalance: rebalance_meta is None exit");
                return;
@@ -565,9 +566,9 @@ impl ECStore {

            let pool_idx = idx;
            let store = self.clone();
-            let rx = rx.resubscribe();
+            let rx_clone = rx.clone();
            tokio::spawn(async move {
-                if let Err(err) = store.rebalance_buckets(rx, pool_idx).await {
+                if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx).await {
                    error!("Rebalance failed for pool {}: {}", pool_idx, err);
                } else {
                    info!("Rebalance completed for pool {}", pool_idx);
@@ -579,7 +580,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    async fn rebalance_buckets(self: &Arc<Self>, mut rx: B_Receiver<bool>, pool_index: usize) -> Result<()> {
+    async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize) -> Result<()> {
        let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);

        // Save rebalance metadata periodically
@@ -651,7 +652,7 @@ impl ECStore {
        info!("Pool {} rebalancing is started", pool_index);

        loop {
-            if let Ok(true) = rx.try_recv() {
+            if rx.is_cancelled() {
                info!("Pool {} rebalancing is stopped", pool_index);
                done_tx.send(Err(Error::other("rebalance stopped canceled"))).await.ok();
                break;
@@ -660,7 +661,7 @@ impl ECStore {
            if let Some(bucket) = self.next_rebal_bucket(pool_index).await? {
                info!("Rebalancing bucket: start {}", bucket);

-                if let Err(err) = self.rebalance_bucket(rx.resubscribe(), bucket.clone(), pool_index).await {
+                if let Err(err) = self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index).await {
                    if err.to_string().contains("not initialized") {
                        info!("rebalance_bucket: rebalance not initialized, continue");
                        continue;
@@ -1033,7 +1034,7 @@ impl ECStore {
    }

    #[tracing::instrument(skip(self, rx))]
-    async fn rebalance_bucket(self: &Arc<Self>, rx: B_Receiver<bool>, bucket: String, pool_index: usize) -> Result<()> {
+    async fn rebalance_bucket(self: &Arc<Self>, rx: CancellationToken, bucket: String, pool_index: usize) -> Result<()> {
        // Placeholder for actual bucket rebalance logic
        info!("Rebalancing bucket {} in pool {}", bucket, pool_index);

@@ -1072,7 +1073,7 @@ impl ECStore {
            });

            let set = set.clone();
-            let rx = rx.resubscribe();
+            let rx = rx.clone();
            let bucket = bucket.clone();
            // let wk = wk.clone();

@@ -1144,7 +1145,7 @@ impl SetDisks {
    #[tracing::instrument(skip(self, rx, cb))]
    pub async fn list_objects_to_rebalance(
        self: &Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        bucket: String,
        cb: ListCallback,
    ) -> Result<()> {
@@ -16,10 +16,8 @@ mod http_auth;
 mod peer_rest_client;
 mod peer_s3_client;
 mod remote_disk;
-mod tonic_service;

 pub use http_auth::{build_auth_headers, verify_rpc_signature};
 pub use peer_rest_client::PeerRestClient;
 pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, RemotePeerS3Client, S3PeerSys};
 pub use remote_disk::RemoteDisk;
-pub use tonic_service::{NodeService, make_server};
@@ -345,21 +345,43 @@ impl DiskAPI for RemoteDisk {
    }

    #[tracing::instrument(skip(self))]
-    async fn delete_versions(
-        &self,
-        volume: &str,
-        versions: Vec<FileInfoVersions>,
-        opts: DeleteOptions,
-    ) -> Result<Vec<Option<Error>>> {
+    async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
        info!("delete_versions");
-        let opts = serde_json::to_string(&opts)?;
+
+        let opts = match serde_json::to_string(&opts) {
+            Ok(opts) => opts,
+            Err(err) => {
+                let mut errors = Vec::with_capacity(versions.len());
+                for _ in 0..versions.len() {
+                    errors.push(Some(Error::other(err.to_string())));
+                }
+                return errors;
+            }
+        };
        let mut versions_str = Vec::with_capacity(versions.len());
        for file_info_versions in versions.iter() {
-            versions_str.push(serde_json::to_string(file_info_versions)?);
+            versions_str.push(match serde_json::to_string(file_info_versions) {
+                Ok(versions_str) => versions_str,
+                Err(err) => {
+                    let mut errors = Vec::with_capacity(versions.len());
+                    for _ in 0..versions.len() {
+                        errors.push(Some(Error::other(err.to_string())));
+                    }
+                    return errors;
+                }
+            });
        }
-        let mut client = node_service_time_out_client(&self.addr)
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+        let mut client = match node_service_time_out_client(&self.addr).await {
+            Ok(client) => client,
+            Err(err) => {
+                let mut errors = Vec::with_capacity(versions.len());
+                for _ in 0..versions.len() {
+                    errors.push(Some(Error::other(err.to_string())));
+                }
+                return errors;
+            }
+        };
+
        let request = Request::new(DeleteVersionsRequest {
            disk: self.endpoint.to_string(),
            volume: volume.to_string(),
@@ -368,11 +390,27 @@ impl DiskAPI for RemoteDisk {
        });

        // TODO: use Error not string
-        let response = client.delete_versions(request).await?.into_inner();
+
+        let response = match client.delete_versions(request).await {
+            Ok(response) => response,
+            Err(err) => {
+                let mut errors = Vec::with_capacity(versions.len());
+                for _ in 0..versions.len() {
+                    errors.push(Some(Error::other(err.to_string())));
+                }
+                return errors;
+            }
+        };
+
+        let response = response.into_inner();
        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
+            let mut errors = Vec::with_capacity(versions.len());
+            for _ in 0..versions.len() {
+                errors.push(Some(Error::other(response.error.clone().map(|e| e.error_info).unwrap_or_default())));
+            }
+            return errors;
        }
-        let errors = response
+        response
            .errors
            .iter()
            .map(|error| {
@@ -382,9 +420,7 @@ impl DiskAPI for RemoteDisk {
                    Some(Error::other(error.to_string()))
                }
            })
-            .collect();
-
-        Ok(errors)
+            .collect()
    }

    #[tracing::instrument(skip(self))]
@@ -17,7 +17,7 @@ use std::{collections::HashMap, sync::Arc};

 use crate::disk::error_reduce::count_errs;
 use crate::error::{Error, Result};
-use crate::store_api::ListPartsInfo;
+use crate::store_api::{ListPartsInfo, ObjectInfoOrErr, WalkOptions};
 use crate::{
    disk::{
        DiskAPI, DiskInfo, DiskOption, DiskStore,
@@ -48,6 +48,7 @@ use rustfs_filemeta::FileInfo;
 use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
 use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
 use tokio::sync::RwLock;
+use tokio_util::sync::CancellationToken;
 use uuid::Uuid;

 use tokio::sync::broadcast::{Receiver, Sender};
@@ -55,8 +56,6 @@ use tokio::time::Duration;
 use tracing::warn;
 use tracing::{error, info};

-use crate::lock_utils::create_unique_clients;
-
 #[derive(Debug, Clone)]
 pub struct Sets {
    pub id: Uuid,
@@ -163,8 +162,6 @@ impl Sets {
                }
            }

-            let _lock_clients = create_unique_clients(&set_endpoints).await?;
-
            // Note: write_quorum was used for the old lock system, no longer needed with FastLock
            let _write_quorum = set_drive_count - parity_count;
            // Create fast lock manager for high performance
@@ -459,6 +456,17 @@ impl StorageAPI for Sets {
        unimplemented!()
    }

+    async fn walk(
+        self: Arc<Self>,
+        _rx: CancellationToken,
+        _bucket: &str,
+        _prefix: &str,
+        _result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
+        _opts: WalkOptions,
+    ) -> Result<()> {
+        unimplemented!()
+    }
+
    #[tracing::instrument(skip(self))]
    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
        self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
@@ -543,7 +551,7 @@ impl StorageAPI for Sets {
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
-    ) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)> {
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
        // Default return value
        let mut del_objects = vec![DeletedObject::default(); objects.len()];

@@ -576,38 +584,11 @@ impl StorageAPI for Sets {
            }
        }

-        // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
-        // let mut jhs = Vec::with_capacity(semaphore.available_permits());
-
-        // for (k, v) in set_obj_map {
-        //     let disks = self.get_disks(k);
-        //     let semaphore = semaphore.clone();
-        //     let opts = opts.clone();
-        //     let bucket = bucket.to_string();
-
-        //     let jh = tokio::spawn(async move {
-        //         let _permit = semaphore.acquire().await.unwrap();
-        //         let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-        //         disks.delete_objects(&bucket, objs, opts).await
-        //     });
-        //     jhs.push(jh);
-        // }
-
-        // let mut results = Vec::with_capacity(jhs.len());
-        // for jh in jhs {
-        //     results.push(jh.await?.unwrap());
-        // }
-
-        // for (dobjects, errs) in results {
-        //     del_objects.extend(dobjects);
-        //     del_errs.extend(errs);
-        // }
-
-        // TODO: Implement concurrency
+        // TODO: concurrency
        for (k, v) in set_obj_map {
            let disks = self.get_disks(k);
            let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-            let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await?;
+            let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await;

            for (i, err) in errs.into_iter().enumerate() {
                let obj = v.get(i).unwrap();
@@ -618,7 +599,15 @@ impl StorageAPI for Sets {
            }
        }

-        Ok((del_objects, del_errs))
+        (del_objects, del_errs)
+    }
+
+    async fn complete_append(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
+        self.get_disks_by_key(object).complete_append(bucket, object, opts).await
+    }
+
+    async fn abort_append(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
+        self.get_disks_by_key(object).abort_append(bucket, object, opts).await
    }

    async fn list_object_parts(
@@ -34,7 +34,9 @@ use crate::global::{
 use crate::notification_sys::get_global_notification_sys;
 use crate::pools::PoolMeta;
 use crate::rebalance::RebalanceMeta;
-use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, ListPartsInfo, MultipartInfo, ObjectIO};
+use crate::store_api::{
+    ListMultipartsInfo, ListObjectVersionsInfo, ListPartsInfo, MultipartInfo, ObjectIO, ObjectInfoOrErr, WalkOptions,
+};
 use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
 use crate::{
    bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
@@ -68,8 +70,9 @@ use std::time::SystemTime;
 use std::{collections::HashMap, sync::Arc, time::Duration};
 use time::OffsetDateTime;
 use tokio::select;
-use tokio::sync::{RwLock, broadcast};
+use tokio::sync::RwLock;
 use tokio::time::sleep;
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, info};
 use tracing::{error, warn};
 use uuid::Uuid;
@@ -109,7 +112,7 @@ pub struct ECStore {
 impl ECStore {
    #[allow(clippy::new_ret_no_self)]
    #[tracing::instrument(level = "debug", skip(endpoint_pools))]
-    pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools) -> Result<Arc<Self>> {
+    pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools, ctx: CancellationToken) -> Result<Arc<Self>> {
        // let layouts = DisksLayout::from_volumes(endpoints.as_slice())?;

        let mut deployment_id = None;
@@ -251,7 +254,7 @@ impl ECStore {
        let wait_sec = 5;
        let mut exit_count = 0;
        loop {
-            if let Err(err) = ec.init().await {
+            if let Err(err) = ec.init(ctx.clone()).await {
                error!("init err: {}", err);
                error!("retry after  {} second", wait_sec);
                sleep(Duration::from_secs(wait_sec)).await;
@@ -273,7 +276,7 @@ impl ECStore {
        Ok(ec)
    }

-    pub async fn init(self: &Arc<Self>) -> Result<()> {
+    pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
        GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;

        if self.load_rebalance_meta().await.is_ok() {
@@ -317,18 +320,16 @@ impl ECStore {
        if !pool_indices.is_empty() {
            let idx = pool_indices[0];
            if endpoints.as_ref()[idx].endpoints.as_ref()[0].is_local {
-                let (_tx, rx) = broadcast::channel(1);
-
                let store = self.clone();

                tokio::spawn(async move {
                    // wait  3 minutes for cluster init
                    tokio::time::sleep(Duration::from_secs(60 * 3)).await;

-                    if let Err(err) = store.decommission(rx.resubscribe(), pool_indices.clone()).await {
+                    if let Err(err) = store.decommission(rx.clone(), pool_indices.clone()).await {
                        if err == StorageError::DecommissionAlreadyRunning {
                            for i in pool_indices.iter() {
-                                store.do_decommission_in_routine(rx.resubscribe(), *i).await;
+                                store.do_decommission_in_routine(rx.clone(), *i).await;
                            }
                            return;
                        }
@@ -700,9 +701,13 @@ impl ECStore {
        opts: &ObjectOptions,
    ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
        let mut futures = Vec::new();
-
        for pool in self.pools.iter() {
-            futures.push(pool.get_object_info(bucket, object, opts));
+            let mut pool_opts = opts.clone();
+            if !pool_opts.metadata_chg {
+                pool_opts.version_id = None;
+            }
+
+            futures.push(async move { pool.get_object_info(bucket, object, &pool_opts).await });
        }

        let results = join_all(futures).await;
@@ -1351,6 +1356,17 @@ impl StorageAPI for ECStore {
            .await
    }

+    async fn walk(
+        self: Arc<Self>,
+        rx: CancellationToken,
+        bucket: &str,
+        prefix: &str,
+        result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
+        opts: WalkOptions,
+    ) -> Result<()> {
+        self.walk_internal(rx, bucket, prefix, result, opts).await
+    }
+
    #[tracing::instrument(skip(self))]
    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
        check_object_args(bucket, object)?;
@@ -1450,9 +1466,12 @@ impl StorageAPI for ECStore {
        let object = encode_dir_object(object);
        let object = object.as_str();

+        let mut gopts = opts.clone();
+        gopts.no_lock = true;
+
        // 查询在哪个 pool
        let (mut pinfo, errs) = self
-            .get_pool_info_existing_with_opts(bucket, object, &opts)
+            .get_pool_info_existing_with_opts(bucket, object, &gopts)
            .await
            .map_err(|e| {
                if is_err_read_quorum(&e) {
@@ -1513,7 +1532,7 @@ impl StorageAPI for ECStore {
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
-    ) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)> {
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
        // encode object name
        let objects: Vec<ObjectToDelete> = objects
            .iter()
@@ -1534,133 +1553,173 @@ impl StorageAPI for ECStore {

        // TODO: nslock

-        let mut futures = Vec::with_capacity(objects.len());
+        let mut futures = Vec::with_capacity(self.pools.len());

-        for obj in objects.iter() {
-            futures.push(async move {
-                self.internal_get_pool_info_existing_with_opts(
-                    bucket,
-                    &obj.object_name,
-                    &ObjectOptions {
-                        no_lock: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-            });
+        for pool in self.pools.iter() {
+            futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone()));
        }

        let results = join_all(futures).await;

-        // let mut jhs = Vec::new();
-        // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
-        // let pools = Arc::new(self.pools.clone());
+        for idx in 0..del_objects.len() {
+            for (dels, errs) in results.iter() {
+                if errs[idx].is_none() && dels[idx].found {
+                    del_errs[idx] = None;
+                    del_objects[idx] = dels[idx].clone();
+                    break;
+                }
+
+                if del_errs[idx].is_none() {
+                    del_errs[idx] = errs[idx].clone();
+                    del_objects[idx] = dels[idx].clone();
+                }
+            }
+        }
+
+        del_objects.iter_mut().for_each(|v| {
+            v.object_name = decode_dir_object(&v.object_name);
+        });
+
+        (del_objects, del_errs)
+
+        // let mut futures = Vec::with_capacity(objects.len());

        // for obj in objects.iter() {
-        //     let (semaphore, pools, bucket, object_name, opt) = (
-        //         semaphore.clone(),
-        //         pools.clone(),
-        //         bucket.to_string(),
-        //         obj.object_name.to_string(),
-        //         ObjectOptions::default(),
-        //     );
-
-        //     let jh = tokio::spawn(async move {
-        //         let _permit = semaphore.acquire().await.unwrap();
-        //         self.internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt)
-        //             .await
+        //     futures.push(async move {
+        //         self.internal_get_pool_info_existing_with_opts(
+        //             bucket,
+        //             &obj.object_name,
+        //             &ObjectOptions {
+        //                 no_lock: true,
+        //                 ..Default::default()
+        //             },
+        //         )
+        //         .await
        //     });
-        //     jhs.push(jh);
-        // }
-        // let mut results = Vec::new();
-        // for jh in jhs {
-        //     results.push(jh.await.unwrap());
        // }

-        // 记录 pool Index 对应的 objects pool_idx -> objects idx
-        let mut pool_obj_idx_map = HashMap::new();
-        let mut orig_index_map = HashMap::new();
+        // let results = join_all(futures).await;

-        for (i, res) in results.into_iter().enumerate() {
-            match res {
-                Ok((pinfo, _)) => {
-                    if let Some(obj) = objects.get(i) {
-                        if pinfo.object_info.delete_marker && obj.version_id.is_none() {
-                            del_objects[i] = DeletedObject {
-                                delete_marker: pinfo.object_info.delete_marker,
-                                delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
-                                object_name: decode_dir_object(&pinfo.object_info.name),
-                                delete_marker_mtime: pinfo.object_info.mod_time,
-                                ..Default::default()
-                            };
-                            continue;
-                        }
+        // // let mut jhs = Vec::new();
+        // // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
+        // // let pools = Arc::new(self.pools.clone());

-                        if !pool_obj_idx_map.contains_key(&pinfo.index) {
-                            pool_obj_idx_map.insert(pinfo.index, vec![obj.clone()]);
-                        } else if let Some(val) = pool_obj_idx_map.get_mut(&pinfo.index) {
-                            val.push(obj.clone());
-                        }
+        // // for obj in objects.iter() {
+        // //     let (semaphore, pools, bucket, object_name, opt) = (
+        // //         semaphore.clone(),
+        // //         pools.clone(),
+        // //         bucket.to_string(),
+        // //         obj.object_name.to_string(),
+        // //         ObjectOptions::default(),
+        // //     );

-                        if !orig_index_map.contains_key(&pinfo.index) {
-                            orig_index_map.insert(pinfo.index, vec![i]);
-                        } else if let Some(val) = orig_index_map.get_mut(&pinfo.index) {
-                            val.push(i);
-                        }
-                    }
-                }
-                Err(e) => {
-                    if !is_err_object_not_found(&e) && is_err_version_not_found(&e) {
-                        del_errs[i] = Some(e)
-                    }
+        // //     let jh = tokio::spawn(async move {
+        // //         let _permit = semaphore.acquire().await.unwrap();
+        // //         self.internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt)
+        // //             .await
+        // //     });
+        // //     jhs.push(jh);
+        // // }
+        // // let mut results = Vec::new();
+        // // for jh in jhs {
+        // //     results.push(jh.await.unwrap());
+        // // }

-                    if let Some(obj) = objects.get(i) {
-                        del_objects[i] = DeletedObject {
-                            object_name: decode_dir_object(&obj.object_name),
-                            version_id: obj.version_id.map(|v| v.to_string()),
-                            ..Default::default()
-                        }
-                    }
-                }
-            }
-        }
+        // // 记录 pool Index 对应的 objects pool_idx -> objects idx
+        // let mut pool_obj_idx_map = HashMap::new();
+        // let mut orig_index_map = HashMap::new();

-        if !pool_obj_idx_map.is_empty() {
-            for (i, sets) in self.pools.iter().enumerate() {
-                //  取 pool idx 对应的 objects index
-                if let Some(objs) = pool_obj_idx_map.get(&i) {
-                    //  取对应 obj，理论上不会 none
-                    // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();
+        // for (i, res) in results.into_iter().enumerate() {
+        //     match res {
+        //         Ok((pinfo, _)) => {
+        //             if let Some(obj) = objects.get(i) {
+        //                 if pinfo.object_info.delete_marker && obj.version_id.is_none() {
+        //                     del_objects[i] = DeletedObject {
+        //                         delete_marker: pinfo.object_info.delete_marker,
+        //                         delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
+        //                         object_name: decode_dir_object(&pinfo.object_info.name),
+        //                         delete_marker_mtime: pinfo.object_info.mod_time,
+        //                         ..Default::default()
+        //                     };
+        //                     continue;
+        //                 }

-                    if objs.is_empty() {
-                        continue;
-                    }
+        //                 if !pool_obj_idx_map.contains_key(&pinfo.index) {
+        //                     pool_obj_idx_map.insert(pinfo.index, vec![obj.clone()]);
+        //                 } else if let Some(val) = pool_obj_idx_map.get_mut(&pinfo.index) {
+        //                     val.push(obj.clone());
+        //                 }

-                    let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;
+        //                 if !orig_index_map.contains_key(&pinfo.index) {
+        //                     orig_index_map.insert(pinfo.index, vec![i]);
+        //                 } else if let Some(val) = orig_index_map.get_mut(&pinfo.index) {
+        //                     val.push(i);
+        //                 }
+        //             }
+        //         }
+        //         Err(e) => {
+        //             if !is_err_object_not_found(&e) && is_err_version_not_found(&e) {
+        //                 del_errs[i] = Some(e)
+        //             }

-                    // 同时存入不可能为 none
-                    let org_indexes = orig_index_map.get(&i).unwrap();
+        //             if let Some(obj) = objects.get(i) {
+        //                 del_objects[i] = DeletedObject {
+        //                     object_name: decode_dir_object(&obj.object_name),
+        //                     version_id: obj.version_id.map(|v| v.to_string()),
+        //                     ..Default::default()
+        //                 }
+        //             }
+        //         }
+        //     }
+        // }

-                    // perrs 的顺序理论上跟 obj_idxs 顺序一致
-                    for (i, err) in perrs.into_iter().enumerate() {
-                        let obj_idx = org_indexes[i];
+        // if !pool_obj_idx_map.is_empty() {
+        //     for (i, sets) in self.pools.iter().enumerate() {
+        //         //  取 pool idx 对应的 objects index
+        //         if let Some(objs) = pool_obj_idx_map.get(&i) {
+        //             //  取对应 obj，理论上不会 none
+        //             // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();

-                        if err.is_some() {
-                            del_errs[obj_idx] = err;
-                        }
+        //             if objs.is_empty() {
+        //                 continue;
+        //             }

-                        let mut dobj = pdel_objs.get(i).unwrap().clone();
-                        dobj.object_name = decode_dir_object(&dobj.object_name);
+        //             let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;

-                        del_objects[obj_idx] = dobj;
-                    }
-                }
-            }
-        }
+        //             // 同时存入不可能为 none
+        //             let org_indexes = orig_index_map.get(&i).unwrap();

-        Ok((del_objects, del_errs))
+        //             // perrs 的顺序理论上跟 obj_idxs 顺序一致
+        //             for (i, err) in perrs.into_iter().enumerate() {
+        //                 let obj_idx = org_indexes[i];
+
+        //                 if err.is_some() {
+        //                     del_errs[obj_idx] = err;
+        //                 }
+
+        //                 let mut dobj = pdel_objs.get(i).unwrap().clone();
+        //                 dobj.object_name = decode_dir_object(&dobj.object_name);
+
+        //                 del_objects[obj_idx] = dobj;
+        //             }
+        //         }
+        //     }
+        // }
+
+        // Ok((del_objects, del_errs))
    }

+    async fn complete_append(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
+        let object = encode_dir_object(object);
+        let (pinfo, _) = self.internal_get_pool_info_existing_with_opts(bucket, &object, opts).await?;
+        self.pools[pinfo.index].complete_append(bucket, &object, opts).await
+    }
+
+    async fn abort_append(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
+        let object = encode_dir_object(object);
+        let (pinfo, _) = self.internal_get_pool_info_existing_with_opts(bucket, &object, opts).await?;
+        self.pools[pinfo.index].abort_append(bucket, &object, opts).await
+    }
    #[tracing::instrument(skip(self))]
    async fn list_object_parts(
        &self,
@@ -13,8 +13,10 @@
 // limitations under the License.

 use crate::bucket::metadata_sys::get_versioning_config;
+use crate::bucket::replication::REPLICATION_RESET;
+use crate::bucket::replication::REPLICATION_STATUS;
+use crate::bucket::replication::{ReplicateDecision, replication_statuses_map, version_purge_statuses_map};
 use crate::bucket::versioning::VersioningApi as _;
-use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusType};
 use crate::disk::DiskStore;
 use crate::error::{Error, Result};
 use crate::store_utils::clean_metadata;
@@ -25,20 +27,25 @@ use crate::{
 };
 use http::{HeaderMap, HeaderValue};
 use rustfs_common::heal_channel::HealOpts;
-use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER;
-use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING};
+use rustfs_filemeta::{
+    FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
+};
 use rustfs_madmin::heal_commands::HealResultItem;
 use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader};
 use rustfs_utils::CompressionAlgorithm;
+use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER};
 use rustfs_utils::path::decode_dir_object;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::io::Cursor;
+use std::pin::Pin;
 use std::str::FromStr as _;
 use std::sync::Arc;
+use std::task::{Context, Poll};
 use time::OffsetDateTime;
-use tokio::io::{AsyncRead, AsyncReadExt};
+use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
+use tokio_util::sync::CancellationToken;
 use tracing::warn;
 use uuid::Uuid;

@@ -221,6 +228,12 @@ impl GetObjectReader {
    }
 }

+impl AsyncRead for GetObjectReader {
+    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        Pin::new(&mut self.stream).poll_read(cx, buf)
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct HTTPRangeSpec {
    pub is_suffix_length: bool,
@@ -315,6 +328,8 @@ pub struct ObjectOptions {
    pub max_parity: bool,
    pub mod_time: Option<OffsetDateTime>,
    pub part_number: Option<usize>,
+    pub append_object: bool,
+    pub append_position: Option<i64>,

    pub delete_prefix: bool,
    pub delete_prefix_object: bool,
@@ -326,6 +341,7 @@ pub struct ObjectOptions {

    pub skip_decommissioned: bool,
    pub skip_rebalancing: bool,
+    pub skip_free_version: bool,

    pub data_movement: bool,
    pub src_pool_idx: usize,
@@ -334,11 +350,10 @@ pub struct ObjectOptions {
    pub metadata_chg: bool,
    pub http_preconditions: Option<HTTPPreconditions>,

+    pub delete_replication: Option<ReplicationState>,
    pub replication_request: bool,
    pub delete_marker: bool,

-    pub skip_free_version: bool,
-
    pub transition: TransitionOptions,
    pub expiration: ExpirationOptions,
    pub lifecycle_audit_event: LcAuditEvent,
@@ -346,15 +361,66 @@ pub struct ObjectOptions {
    pub eval_metadata: Option<HashMap<String, String>>,
 }

-// impl Default for ObjectOptions {
-//     fn default() -> Self {
-//         Self {
-//             max_parity: Default::default(),
-//             mod_time: OffsetDateTime::UNIX_EPOCH,
-//             part_number: Default::default(),
-//         }
-//     }
-// }
+impl ObjectOptions {
+    pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) {
+        let mut rs = ReplicationState {
+            replicate_decision_str: dsc.to_string(),
+            ..Default::default()
+        };
+        if self.version_id.is_none() {
+            rs.replication_status_internal = dsc.pending_status();
+            rs.targets = replication_statuses_map(rs.replication_status_internal.as_deref().unwrap_or_default());
+        } else {
+            rs.version_purge_status_internal = dsc.pending_status();
+            rs.purge_targets = version_purge_statuses_map(rs.version_purge_status_internal.as_deref().unwrap_or_default());
+        }
+
+        self.delete_replication = Some(rs)
+    }
+
+    pub fn set_replica_status(&mut self, status: ReplicationStatusType) {
+        if let Some(rs) = self.delete_replication.as_mut() {
+            rs.replica_status = status;
+            rs.replica_timestamp = Some(OffsetDateTime::now_utc());
+        } else {
+            self.delete_replication = Some(ReplicationState {
+                replica_status: status,
+                replica_timestamp: Some(OffsetDateTime::now_utc()),
+                ..Default::default()
+            });
+        }
+    }
+
+    pub fn version_purge_status(&self) -> VersionPurgeStatusType {
+        self.delete_replication
+            .as_ref()
+            .map(|v| v.composite_version_purge_status())
+            .unwrap_or(VersionPurgeStatusType::Empty)
+    }
+
+    pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
+        self.delete_replication
+            .as_ref()
+            .map(|v| v.composite_replication_status())
+            .unwrap_or(ReplicationStatusType::Empty)
+    }
+
+    pub fn put_replication_state(&self) -> ReplicationState {
+        let rs = match self
+            .user_defined
+            .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_STATUS}").as_str())
+        {
+            Some(v) => v.to_string(),
+            None => return ReplicationState::default(),
+        };
+
+        ReplicationState {
+            replication_status_internal: Some(rs.to_string()),
+            targets: replication_statuses_map(rs.as_str()),
+            ..Default::default()
+        }
+    }
+}

 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct BucketOptions {
@@ -423,6 +489,7 @@ pub struct ObjectInfo {
    pub is_latest: bool,
    pub content_type: Option<String>,
    pub content_encoding: Option<String>,
+    pub expires: Option<OffsetDateTime>,
    pub num_versions: usize,
    pub successor_mod_time: Option<OffsetDateTime>,
    pub put_object_reader: Option<PutObjReader>,
@@ -430,10 +497,11 @@ pub struct ObjectInfo {
    pub inlined: bool,
    pub metadata_only: bool,
    pub version_only: bool,
-    pub replication_status_internal: String,
+    pub replication_status_internal: Option<String>,
    pub replication_status: ReplicationStatusType,
-    pub version_purge_status_internal: String,
+    pub version_purge_status_internal: Option<String>,
    pub version_purge_status: VersionPurgeStatusType,
+    pub replication_decision: String,
    pub checksum: Vec<u8>,
 }

@@ -470,7 +538,9 @@ impl Clone for ObjectInfo {
            replication_status: self.replication_status.clone(),
            version_purge_status_internal: self.version_purge_status_internal.clone(),
            version_purge_status: self.version_purge_status.clone(),
+            replication_decision: self.replication_decision.clone(),
            checksum: Default::default(),
+            expires: self.expires,
        }
    }
 }
@@ -588,6 +658,15 @@ impl ObjectInfo {
            })
            .collect();

+        let append_state = fi.get_append_state();
+        let pending_length: i64 = append_state.pending_segments.iter().map(|seg| seg.length).sum();
+        let logical_size = append_state.committed_length.saturating_add(pending_length);
+        let actual_size_meta = fi
+            .metadata
+            .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"))
+            .and_then(|o| o.parse::<i64>().ok())
+            .unwrap_or(logical_size);
+
        ObjectInfo {
            bucket: bucket.to_string(),
            name,
@@ -597,7 +676,7 @@ impl ObjectInfo {
            version_id,
            delete_marker: fi.deleted,
            mod_time: fi.mod_time,
-            size: fi.size,
+            size: logical_size,
            parts,
            is_latest: fi.is_latest,
            user_tags,
@@ -609,6 +688,7 @@ impl ObjectInfo {
            inlined,
            user_defined: metadata,
            transitioned_object,
+            actual_size: actual_size_meta,
            ..Default::default()
        }
    }
@@ -665,7 +745,10 @@ impl ObjectInfo {
                };

                for fi in versions.iter() {
-                    // TODO:VersionPurgeStatus
+                    if !fi.version_purge_status().is_empty() {
+                        continue;
+                    }
+
                    let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
                    objects.push(ObjectInfo::from_file_info(fi, bucket, &entry.name, versioned));
                }
@@ -770,6 +853,32 @@ impl ObjectInfo {

        objects
    }
+
+    pub fn replication_state(&self) -> ReplicationState {
+        ReplicationState {
+            replication_status_internal: self.replication_status_internal.clone(),
+            version_purge_status_internal: self.version_purge_status_internal.clone(),
+            replicate_decision_str: self.replication_decision.clone(),
+            targets: replication_statuses_map(self.replication_status_internal.clone().unwrap_or_default().as_str()),
+            purge_targets: version_purge_statuses_map(self.version_purge_status_internal.clone().unwrap_or_default().as_str()),
+            reset_statuses_map: self
+                .user_defined
+                .iter()
+                .filter_map(|(k, v)| {
+                    if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}")) {
+                        Some((
+                            k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-"))
+                                .to_string(),
+                            v.clone(),
+                        ))
+                    } else {
+                        None
+                    }
+                })
+                .collect(),
+            ..Default::default()
+        }
+    }
 }

 #[derive(Debug, Default)]
@@ -927,17 +1036,52 @@ pub struct ListPartsInfo {
 pub struct ObjectToDelete {
    pub object_name: String,
    pub version_id: Option<Uuid>,
+    pub delete_marker_replication_status: Option<String>,
+    pub version_purge_status: Option<VersionPurgeStatusType>,
+    pub version_purge_statuses: Option<String>,
+    pub replicate_decision_str: Option<String>,
 }
+
+impl ObjectToDelete {
+    pub fn replication_state(&self) -> ReplicationState {
+        ReplicationState {
+            replication_status_internal: self.delete_marker_replication_status.clone(),
+            version_purge_status_internal: self.version_purge_statuses.clone(),
+            replicate_decision_str: self.replicate_decision_str.clone().unwrap_or_default(),
+            targets: replication_statuses_map(self.delete_marker_replication_status.as_deref().unwrap_or_default()),
+            purge_targets: version_purge_statuses_map(self.version_purge_statuses.as_deref().unwrap_or_default()),
+            ..Default::default()
+        }
+    }
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct DeletedObject {
    pub delete_marker: bool,
-    pub delete_marker_version_id: Option<String>,
+    pub delete_marker_version_id: Option<Uuid>,
    pub object_name: String,
-    pub version_id: Option<String>,
+    pub version_id: Option<Uuid>,
    // MTime of DeleteMarker on source that needs to be propagated to replica
    pub delete_marker_mtime: Option<OffsetDateTime>,
    // to support delete marker replication
-    // pub replication_state: ReplicationState,
+    pub replication_state: Option<ReplicationState>,
+    pub found: bool,
+}
+
+impl DeletedObject {
+    pub fn version_purge_status(&self) -> VersionPurgeStatusType {
+        self.replication_state
+            .as_ref()
+            .map(|v| v.composite_version_purge_status())
+            .unwrap_or(VersionPurgeStatusType::Empty)
+    }
+
+    pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
+        self.replication_state
+            .as_ref()
+            .map(|v| v.composite_replication_status())
+            .unwrap_or(ReplicationStatusType::Empty)
+    }
 }

 #[derive(Debug, Default, Clone)]
@@ -949,8 +1093,33 @@ pub struct ListObjectVersionsInfo {
    pub prefixes: Vec<String>,
 }

+type WalkFilter = fn(&FileInfo) -> bool;
+
+#[derive(Clone, Default)]
+pub struct WalkOptions {
+    pub filter: Option<WalkFilter>,           // return WalkFilter returns 'true/false'
+    pub marker: Option<String>,               // set to skip until this object
+    pub latest_only: bool,                    // returns only latest versions for all matching objects
+    pub ask_disks: String,                    // dictates how many disks are being listed
+    pub versions_sort: WalkVersionsSortOrder, // sort order for versions of the same object; default: Ascending order in ModTime
+    pub limit: usize,                         // maximum number of items, 0 means no limit
+}
+
+#[derive(Clone, Default, PartialEq, Eq)]
+pub enum WalkVersionsSortOrder {
+    #[default]
+    Ascending,
+    Descending,
+}
+
+#[derive(Debug)]
+pub struct ObjectInfoOrErr {
+    pub item: Option<ObjectInfo>,
+    pub err: Option<Error>,
+}
+
 #[async_trait::async_trait]
-pub trait ObjectIO: Send + Sync + 'static {
+pub trait ObjectIO: Send + Sync + Debug + 'static {
    // GetObjectNInfo FIXME:
    async fn get_object_reader(
        &self,
@@ -966,7 +1135,7 @@ pub trait ObjectIO: Send + Sync + 'static {

 #[async_trait::async_trait]
 #[allow(clippy::too_many_arguments)]
-pub trait StorageAPI: ObjectIO {
+pub trait StorageAPI: ObjectIO + Debug {
    // NewNSLock TODO:
    // Shutdown TODO:
    // NSScanner TODO:
@@ -1000,7 +1169,15 @@ pub trait StorageAPI: ObjectIO {
        delimiter: Option<String>,
        max_keys: i32,
    ) -> Result<ListObjectVersionsInfo>;
-    // Walk TODO:
+
+    async fn walk(
+        self: Arc<Self>,
+        rx: CancellationToken,
+        bucket: &str,
+        prefix: &str,
+        result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
+        opts: WalkOptions,
+    ) -> Result<()>;

    async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
    async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
@@ -1021,7 +1198,11 @@ pub trait StorageAPI: ObjectIO {
        bucket: &str,
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
-    ) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)>;
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>);
+
+    async fn complete_append(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
+
+    async fn abort_append(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;

    // TransitionObject TODO:
    // RestoreTransitionedObject TODO:
@@ -23,20 +23,23 @@ use crate::error::{
 };
 use crate::set_disk::SetDisks;
 use crate::store::check_list_objs_args;
-use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectOptions};
+use crate::store_api::{
+    ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions, WalkVersionsSortOrder,
+};
 use crate::store_utils::is_reserved_or_invalid_bucket;
 use crate::{store::ECStore, store_api::ListObjectsV2Info};
 use futures::future::join_all;
 use rand::seq::SliceRandom;
 use rustfs_filemeta::{
-    FileInfo, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
+    MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
    merge_file_meta_versions,
 };
 use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
 use std::collections::HashMap;
 use std::sync::Arc;
-use tokio::sync::broadcast::{self, Receiver as B_Receiver};
+use tokio::sync::broadcast::{self};
 use tokio::sync::mpsc::{self, Receiver, Sender};
+use tokio_util::sync::CancellationToken;
 use tracing::{error, info};
 use uuid::Uuid;

@@ -529,14 +532,15 @@ impl ECStore {
        }

        // cancel channel
-        let (cancel_tx, cancel_rx) = broadcast::channel(1);
+        let cancel = CancellationToken::new();
+
        let (err_tx, mut err_rx) = broadcast::channel::<Arc<Error>>(1);

        let (sender, recv) = mpsc::channel(o.limit as usize);

        let store = self.clone();
        let opts = o.clone();
-        let cancel_rx1 = cancel_rx.resubscribe();
+        let cancel_rx1 = cancel.clone();
        let err_tx1 = err_tx.clone();
        let job1 = tokio::spawn(async move {
            let mut opts = opts;
@@ -547,7 +551,7 @@ impl ECStore {
            }
        });

-        let cancel_rx2 = cancel_rx.resubscribe();
+        let cancel_rx2 = cancel.clone();

        let (result_tx, mut result_rx) = mpsc::channel(1);
        let err_tx2 = err_tx.clone();
@@ -559,7 +563,7 @@ impl ECStore {
            }

            // cancel call exit spawns
-            let _ = cancel_tx.send(true);
+            cancel.cancel();
        });

        let mut result = {
@@ -615,7 +619,7 @@ impl ECStore {
    // Read all
    async fn list_merged(
        &self,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        opts: ListPathOptions,
        sender: Sender<MetaCacheEntry>,
    ) -> Result<Vec<ObjectInfo>> {
@@ -631,9 +635,8 @@ impl ECStore {

                inputs.push(recv);
                let opts = opts.clone();
-
-                let rx = rx.resubscribe();
-                futures.push(set.list_path(rx, opts, send));
+                let rx_clone = rx.clone();
+                futures.push(set.list_path(rx_clone, opts, send));
            }
        }

@@ -695,9 +698,9 @@ impl ECStore {
    }

    #[allow(unused_assignments)]
-    pub async fn walk(
+    pub async fn walk_internal(
        self: Arc<Self>,
-        rx: B_Receiver<bool>,
+        rx: CancellationToken,
        bucket: &str,
        prefix: &str,
        result: Sender<ObjectInfoOrErr>,
@@ -711,11 +714,11 @@ impl ECStore {
        for eset in self.pools.iter() {
            for set in eset.disk_set.iter() {
                let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
-                let rx = rx.resubscribe();
                let opts = opts.clone();

                let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
                inputs.push(list_out_rx);
+                let rx_clone = rx.clone();
                futures.push(async move {
                    let mut ask_disks = get_list_quorum(&opts.ask_disks, set.set_drive_count as i32);
                    if ask_disks == -1 {
@@ -770,7 +773,7 @@ impl ECStore {
                    let tx2 = sender.clone();

                    list_path_raw(
-                        rx.resubscribe(),
+                        rx_clone,
                        ListPathRawOptions {
                            disks: disks.iter().cloned().map(Some).collect(),
                            fallback_disks: fallback_disks.iter().cloned().map(Some).collect(),
@@ -936,33 +939,8 @@ impl ECStore {
    }
 }

-type WalkFilter = fn(&FileInfo) -> bool;
-
-#[derive(Clone, Default)]
-pub struct WalkOptions {
-    pub filter: Option<WalkFilter>,           // return WalkFilter returns 'true/false'
-    pub marker: Option<String>,               // set to skip until this object
-    pub latest_only: bool,                    // returns only latest versions for all matching objects
-    pub ask_disks: String,                    // dictates how many disks are being listed
-    pub versions_sort: WalkVersionsSortOrder, // sort order for versions of the same object; default: Ascending order in ModTime
-    pub limit: usize,                         // maximum number of items, 0 means no limit
-}
-
-#[derive(Clone, Default, PartialEq, Eq)]
-pub enum WalkVersionsSortOrder {
-    #[default]
-    Ascending,
-    Descending,
-}
-
-#[derive(Debug)]
-pub struct ObjectInfoOrErr {
-    pub item: Option<ObjectInfo>,
-    pub err: Option<Error>,
-}
-
 async fn gather_results(
-    _rx: B_Receiver<bool>,
+    _rx: CancellationToken,
    opts: ListPathOptions,
    recv: Receiver<MetaCacheEntry>,
    results_tx: Sender<MetaCacheEntriesSortedResult>,
@@ -1067,12 +1045,11 @@ async fn select_from(

 // TODO: exit when cancel
 async fn merge_entry_channels(
-    rx: B_Receiver<bool>,
+    rx: CancellationToken,
    in_channels: Vec<Receiver<MetaCacheEntry>>,
    out_channel: Sender<MetaCacheEntry>,
    read_quorum: usize,
 ) -> Result<()> {
-    let mut rx = rx;
    let mut in_channels = in_channels;
    if in_channels.len() == 1 {
        loop {
@@ -1085,7 +1062,7 @@ async fn merge_entry_channels(
                        return Ok(())
                    }
                },
-                _ = rx.recv()=>{
+                _ = rx.cancelled()=>{
                    info!("merge_entry_channels rx.recv() cancel");
                    return Ok(())
                },
@@ -1228,7 +1205,7 @@ async fn merge_entry_channels(
 }

 impl SetDisks {
-    pub async fn list_path(&self, rx: B_Receiver<bool>, opts: ListPathOptions, sender: Sender<MetaCacheEntry>) -> Result<()> {
+    pub async fn list_path(&self, rx: CancellationToken, opts: ListPathOptions, sender: Sender<MetaCacheEntry>) -> Result<()> {
        let (mut disks, infos, _) = self.get_online_disks_with_healing_and_info(true).await;

        let mut ask_disks = get_list_quorum(&opts.ask_disks, self.set_drive_count as i32);
@@ -15,8 +15,8 @@
 use crate::config::storageclass::STANDARD;
 use crate::disk::RUSTFS_META_BUCKET;
 use regex::Regex;
-use rustfs_filemeta::headers::AMZ_OBJECT_TAGGING;
-use rustfs_filemeta::headers::AMZ_STORAGE_CLASS;
+use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
+use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
 use std::collections::HashMap;
 use std::io::{Error, Result};

@@ -30,12 +30,13 @@ crc32fast = { workspace = true }
 rmp.workspace = true
 rmp-serde.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 time.workspace = true
 uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
 tokio = { workspace = true, features = ["io-util", "macros", "sync"] }
 xxhash-rust = { workspace = true, features = ["xxh64"] }
 bytes.workspace = true
-rustfs-utils = { workspace = true, features = ["hash"] }
+rustfs-utils = { workspace = true, features = ["hash","http"] }
 byteorder = { workspace = true }
 tracing.workspace = true
 thiserror.workspace = true
@@ -0,0 +1,541 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use uuid::Uuid;
+
+const APPEND_STATE_META_KEY: &str = "x-rustfs-internal-append-state";
+
+/// Tracks the state of append-enabled objects.
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct AppendState {
+    pub state: AppendStateKind,
+    pub epoch: u64,
+    pub committed_length: i64,
+    pub pending_segments: Vec<AppendSegment>,
+}
+
+/// Represents individual append segments that still need consolidation.
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct AppendSegment {
+    pub offset: i64,
+    pub length: i64,
+    pub data_dir: Option<Uuid>,
+    pub etag: Option<String>,
+    pub epoch: u64,
+}
+
+/// Possible append lifecycle states for an object version.
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub enum AppendStateKind {
+    #[default]
+    Disabled,
+    Inline,
+    InlinePendingSpill,
+    SegmentedActive,
+    SegmentedSealed,
+}
+
+/// Persist the provided append state into object metadata.
+pub fn set_append_state(metadata: &mut HashMap<String, String>, state: &AppendState) -> Result<()> {
+    let encoded = serde_json::to_string(state).map_err(Error::other)?;
+    metadata.insert(APPEND_STATE_META_KEY.to_string(), encoded);
+    Ok(())
+}
+
+/// Remove the append state marker from metadata.
+pub fn clear_append_state(metadata: &mut HashMap<String, String>) {
+    metadata.remove(APPEND_STATE_META_KEY);
+}
+
+/// Load append state stored in metadata, if any.
+pub fn get_append_state(metadata: &HashMap<String, String>) -> Result<Option<AppendState>> {
+    let raw = match metadata.get(APPEND_STATE_META_KEY) {
+        Some(val) if !val.is_empty() => val,
+        _ => return Ok(None),
+    };
+
+    let decoded = serde_json::from_str(raw).map_err(Error::other)?;
+    Ok(Some(decoded))
+}
+
+/// Complete append operations by consolidating pending segments and sealing the object
+pub fn complete_append_operation(state: &mut AppendState) -> Result<()> {
+    match state.state {
+        AppendStateKind::SegmentedActive => {
+            // Move all pending segments data to main parts and seal
+            state.committed_length += state.pending_segments.iter().map(|s| s.length).sum::<i64>();
+            state.pending_segments.clear();
+            state.state = AppendStateKind::SegmentedSealed;
+            state.epoch = state.epoch.saturating_add(1);
+            Ok(())
+        }
+        AppendStateKind::Inline => {
+            // Inline objects are always immediately committed, just seal them
+            state.state = AppendStateKind::SegmentedSealed; // Transition to sealed
+            state.epoch = state.epoch.saturating_add(1);
+            Ok(())
+        }
+        AppendStateKind::InlinePendingSpill => {
+            // Wait for spill to complete, then seal
+            // In practice, this might need to trigger the spill completion first
+            state.state = AppendStateKind::SegmentedSealed;
+            state.pending_segments.clear();
+            state.epoch = state.epoch.saturating_add(1);
+            Ok(())
+        }
+        AppendStateKind::SegmentedSealed | AppendStateKind::Disabled => {
+            // Already sealed or disabled
+            Err(Error::other("Cannot complete append on sealed or disabled object"))
+        }
+    }
+}
+
+/// Abort append operations by discarding pending segments and returning to sealed state
+pub fn abort_append_operation(state: &mut AppendState) -> Result<()> {
+    match state.state {
+        AppendStateKind::SegmentedActive => {
+            // Discard all pending segments and seal
+            state.pending_segments.clear();
+            state.state = AppendStateKind::SegmentedSealed;
+            state.epoch = state.epoch.saturating_add(1);
+            Ok(())
+        }
+        AppendStateKind::Inline => {
+            // Inline data is already committed, just seal
+            state.state = AppendStateKind::SegmentedSealed;
+            state.epoch = state.epoch.saturating_add(1);
+            Ok(())
+        }
+        AppendStateKind::InlinePendingSpill => {
+            // Cancel spill and keep inline data, then seal
+            state.state = AppendStateKind::SegmentedSealed;
+            state.pending_segments.clear();
+            state.epoch = state.epoch.saturating_add(1);
+            Ok(())
+        }
+        AppendStateKind::SegmentedSealed | AppendStateKind::Disabled => {
+            // Already sealed or disabled
+            Err(Error::other("Cannot abort append on sealed or disabled object"))
+        }
+    }
+}
+
+/// Check if an append operation can be completed
+pub fn can_complete_append(state: &AppendState) -> bool {
+    matches!(
+        state.state,
+        AppendStateKind::Inline | AppendStateKind::InlinePendingSpill | AppendStateKind::SegmentedActive
+    )
+}
+
+/// Check if an append operation can be aborted
+pub fn can_abort_append(state: &AppendState) -> bool {
+    matches!(
+        state.state,
+        AppendStateKind::Inline | AppendStateKind::InlinePendingSpill | AppendStateKind::SegmentedActive
+    )
+}
+
+/// Verify epoch for optimistic concurrency control
+pub fn verify_append_epoch(current_state: &AppendState, expected_epoch: u64) -> Result<()> {
+    if current_state.epoch != expected_epoch {
+        Err(Error::other(format!(
+            "Append operation conflict: expected epoch {}, found {}",
+            expected_epoch, current_state.epoch
+        )))
+    } else {
+        Ok(())
+    }
+}
+
+/// Prepare next append operation by incrementing epoch
+pub fn prepare_next_append(state: &mut AppendState) {
+    state.epoch = state.epoch.saturating_add(1);
+}
+
+/// Validate that a new append segment doesn't conflict with existing segments
+pub fn validate_new_segment(state: &AppendState, new_offset: i64, new_length: i64) -> Result<()> {
+    let new_end = new_offset + new_length;
+
+    // Check it doesn't overlap with committed data
+    if new_offset < state.committed_length {
+        return Err(Error::other(format!(
+            "New segment overlaps with committed data: offset {} < committed_length {}",
+            new_offset, state.committed_length
+        )));
+    }
+
+    // Check it doesn't overlap with existing pending segments
+    for existing in &state.pending_segments {
+        let existing_start = existing.offset;
+        let existing_end = existing.offset + existing.length;
+
+        // Check for any overlap
+        if new_offset < existing_end && new_end > existing_start {
+            return Err(Error::other(format!(
+                "New segment [{}, {}) overlaps with existing segment [{}, {})",
+                new_offset, new_end, existing_start, existing_end
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::fileinfo::FileInfo;
+
+    #[test]
+    fn append_state_roundtrip_in_metadata() {
+        let mut metadata = HashMap::new();
+        let state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 42,
+            committed_length: 2048,
+            pending_segments: vec![AppendSegment {
+                offset: 2048,
+                length: 512,
+                data_dir: Some(Uuid::new_v4()),
+                etag: Some("abc123".to_string()),
+                epoch: 0,
+            }],
+        };
+
+        set_append_state(&mut metadata, &state).expect("persist append state");
+        assert!(metadata.contains_key(APPEND_STATE_META_KEY));
+
+        let decoded = get_append_state(&metadata)
+            .expect("decode append state")
+            .expect("state present");
+        assert_eq!(decoded, state);
+
+        clear_append_state(&mut metadata);
+        assert!(!metadata.contains_key(APPEND_STATE_META_KEY));
+        assert!(get_append_state(&metadata).unwrap().is_none());
+    }
+
+    #[test]
+    fn fileinfo_append_state_migration_compatibility() {
+        // Test old inline data object
+        let mut inline_fi = FileInfo {
+            size: 1024,
+            ..Default::default()
+        };
+        inline_fi.set_inline_data();
+
+        let state = inline_fi.get_append_state();
+        assert_eq!(state.state, AppendStateKind::Inline);
+        assert_eq!(state.committed_length, 1024);
+        assert!(state.pending_segments.is_empty());
+        assert!(inline_fi.is_appendable());
+        assert!(!inline_fi.has_pending_appends());
+
+        // Test old regular object
+        let regular_fi = FileInfo {
+            size: 2048,
+            ..Default::default()
+        };
+        // No inline_data marker
+
+        let state = regular_fi.get_append_state();
+        assert_eq!(state.state, AppendStateKind::SegmentedSealed);
+        assert_eq!(state.committed_length, 2048);
+        assert!(state.pending_segments.is_empty());
+        assert!(!regular_fi.is_appendable());
+        assert!(!regular_fi.has_pending_appends());
+
+        // Test explicit append state
+        let mut append_fi = FileInfo::default();
+        let explicit_state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 5,
+            committed_length: 1500,
+            pending_segments: vec![AppendSegment {
+                offset: 1500,
+                length: 300,
+                data_dir: Some(Uuid::new_v4()),
+                etag: Some("def456".to_string()),
+                epoch: 0,
+            }],
+        };
+
+        append_fi.set_append_state(&explicit_state).expect("set explicit state");
+        let retrieved_state = append_fi.get_append_state();
+        assert_eq!(retrieved_state, explicit_state);
+        assert!(append_fi.is_appendable());
+        assert!(append_fi.has_pending_appends());
+    }
+
+    #[test]
+    fn append_state_transitions() {
+        // Test state transition validation
+        assert_eq!(AppendStateKind::default(), AppendStateKind::Disabled);
+
+        let inline_state = AppendState {
+            state: AppendStateKind::Inline,
+            ..Default::default()
+        };
+
+        let spill_state = AppendState {
+            state: AppendStateKind::InlinePendingSpill,
+            ..Default::default()
+        };
+
+        let active_state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            ..Default::default()
+        };
+
+        let sealed_state = AppendState {
+            state: AppendStateKind::SegmentedSealed,
+            ..Default::default()
+        };
+
+        // Verify serialization works for all states
+        for state in [inline_state, spill_state, active_state, sealed_state] {
+            let mut metadata = HashMap::new();
+            set_append_state(&mut metadata, &state).expect("serialize state");
+            let decoded = get_append_state(&metadata).unwrap().unwrap();
+            assert_eq!(decoded, state);
+        }
+    }
+
+    #[test]
+    fn complete_append_transitions() {
+        // Test completing SegmentedActive with pending segments
+        let mut active_state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 5,
+            committed_length: 1000,
+            pending_segments: vec![
+                AppendSegment {
+                    offset: 1000,
+                    length: 200,
+                    data_dir: Some(Uuid::new_v4()),
+                    etag: Some("abc123".to_string()),
+                    epoch: 0,
+                },
+                AppendSegment {
+                    offset: 1200,
+                    length: 300,
+                    data_dir: Some(Uuid::new_v4()),
+                    etag: Some("def456".to_string()),
+                    epoch: 0,
+                },
+            ],
+        };
+
+        assert!(can_complete_append(&active_state));
+        complete_append_operation(&mut active_state).expect("complete should succeed");
+
+        assert_eq!(active_state.state, AppendStateKind::SegmentedSealed);
+        assert_eq!(active_state.committed_length, 1500); // 1000 + 200 + 300
+        assert!(active_state.pending_segments.is_empty());
+        assert_eq!(active_state.epoch, 6);
+
+        // Test completing Inline state
+        let mut inline_state = AppendState {
+            state: AppendStateKind::Inline,
+            epoch: 2,
+            committed_length: 500,
+            ..Default::default()
+        };
+
+        assert!(can_complete_append(&inline_state));
+        complete_append_operation(&mut inline_state).expect("complete should succeed");
+
+        assert_eq!(inline_state.state, AppendStateKind::SegmentedSealed);
+        assert_eq!(inline_state.committed_length, 500); // Unchanged
+        assert_eq!(inline_state.epoch, 3);
+
+        // Test completing already sealed state should fail
+        let mut sealed_state = AppendState {
+            state: AppendStateKind::SegmentedSealed,
+            ..Default::default()
+        };
+
+        assert!(!can_complete_append(&sealed_state));
+        assert!(complete_append_operation(&mut sealed_state).is_err());
+    }
+
+    #[test]
+    fn abort_append_transitions() {
+        // Test aborting SegmentedActive with pending segments
+        let mut active_state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 3,
+            committed_length: 800,
+            pending_segments: vec![AppendSegment {
+                offset: 800,
+                length: 400,
+                data_dir: Some(Uuid::new_v4()),
+                etag: Some("xyz789".to_string()),
+                epoch: 0,
+            }],
+        };
+
+        assert!(can_abort_append(&active_state));
+        abort_append_operation(&mut active_state).expect("abort should succeed");
+
+        assert_eq!(active_state.state, AppendStateKind::SegmentedSealed);
+        assert_eq!(active_state.committed_length, 800); // Unchanged, pending discarded
+        assert!(active_state.pending_segments.is_empty());
+        assert_eq!(active_state.epoch, 4);
+
+        // Test aborting InlinePendingSpill
+        let mut spill_state = AppendState {
+            state: AppendStateKind::InlinePendingSpill,
+            epoch: 1,
+            committed_length: 100,
+            pending_segments: vec![],
+        };
+
+        assert!(can_abort_append(&spill_state));
+        abort_append_operation(&mut spill_state).expect("abort should succeed");
+
+        assert_eq!(spill_state.state, AppendStateKind::SegmentedSealed);
+        assert_eq!(spill_state.committed_length, 100);
+        assert_eq!(spill_state.epoch, 2);
+
+        // Test aborting disabled state should fail
+        let mut disabled_state = AppendState {
+            state: AppendStateKind::Disabled,
+            ..Default::default()
+        };
+
+        assert!(!can_abort_append(&disabled_state));
+        assert!(abort_append_operation(&mut disabled_state).is_err());
+    }
+
+    #[test]
+    fn epoch_validation() {
+        let state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 10,
+            committed_length: 1000,
+            pending_segments: vec![],
+        };
+
+        // Valid epoch should succeed
+        assert!(verify_append_epoch(&state, 10).is_ok());
+
+        // Invalid epoch should fail
+        assert!(verify_append_epoch(&state, 9).is_err());
+        assert!(verify_append_epoch(&state, 11).is_err());
+
+        // Error message should contain epoch information
+        let error = verify_append_epoch(&state, 5).unwrap_err();
+        let error_msg = error.to_string();
+        assert!(error_msg.contains("expected epoch 5"));
+        assert!(error_msg.contains("found 10"));
+    }
+
+    #[test]
+    fn next_append_preparation() {
+        let mut state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 5,
+            committed_length: 1000,
+            pending_segments: vec![],
+        };
+
+        prepare_next_append(&mut state);
+        assert_eq!(state.epoch, 6);
+
+        // Test saturation behavior
+        let mut max_state = AppendState {
+            epoch: u64::MAX,
+            ..Default::default()
+        };
+
+        prepare_next_append(&mut max_state);
+        assert_eq!(max_state.epoch, u64::MAX); // Should saturate, not overflow
+    }
+
+    #[test]
+    fn segment_validation() {
+        let state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 3,
+            committed_length: 1000,
+            pending_segments: vec![
+                AppendSegment {
+                    offset: 1000,
+                    length: 200,
+                    data_dir: Some(Uuid::new_v4()),
+                    etag: Some("abc123".to_string()),
+                    epoch: 0,
+                },
+                AppendSegment {
+                    offset: 1300,
+                    length: 300,
+                    data_dir: Some(Uuid::new_v4()),
+                    etag: Some("def456".to_string()),
+                    epoch: 0,
+                },
+            ],
+        };
+
+        // Valid segment after existing segments
+        assert!(validate_new_segment(&state, 1600, 100).is_ok());
+
+        // Valid segment filling gap between committed and first pending
+        assert!(validate_new_segment(&state, 1200, 100).is_ok());
+
+        // Invalid segment overlapping with committed data
+        assert!(validate_new_segment(&state, 900, 200).is_err());
+        let error = validate_new_segment(&state, 900, 200).unwrap_err();
+        assert!(error.to_string().contains("overlaps with committed data"));
+
+        // Invalid segment overlapping with first pending segment
+        assert!(validate_new_segment(&state, 1100, 100).is_err());
+        let error = validate_new_segment(&state, 1100, 100).unwrap_err();
+        assert!(error.to_string().contains("overlaps with existing segment"));
+
+        // Invalid segment overlapping with second pending segment
+        assert!(validate_new_segment(&state, 1400, 100).is_err());
+
+        // Edge case: segment exactly touching committed data (should be valid)
+        assert!(validate_new_segment(&state, 1000, 0).is_ok());
+
+        // Edge case: segment exactly touching existing segment (should be valid)
+        assert!(validate_new_segment(&state, 1200, 0).is_ok());
+    }
+
+    #[test]
+    fn segment_validation_edge_cases() {
+        let empty_state = AppendState {
+            state: AppendStateKind::SegmentedActive,
+            epoch: 1,
+            committed_length: 500,
+            pending_segments: vec![],
+        };
+
+        // First segment after committed data
+        assert!(validate_new_segment(&empty_state, 500, 100).is_ok());
+        assert!(validate_new_segment(&empty_state, 600, 200).is_ok());
+
+        // Zero-length segments (edge case)
+        assert!(validate_new_segment(&empty_state, 500, 0).is_ok());
+
+        // Segment exactly at committed boundary
+        assert!(validate_new_segment(&empty_state, 499, 1).is_err());
+        assert!(validate_new_segment(&empty_state, 500, 1).is_ok());
+    }
+}
@@ -13,11 +13,11 @@
 // limitations under the License.

 use crate::error::{Error, Result};
-use crate::headers::RESERVED_METADATA_PREFIX_LOWER;
-use crate::headers::RUSTFS_HEALING;
+use crate::{ReplicationState, ReplicationStatusType, VersionPurgeStatusType};
 use bytes::Bytes;
 use rmp_serde::Serializer;
 use rustfs_utils::HashAlgorithm;
+use rustfs_utils::http::headers::{RESERVED_METADATA_PREFIX_LOWER, RUSTFS_HEALING};
 use serde::Deserialize;
 use serde::Serialize;
 use std::collections::HashMap;
@@ -202,7 +202,7 @@ pub struct FileInfo {
    // MarkDeleted marks this version as deleted
    pub mark_deleted: bool,
    // ReplicationState - Internal replication state to be passed back in ObjectInfo
-    // pub replication_state: Option<ReplicationState>, // TODO: implement ReplicationState
+    pub replication_state_internal: Option<ReplicationState>,
    pub data: Option<Bytes>,
    pub num_versions: usize,
    pub successor_mod_time: Option<OffsetDateTime>,
@@ -471,6 +471,119 @@ impl FileInfo {
        // TODO: Add replication_state comparison when implemented
        // && self.replication_state == other.replication_state
    }
+
+    pub fn version_purge_status(&self) -> VersionPurgeStatusType {
+        self.replication_state_internal
+            .as_ref()
+            .map(|v| v.composite_version_purge_status())
+            .unwrap_or(VersionPurgeStatusType::Empty)
+    }
+    pub fn replication_status(&self) -> ReplicationStatusType {
+        self.replication_state_internal
+            .as_ref()
+            .map(|v| v.composite_replication_status())
+            .unwrap_or(ReplicationStatusType::Empty)
+    }
+    pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
+        if self.deleted {
+            self.replication_state_internal
+                .as_ref()
+                .map(|v| v.composite_replication_status())
+                .unwrap_or(ReplicationStatusType::Empty)
+        } else {
+            ReplicationStatusType::Empty
+        }
+    }
+    /// Get the append state for this FileInfo, with migration compatibility
+    pub fn get_append_state(&self) -> crate::append::AppendState {
+        use crate::append::{AppendState, AppendStateKind, get_append_state};
+
+        // Try to load from metadata first
+        if let Ok(Some(state)) = get_append_state(&self.metadata) {
+            return state;
+        }
+
+        // Migration compatibility: determine state based on existing data
+        if self.inline_data() {
+            // Has inline data, treat as Inline state
+            AppendState {
+                state: AppendStateKind::Inline,
+                epoch: 0,
+                committed_length: self.size,
+                pending_segments: Vec::new(),
+            }
+        } else {
+            // No inline data, treat as SegmentedSealed (traditional object)
+            AppendState {
+                state: AppendStateKind::SegmentedSealed,
+                epoch: 0,
+                committed_length: self.size,
+                pending_segments: Vec::new(),
+            }
+        }
+    }
+
+    /// Set the append state for this FileInfo
+    pub fn set_append_state(&mut self, state: &crate::append::AppendState) -> crate::error::Result<()> {
+        crate::append::set_append_state(&mut self.metadata, state)
+    }
+
+    /// Check if this object supports append operations
+    pub fn is_appendable(&self) -> bool {
+        use crate::append::AppendStateKind;
+        match self.get_append_state().state {
+            AppendStateKind::Disabled => false,
+            AppendStateKind::Inline | AppendStateKind::InlinePendingSpill | AppendStateKind::SegmentedActive => true,
+            AppendStateKind::SegmentedSealed => false,
+        }
+    }
+
+    /// Check if this object has pending append operations
+    pub fn has_pending_appends(&self) -> bool {
+        use crate::append::AppendStateKind;
+        matches!(
+            self.get_append_state().state,
+            AppendStateKind::InlinePendingSpill | AppendStateKind::SegmentedActive
+        )
+    }
+
+    /// Complete all pending append operations and seal the object
+    pub fn complete_append(&mut self) -> crate::error::Result<()> {
+        let mut append_state = self.get_append_state();
+        crate::append::complete_append_operation(&mut append_state)?;
+        self.set_append_state(&append_state)?;
+
+        // Update file size to reflect completed operation
+        if append_state.state == crate::append::AppendStateKind::SegmentedSealed {
+            self.size = append_state.committed_length;
+        }
+
+        Ok(())
+    }
+
+    /// Abort all pending append operations and seal the object
+    pub fn abort_append(&mut self) -> crate::error::Result<()> {
+        let mut append_state = self.get_append_state();
+        crate::append::abort_append_operation(&mut append_state)?;
+        self.set_append_state(&append_state)?;
+
+        // Update file size to only include committed data
+        if append_state.state == crate::append::AppendStateKind::SegmentedSealed {
+            self.size = append_state.committed_length;
+        }
+
+        Ok(())
+    }
+
+    /// Check if append operations can be completed for this object
+    pub fn can_complete_append(&self) -> bool {
+        crate::append::can_complete_append(&self.get_append_state())
+    }
+
+    /// Check if append operations can be aborted for this object
+    pub fn can_abort_append(&self) -> bool {
+        crate::append::can_abort_append(&self.get_append_state())
+    }
 }

 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -15,12 +15,13 @@
 use crate::error::{Error, Result};
 use crate::fileinfo::{ErasureAlgo, ErasureInfo, FileInfo, FileInfoVersions, ObjectPartInfo, RawFileInfo};
 use crate::filemeta_inline::InlineData;
-use crate::headers::{
+use crate::{ReplicationStatusType, VersionPurgeStatusType};
+use byteorder::ByteOrder;
+use bytes::Bytes;
+use rustfs_utils::http::headers::{
    self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX,
    RESERVED_METADATA_PREFIX_LOWER, VERSION_PURGE_STATUS_KEY,
 };
-use byteorder::ByteOrder;
-use bytes::Bytes;
 use s3s::header::X_AMZ_RESTORE;
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
@@ -30,6 +31,7 @@ use std::io::{Read, Write};
 use std::{collections::HashMap, io::Cursor};
 use time::OffsetDateTime;
 use tokio::io::AsyncRead;
+use tracing::error;
 use uuid::Uuid;
 use xxhash_rust::xxh64;

@@ -159,39 +161,57 @@ impl FileMeta {
        let i = buf.len() as u64;

        // check version, buf = buf[8..]
-        let (buf, _, _) = Self::check_xl2_v1(buf)?;
+        let (buf, _, _) = Self::check_xl2_v1(buf).map_err(|e| {
+            error!("failed to check XL2 v1 format: {}", e);
+            e
+        })?;

        let (mut size_buf, buf) = buf.split_at(5);

        // Get meta data, buf = crc + data
-        let bin_len = rmp::decode::read_bin_len(&mut size_buf)?;
+        let bin_len = rmp::decode::read_bin_len(&mut size_buf).map_err(|e| {
+            error!("failed to read binary length for metadata: {}", e);
+            Error::other(format!("failed to read binary length for metadata: {e}"))
+        })?;

        if buf.len() < bin_len as usize {
+            error!("insufficient data for metadata: expected {} bytes, got {} bytes", bin_len, buf.len());
            return Err(Error::other("insufficient data for metadata"));
        }
        let (meta, buf) = buf.split_at(bin_len as usize);

        if buf.len() < 5 {
+            error!("insufficient data for CRC: expected 5 bytes, got {} bytes", buf.len());
            return Err(Error::other("insufficient data for CRC"));
        }
        let (mut crc_buf, buf) = buf.split_at(5);

        // crc check
-        let crc = rmp::decode::read_u32(&mut crc_buf)?;
+        let crc = rmp::decode::read_u32(&mut crc_buf).map_err(|e| {
+            error!("failed to read CRC value: {}", e);
+            Error::other(format!("failed to read CRC value: {e}"))
+        })?;
        let meta_crc = xxh64::xxh64(meta, XXHASH_SEED) as u32;

        if crc != meta_crc {
+            error!("xl file crc check failed: expected CRC {:#x}, got {:#x}", meta_crc, crc);
            return Err(Error::other("xl file crc check failed"));
        }

        if !buf.is_empty() {
            self.data.update(buf);
-            self.data.validate()?;
+            self.data.validate().map_err(|e| {
+                error!("data validation failed: {}", e);
+                e
+            })?;
        }

        // Parse meta
        if !meta.is_empty() {
-            let (versions_len, _, meta_ver, meta) = Self::decode_xl_headers(meta)?;
+            let (versions_len, _, meta_ver, meta) = Self::decode_xl_headers(meta).map_err(|e| {
+                error!("failed to decode XL headers: {}", e);
+                e
+            })?;

            // let (_, meta) = meta.split_at(read_size as usize);

@@ -201,24 +221,30 @@ impl FileMeta {

            let mut cur: Cursor<&[u8]> = Cursor::new(meta);
            for _ in 0..versions_len {
-                let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
-                let start = cur.position() as usize;
-                let end = start + bin_len;
-                let header_buf = &meta[start..end];
+                let bin_len = rmp::decode::read_bin_len(&mut cur).map_err(|e| {
+                    error!("failed to read binary length for version header: {}", e);
+                    Error::other(format!("failed to read binary length for version header: {e}"))
+                })? as usize;
+
+                let mut header_buf = vec![0u8; bin_len];
+
+                cur.read_exact(&mut header_buf)?;

                let mut ver = FileMetaShallowVersion::default();
-                ver.header.unmarshal_msg(header_buf)?;
+                ver.header.unmarshal_msg(&header_buf).map_err(|e| {
+                    error!("failed to unmarshal version header: {}", e);
+                    e
+                })?;

-                cur.set_position(end as u64);
+                let bin_len = rmp::decode::read_bin_len(&mut cur).map_err(|e| {
+                    error!("failed to read binary length for version metadata: {}", e);
+                    Error::other(format!("failed to read binary length for version metadata: {e}"))
+                })? as usize;

-                let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
-                let start = cur.position() as usize;
-                let end = start + bin_len;
-                let ver_meta_buf = &meta[start..end];
+                let mut ver_meta_buf = vec![0u8; bin_len];
+                cur.read_exact(&mut ver_meta_buf)?;

-                ver.meta.extend_from_slice(ver_meta_buf);
-
-                cur.set_position(end as u64);
+                ver.meta.extend_from_slice(&ver_meta_buf);

                self.versions.push(ver);
            }
@@ -487,39 +513,39 @@ impl FileMeta {

        let version = FileMetaVersion::from(fi);

+        self.add_version_filemata(version)
+    }
+
+    pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> {
        if !version.valid() {
            return Err(Error::other("file meta version invalid"));
        }

-        // should replace
-        for (idx, ver) in self.versions.iter().enumerate() {
-            if ver.header.version_id != vid {
-                continue;
-            }
-
-            return self.set_idx(idx, version);
+        // 1000 is the limit of versions TODO: make it configurable
+        if self.versions.len() + 1 > 1000 {
+            return Err(Error::other(
+                "You've exceeded the limit on the number of versions you can create on this object",
+            ));
        }

-        // TODO: version count limit !
+        if self.versions.is_empty() {
+            self.versions.push(FileMetaShallowVersion::try_from(version)?);
+            return Ok(());
+        }
+
+        let vid = version.get_version_id();
+
+        if let Some(fidx) = self.versions.iter().position(|v| v.header.version_id == vid) {
+            return self.set_idx(fidx, version);
+        }

        let mod_time = version.get_mod_time();

-        // puth a -1 mod time value , so we can relplace this
-        self.versions.push(FileMetaShallowVersion {
-            header: FileMetaVersionHeader {
-                mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?),
-                ..Default::default()
-            },
-            ..Default::default()
-        });
-
        for (idx, exist) in self.versions.iter().enumerate() {
            if let Some(ref ex_mt) = exist.header.mod_time {
                if let Some(ref in_md) = mod_time {
                    if ex_mt <= in_md {
-                        // insert
                        self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?);
-                        self.versions.pop();
                        return Ok(());
                    }
                }
@@ -527,35 +553,33 @@ impl FileMeta {
        }

        Err(Error::other("add_version failed"))
-    }

-    pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> {
-        if !ver.valid() {
-            return Err(Error::other("attempted to add invalid version"));
-        }
+        // if !ver.valid() {
+        //     return Err(Error::other("attempted to add invalid version"));
+        // }

-        if self.versions.len() + 1 >= 100 {
-            return Err(Error::other(
-                "You've exceeded the limit on the number of versions you can create on this object",
-            ));
-        }
+        // if self.versions.len() + 1 >= 100 {
+        //     return Err(Error::other(
+        //         "You've exceeded the limit on the number of versions you can create on this object",
+        //     ));
+        // }

-        let mod_time = ver.get_mod_time();
-        let encoded = ver.marshal_msg()?;
-        let new_version = FileMetaShallowVersion {
-            header: ver.header(),
-            meta: encoded,
-        };
+        // let mod_time = ver.get_mod_time();
+        // let encoded = ver.marshal_msg()?;
+        // let new_version = FileMetaShallowVersion {
+        //     header: ver.header(),
+        //     meta: encoded,
+        // };

-        // Find the insertion position: insert before the first element with mod_time >= new mod_time
-        // This maintains descending order by mod_time (newest first)
-        let insert_pos = self
-            .versions
-            .iter()
-            .position(|existing| existing.header.mod_time <= mod_time)
-            .unwrap_or(self.versions.len());
-        self.versions.insert(insert_pos, new_version);
-        Ok(())
+        // // Find the insertion position: insert before the first element with mod_time >= new mod_time
+        // // This maintains descending order by mod_time (newest first)
+        // let insert_pos = self
+        //     .versions
+        //     .iter()
+        //     .position(|existing| existing.header.mod_time <= mod_time)
+        //     .unwrap_or(self.versions.len());
+        // self.versions.insert(insert_pos, new_version);
+        // Ok(())
    }

    // delete_version deletes version, returns data_dir
@@ -575,10 +599,97 @@ impl FileMeta {
        }

        let mut update_version = fi.mark_deleted;
-        /*if fi.version_purge_status().is_empty()
+        if fi.version_purge_status().is_empty()
+            && (fi.delete_marker_replication_status() == ReplicationStatusType::Replica
+                || fi.delete_marker_replication_status() == ReplicationStatusType::Empty)
        {
            update_version = fi.mark_deleted;
-        }*/
+        } else {
+            if fi.deleted
+                && fi.version_purge_status() != VersionPurgeStatusType::Complete
+                && (!fi.version_purge_status().is_empty() || fi.delete_marker_replication_status().is_empty())
+            {
+                update_version = true;
+            }
+
+            if !fi.version_purge_status().is_empty() && fi.version_purge_status() != VersionPurgeStatusType::Complete {
+                update_version = true;
+            }
+        }
+
+        if fi.deleted {
+            if !fi.delete_marker_replication_status().is_empty() {
+                if let Some(delete_marker) = ventry.delete_marker.as_mut() {
+                    if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
+                        delete_marker.meta_sys.insert(
+                            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
+                            fi.replication_state_internal
+                                .as_ref()
+                                .map(|v| v.replica_status.clone())
+                                .unwrap_or_default()
+                                .as_str()
+                                .as_bytes()
+                                .to_vec(),
+                        );
+                        delete_marker.meta_sys.insert(
+                            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"),
+                            fi.replication_state_internal
+                                .as_ref()
+                                .map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
+                                .unwrap_or_default()
+                                .as_bytes()
+                                .to_vec(),
+                        );
+                    } else {
+                        delete_marker.meta_sys.insert(
+                            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"),
+                            fi.replication_state_internal
+                                .as_ref()
+                                .map(|v| v.replication_status_internal.clone().unwrap_or_default())
+                                .unwrap_or_default()
+                                .as_bytes()
+                                .to_vec(),
+                        );
+                        delete_marker.meta_sys.insert(
+                            format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"),
+                            fi.replication_state_internal
+                                .as_ref()
+                                .map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
+                                .unwrap_or_default()
+                                .as_bytes()
+                                .to_vec(),
+                        );
+                    }
+                }
+            }
+
+            if !fi.version_purge_status().is_empty() {
+                if let Some(delete_marker) = ventry.delete_marker.as_mut() {
+                    delete_marker.meta_sys.insert(
+                        VERSION_PURGE_STATUS_KEY.to_string(),
+                        fi.replication_state_internal
+                            .as_ref()
+                            .map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
+                            .unwrap_or_default()
+                            .as_bytes()
+                            .to_vec(),
+                    );
+                }
+            }
+
+            if let Some(delete_marker) = ventry.delete_marker.as_mut() {
+                for (k, v) in fi
+                    .replication_state_internal
+                    .as_ref()
+                    .map(|v| v.reset_statuses_map.clone())
+                    .unwrap_or_default()
+                {
+                    delete_marker.meta_sys.insert(k.clone(), v.clone().as_bytes().to_vec());
+                }
+            }
+        }
+
+        // ???
        if fi.transition_status == TRANSITION_COMPLETE {
            update_version = false;
        }
@@ -591,22 +702,111 @@ impl FileMeta {
            match ver.header.version_type {
                VersionType::Invalid | VersionType::Legacy => return Err(Error::other("invalid file meta version")),
                VersionType::Delete => {
-                    self.versions.remove(i);
-                    if fi.deleted && fi.version_id.is_none() {
-                        self.add_version_filemata(ventry)?;
+                    if update_version {
+                        let mut v = self.get_idx(i)?;
+                        if v.delete_marker.is_none() {
+                            v.delete_marker = Some(MetaDeleteMarker {
+                                version_id: fi.version_id,
+                                mod_time: fi.mod_time,
+                                meta_sys: HashMap::new(),
+                            });
+                        }
+
+                        if let Some(delete_marker) = v.delete_marker.as_mut() {
+                            if !fi.delete_marker_replication_status().is_empty() {
+                                if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
+                                    delete_marker.meta_sys.insert(
+                                        format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
+                                        fi.replication_state_internal
+                                            .as_ref()
+                                            .map(|v| v.replica_status.clone())
+                                            .unwrap_or_default()
+                                            .as_str()
+                                            .as_bytes()
+                                            .to_vec(),
+                                    );
+                                    delete_marker.meta_sys.insert(
+                                        format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"),
+                                        fi.replication_state_internal
+                                            .as_ref()
+                                            .map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
+                                            .unwrap_or_default()
+                                            .as_bytes()
+                                            .to_vec(),
+                                    );
+                                } else {
+                                    delete_marker.meta_sys.insert(
+                                        format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"),
+                                        fi.replication_state_internal
+                                            .as_ref()
+                                            .map(|v| v.replication_status_internal.clone().unwrap_or_default())
+                                            .unwrap_or_default()
+                                            .as_bytes()
+                                            .to_vec(),
+                                    );
+                                    delete_marker.meta_sys.insert(
+                                        format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"),
+                                        fi.replication_state_internal
+                                            .as_ref()
+                                            .map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
+                                            .unwrap_or_default()
+                                            .as_bytes()
+                                            .to_vec(),
+                                    );
+                                }
+                            }
+
+                            for (k, v) in fi
+                                .replication_state_internal
+                                .as_ref()
+                                .map(|v| v.reset_statuses_map.clone())
+                                .unwrap_or_default()
+                            {
+                                delete_marker.meta_sys.insert(k.clone(), v.clone().as_bytes().to_vec());
+                            }
+                        }
+
+                        self.set_idx(i, v)?;
                        return Ok(None);
                    }
+                    self.versions.remove(i);
+
+                    if (fi.mark_deleted && fi.version_purge_status() != VersionPurgeStatusType::Complete)
+                        || (fi.deleted && fi.version_id.is_none())
+                    {
+                        self.add_version_filemata(ventry)?;
+                    }

                    return Ok(None);
                }
                VersionType::Object => {
                    if update_version && !fi.deleted {
-                        let v = self.get_idx(i)?;
+                        let mut v = self.get_idx(i)?;

-                        self.versions.remove(i);
+                        if let Some(obj) = v.object.as_mut() {
+                            obj.meta_sys.insert(
+                                VERSION_PURGE_STATUS_KEY.to_string(),
+                                fi.replication_state_internal
+                                    .as_ref()
+                                    .map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
+                                    .unwrap_or_default()
+                                    .as_bytes()
+                                    .to_vec(),
+                            );
+                            for (k, v) in fi
+                                .replication_state_internal
+                                .as_ref()
+                                .map(|v| v.reset_statuses_map.clone())
+                                .unwrap_or_default()
+                            {
+                                obj.meta_sys.insert(k.clone(), v.clone().as_bytes().to_vec());
+                            }
+                        }

-                        let a = v.object.map(|v| v.data_dir).unwrap_or_default();
-                        return Ok(a);
+                        let old_dir = v.object.as_ref().map(|v| v.data_dir).unwrap_or_default();
+                        self.set_idx(i, v)?;
+
+                        return Ok(old_dir);
                    }
                }
            }
@@ -641,31 +841,37 @@ impl FileMeta {
        let obj_version_id = obj.version_id;
        let obj_data_dir = obj.data_dir;

-        if fi.expire_restored {
+        let mut err = if fi.expire_restored {
            obj.remove_restore_hdrs();
-            self.set_idx(i, ver)?;
+            self.set_idx(i, ver).err()
        } else if fi.transition_status == TRANSITION_COMPLETE {
            obj.set_transition(fi);
            obj.reset_inline_data();
-            self.set_idx(i, ver)?;
+            self.set_idx(i, ver).err()
        } else {
            self.versions.remove(i);

            let (free_version, to_free) = obj.init_free_version(fi);

            if to_free {
-                self.add_version_filemata(free_version)?;
+                self.add_version_filemata(free_version).err()
+            } else {
+                None
            }
-        }
+        };

        if fi.deleted {
-            self.add_version_filemata(ventry)?;
+            err = self.add_version_filemata(ventry).err();
        }

        if self.shared_data_dir_count(obj_version_id, obj_data_dir) > 0 {
            return Ok(None);
        }

+        if let Some(e) = err {
+            return Err(e);
+        }
+
        Ok(obj_data_dir)
    }

@@ -1642,17 +1848,15 @@ impl MetaObject {
                free_entry.delete_marker = Some(MetaDeleteMarker {
                    version_id: Some(vid),
                    mod_time: self.mod_time,
-                    meta_sys: Some(HashMap::<String, Vec<u8>>::new()),
+                    meta_sys: HashMap::<String, Vec<u8>>::new(),
                });

-                free_entry
-                    .delete_marker
-                    .as_mut()
-                    .unwrap()
+                let delete_marker = free_entry.delete_marker.as_mut().unwrap();
+
+                delete_marker
                    .meta_sys
-                    .as_mut()
-                    .unwrap()
                    .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{FREE_VERSION}"), vec![]);
+
                let tier_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}");
                let tier_obj_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}");
                let tier_obj_vid_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}");
@@ -1660,14 +1864,7 @@ impl MetaObject {
                let aa = [tier_key, tier_obj_key, tier_obj_vid_key];
                for (k, v) in &self.meta_sys {
                    if aa.contains(k) {
-                        free_entry
-                            .delete_marker
-                            .as_mut()
-                            .unwrap()
-                            .meta_sys
-                            .as_mut()
-                            .unwrap()
-                            .insert(k.clone(), v.clone());
+                        delete_marker.meta_sys.insert(k.clone(), v.clone());
                    }
                }
                return (free_entry, true);
@@ -1737,19 +1934,16 @@ pub struct MetaDeleteMarker {
    #[serde(rename = "MTime")]
    pub mod_time: Option<OffsetDateTime>, // Object delete marker modified time
    #[serde(rename = "MetaSys")]
-    pub meta_sys: Option<HashMap<String, Vec<u8>>>, // Delete marker internal metadata
+    pub meta_sys: HashMap<String, Vec<u8>>, // Delete marker internal metadata
 }

 impl MetaDeleteMarker {
    pub fn free_version(&self) -> bool {
-        self.meta_sys
-            .as_ref()
-            .map(|v| v.get(FREE_VERSION_META_HEADER).is_some())
-            .unwrap_or_default()
+        self.meta_sys.contains_key(FREE_VERSION_META_HEADER)
    }

    pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo {
-        let metadata = self.meta_sys.clone().unwrap_or_default();
+        let metadata = self.meta_sys.clone();

        FileInfo {
            version_id: self.version_id.filter(|&vid| !vid.is_nil()),
@@ -1895,7 +2089,7 @@ impl From<FileInfo> for MetaDeleteMarker {
        Self {
            version_id: value.version_id,
            mod_time: value.mod_time,
-            meta_sys: None,
+            meta_sys: HashMap::new(),
        }
    }
 }
@@ -2794,7 +2988,7 @@ mod test {
            let delete_marker = MetaDeleteMarker {
                version_id: Some(Uuid::new_v4()),
                mod_time: Some(OffsetDateTime::now_utc()),
-                meta_sys: None,
+                meta_sys: HashMap::new(),
            };

            let delete_version = FileMetaVersion {
@@ -12,17 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+mod append;
 mod error;
 pub mod fileinfo;
 mod filemeta;
 mod filemeta_inline;
-pub mod headers;
-pub mod metacache;
+// pub mod headers;
+mod metacache;
+mod replication;

 pub mod test_data;

+pub use append::*;
 pub use error::*;
 pub use fileinfo::*;
 pub use filemeta::*;
 pub use filemeta_inline::*;
 pub use metacache::*;
+pub use replication::*;
@@ -0,0 +1,494 @@
+use core::fmt;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::time::Duration;
+use time::OffsetDateTime;
+
+/// StatusType of Replication for x-amz-replication-status header
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Hash)]
+pub enum ReplicationStatusType {
+    /// Pending - replication is pending.
+    Pending,
+    /// Completed - replication completed ok.
+    Completed,
+    /// CompletedLegacy was called "COMPLETE" incorrectly.
+    CompletedLegacy,
+    /// Failed - replication failed.
+    Failed,
+    /// Replica - this is a replica.
+    Replica,
+    #[default]
+    Empty,
+}
+
+impl ReplicationStatusType {
+    /// Returns string representation of status
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReplicationStatusType::Pending => "PENDING",
+            ReplicationStatusType::Completed => "COMPLETED",
+            ReplicationStatusType::CompletedLegacy => "COMPLETE",
+            ReplicationStatusType::Failed => "FAILED",
+            ReplicationStatusType::Replica => "REPLICA",
+            ReplicationStatusType::Empty => "",
+        }
+    }
+    pub fn is_empty(&self) -> bool {
+        matches!(self, ReplicationStatusType::Empty)
+    }
+}
+
+impl fmt::Display for ReplicationStatusType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+
+impl From<&str> for ReplicationStatusType {
+    fn from(s: &str) -> Self {
+        match s {
+            "PENDING" => ReplicationStatusType::Pending,
+            "COMPLETED" => ReplicationStatusType::Completed,
+            "COMPLETE" => ReplicationStatusType::CompletedLegacy,
+            "FAILED" => ReplicationStatusType::Failed,
+            "REPLICA" => ReplicationStatusType::Replica,
+            _ => ReplicationStatusType::Empty,
+        }
+    }
+}
+
+impl From<VersionPurgeStatusType> for ReplicationStatusType {
+    fn from(status: VersionPurgeStatusType) -> Self {
+        match status {
+            VersionPurgeStatusType::Pending => ReplicationStatusType::Pending,
+            VersionPurgeStatusType::Complete => ReplicationStatusType::Completed,
+            VersionPurgeStatusType::Failed => ReplicationStatusType::Failed,
+            VersionPurgeStatusType::Empty => ReplicationStatusType::Empty,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub enum VersionPurgeStatusType {
+    Pending,
+    Complete,
+    Failed,
+    #[default]
+    Empty,
+}
+
+impl VersionPurgeStatusType {
+    /// Returns string representation of version purge status
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            VersionPurgeStatusType::Pending => "PENDING",
+            VersionPurgeStatusType::Complete => "COMPLETE",
+            VersionPurgeStatusType::Failed => "FAILED",
+            VersionPurgeStatusType::Empty => "",
+        }
+    }
+
+    /// Returns true if the version is pending purge.
+    pub fn is_pending(&self) -> bool {
+        matches!(self, VersionPurgeStatusType::Pending | VersionPurgeStatusType::Failed)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        matches!(self, VersionPurgeStatusType::Empty)
+    }
+}
+
+impl fmt::Display for VersionPurgeStatusType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+
+impl From<&str> for VersionPurgeStatusType {
+    fn from(s: &str) -> Self {
+        match s {
+            "PENDING" => VersionPurgeStatusType::Pending,
+            "COMPLETE" => VersionPurgeStatusType::Complete,
+            "FAILED" => VersionPurgeStatusType::Failed,
+            _ => VersionPurgeStatusType::Empty,
+        }
+    }
+}
+
+/// Type - replication type enum
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub enum ReplicationType {
+    #[default]
+    Unset,
+    Object,
+    Delete,
+    Metadata,
+    Heal,
+    ExistingObject,
+    Resync,
+    All,
+}
+
+impl ReplicationType {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReplicationType::Unset => "",
+            ReplicationType::Object => "OBJECT",
+            ReplicationType::Delete => "DELETE",
+            ReplicationType::Metadata => "METADATA",
+            ReplicationType::Heal => "HEAL",
+            ReplicationType::ExistingObject => "EXISTING_OBJECT",
+            ReplicationType::Resync => "RESYNC",
+            ReplicationType::All => "ALL",
+        }
+    }
+
+    pub fn is_valid(&self) -> bool {
+        matches!(
+            self,
+            ReplicationType::Object
+                | ReplicationType::Delete
+                | ReplicationType::Metadata
+                | ReplicationType::Heal
+                | ReplicationType::ExistingObject
+                | ReplicationType::Resync
+                | ReplicationType::All
+        )
+    }
+
+    pub fn is_data_replication(&self) -> bool {
+        matches!(self, ReplicationType::Object | ReplicationType::Delete | ReplicationType::Heal)
+    }
+}
+
+impl fmt::Display for ReplicationType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+
+impl From<&str> for ReplicationType {
+    fn from(s: &str) -> Self {
+        match s {
+            "UNSET" => ReplicationType::Unset,
+            "OBJECT" => ReplicationType::Object,
+            "DELETE" => ReplicationType::Delete,
+            "METADATA" => ReplicationType::Metadata,
+            "HEAL" => ReplicationType::Heal,
+            "EXISTING_OBJECT" => ReplicationType::ExistingObject,
+            "RESYNC" => ReplicationType::Resync,
+            "ALL" => ReplicationType::All,
+            _ => ReplicationType::Unset,
+        }
+    }
+}
+
+/// ReplicationState represents internal replication state
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
+pub struct ReplicationState {
+    pub replica_timestamp: Option<OffsetDateTime>,
+    pub replica_status: ReplicationStatusType,
+    pub delete_marker: bool,
+    pub replication_timestamp: Option<OffsetDateTime>,
+    pub replication_status_internal: Option<String>,
+    pub version_purge_status_internal: Option<String>,
+    pub replicate_decision_str: String,
+    pub targets: HashMap<String, ReplicationStatusType>,
+    pub purge_targets: HashMap<String, VersionPurgeStatusType>,
+    pub reset_statuses_map: HashMap<String, String>,
+}
+
+impl ReplicationState {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Returns true if replication state is identical for version purge statuses and replication statuses
+    pub fn equal(&self, other: &ReplicationState) -> bool {
+        self.replica_status == other.replica_status
+            && self.replication_status_internal == other.replication_status_internal
+            && self.version_purge_status_internal == other.version_purge_status_internal
+    }
+
+    /// Returns overall replication status for the object version being replicated
+    pub fn composite_replication_status(&self) -> ReplicationStatusType {
+        if let Some(replication_status_internal) = &self.replication_status_internal {
+            match ReplicationStatusType::from(replication_status_internal.as_str()) {
+                ReplicationStatusType::Pending
+                | ReplicationStatusType::Completed
+                | ReplicationStatusType::Failed
+                | ReplicationStatusType::Replica => {
+                    return ReplicationStatusType::from(replication_status_internal.as_str());
+                }
+                _ => {
+                    let repl_status = get_composite_replication_status(&self.targets);
+
+                    if self.replica_timestamp.is_none() {
+                        return repl_status;
+                    }
+
+                    if repl_status == ReplicationStatusType::Completed {
+                        if let (Some(replica_timestamp), Some(replication_timestamp)) =
+                            (self.replica_timestamp, self.replication_timestamp)
+                        {
+                            if replica_timestamp > replication_timestamp {
+                                return self.replica_status.clone();
+                            }
+                        }
+                    }
+
+                    return repl_status;
+                }
+            }
+        } else if self.replica_status != ReplicationStatusType::default() {
+            return self.replica_status.clone();
+        }
+
+        ReplicationStatusType::default()
+    }
+
+    /// Returns overall replication purge status for the permanent delete being replicated
+    pub fn composite_version_purge_status(&self) -> VersionPurgeStatusType {
+        match VersionPurgeStatusType::from(self.version_purge_status_internal.clone().unwrap_or_default().as_str()) {
+            VersionPurgeStatusType::Pending | VersionPurgeStatusType::Complete | VersionPurgeStatusType::Failed => {
+                VersionPurgeStatusType::from(self.version_purge_status_internal.clone().unwrap_or_default().as_str())
+            }
+            _ => get_composite_version_purge_status(&self.purge_targets),
+        }
+    }
+
+    /// Returns replicatedInfos struct initialized with the previous state of replication
+    pub fn target_state(&self, arn: &str) -> ReplicatedTargetInfo {
+        ReplicatedTargetInfo {
+            arn: arn.to_string(),
+            prev_replication_status: self.targets.get(arn).cloned().unwrap_or_default(),
+            version_purge_status: self.purge_targets.get(arn).cloned().unwrap_or_default(),
+            resync_timestamp: self.reset_statuses_map.get(arn).cloned().unwrap_or_default(),
+            ..Default::default()
+        }
+    }
+}
+
+pub fn get_composite_replication_status(targets: &HashMap<String, ReplicationStatusType>) -> ReplicationStatusType {
+    if targets.is_empty() {
+        return ReplicationStatusType::Empty;
+    }
+
+    let mut completed = 0;
+    for status in targets.values() {
+        match status {
+            ReplicationStatusType::Failed => return ReplicationStatusType::Failed,
+            ReplicationStatusType::Completed => completed += 1,
+            _ => {}
+        }
+    }
+
+    if completed == targets.len() {
+        ReplicationStatusType::Completed
+    } else {
+        ReplicationStatusType::Pending
+    }
+}
+
+pub fn get_composite_version_purge_status(targets: &HashMap<String, VersionPurgeStatusType>) -> VersionPurgeStatusType {
+    if targets.is_empty() {
+        return VersionPurgeStatusType::default();
+    }
+
+    let mut completed = 0;
+    for status in targets.values() {
+        match status {
+            VersionPurgeStatusType::Failed => return VersionPurgeStatusType::Failed,
+            VersionPurgeStatusType::Complete => completed += 1,
+            _ => {}
+        }
+    }
+
+    if completed == targets.len() {
+        VersionPurgeStatusType::Complete
+    } else {
+        VersionPurgeStatusType::Pending
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub enum ReplicationAction {
+    /// Replicate all data
+    All,
+    /// Replicate only metadata
+    Metadata,
+    /// Do not replicate
+    #[default]
+    None,
+}
+
+impl ReplicationAction {
+    /// Returns string representation of replication action
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReplicationAction::All => "all",
+            ReplicationAction::Metadata => "metadata",
+            ReplicationAction::None => "none",
+        }
+    }
+}
+
+impl fmt::Display for ReplicationAction {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+
+impl From<&str> for ReplicationAction {
+    fn from(s: &str) -> Self {
+        match s {
+            "all" => ReplicationAction::All,
+            "metadata" => ReplicationAction::Metadata,
+            "none" => ReplicationAction::None,
+            _ => ReplicationAction::None,
+        }
+    }
+}
+
+/// ReplicatedTargetInfo struct represents replication info on a target
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct ReplicatedTargetInfo {
+    pub arn: String,
+    pub size: i64,
+    pub duration: Duration,
+    pub replication_action: ReplicationAction,
+    pub op_type: ReplicationType,
+    pub replication_status: ReplicationStatusType,
+    pub prev_replication_status: ReplicationStatusType,
+    pub version_purge_status: VersionPurgeStatusType,
+    pub resync_timestamp: String,
+    pub replication_resynced: bool,
+    pub endpoint: String,
+    pub secure: bool,
+    pub error: Option<String>,
+}
+
+impl ReplicatedTargetInfo {
+    /// Returns true for a target if arn is empty
+    pub fn is_empty(&self) -> bool {
+        self.arn.is_empty()
+    }
+}
+
+/// ReplicatedInfos struct contains replication information for multiple targets
+#[derive(Debug, Clone)]
+pub struct ReplicatedInfos {
+    pub replication_timestamp: Option<OffsetDateTime>,
+    pub targets: Vec<ReplicatedTargetInfo>,
+}
+
+impl ReplicatedInfos {
+    /// Returns the total size of completed replications
+    pub fn completed_size(&self) -> i64 {
+        let mut sz = 0i64;
+        for target in &self.targets {
+            if target.is_empty() {
+                continue;
+            }
+            if target.replication_status == ReplicationStatusType::Completed
+                && target.prev_replication_status != ReplicationStatusType::Completed
+            {
+                sz += target.size;
+            }
+        }
+        sz
+    }
+
+    /// Returns true if replication was attempted on any of the targets for the object version queued
+    pub fn replication_resynced(&self) -> bool {
+        for target in &self.targets {
+            if target.is_empty() || !target.replication_resynced {
+                continue;
+            }
+            return true;
+        }
+        false
+    }
+
+    /// Returns internal representation of replication status for all targets
+    pub fn replication_status_internal(&self) -> Option<String> {
+        let mut result = String::new();
+        for target in &self.targets {
+            if target.is_empty() {
+                continue;
+            }
+            result.push_str(&format!("{}={};", target.arn, target.replication_status));
+        }
+        if result.is_empty() { None } else { Some(result) }
+    }
+
+    /// Returns overall replication status across all targets
+    pub fn replication_status(&self) -> ReplicationStatusType {
+        if self.targets.is_empty() {
+            return ReplicationStatusType::Empty;
+        }
+
+        let mut completed = 0;
+        for target in &self.targets {
+            match target.replication_status {
+                ReplicationStatusType::Failed => return ReplicationStatusType::Failed,
+                ReplicationStatusType::Completed => completed += 1,
+                _ => {}
+            }
+        }
+
+        if completed == self.targets.len() {
+            ReplicationStatusType::Completed
+        } else {
+            ReplicationStatusType::Pending
+        }
+    }
+
+    /// Returns overall version purge status across all targets
+    pub fn version_purge_status(&self) -> VersionPurgeStatusType {
+        if self.targets.is_empty() {
+            return VersionPurgeStatusType::Empty;
+        }
+
+        let mut completed = 0;
+        for target in &self.targets {
+            match target.version_purge_status {
+                VersionPurgeStatusType::Failed => return VersionPurgeStatusType::Failed,
+                VersionPurgeStatusType::Complete => completed += 1,
+                _ => {}
+            }
+        }
+
+        if completed == self.targets.len() {
+            VersionPurgeStatusType::Complete
+        } else {
+            VersionPurgeStatusType::Pending
+        }
+    }
+
+    /// Returns internal representation of version purge status for all targets
+    pub fn version_purge_status_internal(&self) -> Option<String> {
+        let mut result = String::new();
+        for target in &self.targets {
+            if target.is_empty() || target.version_purge_status.is_empty() {
+                continue;
+            }
+            result.push_str(&format!("{}={};", target.arn, target.version_purge_status));
+        }
+        if result.is_empty() { None } else { Some(result) }
+    }
+
+    /// Returns replication action based on target that actually performed replication
+    pub fn action(&self) -> ReplicationAction {
+        for target in &self.targets {
+            if target.is_empty() {
+                continue;
+            }
+            // rely on replication action from target that actually performed replication now.
+            if target.prev_replication_status != ReplicationStatusType::Completed {
+                return target.replication_action;
+            }
+        }
+        ReplicationAction::None
+    }
+}
@@ -67,7 +67,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
    let delete_marker = MetaDeleteMarker {
        version_id: Some(delete_version_id),
        mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312260)?), // 1 minute later
-        meta_sys: None,
+        meta_sys: HashMap::new(),
    };

    let delete_file_version = FileMetaVersion {
@@ -151,7 +151,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
            let delete_marker = MetaDeleteMarker {
                version_id: Some(delete_version_id),
                mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200 + i * 60 + 30)?),
-                meta_sys: None,
+                meta_sys: HashMap::new(),
            };

            let delete_file_version = FileMetaVersion {
@@ -46,5 +46,6 @@ jsonwebtoken = { workspace = true }
 tracing.workspace = true
 rustfs-madmin.workspace = true
 rustfs-utils = { workspace = true, features = ["path"] }
+tokio-util.workspace = true

 [dev-dependencies]
@@ -43,3 +43,7 @@ pub async fn init_iam_sys(ecstore: Arc<ECStore>) -> Result<()> {
 pub fn get() -> Result<Arc<IamSys<ObjectStore>>> {
    IAM_SYS.get().map(Arc::clone).ok_or(Error::IamSysNotInitialized)
 }
+
+pub fn get_global_iam_sys() -> Option<Arc<IamSys<ObjectStore>>> {
+    IAM_SYS.get().cloned()
+}
@@ -23,6 +23,7 @@ use time::OffsetDateTime;

 #[async_trait::async_trait]
 pub trait Store: Clone + Send + Sync + 'static {
+    fn has_watcher(&self) -> bool;
    async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()>;
    async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item>;
    async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
@@ -89,6 +90,24 @@ impl UserType {
            UserType::None => "",
        }
    }
+    pub fn to_u64(&self) -> u64 {
+        match self {
+            UserType::Svc => 1,
+            UserType::Sts => 2,
+            UserType::Reg => 3,
+            UserType::None => 0,
+        }
+    }
+
+    pub fn from_u64(u64: u64) -> Option<Self> {
+        match u64 {
+            1 => Some(UserType::Svc),
+            2 => Some(UserType::Sts),
+            3 => Some(UserType::Reg),
+            0 => Some(UserType::None),
+            _ => None,
+        }
+    }
 }

 #[derive(Serialize, Deserialize, Clone)]
@@ -20,6 +20,8 @@ use crate::{
    manager::{extract_jwt_claims, get_default_policyes},
 };
 use futures::future::join_all;
+use rustfs_ecstore::StorageAPI as _;
+use rustfs_ecstore::store_api::{ObjectInfoOrErr, WalkOptions};
 use rustfs_ecstore::{
    config::{
        RUSTFS_CONFIG_PREFIX,
@@ -28,15 +30,14 @@ use rustfs_ecstore::{
    global::get_global_action_cred,
    store::ECStore,
    store_api::{ObjectInfo, ObjectOptions},
-    store_list_objects::{ObjectInfoOrErr, WalkOptions},
 };
 use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
 use serde::{Serialize, de::DeserializeOwned};
 use std::sync::LazyLock;
 use std::{collections::HashMap, sync::Arc};
-use tokio::sync::broadcast::{self, Receiver as B_Receiver};
 use tokio::sync::mpsc::{self, Sender};
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};

 pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam"));
@@ -144,7 +145,7 @@ impl ObjectStore {
        Ok((Self::decrypt_data(&data)?, obj))
    }

-    async fn list_iam_config_items(&self, prefix: &str, ctx_rx: B_Receiver<bool>, sender: Sender<StringOrErr>) {
+    async fn list_iam_config_items(&self, prefix: &str, ctx: CancellationToken, sender: Sender<StringOrErr>) {
        // debug!("list iam config items, prefix: {}", &prefix);

        // TODO: Implement walk, use walk
@@ -156,7 +157,11 @@ impl ObjectStore {
        let (tx, mut rx) = mpsc::channel::<ObjectInfoOrErr>(100);

        let path = prefix.to_owned();
-        tokio::spawn(async move { store.walk(ctx_rx, Self::BUCKET_NAME, &path, tx, WalkOptions::default()).await });
+        tokio::spawn(async move {
+            store
+                .walk(ctx.clone(), Self::BUCKET_NAME, &path, tx, WalkOptions::default())
+                .await
+        });

        let prefix = prefix.to_owned();
        tokio::spawn(async move {
@@ -190,10 +195,11 @@ impl ObjectStore {
    }

    async fn list_all_iamconfig_items(&self) -> Result<HashMap<String, Vec<String>>> {
-        let (ctx_tx, ctx_rx) = broadcast::channel(1);
        let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);

-        self.list_iam_config_items(format!("{}/", *IAM_CONFIG_PREFIX).as_str(), ctx_rx, tx)
+        let ctx = CancellationToken::new();
+
+        self.list_iam_config_items(format!("{}/", *IAM_CONFIG_PREFIX).as_str(), ctx.clone(), tx)
            .await;

        let mut res = HashMap::new();
@@ -201,7 +207,7 @@ impl ObjectStore {
        while let Some(v) = rx.recv().await {
            if let Some(err) = v.err {
                warn!("list_iam_config_items {:?}", err);
-                let _ = ctx_tx.send(true);
+                ctx.cancel();

                return Err(err);
            }
@@ -215,7 +221,7 @@ impl ObjectStore {
            }
        }

-        let _ = ctx_tx.send(true);
+        ctx.cancel();

        Ok(res)
    }
@@ -374,6 +380,9 @@ impl ObjectStore {

 #[async_trait::async_trait]
 impl Store for ObjectStore {
+    fn has_watcher(&self) -> bool {
+        false
+    }
    async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
        let mut data = read_config(self.object_api.clone(), path.as_ref()).await?;

@@ -477,15 +486,15 @@ impl Store for ObjectStore {
            UserType::None => "",
        };

-        let (ctx_tx, ctx_rx) = broadcast::channel(1);
+        let ctx = CancellationToken::new();
        let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);

-        self.list_iam_config_items(base_prefix, ctx_rx, tx).await;
+        self.list_iam_config_items(base_prefix, ctx.clone(), tx).await;

        while let Some(v) = rx.recv().await {
            if let Some(err) = v.err {
                warn!("list_iam_config_items {:?}", err);
-                let _ = ctx_tx.send(true);
+                let _ = ctx.cancel();

                return Err(err);
            }
@@ -495,7 +504,7 @@ impl Store for ObjectStore {
                self.load_user(&name, user_type, m).await?;
            }
        }
-        let _ = ctx_tx.send(true);
+        let _ = ctx.cancel();
        Ok(())
    }
    async fn load_secret_key(&self, name: &str, user_type: UserType) -> Result<String> {
@@ -539,15 +548,15 @@ impl Store for ObjectStore {
        Ok(())
    }
    async fn load_groups(&self, m: &mut HashMap<String, GroupInfo>) -> Result<()> {
-        let (ctx_tx, ctx_rx) = broadcast::channel(1);
+        let ctx = CancellationToken::new();
        let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);

-        self.list_iam_config_items(&IAM_CONFIG_GROUPS_PREFIX, ctx_rx, tx).await;
+        self.list_iam_config_items(&IAM_CONFIG_GROUPS_PREFIX, ctx.clone(), tx).await;

        while let Some(v) = rx.recv().await {
            if let Some(err) = v.err {
                warn!("list_iam_config_items {:?}", err);
-                let _ = ctx_tx.send(true);
+                let _ = ctx.cancel();

                return Err(err);
            }
@@ -557,7 +566,7 @@ impl Store for ObjectStore {
                self.load_group(&name, m).await?;
            }
        }
-        let _ = ctx_tx.send(true);
+        let _ = ctx.cancel();
        Ok(())
    }

@@ -603,15 +612,15 @@ impl Store for ObjectStore {
        Ok(())
    }
    async fn load_policy_docs(&self, m: &mut HashMap<String, PolicyDoc>) -> Result<()> {
-        let (ctx_tx, ctx_rx) = broadcast::channel(1);
+        let ctx = CancellationToken::new();
        let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);

-        self.list_iam_config_items(&IAM_CONFIG_POLICIES_PREFIX, ctx_rx, tx).await;
+        self.list_iam_config_items(&IAM_CONFIG_POLICIES_PREFIX, ctx.clone(), tx).await;

        while let Some(v) = rx.recv().await {
            if let Some(err) = v.err {
                warn!("list_iam_config_items {:?}", err);
-                let _ = ctx_tx.send(true);
+                let _ = ctx.cancel();

                return Err(err);
            }
@@ -621,7 +630,7 @@ impl Store for ObjectStore {
                self.load_policy_doc(&name, m).await?;
            }
        }
-        let _ = ctx_tx.send(true);
+        let _ = ctx.cancel();
        Ok(())
    }

@@ -678,15 +687,15 @@ impl Store for ObjectStore {
                }
            }
        };
-        let (ctx_tx, ctx_rx) = broadcast::channel(1);
+        let ctx = CancellationToken::new();
        let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);

-        self.list_iam_config_items(base_path, ctx_rx, tx).await;
+        self.list_iam_config_items(base_path, ctx.clone(), tx).await;

        while let Some(v) = rx.recv().await {
            if let Some(err) = v.err {
                warn!("list_iam_config_items {:?}", err);
-                let _ = ctx_tx.send(true);
+                let _ = ctx.cancel();

                return Err(err);
            }
@@ -696,7 +705,7 @@ impl Store for ObjectStore {
                self.load_mapped_policy(name, user_type, is_group, m).await?;
            }
        }
-        let _ = ctx_tx.send(true);
+        let _ = ctx.cancel(); // TODO: check if this is needed
        Ok(())
    }

@@ -25,6 +25,7 @@ use crate::store::Store;
 use crate::store::UserType;
 use crate::utils::extract_claims;
 use rustfs_ecstore::global::get_global_action_cred;
+use rustfs_ecstore::notification_sys::get_global_notification_sys;
 use rustfs_madmin::AddOrUpdateUserReq;
 use rustfs_madmin::GroupDesc;
 use rustfs_policy::arn::ARN;
@@ -41,6 +42,7 @@ use serde_json::json;
 use std::collections::HashMap;
 use std::sync::Arc;
 use time::OffsetDateTime;
+use tracing::warn;

 pub const MAX_SVCSESSION_POLICY_SIZE: usize = 4096;

@@ -63,6 +65,9 @@ impl<T: Store> IamSys<T> {
            roles_map: HashMap::new(),
        }
    }
+    pub fn has_watcher(&self) -> bool {
+        self.store.api.has_watcher()
+    }

    pub async fn load_group(&self, name: &str) -> Result<()> {
        self.store.group_notification_handler(name).await
@@ -104,8 +109,17 @@ impl<T: Store> IamSys<T> {

        self.store.delete_policy(name, notify).await?;

-        if notify {
-            // TODO: implement notification
+        if !notify || self.has_watcher() {
+            return Ok(());
+        }
+
+        if let Some(notification_sys) = get_global_notification_sys() {
+            let resp = notification_sys.delete_policy(name).await;
+            for r in resp {
+                if let Some(err) = r.err {
+                    warn!("notify delete_policy failed: {}", err);
+                }
+            }
        }

        Ok(())
@@ -142,9 +156,20 @@ impl<T: Store> IamSys<T> {
    }

    pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
-        self.store.set_policy(name, policy).await
+        let updated_at = self.store.set_policy(name, policy).await?;

-        // TODO: notification
+        if !self.has_watcher() {
+            if let Some(notification_sys) = get_global_notification_sys() {
+                let resp = notification_sys.load_policy(name).await;
+                for r in resp {
+                    if let Some(err) = r.err {
+                        warn!("notify load_policy failed: {}", err);
+                    }
+                }
+            }
+        }
+
+        Ok(updated_at)
    }

    pub async fn get_role_policy(&self, arn_str: &str) -> Result<(ARN, String)> {
@@ -159,9 +184,51 @@ impl<T: Store> IamSys<T> {
        Ok((arn, policy.clone()))
    }

-    pub async fn delete_user(&self, name: &str, _notify: bool) -> Result<()> {
-        self.store.delete_user(name, UserType::Reg).await
-        // TODO: notification
+    pub async fn delete_user(&self, name: &str, notify: bool) -> Result<()> {
+        self.store.delete_user(name, UserType::Reg).await?;
+
+        if notify && !self.has_watcher() {
+            if let Some(notification_sys) = get_global_notification_sys() {
+                let resp = notification_sys.delete_user(name).await;
+                for r in resp {
+                    if let Some(err) = r.err {
+                        warn!("notify delete_user failed: {}", err);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn notify_for_user(&self, name: &str, is_temp: bool) {
+        if self.has_watcher() {
+            return;
+        }
+
+        if let Some(notification_sys) = get_global_notification_sys() {
+            let resp = notification_sys.load_user(name, is_temp).await;
+            for r in resp {
+                if let Some(err) = r.err {
+                    warn!("notify load_user failed: {}", err);
+                }
+            }
+        }
+    }
+
+    async fn notify_for_service_account(&self, name: &str) {
+        if self.has_watcher() {
+            return;
+        }
+
+        if let Some(notification_sys) = get_global_notification_sys() {
+            let resp = notification_sys.load_service_account(name).await;
+            for r in resp {
+                if let Some(err) = r.err {
+                    warn!("notify load_service_account failed: {}", err);
+                }
+            }
+        }
    }

    pub async fn current_policies(&self, name: &str) -> String {
@@ -177,8 +244,11 @@ impl<T: Store> IamSys<T> {
    }

    pub async fn set_temp_user(&self, name: &str, cred: &Credentials, policy_name: Option<&str>) -> Result<OffsetDateTime> {
-        self.store.set_temp_user(name, cred, policy_name).await
-        // TODO: notification
+        let updated_at = self.store.set_temp_user(name, cred, policy_name).await?;
+
+        self.notify_for_user(&cred.access_key, true).await;
+
+        Ok(updated_at)
    }

    pub async fn is_temp_user(&self, name: &str) -> Result<(bool, String)> {
@@ -208,8 +278,11 @@ impl<T: Store> IamSys<T> {
    }

    pub async fn set_user_status(&self, name: &str, status: rustfs_madmin::AccountStatus) -> Result<OffsetDateTime> {
-        self.store.set_user_status(name, status).await
-        // TODO: notification
+        let updated_at = self.store.set_user_status(name, status).await?;
+
+        self.notify_for_user(name, false).await;
+
+        Ok(updated_at)
    }

    pub async fn new_service_account(
@@ -294,14 +367,17 @@ impl<T: Store> IamSys<T> {

        let create_at = self.store.add_service_account(cred.clone()).await?;

+        self.notify_for_service_account(&cred.access_key).await;
+
        Ok((cred, create_at))
-        // TODO: notification
    }

    pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
-        self.store.update_service_account(name, opts).await
+        let updated_at = self.store.update_service_account(name, opts).await?;

-        // TODO: notification
+        self.notify_for_service_account(name).await;
+
+        Ok(updated_at)
    }

    pub async fn list_service_accounts(&self, access_key: &str) -> Result<Vec<Credentials>> {
@@ -424,7 +500,7 @@ impl<T: Store> IamSys<T> {
        extract_jwt_claims(&u)
    }

-    pub async fn delete_service_account(&self, access_key: &str, _notify: bool) -> Result<()> {
+    pub async fn delete_service_account(&self, access_key: &str, notify: bool) -> Result<()> {
        let Some(u) = self.store.get_user(access_key).await else {
            return Ok(());
        };
@@ -433,9 +509,35 @@ impl<T: Store> IamSys<T> {
            return Ok(());
        }

-        self.store.delete_user(access_key, UserType::Svc).await
+        self.store.delete_user(access_key, UserType::Svc).await?;

-        // TODO: notification
+        if notify && !self.has_watcher() {
+            if let Some(notification_sys) = get_global_notification_sys() {
+                let resp = notification_sys.delete_service_account(access_key).await;
+                for r in resp {
+                    if let Some(err) = r.err {
+                        warn!("notify delete_service_account failed: {}", err);
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn notify_for_group(&self, group: &str) {
+        if self.has_watcher() {
+            return;
+        }
+
+        if let Some(notification_sys) = get_global_notification_sys() {
+            let resp = notification_sys.load_group(group).await;
+            for r in resp {
+                if let Some(err) = r.err {
+                    warn!("notify load_group failed: {}", err);
+                }
+            }
+        }
    }

    pub async fn create_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
@@ -451,8 +553,11 @@ impl<T: Store> IamSys<T> {
            return Err(IamError::InvalidSecretKeyLength);
        }

-        self.store.add_user(access_key, args).await
-        // TODO: notification
+        let updated_at = self.store.add_user(access_key, args).await?;
+
+        self.notify_for_user(access_key, false).await;
+
+        Ok(updated_at)
    }

    pub async fn set_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<()> {
@@ -495,18 +600,27 @@ impl<T: Store> IamSys<T> {
        if contains_reserved_chars(group) {
            return Err(IamError::GroupNameContainsReservedChars);
        }
-        self.store.add_users_to_group(group, users).await
-        // TODO: notification
+        let updated_at = self.store.add_users_to_group(group, users).await?;
+
+        self.notify_for_group(group).await;
+
+        Ok(updated_at)
    }

    pub async fn remove_users_from_group(&self, group: &str, users: Vec<String>) -> Result<OffsetDateTime> {
-        self.store.remove_users_from_group(group, users).await
-        // TODO: notification
+        let updated_at = self.store.remove_users_from_group(group, users).await?;
+
+        self.notify_for_group(group).await;
+
+        Ok(updated_at)
    }

    pub async fn set_group_status(&self, group: &str, enable: bool) -> Result<OffsetDateTime> {
-        self.store.set_group_status(group, enable).await
-        // TODO: notification
+        let updated_at = self.store.set_group_status(group, enable).await?;
+
+        self.notify_for_group(group).await;
+
+        Ok(updated_at)
    }
    pub async fn get_group_description(&self, group: &str) -> Result<GroupDesc> {
        self.store.get_group_description(group).await
@@ -517,8 +631,20 @@ impl<T: Store> IamSys<T> {
    }

    pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
-        self.store.policy_db_set(name, user_type, is_group, policy).await
-        // TODO: notification
+        let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?;
+
+        if !self.has_watcher() {
+            if let Some(notification_sys) = get_global_notification_sys() {
+                let resp = notification_sys.load_policy_mapping(name, user_type.to_u64(), is_group).await;
+                for r in resp {
+                    if let Some(err) = r.err {
+                        warn!("notify load_policy failed: {}", err);
+                    }
+                }
+            }
+        }
+
+        Ok(updated_at)
    }

    pub async fn policy_db_get(&self, name: &str, groups: &Option<Vec<String>>) -> Result<Vec<String>> {
@@ -111,7 +111,7 @@ impl LocalKmsClient {

    /// Get the file path for a master key
    fn master_key_path(&self, key_id: &str) -> PathBuf {
-        self.config.key_dir.join(format!("{}.key", key_id))
+        self.config.key_dir.join(format!("{key_id}.key"))
    }

    /// Load a master key from disk
@@ -334,12 +334,11 @@ impl KmsClient for LocalKmsClient {
                if let Some(actual_value) = envelope.encryption_context.get(key) {
                    if actual_value != expected_value {
                        return Err(KmsError::context_mismatch(format!(
-                            "Context mismatch for key '{}': expected '{}', got '{}'",
-                            key, expected_value, actual_value
+                            "Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'"
                        )));
                    }
                } else {
-                    return Err(KmsError::context_mismatch(format!("Missing context key '{}'", key)));
+                    return Err(KmsError::context_mismatch(format!("Missing context key '{key}'")));
                }
            }
        }
@@ -720,14 +719,14 @@ impl KmsBackend for LocalKmsBackend {
            .client
            .load_master_key(key_id)
            .await
-            .map_err(|_| crate::error::KmsError::key_not_found(format!("Key {} not found", key_id)))?;
+            .map_err(|_| crate::error::KmsError::key_not_found(format!("Key {key_id} not found")))?;

        let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) {
            // For immediate deletion, actually delete the key from filesystem
            let key_path = self.client.master_key_path(key_id);
            tokio::fs::remove_file(&key_path)
                .await
-                .map_err(|e| crate::error::KmsError::internal_error(format!("Failed to delete key file: {}", e)))?;
+                .map_err(|e| crate::error::KmsError::internal_error(format!("Failed to delete key file: {e}")))?;

            // Remove from cache
            let mut cache = self.client.key_cache.write().await;
@@ -773,9 +772,9 @@ impl KmsBackend for LocalKmsBackend {
        let key_path = self.client.master_key_path(key_id);
        let content = tokio::fs::read(&key_path)
            .await
-            .map_err(|e| crate::error::KmsError::internal_error(format!("Failed to read key file: {}", e)))?;
+            .map_err(|e| crate::error::KmsError::internal_error(format!("Failed to read key file: {e}")))?;
        let stored_key: crate::backends::local::StoredMasterKey = serde_json::from_slice(&content)
-            .map_err(|e| crate::error::KmsError::internal_error(format!("Failed to parse stored key: {}", e)))?;
+            .map_err(|e| crate::error::KmsError::internal_error(format!("Failed to parse stored key: {e}")))?;

        // Decrypt the existing key material to preserve it
        let existing_key_material = if let Some(ref cipher) = self.client.master_cipher {
@@ -821,13 +820,10 @@ impl KmsBackend for LocalKmsBackend {
            .client
            .load_master_key(key_id)
            .await
-            .map_err(|_| crate::error::KmsError::key_not_found(format!("Key {} not found", key_id)))?;
+            .map_err(|_| crate::error::KmsError::key_not_found(format!("Key {key_id} not found")))?;

        if master_key.status != KeyStatus::PendingDeletion {
-            return Err(crate::error::KmsError::invalid_key_state(format!(
-                "Key {} is not pending deletion",
-                key_id
-            )));
+            return Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
        }

        // Cancel the deletion by resetting the state
@@ -89,10 +89,10 @@ impl VaultKmsClient {

        let settings = settings_builder
            .build()
-            .map_err(|e| KmsError::backend_error(format!("Failed to build Vault client settings: {}", e)))?;
+            .map_err(|e| KmsError::backend_error(format!("Failed to build Vault client settings: {e}")))?;

        let client =
-            VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {}", e)))?;
+            VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?;

        info!("Successfully connected to Vault at {}", config.address);

@@ -144,7 +144,7 @@ impl VaultKmsClient {

        kv2::set(&self.client, &self.kv_mount, &path, key_data)
            .await
-            .map_err(|e| KmsError::backend_error(format!("Failed to store key in Vault: {}", e)))?;
+            .map_err(|e| KmsError::backend_error(format!("Failed to store key in Vault: {e}")))?;

        debug!("Stored key {} in Vault at path {}", key_id, path);
        Ok(())
@@ -176,7 +176,7 @@ impl VaultKmsClient {
        let secret: VaultKeyData = kv2::read(&self.client, &self.kv_mount, &path).await.map_err(|e| match e {
            vaultrs::error::ClientError::ResponseWrapError => KmsError::key_not_found(key_id),
            vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id),
-            _ => KmsError::backend_error(format!("Failed to read key from Vault: {}", e)),
+            _ => KmsError::backend_error(format!("Failed to read key from Vault: {e}")),
        })?;

        debug!("Retrieved key {} from Vault, tags: {:?}", key_id, secret.tags);
@@ -200,7 +200,7 @@ impl VaultKmsClient {
                debug!("Key path doesn't exist in Vault (404), returning empty list");
                Ok(Vec::new())
            }
-            Err(e) => Err(KmsError::backend_error(format!("Failed to list keys in Vault: {}", e))),
+            Err(e) => Err(KmsError::backend_error(format!("Failed to list keys in Vault: {e}"))),
        }
    }

@@ -214,7 +214,7 @@ impl VaultKmsClient {
            .await
            .map_err(|e| match e {
                vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id),
-                _ => KmsError::backend_error(format!("Failed to delete key metadata from Vault: {}", e)),
+                _ => KmsError::backend_error(format!("Failed to delete key metadata from Vault: {e}")),
            })?;

        debug!("Permanently deleted key {} metadata from Vault at path {}", key_id, path);
@@ -649,7 +649,7 @@ impl KmsBackend for VaultKmsBackend {
        let mut key_metadata = match self.describe_key(describe_request).await {
            Ok(response) => response.key_metadata,
            Err(_) => {
-                return Err(crate::error::KmsError::key_not_found(format!("Key {} not found", key_id)));
+                return Err(crate::error::KmsError::key_not_found(format!("Key {key_id} not found")));
            }
        };

@@ -705,15 +705,12 @@ impl KmsBackend for VaultKmsBackend {
        let mut key_metadata = match self.describe_key(describe_request).await {
            Ok(response) => response.key_metadata,
            Err(_) => {
-                return Err(crate::error::KmsError::key_not_found(format!("Key {} not found", key_id)));
+                return Err(crate::error::KmsError::key_not_found(format!("Key {key_id} not found")));
            }
        };

        if key_metadata.key_state != KeyState::PendingDeletion {
-            return Err(crate::error::KmsError::invalid_key_state(format!(
-                "Key {} is not pending deletion",
-                key_id
-            )));
+            return Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
        }

        // Cancel the deletion by resetting the state
@@ -315,7 +315,7 @@ impl KmsConfig {
            config.backend = match backend_type.to_lowercase().as_str() {
                "local" => KmsBackend::Local,
                "vault" => KmsBackend::Vault,
-                _ => return Err(KmsError::configuration_error(format!("Unknown KMS backend: {}", backend_type))),
+                _ => return Err(KmsError::configuration_error(format!("Unknown KMS backend: {backend_type}"))),
            };
        }

@@ -227,7 +227,7 @@ impl ObjectEncryptionService {
                self.kms_manager
                    .create_key(create_req)
                    .await
-                    .map_err(|e| KmsError::backend_error(format!("Failed to auto-create SSE-S3 key {}: {}", actual_key_id, e)))?;
+                    .map_err(|e| KmsError::backend_error(format!("Failed to auto-create SSE-S3 key {actual_key_id}: {e}")))?;
            }
        } else {
            // For SSE-KMS, key must exist
@@ -235,7 +235,7 @@ impl ObjectEncryptionService {
                key_id: actual_key_id.to_string(),
            };
            self.kms_manager.describe_key(describe_req).await.map_err(|_| {
-                KmsError::invalid_operation(format!("SSE-KMS key '{}' not found. Please create it first.", actual_key_id))
+                KmsError::invalid_operation(format!("SSE-KMS key '{actual_key_id}' not found. Please create it first."))
            })?;
        }

@@ -250,7 +250,7 @@ impl ObjectEncryptionService {
            .kms_manager
            .generate_data_key(request)
            .await
-            .map_err(|e| KmsError::backend_error(format!("Failed to generate data key: {}", e)))?;
+            .map_err(|e| KmsError::backend_error(format!("Failed to generate data key: {e}")))?;

        let plaintext_key = data_key.plaintext_key;

@@ -325,7 +325,7 @@ impl ObjectEncryptionService {
            .kms_manager
            .decrypt(decrypt_request)
            .await
-            .map_err(|e| KmsError::backend_error(format!("Failed to decrypt data key: {}", e)))?;
+            .map_err(|e| KmsError::backend_error(format!("Failed to decrypt data key: {e}")))?;

        // Create cipher
        let cipher = create_cipher(&algorithm, &decrypt_response.plaintext)?;
@@ -379,7 +379,7 @@ impl ObjectEncryptionService {
        // Validate key MD5 if provided
        if let Some(expected_md5) = customer_key_md5 {
            let actual_md5 = md5::compute(customer_key);
-            let actual_md5_hex = format!("{:x}", actual_md5);
+            let actual_md5_hex = format!("{actual_md5:x}");
            if actual_md5_hex != expected_md5.to_lowercase() {
                return Err(KmsError::validation_error("Customer key MD5 mismatch"));
            }
@@ -487,12 +487,11 @@ impl ObjectEncryptionService {
                Some(actual_value) if actual_value == expected_value => continue,
                Some(actual_value) => {
                    return Err(KmsError::context_mismatch(format!(
-                        "Context mismatch for '{}': expected '{}', got '{}'",
-                        key, expected_value, actual_value
+                        "Context mismatch for '{key}': expected '{expected_value}', got '{actual_value}'"
                    )));
                }
                None => {
-                    return Err(KmsError::context_mismatch(format!("Missing context key '{}'", key)));
+                    return Err(KmsError::context_mismatch(format!("Missing context key '{key}'")));
                }
            }
        }
@@ -562,13 +561,13 @@ impl ObjectEncryptionService {
            .ok_or_else(|| KmsError::validation_error("Missing IV header"))?;
        let iv = base64::engine::general_purpose::STANDARD
            .decode(iv)
-            .map_err(|e| KmsError::validation_error(format!("Invalid IV: {}", e)))?;
+            .map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?;

        let tag = if let Some(tag_str) = headers.get("x-rustfs-encryption-tag") {
            Some(
                base64::engine::general_purpose::STANDARD
                    .decode(tag_str)
-                    .map_err(|e| KmsError::validation_error(format!("Invalid tag: {}", e)))?,
+                    .map_err(|e| KmsError::validation_error(format!("Invalid tag: {e}")))?,
            )
        } else {
            None
@@ -577,14 +576,14 @@ impl ObjectEncryptionService {
        let encrypted_data_key = if let Some(key_str) = headers.get("x-rustfs-encryption-key") {
            base64::engine::general_purpose::STANDARD
                .decode(key_str)
-                .map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {}", e)))?
+                .map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))?
        } else {
            Vec::new() // Empty for SSE-C
        };

        let encryption_context = if let Some(context_str) = headers.get("x-rustfs-encryption-context") {
            serde_json::from_str(context_str)
-                .map_err(|e| KmsError::validation_error(format!("Invalid encryption context: {}", e)))?
+                .map_err(|e| KmsError::validation_error(format!("Invalid encryption context: {e}")))?
        } else {
            HashMap::new()
        };
@@ -225,7 +225,7 @@ impl KmsError {
 impl From<url::ParseError> for KmsError {
    fn from(error: url::ParseError) -> Self {
        Self::ConfigurationError {
-            message: format!("Invalid URL: {}", error),
+            message: format!("Invalid URL: {error}"),
        }
    }
 }
@@ -233,7 +233,7 @@ impl From<url::ParseError> for KmsError {
 impl From<reqwest::Error> for KmsError {
    fn from(error: reqwest::Error) -> Self {
        Self::BackendError {
-            message: format!("HTTP request failed: {}", error),
+            message: format!("HTTP request failed: {error}"),
        }
    }
 }
@@ -137,7 +137,7 @@ impl KmsServiceManager {
                Ok(())
            }
            Err(e) => {
-                let err_msg = format!("Failed to create KMS backend: {}", e);
+                let err_msg = format!("Failed to create KMS backend: {e}");
                error!("{}", err_msg);
                let mut status = self.status.write().await;
                *status = KmsServiceStatus::Error(err_msg.clone());
@@ -218,7 +218,7 @@ impl KmsServiceManager {
                        error!("KMS health check error: {}", e);
                        // Update status to error
                        let mut status = self.status.write().await;
-                        *status = KmsServiceStatus::Error(format!("Health check failed: {}", e));
+                        *status = KmsServiceStatus::Error(format!("Health check failed: {e}"));
                        Err(e)
                    }
                }
@@ -98,12 +98,18 @@ impl DisabledLockManager {

    /// Always succeeds - all locks acquired
    pub async fn acquire_locks_batch(&self, batch_request: BatchLockRequest) -> BatchLockResult {
-        let successful_locks: Vec<ObjectKey> = batch_request.requests.into_iter().map(|req| req.key).collect();
+        let successful_locks: Vec<ObjectKey> = batch_request.requests.iter().map(|req| req.key.clone()).collect();
+        let guards = batch_request
+            .requests
+            .into_iter()
+            .map(|req| FastLockGuard::new_disabled(req.key, req.mode, req.owner))
+            .collect();

        BatchLockResult {
            successful_locks,
            failed_locks: Vec::new(),
            all_acquired: true,
+            guards,
        }
    }

@@ -106,6 +106,10 @@ impl FastObjectLockManager {
        object: impl Into<Arc<str>>,
        owner: impl Into<Arc<str>>,
    ) -> Result<FastLockGuard, LockResult> {
+        // let bucket = bucket.into();
+        // let object = object.into();
+        // let owner = owner.into();
+        // error!("acquire_write_lock: bucket={:?}, object={:?}, owner={:?}", bucket, object, owner);
        let request = ObjectLockRequest::new_write(bucket, object, owner);
        self.acquire_lock(request).await
    }
@@ -213,20 +217,33 @@ impl FastObjectLockManager {
    ) -> BatchLockResult {
        let mut all_successful = Vec::new();
        let mut all_failed = Vec::new();
+        let mut guards = Vec::new();

        for (&shard_id, requests) in shard_groups {
-            let shard = &self.shards[shard_id];
+            let shard = self.shards[shard_id].clone();

-            // Try fast path first for each request
            for request in requests {
-                if shard.try_fast_path_only(request) {
-                    all_successful.push(request.key.clone());
+                let key = request.key.clone();
+                let owner = request.owner.clone();
+                let mode = request.mode;
+
+                let acquired = if shard.try_fast_path_only(request) {
+                    true
                } else {
-                    // Fallback to slow path
                    match shard.acquire_lock(request).await {
-                        Ok(()) => all_successful.push(request.key.clone()),
-                        Err(err) => all_failed.push((request.key.clone(), err)),
+                        Ok(()) => true,
+                        Err(err) => {
+                            all_failed.push((key.clone(), err));
+                            false
+                        }
                    }
+                };
+
+                if acquired {
+                    let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
+                    shard.register_guard(guard.guard_id());
+                    all_successful.push(key);
+                    guards.push(guard);
                }
            }
        }
@@ -236,6 +253,7 @@ impl FastObjectLockManager {
            successful_locks: all_successful,
            failed_locks: all_failed,
            all_acquired,
+            guards,
        }
    }

@@ -245,16 +263,18 @@ impl FastObjectLockManager {
        shard_groups: &std::collections::HashMap<usize, Vec<ObjectLockRequest>>,
    ) -> BatchLockResult {
        // Phase 1: Try to acquire all locks
-        let mut acquired_locks = Vec::new();
+        let mut acquired_guards = Vec::new();
        let mut failed_locks = Vec::new();

        'outer: for (&shard_id, requests) in shard_groups {
-            let shard = &self.shards[shard_id];
+            let shard = self.shards[shard_id].clone();

            for request in requests {
                match shard.acquire_lock(request).await {
                    Ok(()) => {
-                        acquired_locks.push((request.key.clone(), request.mode, request.owner.clone()));
+                        let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
+                        shard.register_guard(guard.guard_id());
+                        acquired_guards.push(guard);
                    }
                    Err(err) => {
                        failed_locks.push((request.key.clone(), err));
@@ -266,35 +286,22 @@ impl FastObjectLockManager {

        // Phase 2: If any failed, release all acquired locks with error tracking
        if !failed_locks.is_empty() {
-            let mut cleanup_failures = 0;
-            for (key, mode, owner) in acquired_locks {
-                let shard = self.get_shard(&key);
-                if !shard.release_lock(&key, &owner, mode) {
-                    cleanup_failures += 1;
-                    tracing::warn!(
-                        "Failed to release lock during batch cleanup: bucket={}, object={}",
-                        key.bucket,
-                        key.object
-                    );
-                }
-            }
-
-            if cleanup_failures > 0 {
-                tracing::error!("Batch lock cleanup had {} failures", cleanup_failures);
-            }
-
+            // Drop guards to release any acquired locks.
+            drop(acquired_guards);
            return BatchLockResult {
                successful_locks: Vec::new(),
                failed_locks,
                all_acquired: false,
+                guards: Vec::new(),
            };
        }

-        // All successful
+        let successful_locks = acquired_guards.iter().map(|guard| guard.key().clone()).collect();
        BatchLockResult {
-            successful_locks: acquired_locks.into_iter().map(|(key, _, _)| key).collect(),
+            successful_locks,
            failed_locks: Vec::new(),
            all_acquired: true,
+            guards: acquired_guards,
        }
    }

@@ -111,6 +111,9 @@ impl ObjectLockState {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::fast_lock::state::{ExclusiveOwnerInfo, SharedOwnerEntry};
+    use std::sync::Arc;
+    use std::time::{Duration, SystemTime};

    #[test]
    fn test_object_pool() {
@@ -142,8 +145,17 @@ mod tests {
        let mut state = ObjectLockState::new();

        // Modify state
-        *state.current_owner.write() = Some("test_owner".into());
-        state.shared_owners.write().push("shared_owner".into());
+        *state.current_owner.write() = Some(ExclusiveOwnerInfo {
+            owner: Arc::from("test_owner"),
+            acquired_at: SystemTime::now(),
+            lock_timeout: Duration::from_secs(30),
+        });
+        state.shared_owners.write().push(SharedOwnerEntry {
+            owner: Arc::from("shared_owner"),
+            count: 1,
+            acquired_at: SystemTime::now(),
+            lock_timeout: Duration::from_secs(30),
+        });

        // Reset
        state.reset_for_reuse();
@@ -88,8 +88,8 @@ impl LockShard {

                // Try atomic acquisition
                let success = match request.mode {
-                    LockMode::Shared => state.try_acquire_shared_fast(&request.owner),
-                    LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner),
+                    LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
+                    LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
                };

                if success {
@@ -108,14 +108,14 @@ impl LockShard {
                let state = state.clone();
                drop(objects);

-                if state.try_acquire_exclusive_fast(&request.owner) {
+                if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
                    return Some(state);
                }
            } else {
                // Create new state from pool and acquire immediately
                let state_box = self.object_pool.acquire();
                let state = Arc::new(*state_box);
-                if state.try_acquire_exclusive_fast(&request.owner) {
+                if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
                    objects.insert(request.key.clone(), state.clone());
                    return Some(state);
                }
@@ -151,8 +151,8 @@ impl LockShard {

            // Try acquisition again
            let success = match request.mode {
-                LockMode::Shared => state.try_acquire_shared_fast(&request.owner),
-                LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner),
+                LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
+                LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
            };

            if success {
@@ -443,22 +443,24 @@ impl LockShard {
        let objects = self.objects.read();
        if let Some(state) = objects.get(key) {
            if let Some(mode) = state.current_mode() {
-                let owner = match mode {
+                let (owner, acquired_at, lock_timeout) = match mode {
                    LockMode::Exclusive => {
                        let current_owner = state.current_owner.read();
-                        current_owner.clone()?
+                        let info = current_owner.clone()?;
+                        (info.owner, info.acquired_at, info.lock_timeout)
                    }
                    LockMode::Shared => {
                        let shared_owners = state.shared_owners.read();
-                        shared_owners.first()?.clone()
+                        let entry = shared_owners.first()?.clone();
+                        (entry.owner, entry.acquired_at, entry.lock_timeout)
                    }
                };

                let priority = *state.priority.read();

-                // Estimate acquisition time (approximate)
-                let acquired_at = SystemTime::now() - Duration::from_secs(60);
-                let expires_at = acquired_at + Duration::from_secs(300);
+                let expires_at = acquired_at
+                    .checked_add(lock_timeout)
+                    .unwrap_or_else(|| acquired_at + crate::fast_lock::DEFAULT_LOCK_TIMEOUT);

                return Some(crate::fast_lock::types::ObjectLockInfo {
                    key: key.clone(),
@@ -308,13 +308,28 @@ pub struct ObjectLockState {

    // Third cache line: Less frequently accessed data
    /// Current owner of exclusive lock (if any)
-    pub current_owner: parking_lot::RwLock<Option<Arc<str>>>,
+    pub current_owner: parking_lot::RwLock<Option<ExclusiveOwnerInfo>>,
    /// Shared owners - optimized for small number of readers
-    pub shared_owners: parking_lot::RwLock<smallvec::SmallVec<[Arc<str>; 4]>>,
+    pub shared_owners: parking_lot::RwLock<smallvec::SmallVec<[SharedOwnerEntry; 4]>>,
    /// Lock priority for conflict resolution
    pub priority: parking_lot::RwLock<LockPriority>,
 }

+#[derive(Clone, Debug)]
+pub struct ExclusiveOwnerInfo {
+    pub owner: Arc<str>,
+    pub acquired_at: SystemTime,
+    pub lock_timeout: Duration,
+}
+
+#[derive(Clone, Debug)]
+pub struct SharedOwnerEntry {
+    pub owner: Arc<str>,
+    pub count: u32,
+    pub acquired_at: SystemTime,
+    pub lock_timeout: Duration,
+}
+
 impl Default for ObjectLockState {
    fn default() -> Self {
        Self::new()
@@ -335,60 +350,87 @@ impl ObjectLockState {
    }

    /// Try fast path shared lock acquisition
-    pub fn try_acquire_shared_fast(&self, owner: &Arc<str>) -> bool {
-        if self.atomic_state.try_acquire_shared() {
-            self.atomic_state.update_access_time();
-            let mut shared = self.shared_owners.write();
-            if !shared.contains(owner) {
-                shared.push(owner.clone());
-            }
-            true
-        } else {
-            false
+    pub fn try_acquire_shared_fast(&self, owner: &Arc<str>, lock_timeout: Duration) -> bool {
+        if !self.atomic_state.try_acquire_shared() {
+            return false;
        }
+
+        self.atomic_state.update_access_time();
+        let mut shared = self.shared_owners.write();
+        if let Some(entry) = shared.iter_mut().find(|entry| entry.owner.as_ref() == owner.as_ref()) {
+            entry.count = entry.count.saturating_add(1);
+            entry.acquired_at = SystemTime::now();
+            entry.lock_timeout = lock_timeout;
+        } else {
+            shared.push(SharedOwnerEntry {
+                owner: owner.clone(),
+                count: 1,
+                acquired_at: SystemTime::now(),
+                lock_timeout,
+            });
+        }
+        true
    }

    /// Try fast path exclusive lock acquisition
-    pub fn try_acquire_exclusive_fast(&self, owner: &Arc<str>) -> bool {
-        if self.atomic_state.try_acquire_exclusive() {
-            self.atomic_state.update_access_time();
-            let mut current = self.current_owner.write();
-            *current = Some(owner.clone());
-            true
-        } else {
-            false
+    pub fn try_acquire_exclusive_fast(&self, owner: &Arc<str>, lock_timeout: Duration) -> bool {
+        if !self.atomic_state.try_acquire_exclusive() {
+            return false;
        }
+
+        self.atomic_state.update_access_time();
+        let mut current = self.current_owner.write();
+        *current = Some(ExclusiveOwnerInfo {
+            owner: owner.clone(),
+            acquired_at: SystemTime::now(),
+            lock_timeout,
+        });
+        true
    }

    /// Release shared lock
    pub fn release_shared(&self, owner: &Arc<str>) -> bool {
        let mut shared = self.shared_owners.write();
-        if let Some(pos) = shared.iter().position(|x| x.as_ref() == owner.as_ref()) {
-            shared.remove(pos);
+        if let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) {
+            let original_entry = shared[pos].clone();
+            let removed_entry = if shared[pos].count > 1 {
+                shared[pos].count -= 1;
+                None
+            } else {
+                Some(shared.remove(pos))
+            };
            if self.atomic_state.release_shared() {
-                // Notify waiting writers if no more readers
                if shared.is_empty() {
                    drop(shared);
                    self.optimized_notify.notify_writer();
                }
                true
            } else {
-                // Inconsistency detected - atomic state shows no shared lock but owner was found
                tracing::warn!(
-                    "Atomic state inconsistency during shared lock release: owner={}, remaining_owners={}",
+                    "Atomic state inconsistency during shared lock release: owner={}, remaining_entries={}",
                    owner,
                    shared.len()
                );
-                // Re-add owner to maintain consistency
-                shared.push(owner.clone());
+                // Re-add owner entry to maintain consistency when release failed
+                match removed_entry {
+                    Some(entry) => {
+                        shared.push(entry);
+                    }
+                    None => {
+                        if let Some(existing) = shared.iter_mut().find(|existing| existing.owner.as_ref() == owner.as_ref()) {
+                            existing.count = existing.count.saturating_add(1);
+                        } else {
+                            shared.push(original_entry);
+                        }
+                    }
+                }
                false
            }
        } else {
-            // Owner not found in shared owners list
            tracing::debug!(
-                "Shared lock release failed - owner not found: owner={}, current_owners={:?}",
+                "Shared lock release failed - owner not found: owner={}, current_entries={:?}",
                owner,
-                shared.iter().map(|s| s.as_ref()).collect::<Vec<_>>()
+                shared.iter().map(|s| s.owner.as_ref()).collect::<Vec<_>>()
            );
            false
        }
@@ -397,7 +439,7 @@ impl ObjectLockState {
    /// Release exclusive lock
    pub fn release_exclusive(&self, owner: &Arc<str>) -> bool {
        let mut current = self.current_owner.write();
-        if current.as_ref() == Some(owner) {
+        if current.as_ref().is_some_and(|info| info.owner.as_ref() == owner.as_ref()) {
            if self.atomic_state.release_exclusive() {
                *current = None;
                drop(current);
@@ -426,7 +468,7 @@ impl ObjectLockState {
            tracing::debug!(
                "Exclusive lock release failed - owner mismatch: expected_owner={}, actual_owner={:?}",
                owner,
-                current.as_ref().map(|s| s.as_ref())
+                current.as_ref().map(|s| s.owner.as_ref())
            );
            false
        }
@@ -483,16 +525,18 @@ mod tests {
        let owner2 = Arc::from("owner2");

        // Test shared locks
-        assert!(state.try_acquire_shared_fast(&owner1));
-        assert!(state.try_acquire_shared_fast(&owner2));
-        assert!(!state.try_acquire_exclusive_fast(&owner1));
+        let timeout = Duration::from_secs(30);
+
+        assert!(state.try_acquire_shared_fast(&owner1, timeout));
+        assert!(state.try_acquire_shared_fast(&owner2, timeout));
+        assert!(!state.try_acquire_exclusive_fast(&owner1, timeout));

        assert!(state.release_shared(&owner1));
        assert!(state.release_shared(&owner2));

        // Test exclusive lock
-        assert!(state.try_acquire_exclusive_fast(&owner1));
-        assert!(!state.try_acquire_shared_fast(&owner2));
+        assert!(state.try_acquire_exclusive_fast(&owner1, timeout));
+        assert!(!state.try_acquire_shared_fast(&owner2, timeout));
        assert!(state.release_exclusive(&owner1));
    }
 }
@@ -19,6 +19,8 @@ use std::hash::{Hash, Hasher};
 use std::sync::Arc;
 use std::time::{Duration, SystemTime};

+use crate::fast_lock::guard::FastLockGuard;
+
 /// Object key for version-aware locking
 #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
 pub struct ObjectKey {
@@ -340,6 +342,7 @@ pub struct BatchLockResult {
    pub successful_locks: Vec<ObjectKey>,
    pub failed_locks: Vec<(ObjectKey, LockResult)>,
    pub all_acquired: bool,
+    pub guards: Vec<FastLockGuard>,
 }

 #[cfg(test)]
@@ -296,11 +296,11 @@ impl S3Client {
            .context(format!("Failed to resolve file path: {local_path}"))?;

        if !canonical_path.exists() {
-            anyhow::bail!("File does not exist: {}", local_path);
+            anyhow::bail!("File does not exist: {local_path}");
        }

        if !canonical_path.is_file() {
-            anyhow::bail!("Path is not a file: {}", local_path);
+            anyhow::bail!("Path is not a file: {local_path}");
        }

        let metadata = tokio::fs::metadata(&canonical_path)
@@ -432,7 +432,7 @@ impl S3Client {

        while let Some(bytes_result) = byte_stream.try_next().await.context("Failed to read object content")? {
            if total_read + bytes_result.len() > max_size {
-                anyhow::bail!("Object size exceeds maximum allowed size of {} bytes", max_size);
+                anyhow::bail!("Object size exceeds maximum allowed size of {max_size} bytes");
            }
            content.extend_from_slice(&bytes_result);
            total_read += bytes_result.len();
@@ -82,6 +82,7 @@ pub trait HashReaderDetector {
 impl Reader for crate::HashReader {}
 impl Reader for crate::HardLimitReader {}
 impl Reader for crate::EtagReader {}
+impl<R> Reader for crate::LimitReader<R> where R: Reader {}
 impl<R> Reader for crate::CompressReader<R> where R: Reader {}
 impl<R> Reader for crate::EncryptReader<R> where R: Reader {}
 impl<R> Reader for crate::DecryptReader<R> where R: Reader {}
@@ -37,7 +37,7 @@ use std::pin::Pin;
 use std::task::{Context, Poll};
 use tokio::io::{AsyncRead, ReadBuf};

-use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
+use crate::{EtagResolvable, HashReaderDetector, HashReaderMut, TryGetIndex};

 pin_project! {
    #[derive(Debug)]
@@ -118,6 +118,8 @@ where
    }
 }

+impl<R> TryGetIndex for LimitReader<R> where R: AsyncRead + Unpin + Send + Sync {}
+
 #[cfg(test)]
 mod tests {
    use std::io::Cursor;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
安正超	007d9c0b21	fix: normalize ETag comparison in multipart upload and replication (#627 ) - Normalize ETags by removing quotes before comparison in complete_multipart_upload - Fix ETag comparison in replication logic to handle quoted ETags from API responses - Fix ETag comparison in transition object logic - Add unit tests for trim_etag function This fixes the ETag mismatch error when uploading large files (5GB+) via multipart upload, which was caused by PR #592 adding quotes to ETag responses while internal storage remains unquoted. Fixes #625	2025-10-08 21:19:57 +08:00
Nugine	626c7ed34a	fix: CompleteMultipartUpload encryption (#626 )	2025-10-08 20:27:40 +08:00
houseme	0e680eae31	fix typos and bump the dependencies group with 9 updates (#614 ) * fix typos * build(deps): bump the dependencies group with 9 updates (#613) Bumps the dependencies group with 9 updates: \| Package \| From \| To \| \| --- \| --- \| --- \| \| [axum](https://github.com/tokio-rs/axum) \| `0.8.4` \| `0.8.6` \| \| [axum-extra](https://github.com/tokio-rs/axum) \| `0.10.1` \| `0.10.3` \| \| [regex](https://github.com/rust-lang/regex) \| `1.11.2` \| `1.11.3` \| \| [serde](https://github.com/serde-rs/serde) \| `1.0.226` \| `1.0.228` \| \| [shadow-rs](https://github.com/baoyachi/shadow-rs) \| `1.3.0` \| `1.4.0` \| \| [sysinfo](https://github.com/GuillaumeGomez/sysinfo) \| `0.37.0` \| `0.37.1` \| \| [thiserror](https://github.com/dtolnay/thiserror) \| `2.0.16` \| `2.0.17` \| \| [tokio-rustls](https://github.com/rustls/tokio-rustls) \| `0.26.3` \| `0.26.4` \| \| [zeroize](https://github.com/RustCrypto/utils) \| `1.8.1` \| `1.8.2` \| Updates `axum` from 0.8.4 to 0.8.6 - [Release notes](https://github.com/tokio-rs/axum/releases) - [Changelog](https://github.com/tokio-rs/axum/blob/main/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/axum/compare/axum-v0.8.4...axum-v0.8.6) Updates `axum-extra` from 0.10.1 to 0.10.3 - [Release notes](https://github.com/tokio-rs/axum/releases) - [Changelog](https://github.com/tokio-rs/axum/blob/main/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/axum/compare/axum-extra-v0.10.1...axum-extra-v0.10.3) Updates `regex` from 1.11.2 to 1.11.3 - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.11.2...1.11.3) Updates `serde` from 1.0.226 to 1.0.228 - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.226...v1.0.228) Updates `shadow-rs` from 1.3.0 to 1.4.0 - [Release notes](https://github.com/baoyachi/shadow-rs/releases) - [Commits](https://github.com/baoyachi/shadow-rs/compare/1.3.0...v1.4.0) Updates `sysinfo` from 0.37.0 to 0.37.1 - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/compare/v0.37.0...v0.37.1) Updates `thiserror` from 2.0.16 to 2.0.17 - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/2.0.16...2.0.17) Updates `tokio-rustls` from 0.26.3 to 0.26.4 - [Release notes](https://github.com/rustls/tokio-rustls/releases) - [Commits](https://github.com/rustls/tokio-rustls/compare/v/0.26.3...v/0.26.4) Updates `zeroize` from 1.8.1 to 1.8.2 - [Commits](https://github.com/RustCrypto/utils/compare/zeroize-v1.8.1...zeroize-v1.8.2) --- updated-dependencies: - dependency-name: axum dependency-version: 0.8.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: axum-extra dependency-version: 0.10.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: regex dependency-version: 1.11.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: serde dependency-version: 1.0.228 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: shadow-rs dependency-version: 1.4.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: dependencies - dependency-name: sysinfo dependency-version: 0.37.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: thiserror dependency-version: 2.0.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: tokio-rustls dependency-version: 0.26.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies - dependency-name: zeroize dependency-version: 1.8.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: dependencies ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2025-10-02 23:29:18 +08:00
weisd	7622b37f7b	add iam notification (#604 ) move tonic service to rustfs	2025-09-30 17:32:23 +08:00
Nugine	f1dd3a982e	build(deps): upgrade s3s (#595 ) Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>	2025-09-28 21:10:42 +08:00
guojidan	4f73760a45	feat(append): implement object append operations with state tracking (#599 ) * feat(append): implement object append operations with state tracking Signed-off-by: junxiang Mu <1948535941@qq.com> * chore: rebase Signed-off-by: junxiang Mu <1948535941@qq.com> --------- Signed-off-by: junxiang Mu <1948535941@qq.com>	2025-09-27 20:06:26 -07:00
guojidan	be66cf8bd3	Improve lock (#596 ) * improve lock Signed-off-by: Mu junxiang <1948535941@qq.com> * feat(tests): add wait_for_object_absence helper and improve lifecycle test reliability Signed-off-by: Mu junxiang <1948535941@qq.com> * chore: remove dirty docs Signed-off-by: Mu junxiang <1948535941@qq.com> --------- Signed-off-by: Mu junxiang <1948535941@qq.com>	2025-09-27 17:57:56 -07:00
Copilot	23b40d398f	Fix ETag format to comply with HTTP standards by wrapping with quotes (#592 ) * Initial plan * Fix ETag format to comply with HTTP standards by wrapping with quotes Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> * bufigx --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com> Co-authored-by: overtrue <anzhengchao@gmail.com>	2025-09-27 10:03:05 +08:00
weisd	90f21a9102	refactor: Reimplement bucket replication system with enhanced architecture (#590 ) * feat:refactor replication * use aws sdk for replication client * refactor/replication * merge main * fix lifecycle test	2025-09-26 14:27:53 +08:00
guojidan	9b029d18b2	feat(lock): enhance lock management with timeout and ownership tracking (#589 ) - Add lock timeout support and track acquisition time in lock state - Improve lock conflict handling with detailed error messages - Optimize lock reuse when already held by same owner - Refactor lock state to store owner info and timeout duration - Update all lock operations to handle new state structure Signed-off-by: junxiang Mu <1948535941@qq.com>	2025-09-25 20:21:53 -07:00