fix(ecstore): log missing local paths during physical disk independence checks (#2680)

Co-authored-by: houseme <housemecn@gmail.com>

.docker/observability/README.md

@@ -13,6 +13,9 @@ The stack is composed of the following best-in-class open-source components:
 - **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
 - **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
 
+By default, this stack uses Tempo in single-binary mode and does not require Kafka/Redpanda.
+If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rustfs.yml` together with `docker-compose-tempo-ha-override.yml`.
+
 ## Architecture
 
 1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
@@ -46,6 +49,15 @@ Run the following command to start the entire stack:
 docker compose up -d
 ```
 
+### High Availability Tempo
+
+The default `docker-compose.yml` is the single-node stack.
+If you need the Kafka-backed HA Tempo configuration, start it with:
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### Access Dashboards
 
 | Service        | URL                                              | Credentials       | Description                    |

.docker/observability/README_ZH.md

@@ -13,6 +13,9 @@
 - **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
 - **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
 
+默认情况下，这套技术栈使用 Tempo 单二进制模式，不依赖 Kafka/Redpanda。
+如果需要基于 Kafka 的 HA Tempo 路径，请使用 `docker-compose-example-for-rustfs.yml` 配合 `docker-compose-tempo-ha-override.yml`。
+
 ## 架构
 
 1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
@@ -46,6 +49,15 @@
 docker compose up -d
 ```
 
+### Tempo 高可用模式
+
+默认的 `docker-compose.yml` 对应单机栈。
+如果需要基于 Kafka 的 HA Tempo 配置，请使用：
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### 访问仪表盘
 
 | 服务 | URL | 凭据 | 描述 |

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -85,10 +85,8 @@ services:
     networks:
       - otel-network
     restart: unless-stopped
-    depends_on:
-      - redpanda
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
@@ -205,7 +203,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/docker-compose.yml

@@ -16,10 +16,8 @@ services:
 
   # --- Tracing ---
   tempo:
-    image: grafana/tempo:2.10.3
+    image: grafana/tempo:2.10.5
     container_name: tempo
-    depends_on:
-      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -33,31 +31,11 @@ services:
       - otel-network
     restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
-  redpanda:
-    image: redpandadata/redpanda:latest
-    ports:
-      - "9092:9092" # Kafka API for clients
-    command: >
-      redpanda start --overprovisioned
-      --mode=dev-container
-      --kafka-addr=PLAINTEXT://0.0.0.0:9092
-      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
-
-  redpanda-console:
-    image: docker.redpanda.com/redpandadata/console:latest
-    environment:
-      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
-    volumes:
-      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
-    ports:
-      - "8080:8080"
-    depends_on:
-      - redpanda
 
   vulture:
     image: grafana/tempo-vulture:latest
@@ -168,7 +146,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/grafana/provisioning/datasources.yaml

@@ -1,97 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    uid: prometheus
-    access: proxy
-    orgId: 1
-    url: http://prometheus:9090
-    isDefault: true
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      exemplarTraceIdDestinations:
-        - name: trace_id
-          datasourceUid: tempo
-
-  - name: Tempo
-    type: tempo
-    uid: tempo
-    access: proxy
-    orgId: 1
-    url: http://tempo:3200
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      serviceMap:
-        datasourceUid: prometheus
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false
-      tracesToMetrics:
-        datasourceUid: prometheus
-        tags: [ { key: 'service.name' }, { key: 'job' } ]
-        queries:
-          - name: 'Service-Level Latency'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
-          - name: 'Service-Level Calls'
-            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
-          - name: 'Service-Level Errors'
-            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
-      nodeGraph:
-        enabled: true
-
-  - name: Loki
-    type: loki
-    uid: loki
-    orgId: 1
-    url: http://loki:3100
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      derivedFields:
-        - datasourceUid: tempo
-          matcherRegex: 'trace_id=(\w+)'
-          name: 'TraceID'
-          url: '$${__value.raw}'
-
-  - name: Jaeger
-    type: jaeger
-    uid: jaeger
-    url: http://jaeger:16686
-    access: proxy
-    isDefault: false
-    editable: false
-    jsonData:
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false

.docker/observability/tempo.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-partition_ring_live_store: true
 stream_over_http_enabled: true
 
 server:
@@ -33,33 +32,17 @@ distributor:
           endpoint: "0.0.0.0:4317"
         http:
           endpoint: "0.0.0.0:4318"
-  #log_received_spans:
-  #    enabled: true
-  #  log_discarded_spans:
-  #    enabled: true
 
-backend_scheduler:
-  provider:
-    compaction:
-      compaction:
-        block_retention: 1h
-
-backend_worker:
-  backend_scheduler_addr: localhost:3200
-  compaction:
-    block_retention: 1h
-  ring:
-    kvstore:
-      store: memberlist
-
-querier:
-  query_live_store: true
+ingester:
+  max_block_duration: 5m
 
 metrics_generator:
   registry:
     external_labels:
       source: tempo
       cluster: docker-compose
+  traces_storage:
+    path: /var/tempo/generator/traces
   storage:
     path: /var/tempo/generator/wal
     remote_write:
@@ -85,15 +68,5 @@ overrides:
       processors: [ "span-metrics", "service-graphs", "local-blocks" ]
       generate_native_histograms: both
 
-ingest:
-  enabled: true
-  kafka:
-    address: redpanda:9092
-    topic: tempo-ingest
-
-block_builder:
-  consume_cycle_duration: 30s
-
 usage_report:
   reporting_enabled: false
-

Cargo.lock

@@ -557,9 +557,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.41"
+version = "0.4.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1"
+checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac"
 dependencies = [
  "compression-codecs",
  "compression-core",
@@ -690,9 +690,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
 
 [[package]]
 name = "aws-config"
-version = "1.8.15"
+version = "1.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc"
+checksum = "50f156acdd2cf55f5aa53ee416c4ac851cf1222694506c0b1f78c85695e9ca9d"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -755,9 +755,9 @@ dependencies = [
 
 [[package]]
 name = "aws-runtime"
-version = "1.7.2"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17"
+checksum = "5dcd93c82209ac7413532388067dce79be5a8780c1786e5fae3df22e4dee2864"
 dependencies = [
  "aws-credential-types",
  "aws-sigv4",
@@ -783,9 +783,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "1.129.0"
+version = "1.131.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d4e8410fadbc0ee453145dd77a4958227b18b05bf67c2795d0a8b8596c9aa0f"
+checksum = "fe1b8c5282bf859170836045296b3cd710b7573aceb909498366bb508a41058e"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -804,23 +804,23 @@ dependencies = [
  "bytes",
  "fastrand",
  "hex",
- "hmac 0.12.1",
+ "hmac 0.13.0",
  "http 0.2.12",
  "http 1.4.0",
  "http-body 1.0.1",
  "lru 0.16.4",
  "percent-encoding",
  "regex-lite",
- "sha2 0.10.9",
+ "sha2 0.11.0",
  "tracing",
  "url",
 ]
 
 [[package]]
 name = "aws-sdk-sso"
-version = "1.97.0"
+version = "1.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567"
+checksum = "d69c77aafa20460c68b6b3213c84f6423b6e76dbf89accd3e1789a686ffd9489"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -842,9 +842,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.99.0"
+version = "1.100.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8"
+checksum = "1c7e7b09346d5ca22a2a08267555843a6a0127fb20d8964cb6ecfb8fdb190225"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -866,9 +866,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sts"
-version = "1.102.0"
+version = "1.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fc35b7a14cabdad13795fbbbd26d5ddec0882c01492ceedf2af575aad5f37dd"
+checksum = "c2249b81a2e73a8027c41c378463a81ec39b8510f184f2caab87de912af0f49b"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -891,9 +891,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sigv4"
-version = "1.4.2"
+version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4"
+checksum = "68dc0b907359b120170613b5c09ccc61304eac3998ff6274b97d93ee6490115a"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-eventstream",
@@ -904,13 +904,13 @@ dependencies = [
  "crypto-bigint 0.5.5",
  "form_urlencoded",
  "hex",
- "hmac 0.12.1",
+ "hmac 0.13.0",
  "http 0.2.12",
  "http 1.4.0",
  "p256 0.11.1",
  "percent-encoding",
  "ring",
- "sha2 0.10.9",
+ "sha2 0.11.0",
  "subtle",
  "time",
  "tracing",
@@ -1125,9 +1125,9 @@ dependencies = [
 
 [[package]]
 name = "aws-types"
-version = "1.3.14"
+version = "1.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9"
+checksum = "2f4bbcaa9304ea40902d3d5f42a0428d1bd895a2b0f6999436fb279ffddc58ac"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-async",
@@ -1295,9 +1295,9 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.8.4"
+version = "1.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e"
+checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -1443,9 +1443,9 @@ dependencies = [
 
 [[package]]
 name = "cargo-platform"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87a0c0e6148f11f01f32650a2ea02d532b2ad4e81d8bd41e6e565b5adc5e6082"
+checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba"
 dependencies = [
  "serde",
  "serde_core",
@@ -1473,9 +1473,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
-version = "1.2.60"
+version = "1.2.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
+checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1691,9 +1691,9 @@ dependencies = [
 
 [[package]]
 name = "compression-codecs"
-version = "0.4.37"
+version = "0.4.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7"
+checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf"
 dependencies = [
  "brotli",
  "bzip2",
@@ -1707,9 +1707,9 @@ dependencies = [
 
 [[package]]
 name = "compression-core"
-version = "0.4.31"
+version = "0.4.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d"
+checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
 
 [[package]]
 name = "concurrent-queue"
@@ -1845,9 +1845,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "cpp_demangle"
-version = "0.5.1"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def"
+checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253"
 dependencies = [
  "cfg-if",
 ]
@@ -1887,9 +1887,9 @@ dependencies = [
 
 [[package]]
 name = "crc-catalog"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
+checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
 
 [[package]]
 name = "crc-fast"
@@ -2266,9 +2266,9 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.10.0"
+version = "2.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
+checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
 
 [[package]]
 name = "datafusion"
@@ -3104,9 +3104,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-macro"
-version = "0.3.0"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877b67d2a742d9c722961c468e8b9cfe87d78e53f2e1d61125c8d388bb8a3bba"
+checksum = "5e920f265b9cf1170b472f1ee1165ec80554b306e1e7aee36ce99913585159e1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3115,9 +3115,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-tokio-telemetry"
-version = "0.3.0"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1edba70ebc611e0ebc889d7f6b3d08ce742e48cb188da1f298ec6378b0b4f86b"
+checksum = "0d401f6c67fdd6e05bed120acf7f59960e5b7a3ada0768897f7847f2fb4304b0"
 dependencies = [
  "arc-swap",
  "bon",
@@ -3142,9 +3142,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-trace-format"
-version = "0.3.0"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c9d20951a4584f9ea544ff27143709821c4709f27d30b52395300f020392475"
+checksum = "9ebf6e8b29800b2e9aab3012cc042a1fefffb3a2a0093ae7a027e03e332c0323"
 dependencies = [
  "dial9-trace-format-derive",
  "serde",
@@ -3152,9 +3152,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-trace-format-derive"
-version = "0.3.0"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5800a5a155746fe428681cdc30570f33d961a31ed800e6df41b78d0af64529ed"
+checksum = "4593cd0dcf9c490a0d460821f1d68ab3805190f1e7f2258824c44dfaed523336"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3255,6 +3255,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "chrono",
+ "clap",
  "flatbuffers",
  "flate2",
  "futures",
@@ -4403,9 +4404,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
 
 [[package]]
 name = "hybrid-array"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214"
+checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5"
 dependencies = [
  "typenum",
 ]
@@ -4725,9 +4726,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
 [[package]]
 name = "io-uring"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344"
+checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
 dependencies = [
  "bitflags 2.11.1",
  "cfg-if",
@@ -4834,9 +4835,9 @@ dependencies = [
 
 [[package]]
 name = "jiff"
-version = "0.2.23"
+version = "0.2.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359"
+checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d"
 dependencies = [
  "jiff-static",
  "jiff-tzdb-platform",
@@ -4849,9 +4850,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.23"
+version = "0.2.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4"
+checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4956,6 +4957,21 @@ dependencies = [
  "simple_asn1",
 ]
 
+[[package]]
+name = "kafka-protocol"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2"
+dependencies = [
+ "anyhow",
+ "bytes",
+ "crc",
+ "crc32c",
+ "indexmap 2.14.0",
+ "paste",
+ "uuid",
+]
+
 [[package]]
 name = "konst"
 version = "0.2.20"
@@ -5074,9 +5090,9 @@ checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f"
 
 [[package]]
 name = "libc"
-version = "0.2.185"
+version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
 [[package]]
 name = "libflate"
@@ -5140,12 +5156,11 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
 [[package]]
 name = "libmimalloc-sys"
-version = "0.1.44"
+version = "0.1.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
+checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
 dependencies = [
  "cc",
- "libc",
 ]
 
 [[package]]
@@ -5230,9 +5245,9 @@ checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
 
 [[package]]
 name = "local-ip-address"
-version = "0.6.11"
+version = "0.6.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4a59a0cb1c7f84471ad5cd38d768c2a29390d17f1ff2827cdf49bc53e8ac70b"
+checksum = "d7b0187df4e614e42405b49511b82ff7a1774fbd9a816060ee465067847cac22"
 dependencies = [
  "libc",
  "neli",
@@ -5451,9 +5466,9 @@ dependencies = [
 
 [[package]]
 name = "metrique"
-version = "0.1.24"
+version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2cb1b5bc8898d2cb0da392c35f91826182c4cced80dddf061abd63bb7e34318"
+checksum = "2638fb6c2325d3e35c7b0d1ce49cf8ce74b89b7439d4dc5ce1c9fff0f7fa18a1"
 dependencies = [
  "itoa",
  "jiff",
@@ -5471,9 +5486,9 @@ dependencies = [
 
 [[package]]
 name = "metrique-core"
-version = "0.1.19"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67e3d9e65d3978d38102e3c4bd4142b91351467bb9c698db99654375e8f3858b"
+checksum = "16b99ea3484ab9afe6337413bacc842d516e65623a476699f9873803028e6a9e"
 dependencies = [
  "itertools 0.14.0",
  "metrique-writer-core",
@@ -5481,9 +5496,9 @@ dependencies = [
 
 [[package]]
 name = "metrique-macro"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07c50c41313eaa762e16c251aa3aa7f1eff170ab403fac8fc688840e93f39b18"
+checksum = "0a02c5cb6e55c1ec5eb3ef2d10d137805547c486ad54b587d42d0a49713e6ef9"
 dependencies = [
  "Inflector",
  "darling 0.23.0",
@@ -5494,9 +5509,9 @@ dependencies = [
 
 [[package]]
 name = "metrique-service-metrics"
-version = "0.1.20"
+version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd93adb879c2d72ea8a23178deb5c9b379a6e459bfa08eb2c86a9c8c213cb3a6"
+checksum = "bb71714973a9ba53d609a577311aae4d8beb7df37869b5a92cb777a3f4c905b8"
 dependencies = [
  "metrique-writer",
 ]
@@ -5509,9 +5524,9 @@ checksum = "d607939211e4eaaa8cd35394fa5e57faffb7390d0ac513b39992edcaf3cc526c"
 
 [[package]]
 name = "metrique-writer"
-version = "0.1.21"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7885027d11fce09ba86f50b11ed8c34166a76e68b1568de3e9cb6689c18ea39"
+checksum = "04f770fe8a45a7c2f285c7ce81b9364e0e5604cbb7daf693fef1f624bf8f0bf8"
 dependencies = [
  "ahash 0.8.12",
  "crossbeam-queue",
@@ -5530,9 +5545,9 @@ dependencies = [
 
 [[package]]
 name = "metrique-writer-core"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2aa7ddd300b7405e6fc576fa57026b45097b5bf7b7c88c9be180dce6f4a1e09"
+checksum = "3a0b67a72a8849987122ca8ceacaf81d47eaba36cd673767fc3742a086dd04ac"
 dependencies = [
  "derive-where",
  "itertools 0.14.0",
@@ -5542,9 +5557,9 @@ dependencies = [
 
 [[package]]
 name = "metrique-writer-macro"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7417c002f7b01c3d96792ff553b0b7e333059048a9e40d4f8d4bab23f570773e"
+checksum = "5891f02fdba5bd734992ea074502934344e4138525bf299ee4984f16160c3e6e"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
@@ -5556,9 +5571,9 @@ dependencies = [
 
 [[package]]
 name = "mimalloc"
-version = "0.1.48"
+version = "0.1.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
+checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640"
 dependencies = [
  "libmimalloc-sys",
 ]
@@ -6435,9 +6450,9 @@ dependencies = [
 
 [[package]]
 name = "pbkdf2"
-version = "0.13.0-rc.10"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f24f3eb2f4471b1730d59e4b730b747939960a8c7eb0c33c5a9076f2d3dddea"
+checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629"
 dependencies = [
  "digest 0.11.2",
  "hmac 0.13.0",
@@ -7020,9 +7035,9 @@ dependencies = [
 
 [[package]]
 name = "psm"
-version = "0.1.30"
+version = "0.1.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8"
+checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea"
 dependencies = [
  "ar_archive_writer",
  "cc",
@@ -7080,9 +7095,9 @@ dependencies = [
 
 [[package]]
 name = "pyroscope"
-version = "2.0.0"
+version = "2.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85eebd4bcbf45db75f67d2ba20ea0207bd111d2029c07a7db3229289173d4387"
+checksum = "8ef335beaacecb830dc21ab54c8b4a75677874bb78a375ad880715ce385ccb35"
 dependencies = [
  "lazy_static",
  "libc",
@@ -7870,6 +7885,7 @@ dependencies = [
  "mimalloc",
  "mime_guess",
  "opentelemetry",
+ "opentelemetry_sdk",
  "percent-encoding",
  "pin-project-lite",
  "pprof-pyroscope-fork",
@@ -7937,6 +7953,7 @@ dependencies = [
  "tower-http",
  "tracing",
  "tracing-opentelemetry",
+ "tracing-subscriber",
  "url",
  "urlencoding",
  "uuid",
@@ -7964,7 +7981,6 @@ dependencies = [
  "futures",
  "hashbrown 0.17.0",
  "metrics",
- "rumqttc-next",
  "rustfs-config",
  "rustfs-ecstore",
  "rustfs-s3-common",
@@ -8050,7 +8066,7 @@ dependencies = [
  "cfg-if",
  "chacha20poly1305",
  "jsonwebtoken",
- "pbkdf2 0.13.0-rc.10",
+ "pbkdf2 0.13.0",
  "rand 0.10.1",
  "serde_json",
  "sha2 0.11.0",
@@ -8100,6 +8116,8 @@ dependencies = [
  "memmap2 0.9.10",
  "metrics",
  "num_cpus",
+ "opentelemetry",
+ "opentelemetry_sdk",
  "parking_lot 0.12.5",
  "path-absolutize",
  "pin-project-lite",
@@ -8146,6 +8164,7 @@ dependencies = [
  "tonic",
  "tower",
  "tracing",
+ "tracing-opentelemetry",
  "tracing-subscriber",
  "url",
  "urlencoding",
@@ -8262,6 +8281,54 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "rustfs-kafka"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eee0644a99743fb2f51db7fbae1a6ca2d85f064daf7595784eaa52834a68c96"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "fnv",
+ "hmac 0.13.0",
+ "indexmap 2.14.0",
+ "kafka-protocol",
+ "metrics",
+ "pbkdf2 0.13.0",
+ "rand 0.10.1",
+ "rustls",
+ "rustls-native-certs",
+ "sha2 0.11.0",
+ "socket2",
+ "thiserror 2.0.18",
+ "tracing",
+ "twox-hash",
+ "webpki-roots 1.0.7",
+]
+
+[[package]]
+name = "rustfs-kafka-async"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd1997c3116cb94ede80d9a0b828f46dd27386cc93825fce141a92eb3aa9630"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "hmac 0.13.0",
+ "kafka-protocol",
+ "metrics",
+ "pbkdf2 0.13.0",
+ "rand 0.10.1",
+ "rustfs-kafka",
+ "rustls",
+ "rustls-native-certs",
+ "sha2 0.11.0",
+ "tokio",
+ "tokio-rustls",
+ "tracing",
+ "webpki-roots 1.0.7",
+]
+
 [[package]]
 name = "rustfs-keystone"
 version = "0.0.5"
@@ -8363,7 +8430,6 @@ dependencies = [
  "hashbrown 0.17.0",
  "quick-xml 0.39.2",
  "rayon",
- "rumqttc-next",
  "rustc-hash",
  "rustfs-config",
  "rustfs-ecstore",
@@ -8421,6 +8487,7 @@ dependencies = [
  "opentelemetry-semantic-conventions",
  "opentelemetry-stdout",
  "opentelemetry_sdk",
+ "percent-encoding",
  "pyroscope",
  "rustfs-audit",
  "rustfs-config",
@@ -8679,6 +8746,7 @@ dependencies = [
  "rumqttc-next",
  "rustfs-config",
  "rustfs-ecstore",
+ "rustfs-kafka-async",
  "rustfs-s3-common",
  "rustfs-utils",
  "rustls",
@@ -8851,9 +8919,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.38"
+version = "0.23.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"
+checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e"
 dependencies = [
  "aws-lc-rs",
  "log",
@@ -8879,9 +8947,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.14.0"
+version = "1.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
+checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
 dependencies = [
  "web-time",
  "zeroize",
@@ -8916,9 +8984,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.12"
+version = "0.103.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -9337,16 +9405,15 @@ dependencies = [
 
 [[package]]
 name = "shadow-rs"
-version = "1.7.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c798acfc78a69c7b038adde44084d8df875555b091da42c90ae46257cdcc41a"
+checksum = "1dd39b4b2077bd36e60ca28c31d494046e747759cb9b507a7d177bb64787c39e"
 dependencies = [
  "cargo_metadata",
  "const_format",
  "is_debug",
+ "jiff",
  "serde_json",
- "time",
- "tzdb",
 ]
 
 [[package]]
@@ -9658,15 +9725,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
 [[package]]
 name = "stacker"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013"
+checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190"
 dependencies = [
  "cc",
  "cfg-if",
  "libc",
  "psm",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -9751,9 +9818,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "suppaftp"
-version = "8.0.2"
+version = "8.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d3da253d7e9993de86df41eb89e8cb1b6f567abe215798645651fca4148d0aa"
+checksum = "4275c142b5be3af2eeadd70dd368caf3b65546c8af1035839372dd7a1436127d"
 dependencies = [
  "async-trait",
  "chrono",
@@ -9770,9 +9837,9 @@ dependencies = [
 
 [[package]]
 name = "symbolic-common"
-version = "12.18.1"
+version = "12.18.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1f3cdeaae6779ecba2567f20bf7716718b8c4ce6717c9def4ced18786bb11ea"
+checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd"
 dependencies = [
  "debugid",
  "memmap2 0.9.10",
@@ -9782,9 +9849,9 @@ dependencies = [
 
 [[package]]
 name = "symbolic-demangle"
-version = "12.18.1"
+version = "12.18.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "672c6ad9cb8fce6a1283cc9df9070073cccad00ae241b80e3686328a64e3523b"
+checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471"
 dependencies = [
  "cpp_demangle",
  "rustc-demangle",
@@ -10515,35 +10582,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
 
 [[package]]
 name = "typenum"
-version = "1.19.0"
+version = "1.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
-
-[[package]]
-name = "tz-rs"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fc6c929ffa10fb34f4a3c7e9a73620a83ef2e85e47f9ec3381b8289e6762f42"
-
-[[package]]
-name = "tzdb"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56d4e985b6dda743ae7fd4140c28105316ffd75bc58258ee6cc12934e3eb7a0c"
-dependencies = [
- "iana-time-zone",
- "tz-rs",
- "tzdb_data",
-]
-
-[[package]]
-name = "tzdb_data"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "125a0a63c4bd75c73f61863463cb400db4b1aa5039b203b0ee1d628a7e3dabb2"
-dependencies = [
- "tz-rs",
-]
+checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
 
 [[package]]
 name = "unftp-core"

Cargo.toml

@@ -115,7 +115,7 @@ rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.41" }
+async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
 async-nats = "0.47.0"
@@ -132,6 +132,7 @@ http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+rustfs-kafka-async = { version = "1.2.0" }
 socket2 = { version = "0.6.3", features = ["all"] }
 tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
@@ -167,10 +168,10 @@ crc-fast = "1.9.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
-pbkdf2 = "0.13.0-rc.10"
+pbkdf2 = "0.13.0"
 rsa = { version = "0.10.0-rc.17" }
-rustls = { version = "0.23.38", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pki-types = "1.14.0"
+rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls-pki-types = "1.14.1"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"
@@ -179,7 +180,7 @@ zeroize = { version = "1.8.2", features = ["derive"] }
 # Time and Date
 chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.23", features = ["serde"] }
+jiff = { version = "0.2.24", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
@@ -188,9 +189,9 @@ arc-swap = "1.9.1"
 astral-tokio-tar = "0.6.0"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.15" }
+aws-config = { version = "1.8.16" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.131.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
@@ -220,9 +221,9 @@ hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.185"
+libc = "0.2.186"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.11"
+local-ip-address = "0.6.12"
 memmap2 = "0.9.10"
 lz4 = "1.28.1"
 matchit = "0.9.2"
@@ -252,7 +253,7 @@ rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
 s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
 serial_test = "3.4.0"
-shadow-rs = { version = "1.7.1", default-features = false }
+shadow-rs = { version = "2.0.0", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
@@ -291,12 +292,12 @@ opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rus
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }
-pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.0.2", features = ["backend-pprof-rs"] }
 
 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }
 unftp-core = "0.1.0"
-suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"
 
 # WebDAV

crates/audit/Cargo.toml

@@ -41,11 +41,10 @@ serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
-url = { workspace = true }
-rumqttc = { workspace = true }
 
 [dev-dependencies]
 temp-env = { workspace = true }
+url = { workspace = true }
 
 [lints]
 workspace = true

crates/audit/src/factory.rs

@@ -15,13 +15,13 @@
 use crate::AuditEntry;
 use async_trait::async_trait;
 use rustfs_config::AUDIT_DEFAULT_DIR;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
+use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
     config::{
-        build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
-        validate_pulsar_config, validate_webhook_config,
+        build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+        validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
     },
     error::TargetError,
     target::TargetType,
@@ -119,3 +119,22 @@ impl TargetFactory for PulsarTargetFactory {
         AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }
+
+pub struct KafkaTargetFactory;
+
+#[async_trait]
+impl TargetFactory for KafkaTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_kafka_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}

crates/audit/src/registry.rs

@@ -14,7 +14,9 @@
 
 use crate::{
     AuditEntry, AuditError, AuditResult,
-    factory::{MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory},
+    factory::{
+        KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
+    },
 };
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
@@ -53,6 +55,7 @@ impl AuditRegistry {
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
         registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
         registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
+        registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
 
         registry
     }
@@ -197,8 +200,8 @@ impl AuditRegistry {
             }
         }
 
-        if !errors.is_empty() {
-            return Err(AuditError::Target(errors.into_iter().next().unwrap()));
+        if let Some(error) = errors.into_iter().next() {
+            return Err(AuditError::Target(error));
         }
 
         Ok(())

crates/config/src/audit/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Kafka Environment Variables
+pub const ENV_AUDIT_KAFKA_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_ENABLE";
+pub const ENV_AUDIT_KAFKA_BROKERS: &str = "RUSTFS_AUDIT_KAFKA_BROKERS";
+pub const ENV_AUDIT_KAFKA_TOPIC: &str = "RUSTFS_AUDIT_KAFKA_TOPIC";
+pub const ENV_AUDIT_KAFKA_ACKS: &str = "RUSTFS_AUDIT_KAFKA_ACKS";
+pub const ENV_AUDIT_KAFKA_TLS_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_TLS_ENABLE";
+pub const ENV_AUDIT_KAFKA_TLS_CA: &str = "RUSTFS_AUDIT_KAFKA_TLS_CA";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_KAFKA_QUEUE_DIR: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_DIR";
+pub const ENV_AUDIT_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_AUDIT_KAFKA_ENABLE,
+    ENV_AUDIT_KAFKA_BROKERS,
+    ENV_AUDIT_KAFKA_TOPIC,
+    ENV_AUDIT_KAFKA_ACKS,
+    ENV_AUDIT_KAFKA_TLS_ENABLE,
+    ENV_AUDIT_KAFKA_TLS_CA,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_CERT,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_KEY,
+    ENV_AUDIT_KAFKA_QUEUE_DIR,
+    ENV_AUDIT_KAFKA_QUEUE_LIMIT,
+];
+
+/// A list of all valid configuration keys for a Kafka audit target.
+pub const AUDIT_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/mod.rs

@@ -16,11 +16,13 @@
 //! This module defines the configuration for audit systems, including
 //! webhook and MQTT audit-related settings.
 
+mod kafka;
 mod mqtt;
 mod nats;
 mod pulsar;
 mod webhook;
 
+pub use kafka::*;
 pub use mqtt::*;
 pub use nats::*;
 pub use pulsar::*;
@@ -33,6 +35,7 @@ pub const AUDIT_PREFIX: &str = "audit";
 pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
 
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
+pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
 pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
 pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
 pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
@@ -40,6 +43,7 @@ pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
 #[allow(dead_code)]
 pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
+    AUDIT_KAFKA_SUB_SYS,
     AUDIT_MQTT_SUB_SYS,
     AUDIT_NATS_SUB_SYS,
     AUDIT_PULSAR_SUB_SYS,

crates/config/src/constants/app.rs

@@ -131,6 +131,18 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable to explicitly bypass local physical disk independence checks.
+pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
+
+/// Compatibility alias used by legacy MinIO CI pipelines.
+///
+/// RustFS keeps this alias for backward compatibility only. Prefer
+/// `ENV_UNSAFE_BYPASS_DISK_CHECK` for explicit bypass control.
+pub const ENV_MINIO_CI: &str = "MINIO_CI";
+
+/// Default flag value for bypassing local physical disk independence checks.
+pub const DEFAULT_UNSAFE_BYPASS_DISK_CHECK: bool = false;
+
 /// Environment variable for server access key.
 pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
 

crates/config/src/constants/env.rs

@@ -26,6 +26,15 @@ pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";
 
+/// Global switch for enabling the audit module.
+pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
+/// Global switch for enabling the notify module.
+pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
+/// Default global audit switch (disabled by default).
+pub const DEFAULT_AUDIT_ENABLE: bool = false;
+/// Default global notify switch (disabled by default).
+pub const DEFAULT_NOTIFY_ENABLE: bool = false;
+
 /// Medium-drawn lines separator
 /// This is used to separate words in environment variable names.
 pub const ENV_WORD_DELIMITER_DASH: &str = "-";
@@ -290,4 +299,10 @@ mod tests {
             assert!(state.is_disabled());
         }
     }
+
+    #[test]
+    fn test_global_audit_notify_switch_constants() {
+        assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
+        assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
+    }
 }

crates/config/src/constants/health.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Enable or disable public `/health` and `/health/ready` endpoints.
+/// When disabled, the routes are not registered and return 404.
+pub const ENV_HEALTH_ENDPOINT_ENABLE: &str = "RUSTFS_HEALTH_ENDPOINT_ENABLE";
+pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
+
+/// Cache TTL for storage readiness runtime-state evaluation (milliseconds).
+/// This reduces storage-layer pressure when probes are called at high frequency.
+pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
+pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
+
+/// Enable minimal health payload mode for GET `/health*` responses.
+/// When enabled, only `status` and `ready` fields are returned.
+pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
+pub const DEFAULT_HEALTH_MINIMAL_RESPONSE_ENABLE: bool = false;

crates/config/src/constants/mod.rs

@@ -20,6 +20,7 @@ pub(crate) mod console;
 pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
+pub(crate) mod health;
 pub(crate) mod object;
 pub(crate) mod oidc;
 pub(crate) mod profiler;

crates/config/src/constants/proxy.rs

@@ -20,6 +20,11 @@ pub const ENV_TRUSTED_PROXY_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_ENABLED";
 /// Trusted proxy middleware is enabled by default.
 pub const DEFAULT_TRUSTED_PROXY_ENABLED: bool = true;
 
+/// Environment variable to select the trusted proxy implementation.
+pub const ENV_TRUSTED_PROXY_IMPLEMENTATION: &str = "RUSTFS_TRUSTED_PROXY_IMPLEMENTATION";
+/// The simplified implementation is used by default.
+pub const DEFAULT_TRUSTED_PROXY_IMPLEMENTATION: &str = "simple";
+
 /// Environment variable for the proxy validation mode.
 pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDATION_MODE";
 /// Default validation mode is "hop_by_hop".

crates/config/src/constants/targets.rs

@@ -40,6 +40,15 @@ pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
 pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
 pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
+pub const KAFKA_BROKERS: &str = "brokers";
+pub const KAFKA_TOPIC: &str = "topic";
+pub const KAFKA_ACKS: &str = "acks";
+pub const KAFKA_QUEUE_DIR: &str = "queue_dir";
+pub const KAFKA_QUEUE_LIMIT: &str = "queue_limit";
+pub const KAFKA_TLS_ENABLE: &str = "tls_enable";
+pub const KAFKA_TLS_CA: &str = "tls_ca";
+pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key";
 
 pub const NATS_ADDRESS: &str = "address";
 pub const NATS_SUBJECT: &str = "subject";

crates/config/src/lib.rs

@@ -31,6 +31,8 @@ pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::heal::*;
 #[cfg(feature = "constants")]
+pub use constants::health::*;
+#[cfg(feature = "constants")]
 pub use constants::object::*;
 #[cfg(feature = "constants")]
 pub use constants::profiler::*;

crates/config/src/notify/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// A list of all valid configuration keys for a Kafka target.
+pub const NOTIFY_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+// Kafka Environment Variables
+pub const ENV_NOTIFY_KAFKA_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_ENABLE";
+pub const ENV_NOTIFY_KAFKA_BROKERS: &str = "RUSTFS_NOTIFY_KAFKA_BROKERS";
+pub const ENV_NOTIFY_KAFKA_TOPIC: &str = "RUSTFS_NOTIFY_KAFKA_TOPIC";
+pub const ENV_NOTIFY_KAFKA_ACKS: &str = "RUSTFS_NOTIFY_KAFKA_ACKS";
+pub const ENV_NOTIFY_KAFKA_TLS_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_TLS_ENABLE";
+pub const ENV_NOTIFY_KAFKA_TLS_CA: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CA";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_KAFKA_QUEUE_DIR: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_DIR";
+pub const ENV_NOTIFY_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_NOTIFY_KAFKA_ENABLE,
+    ENV_NOTIFY_KAFKA_BROKERS,
+    ENV_NOTIFY_KAFKA_TOPIC,
+    ENV_NOTIFY_KAFKA_ACKS,
+    ENV_NOTIFY_KAFKA_TLS_ENABLE,
+    ENV_NOTIFY_KAFKA_TLS_CA,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY,
+    ENV_NOTIFY_KAFKA_QUEUE_DIR,
+    ENV_NOTIFY_KAFKA_QUEUE_LIMIT,
+];

crates/config/src/notify/mod.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod arn;
+mod kafka;
 mod mqtt;
 mod nats;
 mod pulsar;
@@ -20,6 +21,7 @@ mod store;
 mod webhook;
 
 pub use arn::*;
+pub use kafka::*;
 pub use mqtt::*;
 pub use nats::*;
 pub use pulsar::*;
@@ -69,13 +71,13 @@ pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
 
 #[allow(dead_code)]
 pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
+    NOTIFY_KAFKA_SUB_SYS,
     NOTIFY_MQTT_SUB_SYS,
     NOTIFY_NATS_SUB_SYS,
     NOTIFY_PULSAR_SUB_SYS,
     NOTIFY_WEBHOOK_SUB_SYS,
 ];
 
-#[allow(dead_code)]
 pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
 pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
 #[allow(dead_code)]

crates/config/src/observability/mod.rs

@@ -22,6 +22,14 @@ pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
 pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT";
 pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT";
 pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT";
+pub const ENV_OBS_ENDPOINT_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_HEADERS";
+pub const ENV_OBS_ENDPOINT_TRACES_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS";
+pub const ENV_OBS_ENDPOINT_METRICS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS";
+pub const ENV_OBS_ENDPOINT_LOGS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS";
+pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
 pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
@@ -108,6 +116,14 @@ mod tests {
         assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT");
         assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT");
         assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT");
+        assert_eq!(ENV_OBS_ENDPOINT_HEADERS, "RUSTFS_OBS_ENDPOINT_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_HEADERS, "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_HEADERS, "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_HEADERS, "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS");
         assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT");
         assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
         assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");

crates/e2e_test/Cargo.toml

@@ -73,3 +73,4 @@ rcgen.workspace = true
 anyhow.workspace = true
 rustls.workspace = true
 zip.workspace = true
+clap.workspace = true

crates/e2e_test/src/bin/tls_gen.rs

@@ -0,0 +1,22 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Parser;
+use e2e_test::tls_gen::{Args, run};
+
+fn main() -> anyhow::Result<()> {
+    let out_dir = run(Args::parse())?;
+    println!("Generated RustFS TLS bundle in {}", out_dir.display());
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -125,3 +125,5 @@ mod replication_extension_test;
 
 #[cfg(test)]
 mod snowball_auto_extract_test;
+
+pub mod tls_gen;

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -19,6 +19,7 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
     use serial_test::serial;
     use tracing::info;
@@ -179,4 +180,84 @@ mod tests {
             "Delete marker should no longer be latest after the second put"
         );
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_prefix_with_marker_object_returns_children() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns prefix children when a marker object also exists");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-versions-prefix-marker";
+        let marker_key = "data01";
+        let child_keys = [
+            "data01/meta/dump-2026-04-08-053205.json.gz",
+            "data01/meta/dump-2026-04-08-063209.json.gz",
+        ];
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to suspend versioning");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(marker_key)
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to put marker object");
+
+        for key in child_keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(ByteStream::from_static(b"payload"))
+                .send()
+                .await
+                .expect("Failed to put child object");
+        }
+
+        let listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix("data01/")
+            .send()
+            .await
+            .expect("Failed to list object versions by prefix");
+
+        let version_keys: Vec<_> = listing.versions().iter().filter_map(|version| version.key()).collect();
+
+        assert_eq!(
+            version_keys.len(),
+            child_keys.len(),
+            "ListObjectVersions with a trailing slash prefix should include child objects even when the marker object exists"
+        );
+
+        for key in child_keys {
+            assert!(
+                version_keys.contains(&key),
+                "ListObjectVersions(prefix=data01/) should include child object {key}"
+            );
+        }
+    }
 }

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -120,11 +120,6 @@ impl LockClient for GrpcLockClient {
             .map_err(|e| LockError::internal(e.to_string()))?
             .into_inner();
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -134,7 +129,8 @@ impl LockClient for GrpcLockClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -50,6 +50,18 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
     }
 }
 
+fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> GenerallyLockResult {
+    if success {
+        GenerallyLockResult {
+            success: true,
+            error_info: None,
+            lock_info: None,
+        }
+    } else {
+        lock_result_from_error(format!("lock not found for release: {lock_id}"))
+    }
+}
+
 /// Minimal NodeService implementation that only supports Lock RPCs
 /// Used for testing distributed lock scenarios with real gRPC
 #[derive(Debug)]
@@ -102,7 +114,7 @@ impl NodeService for MinimalLockNodeService {
                 let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
                 Ok(Response::new(GenerallyLockResponse {
                     success: result.success,
-                    error_info: None,
+                    error_info: result.error,
                     lock_info: lock_info_json,
                 }))
             }
@@ -131,11 +143,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -161,11 +176,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.force_release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -271,10 +289,9 @@ impl NodeService for MinimalLockNodeService {
                 Ok(batch_results) => {
                     for (result_idx, success) in batch_results.into_iter().enumerate() {
                         if let Some(request_idx) = valid_indices.get(result_idx) {
-                            results[*request_idx] = GenerallyLockResult {
-                                success,
-                                error_info: None,
-                                lock_info: None,
+                            results[*request_idx] = match lock_ids.get(result_idx) {
+                                Some(lock_id) => lock_result_from_release(lock_id, success),
+                                None => lock_result_from_error(format!("unlock response index out of range: {result_idx}")),
                             };
                         }
                     }

crates/e2e_test/src/reliant/head_tls_bodyless_test.rs

@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for TLS/HTTP2 `HEAD` responses on missing objects.
+//!
+//! Before the fix, RustFS returned `404` for a missing object but still wrote
+//! the XML error payload on a `HEAD` request. Under HTTP/2 this emitted DATA
+//! frames after the response headers, which clients surfaced as a protocol
+//! error. This test keeps the request at the raw HTTPS layer so it can validate
+//! the final wire-facing behavior rather than SDK-level error mapping.
+
+#![cfg(test)]
+
+use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use http::Version;
+use http::header::HOST;
+use rcgen::generate_simple_self_signed;
+use reqwest::{Certificate, Client, Response, StatusCode};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+use std::path::Path;
+use std::process::Command;
+use tokio::fs;
+use tokio::time::{Duration, sleep};
+use tracing::info;
+
+const ACCESS_KEY: &str = "rustfsadmin";
+const SECRET_KEY: &str = "rustfsadmin";
+const BUCKET: &str = "test-head-tls-bodyless-bucket";
+
+async fn generate_tls_bundle(tls_dir: &Path) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    fs::create_dir_all(tls_dir).await?;
+    let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
+    let cert_pem = cert.cert.pem();
+    let key_pem = cert.signing_key.serialize_pem();
+
+    fs::write(tls_dir.join("rustfs_cert.pem"), cert_pem.as_bytes()).await?;
+    fs::write(tls_dir.join("rustfs_key.pem"), key_pem.as_bytes()).await?;
+
+    Ok(cert_pem.into_bytes())
+}
+
+fn local_https_h2_client(ca_pem: &[u8]) -> Result<Client, Box<dyn Error + Send + Sync>> {
+    let _ca_cert = Certificate::from_pem(ca_pem)?;
+    Ok(Client::builder()
+        .no_proxy()
+        .no_gzip()
+        .no_brotli()
+        .no_zstd()
+        .no_deflate()
+        .danger_accept_invalid_certs(true)
+        .build()?)
+}
+
+async fn signed_empty_request(
+    client: &Client,
+    method: http::Method,
+    url: &str,
+) -> Result<Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let request = http::Request::builder()
+        .method(method.as_str())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+
+    let signed = sign_v4(request, 0, ACCESS_KEY, SECRET_KEY, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        builder = builder.header(name, value);
+    }
+
+    Ok(builder.send().await?)
+}
+
+async fn ensure_bucket_exists(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let bucket_url = format!("{endpoint}/{BUCKET}/");
+    let response = signed_empty_request(client, http::Method::HEAD, &bucket_url).await?;
+
+    if response.status() == StatusCode::OK {
+        return Ok(());
+    }
+
+    let response = signed_empty_request(client, http::Method::PUT, &bucket_url).await?;
+    match response.status() {
+        StatusCode::OK => Ok(()),
+        StatusCode::CONFLICT => Ok(()),
+        status => Err(format!("unexpected bucket setup status: {status}").into()),
+    }
+}
+
+async fn wait_for_tls_server_ready(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let ready_url = format!("{endpoint}/");
+    for _attempt in 0..60 {
+        match signed_empty_request(client, http::Method::GET, &ready_url).await {
+            Ok(response) if response.status().is_success() => return Ok(()),
+            Ok(_) | Err(_) => sleep(Duration::from_millis(500)).await,
+        }
+    }
+
+    Err("RustFS TLS server failed to become ready within 30 seconds".into())
+}
+
+async fn start_tls_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let binary_path = rustfs_binary_path();
+    let mut command = Command::new(&binary_path);
+    command
+        .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .current_dir(&env.temp_dir);
+
+    for key in [
+        "RUSTFS_ADDRESS",
+        "RUSTFS_VOLUMES",
+        "RUSTFS_ACCESS_KEY",
+        "RUSTFS_SECRET_KEY",
+        "RUSTFS_TLS_PATH",
+        "RUSTFS_OBS_LOG_DIRECTORY",
+    ] {
+        command.env_remove(key);
+    }
+
+    let process = command
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .env("RUSTFS_CONSOLE_ENABLE", "false")
+        .args([
+            "--address",
+            &env.address,
+            "--access-key",
+            &env.access_key,
+            "--secret-key",
+            &env.secret_key,
+            &env.temp_dir,
+        ])
+        .spawn()?;
+
+    env.process = Some(process);
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_head_missing_object_over_tls_http2_is_bodyless() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let tls_dir = std::path::PathBuf::from(&env.temp_dir).join("tls");
+    let ca_pem = generate_tls_bundle(&tls_dir).await?;
+    start_tls_rustfs_server(&mut env, &tls_dir).await?;
+
+    let endpoint = format!("https://{}", env.address);
+    let client = local_https_h2_client(&ca_pem)?;
+    wait_for_tls_server_ready(&client, &endpoint).await?;
+    ensure_bucket_exists(&client, &endpoint).await?;
+
+    let missing_key = "head-does-not-exist.txt";
+    let object_url = format!("{endpoint}/{BUCKET}/{missing_key}");
+
+    let get_response = signed_empty_request(&client, http::Method::GET, &object_url).await?;
+    assert_eq!(get_response.status(), StatusCode::NOT_FOUND);
+    let get_version = get_response.version();
+    let get_body = get_response.bytes().await?;
+    let get_body_text = String::from_utf8_lossy(&get_body);
+    assert!(
+        get_body_text.contains("<Code>NoSuchKey</Code>") || get_body_text.contains("<Code>NoSuchObject</Code>"),
+        "GET missing-object error body should expose NoSuchKey/NoSuchObject, got: {}",
+        get_body_text
+    );
+    info!("GET missing object over TLS used {:?} and returned {} bytes", get_version, get_body.len());
+
+    let head_response = signed_empty_request(&client, http::Method::HEAD, &object_url).await?;
+    assert_eq!(head_response.status(), StatusCode::NOT_FOUND);
+    assert_eq!(head_response.version(), Version::HTTP_2, "HEAD regression test must exercise HTTP/2");
+    let head_body = head_response.bytes().await?;
+    assert!(
+        head_body.is_empty(),
+        "HEAD missing-object response must not send body bytes over TLS/HTTP2, got {} bytes: {:?}",
+        head_body.len(),
+        head_body
+    );
+
+    Ok(())
+}

crates/e2e_test/src/reliant/lock.rs

@@ -275,6 +275,41 @@ async fn test_grpc_lock_client_batch_acquire_and_release() {
     handle.abort();
 }
 
+#[tokio::test]
+async fn test_grpc_lock_client_uses_request_lock_id_and_reports_missing_unlock() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+    let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client = GrpcLockClient::new(addr);
+    let request = LockRequest::new(test_resource(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(2));
+
+    let response = grpc_client.acquire_lock(&request).await.expect("gRPC acquire should succeed");
+    let lock_info = response.lock_info.expect("gRPC acquire should include lock info");
+    assert_eq!(lock_info.id, request.lock_id);
+
+    assert!(
+        grpc_client
+            .release(&request.lock_id)
+            .await
+            .expect("gRPC release should succeed"),
+        "release should find the request lock id"
+    );
+
+    let missing_release = grpc_client
+        .release(&request.lock_id)
+        .await
+        .expect_err("second release should report missing lock");
+    assert!(
+        missing_release.to_string().contains("lock not found for release"),
+        "missing release should preserve server error, got: {missing_release}"
+    );
+
+    handle.abort();
+}
+
 #[tokio::test]
 async fn test_distributed_lock_4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes() {
     let manager1 = Arc::new(GlobalLockManager::new());

crates/e2e_test/src/reliant/mod.rs

@@ -17,6 +17,7 @@ mod get_deleted_object_test;
 mod grpc_lock_client;
 mod grpc_lock_server;
 mod head_deleted_object_versioning_test;
+mod head_tls_bodyless_test;
 mod lifecycle;
 mod lock;
 mod node_interact_test;

crates/e2e_test/src/replication_extension_test.rs

@@ -12,14 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+};
+use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+use aws_sdk_s3::{Client, Config};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
 use rustfs_madmin::{
-    PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus,
-    SRStatusInfo, SiteReplicationInfo, SyncStatus,
+    AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
+    ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
@@ -30,6 +34,8 @@ use std::error::Error;
 use time::Duration as TimeDuration;
 use tokio::time::{Duration, sleep};
 
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
 #[derive(Debug, Clone, serde::Deserialize)]
 struct ReplicationResetStatusResponse {
     #[serde(rename = "Targets", default)]
@@ -79,6 +85,65 @@ async fn signed_request(
     Ok(request_builder.send().await?)
 }
 
+async fn signed_request_with_session_token(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if !session_token.is_empty() {
+        request = request.header("x-amz-security-token", session_token);
+    }
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(
+        request.body(Body::empty())?,
+        content_len,
+        access_key,
+        secret_key,
+        session_token,
+        "us-east-1",
+    );
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
+    let open = format!("<{tag}>");
+    let close = format!("</{tag}>");
+    let start = xml.find(&open)? + open.len();
+    let end = xml[start..].find(&close)? + start;
+    Some(xml[start..end].to_string())
+}
+
+fn parse_assume_role_credentials(xml: &str) -> Result<(String, String, String), Box<dyn Error + Send + Sync>> {
+    let access_key = extract_xml_tag(xml, "AccessKeyId").ok_or("missing AccessKeyId in AssumeRole response")?;
+    let secret_key = extract_xml_tag(xml, "SecretAccessKey").ok_or("missing SecretAccessKey in AssumeRole response")?;
+    let session_token = extract_xml_tag(xml, "SessionToken").ok_or("missing SessionToken in AssumeRole response")?;
+    Ok((access_key, secret_key, session_token))
+}
+
 async fn set_replication_target(
     source_env: &RustFSTestEnvironment,
     source_bucket: &str,
@@ -213,6 +278,143 @@ async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) ->
     Ok(())
 }
 
+fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    let body = serde_json::json!({
+        "secretKey": secret_key,
+        "status": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("create user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_add_canned_policy(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    policy: &serde_json::Value,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(policy.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add canned policy failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_attach_policy_to_user(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    username: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+        env.url, policy_name, username
+    );
+    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("attach policy to user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_update_group_members(
+    env: &RustFSTestEnvironment,
+    group_name: &str,
+    members: &[&str],
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let body = serde_json::json!({
+        "group": group_name,
+        "members": members,
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("update group members failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_attach_policy_to_group(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    group_name: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
+        env.url, policy_name, group_name
+    );
+    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("attach policy to group failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
 async fn run_replication_check(
     env: &RustFSTestEnvironment,
     bucket: &str,
@@ -259,6 +461,182 @@ async fn remove_replication_target_request(
     signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
 }
 
+async fn add_service_account(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    req: &AddServiceAccountReq,
+) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-service-account", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        signer_access_key,
+        signer_secret_key,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add service account failed: {status} {body}").into());
+    }
+
+    let body = response.bytes().await?;
+    let parsed: serde_json::Value = serde_json::from_slice(&body)?;
+    let credentials = parsed
+        .get("credentials")
+        .ok_or("add service account response missing credentials")?;
+    let access_key = credentials
+        .get("accessKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing access key")?
+        .to_string();
+    let secret_key = credentials
+        .get("secretKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing secret key")?
+        .to_string();
+
+    Ok((access_key, secret_key))
+}
+
+async fn add_service_account_with_session_token(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    session_token: &str,
+    req: &AddServiceAccountReq,
+) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-service-account", env.url);
+    let response = signed_request_with_session_token(
+        http::Method::PUT,
+        &url,
+        signer_access_key,
+        signer_secret_key,
+        session_token,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add service account with session token failed: {status} {body}").into());
+    }
+
+    let body = response.bytes().await?;
+    let parsed: serde_json::Value = serde_json::from_slice(&body)?;
+    let credentials = parsed
+        .get("credentials")
+        .ok_or("add service account response missing credentials")?;
+    let access_key = credentials
+        .get("accessKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing access key")?
+        .to_string();
+    let secret_key = credentials
+        .get("secretKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing secret key")?
+        .to_string();
+
+    Ok((access_key, secret_key))
+}
+
+async fn list_service_accounts(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    user: Option<&str>,
+) -> Result<ListServiceAccountsResp, Box<dyn Error + Send + Sync>> {
+    let mut url = format!("{}/rustfs/admin/v3/list-service-accounts", env.url);
+    if let Some(user) = user {
+        url.push_str("?user=");
+        url.push_str(&urlencoding::encode(user));
+    }
+
+    let response = signed_request(http::Method::GET, &url, signer_access_key, signer_secret_key, None, None).await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("list service accounts failed: {status} {body}").into());
+    }
+
+    Ok(response.json().await?)
+}
+
+async fn wait_for_service_accounts(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    user: Option<&str>,
+    expected: &[&str],
+) -> Result<ListServiceAccountsResp, Box<dyn Error + Send + Sync>> {
+    for _ in 0..20 {
+        let resp = list_service_accounts(env, signer_access_key, signer_secret_key, user).await?;
+        let access_keys: Vec<&str> = resp.accounts.iter().map(|account| account.access_key.as_str()).collect();
+        if expected
+            .iter()
+            .all(|expected_key| access_keys.iter().any(|actual| actual == expected_key))
+        {
+            return Ok(resp);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("service accounts did not reach expected keys {expected:?} on {}", env.address).into())
+}
+
+async fn wait_for_object_on_target(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    for _ in 0..40 {
+        match client.get_object().bucket(bucket).key(key).send().await {
+            Ok(output) => {
+                let body = output.body.collect().await?.into_bytes().to_vec();
+                return Ok(body);
+            }
+            Err(err) => {
+                if err.to_string().contains("NoSuchKey") || err.to_string().contains("NotFound") {
+                    sleep(Duration::from_millis(250)).await;
+                    continue;
+                }
+                return Err(err.into());
+            }
+        }
+    }
+
+    Err(format!("object {bucket}/{key} was not replicated in time").into())
+}
+
+async fn wait_for_user_get_object(client: &Client, bucket: &str, key: &str) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    let mut last_error = None;
+    for _ in 0..40 {
+        match client.get_object().bucket(bucket).key(key).send().await {
+            Ok(output) => {
+                let body = output.body.collect().await?.into_bytes().to_vec();
+                return Ok(body);
+            }
+            Err(err) => {
+                last_error = Some(err.to_string());
+                sleep(Duration::from_millis(250)).await;
+            }
+        }
+    }
+
+    Err(format!(
+        "user could not read replicated object {bucket}/{key} in time; last error: {}",
+        last_error.unwrap_or_else(|| "unknown".to_string())
+    )
+    .into())
+}
+
 async fn list_replication_targets_request(
     env: &RustFSTestEnvironment,
     bucket: Option<&str>,
@@ -1546,3 +1924,449 @@ async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Re
 
     Ok(())
 }
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_object_with_bucket_versioning_real_dual_node() -> TestResult {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-versioned";
+    let key = "hello.txt";
+    let payload = b"site replication should replicate after enabling versioning".to_vec();
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    let replication_response = signed_request(
+        http::Method::GET,
+        &format!("{}/{bucket}?replication", source_env.url),
+        &source_env.access_key,
+        &source_env.secret_key,
+        None,
+        None,
+    )
+    .await?;
+    let replication_status = replication_response.status();
+    let replication_body = replication_response.text().await.unwrap_or_default();
+    assert_eq!(
+        replication_status,
+        StatusCode::OK,
+        "source bucket replication config missing after site replication setup: {replication_body}"
+    );
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-policy-user";
+    let key = "seed.txt";
+    let payload = b"site replication policy-backed user access".to_vec();
+    let policy_name = "site-repl-readonly";
+    let username = "site-repl-user";
+    let secret_key = "site-repl-user-secret-key-123456";
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{bucket}/*")]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                "Resource": [format!("arn:aws:s3:::{bucket}")]
+            }
+        ]
+    });
+    admin_add_canned_policy(&source_env, policy_name, &policy).await?;
+    admin_create_user(&source_env, username, secret_key).await?;
+    admin_attach_policy_to_user(&source_env, policy_name, username).await?;
+
+    let target_user_client = create_user_s3_client(&target_env, username, secret_key);
+    let fetched = wait_for_user_get_object(&target_user_client, bucket, key).await?;
+    assert_eq!(fetched, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_group_policy_backed_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>>
+{
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-policy-group";
+    let key = "seed.txt";
+    let payload = b"site replication group-policy-backed user access".to_vec();
+    let policy_name = "site-repl-group-readonly";
+    let group_name = "site-repl-group";
+    let username = "site-repl-group-user";
+    let secret_key = "site-repl-group-user-secret-key-12";
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{bucket}/*")]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                "Resource": [format!("arn:aws:s3:::{bucket}")]
+            }
+        ]
+    });
+    admin_add_canned_policy(&source_env, policy_name, &policy).await?;
+    admin_create_user(&source_env, username, secret_key).await?;
+    admin_update_group_members(&source_env, group_name, &[username]).await?;
+    admin_attach_policy_to_group(&source_env, policy_name, group_name).await?;
+
+    let target_user_client = create_user_s3_client(&target_env, username, secret_key);
+    let fetched = wait_for_user_get_object(&target_user_client, bucket, key).await?;
+    assert_eq!(fetched, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_multiple_service_accounts_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let first_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-alpha".to_string(),
+        secret_key: "svc-alpha-secret-key-1234567890abcdef".to_string(),
+        name: Some("svc-alpha".to_string()),
+        description: Some("first replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let first = add_service_account(&source_env, &source_env.access_key, &source_env.secret_key, &first_req).await?;
+
+    let target_after_first = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-alpha"],
+    )
+    .await?;
+    assert!(
+        target_after_first
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-alpha"),
+        "target accounts missing svc-alpha: {:?}",
+        target_after_first.accounts
+    );
+
+    let second_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-beta".to_string(),
+        secret_key: "svc-beta-secret-key-1234567890abcdef1".to_string(),
+        name: Some("svc-beta".to_string()),
+        description: Some("second replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let _second = add_service_account(&source_env, &first.0, &first.1, &second_req).await?;
+
+    let target_after_second = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-alpha", "svc-beta"],
+    )
+    .await?;
+    assert!(
+        target_after_second
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-beta"),
+        "target accounts missing svc-beta: {:?}",
+        target_after_second.accounts
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
+    init_logging();
+
+    if !awscurl_available() {
+        eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
+        return Ok(());
+    }
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let assume_role_body = "Action=AssumeRole&Version=2011-06-15&DurationSeconds=3600";
+    let sts_xml = awscurl_post_sts_form_urlencoded(
+        &format!("{}/", source_env.url.trim_end_matches('/')),
+        assume_role_body,
+        &source_env.access_key,
+        &source_env.secret_key,
+    )
+    .await?;
+    let (sts_access_key, sts_secret_key, sts_session_token) = parse_assume_role_credentials(&sts_xml)?;
+
+    let first_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-sts-alpha".to_string(),
+        secret_key: "svc-sts-alpha-secret-key-1234567890".to_string(),
+        name: Some("svc-sts-alpha".to_string()),
+        description: Some("sts-created replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let first =
+        add_service_account_with_session_token(&source_env, &sts_access_key, &sts_secret_key, &sts_session_token, &first_req)
+            .await?;
+
+    let target_after_first = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-sts-alpha"],
+    )
+    .await?;
+    assert!(
+        target_after_first
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-sts-alpha"),
+        "target accounts missing svc-sts-alpha: {:?}",
+        target_after_first.accounts
+    );
+
+    let second_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-sts-beta".to_string(),
+        secret_key: "svc-sts-beta-secret-key-1234567890a".to_string(),
+        name: Some("svc-sts-beta".to_string()),
+        description: Some("second replicated service account from sts-created ak".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let _second = add_service_account(&source_env, &first.0, &first.1, &second_req).await?;
+
+    let target_after_second = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-sts-alpha", "svc-sts-beta"],
+    )
+    .await?;
+    assert!(
+        target_after_second
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-sts-beta"),
+        "target accounts missing svc-sts-beta: {:?}",
+        target_after_second.accounts
+    );
+
+    Ok(())
+}

crates/e2e_test/src/tls_gen.rs

@@ -0,0 +1,259 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::{Context, Result, bail};
+use clap::Parser;
+use rcgen::{
+    BasicConstraints, CertificateParams, CertifiedIssuer, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose,
+    SanType,
+};
+use std::fs;
+use std::net::IpAddr;
+use std::path::{Path, PathBuf};
+use time::{Duration, OffsetDateTime};
+
+pub const DEFAULT_OUT_DIR: &str = "target/tls";
+pub const OUTPUT_FILES: [&str; 7] = [
+    "rustfs_cert.pem",
+    "rustfs_key.pem",
+    "ca.crt",
+    "public.crt",
+    "client_ca.crt",
+    "client_cert.pem",
+    "client_key.pem",
+];
+
+#[derive(Debug, Parser)]
+#[command(name = "tls_gen", about = "Generate a full RustFS TLS bundle for local TLS and mTLS tests.")]
+pub struct Args {
+    #[arg(long, default_value = DEFAULT_OUT_DIR)]
+    pub out_dir: PathBuf,
+    #[arg(long, default_value_t = 365)]
+    pub days: i64,
+    #[arg(long)]
+    pub force: bool,
+}
+
+pub fn run(args: Args) -> Result<PathBuf> {
+    if args.days <= 0 {
+        bail!("--days must be a positive integer");
+    }
+
+    write_bundle(&args.out_dir, args.force, args.days)?;
+    Ok(args.out_dir)
+}
+
+pub fn ensure_writable(out_dir: &Path, force: bool) -> Result<()> {
+    if force {
+        return Ok(());
+    }
+
+    let existing: Vec<_> = OUTPUT_FILES
+        .iter()
+        .map(|name| out_dir.join(name))
+        .filter(|path| path.exists())
+        .collect();
+
+    if existing.is_empty() {
+        return Ok(());
+    }
+
+    let existing_list = existing
+        .iter()
+        .map(|path| path.file_name().and_then(|name| name.to_str()).unwrap_or("<unknown>"))
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    bail!(
+        "Refusing to overwrite existing files in {}: {}. Re-run with --force to replace them.",
+        out_dir.display(),
+        existing_list
+    )
+}
+
+fn write_bundle(out_dir: &Path, force: bool, days: i64) -> Result<()> {
+    fs::create_dir_all(out_dir).with_context(|| format!("failed to create output directory {}", out_dir.display()))?;
+    ensure_writable(out_dir, force)?;
+
+    let ca_key = generate_private_key()?;
+    let ca = build_ca_certificate(ca_key, days)?;
+
+    let server_key = generate_private_key()?;
+    let server_cert = build_leaf_certificate(
+        &server_key,
+        "localhost",
+        &[SanType::DnsName("localhost".try_into()?)],
+        &[
+            SanType::IpAddress(IpAddr::V4("127.0.0.1".parse()?)),
+            SanType::IpAddress(IpAddr::V6("::1".parse()?)),
+        ],
+        ExtendedKeyUsagePurpose::ServerAuth,
+        &ca,
+        days,
+    )?;
+
+    let client_key = generate_private_key()?;
+    let client_cert = build_leaf_certificate(
+        &client_key,
+        "rustfs-test-client",
+        &[SanType::DnsName("rustfs-test-client".try_into()?)],
+        &[],
+        ExtendedKeyUsagePurpose::ClientAuth,
+        &ca,
+        days,
+    )?;
+
+    let ca_pem = ca.pem();
+    let bundle = [
+        ("rustfs_cert.pem", server_cert.pem()),
+        ("rustfs_key.pem", server_key.serialize_pem()),
+        ("ca.crt", ca_pem.clone()),
+        ("public.crt", ca_pem.clone()),
+        ("client_ca.crt", ca_pem),
+        ("client_cert.pem", client_cert.pem()),
+        ("client_key.pem", client_key.serialize_pem()),
+    ];
+
+    for (name, content) in bundle {
+        fs::write(out_dir.join(name), content).with_context(|| format!("failed to write {}", out_dir.join(name).display()))?;
+    }
+
+    Ok(())
+}
+
+fn build_ca_certificate(signing_key: KeyPair, days: i64) -> Result<CertifiedIssuer<'static, KeyPair>> {
+    let mut params = base_params("RustFS Test CA", days)?;
+    params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+    params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
+
+    CertifiedIssuer::self_signed(params, signing_key).context("failed to create CA certificate")
+}
+
+fn build_leaf_certificate(
+    signing_key: &KeyPair,
+    common_name: &str,
+    dns_names: &[SanType],
+    ip_addresses: &[SanType],
+    usage: ExtendedKeyUsagePurpose,
+    issuer: &CertifiedIssuer<'_, KeyPair>,
+    days: i64,
+) -> Result<rcgen::Certificate> {
+    let mut params = base_params(common_name, days)?;
+    params.is_ca = IsCa::ExplicitNoCa;
+    params.key_usages = vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment];
+    params.extended_key_usages = vec![usage];
+    params.use_authority_key_identifier_extension = true;
+    params.subject_alt_names.extend_from_slice(dns_names);
+    params.subject_alt_names.extend_from_slice(ip_addresses);
+
+    params
+        .signed_by(signing_key, issuer)
+        .with_context(|| format!("failed to create leaf certificate for {common_name}"))
+}
+
+fn base_params(common_name: &str, days: i64) -> Result<CertificateParams> {
+    let mut params = CertificateParams::default();
+    let issued_at = OffsetDateTime::now_utc() - Duration::minutes(5);
+    params.not_before = issued_at;
+    params.not_after = issued_at + Duration::days(days);
+    params.distinguished_name.push(DnType::CountryName, "US");
+    params.distinguished_name.push(DnType::OrganizationName, "RustFS");
+    params.distinguished_name.push(DnType::CommonName, common_name);
+    Ok(params)
+}
+
+fn generate_private_key() -> Result<KeyPair> {
+    KeyPair::generate().context("failed to generate private key")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Args, OUTPUT_FILES, ensure_writable, run};
+    use std::fs;
+    use std::path::{Path, PathBuf};
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn unique_temp_dir() -> PathBuf {
+        let suffix = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("system time must be after unix epoch")
+            .as_nanos();
+        std::env::temp_dir().join(format!("rustfs-tls-gen-{suffix}"))
+    }
+
+    struct TempDir {
+        path: PathBuf,
+    }
+
+    impl TempDir {
+        fn new() -> Self {
+            let path = unique_temp_dir();
+            fs::create_dir_all(&path).expect("temporary directory should be created");
+            Self { path }
+        }
+
+        fn path(&self) -> &Path {
+            &self.path
+        }
+    }
+
+    impl Drop for TempDir {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.path);
+        }
+    }
+
+    #[test]
+    fn run_writes_full_bundle() {
+        let temp_dir = TempDir::new();
+
+        let out_dir = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 365,
+            force: false,
+        })
+        .expect("bundle generation should succeed");
+
+        for name in OUTPUT_FILES {
+            let content = fs::read(out_dir.join(name)).unwrap_or_else(|error| panic!("{name} should exist: {error}"));
+            assert!(!content.is_empty(), "{name} should not be empty");
+        }
+    }
+
+    #[test]
+    fn ensure_writable_rejects_existing_files_without_force() {
+        let temp_dir = TempDir::new();
+        let existing = temp_dir.path().join(OUTPUT_FILES[0]);
+        fs::write(&existing, "existing").expect("existing file should be created");
+
+        let error = ensure_writable(temp_dir.path(), false).expect_err("existing files must be rejected");
+        let message = format!("{error:#}");
+
+        assert!(message.contains("Refusing to overwrite existing files"));
+        assert!(message.contains(OUTPUT_FILES[0]));
+    }
+
+    #[test]
+    fn run_rejects_non_positive_days() {
+        let temp_dir = TempDir::new();
+        let error = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 0,
+            force: false,
+        })
+        .expect_err("non-positive days must fail");
+
+        assert_eq!(format!("{error:#}"), "--days must be a positive integer");
+    }
+}

crates/ecstore/Cargo.toml

@@ -55,6 +55,7 @@ flatbuffers.workspace = true
 futures.workspace = true
 futures-util.workspace = true
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
 serde.workspace = true
 time.workspace = true
 bytesize.workspace = true
@@ -62,6 +63,7 @@ serde_json.workspace = true
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
 s3s.workspace = true
 http.workspace = true
+opentelemetry.workspace = true
 http-body = { workspace = true }
 http-body-util.workspace = true
 url.workspace = true
@@ -124,9 +126,10 @@ metrics = { workspace = true }
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true }
 serial_test = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 
 [build-dependencies]
 shadow-rs = { workspace = true, features = ["build", "metadata"] }

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -956,6 +956,9 @@ pub type DynReplicationPool = dyn ReplicationPoolTrait + Send + Sync;
 /// Trait that abstracts the replication pool operations
 #[async_trait::async_trait]
 pub trait ReplicationPoolTrait: std::fmt::Debug {
+    fn active_workers(&self) -> i32;
+    fn active_mrf_workers(&self) -> i32;
+    fn active_lrg_workers(&self) -> i32;
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo);
     async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo);
     async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
@@ -972,6 +975,18 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
 // Implement the trait for ReplicationPool
 #[async_trait::async_trait]
 impl<S: StorageAPI> ReplicationPoolTrait for ReplicationPool<S> {
+    fn active_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_workers(self)
+    }
+
+    fn active_mrf_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_mrf_workers(self)
+    }
+
+    fn active_lrg_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_lrg_workers(self)
+    }
+
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo) {
         self.queue_replica_task(ri).await;
     }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -35,7 +35,7 @@ use crate::set_disk::get_lock_acquire_timeout;
 use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
-use aws_sdk_s3::operation::head_object::HeadObjectOutput;
+use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
 use aws_smithy_types::body::SdkBody;
@@ -115,6 +115,41 @@ fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &Res
     state.resync_id.is_empty() || opts.resync_id.is_empty() || state.resync_id == opts.resync_id
 }
 
+fn should_count_head_proxy_failure(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
+    if is_not_found || matches!(code, Some("MethodNotAllowed" | "405")) {
+        return false;
+    }
+    !matches!(raw_status, Some(404 | 405))
+}
+
+fn is_head_proxy_failure(err: &SdkError<HeadObjectError>) -> bool {
+    let (is_not_found, code) = err
+        .as_service_error()
+        .map(|service_err| (service_err.is_not_found(), service_err.code()))
+        .unwrap_or((false, None));
+    let raw_status = err.raw_response().map(|resp| resp.status().as_u16());
+    should_count_head_proxy_failure(is_not_found, code, raw_status)
+}
+
+async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) {
+    if let Some(stats) = GLOBAL_REPLICATION_STATS.get() {
+        stats.inc_proxy(bucket, api, is_err).await;
+    }
+}
+
+async fn head_object_with_proxy_stats(
+    source_bucket: &str,
+    target_client: &TargetClient,
+    target_bucket: &str,
+    object: &str,
+    version_id: Option<String>,
+) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    let result = target_client.head_object(target_bucket, object, version_id).await;
+    let is_err = result.as_ref().err().is_some_and(is_head_proxy_failure);
+    record_proxy_request(source_bucket, "HeadObject", is_err).await;
+    result
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ResyncOpts {
     pub bucket: String,
@@ -748,10 +783,15 @@ impl ReplicationResyncer {
 
                     let reset_id = target_client.reset_id.clone();
 
-                    let (size, err) = if let Err(err) = target_client
-                        .head_object(&target_client.bucket, &roi.name, roi.version_id.map(|v| v.to_string()))
-                        .await
-                    {
+                    let head_result = head_object_with_proxy_stats(
+                        &bucket_name,
+                        target_client.as_ref(),
+                        &target_client.bucket,
+                        &roi.name,
+                        roi.version_id.map(|v| v.to_string()),
+                    )
+                    .await;
+                    let (size, err) = if let Err(err) = head_result {
                         if roi.delete_marker {
                             st.replicated_count += 1;
                         } else {
@@ -2120,9 +2160,14 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
     };
 
     if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
-        match tgt_client
-            .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
-            .await
+        match head_object_with_proxy_stats(
+            &dobj.bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &dobj.delete_object.object_name,
+            version_id.clone(),
+        )
+        .await
         {
             Ok(_) => {}
             Err(e) => {
@@ -2471,9 +2516,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         }
 
         let mut replication_action = replication_action;
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2530,7 +2580,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
 
         if let Some(err) = if is_multipart {
             drop(gr);
-            replicate_object_with_multipart(MultipartReplicationContext {
+            let result = replicate_object_with_multipart(MultipartReplicationContext {
                 storage: storage.clone(),
                 cli: tgt_client.clone(),
                 src_bucket: &bucket,
@@ -2541,16 +2591,18 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 arn: &rinfo.arn,
                 put_opts,
             })
-            .await
-            .err()
+            .await;
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            result.err()
         } else {
             gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
             let byte_stream = async_read_to_bytestream(gr.stream);
-            tgt_client
+            let result = tgt_client
                 .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                 .await
-                .map_err(|e| std::io::Error::other(e.to_string()))
-                .err()
+                .map_err(|e| std::io::Error::other(e.to_string()));
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            result.err()
         } {
             rinfo.replication_status = ReplicationStatusType::Failed;
             rinfo.error = Some(err.to_string());
@@ -2690,9 +2742,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             warn!("failed to set replication tagging directive header: {err}");
         }
 
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2812,7 +2869,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
 
             if let Some(err) = if is_multipart {
                 drop(gr);
-                replicate_object_with_multipart(MultipartReplicationContext {
+                let result = replicate_object_with_multipart(MultipartReplicationContext {
                     storage: storage.clone(),
                     cli: tgt_client.clone(),
                     src_bucket: &bucket,
@@ -2823,16 +2880,18 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     arn: &rinfo.arn,
                     put_opts,
                 })
-                .await
-                .err()
+                .await;
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                result.err()
             } else {
                 gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
                 let byte_stream = async_read_to_bytestream(gr.stream);
-                tgt_client
+                let result = tgt_client
                     .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                     .await
-                    .map_err(|e| std::io::Error::other(e.to_string()))
-                    .err()
+                    .map_err(|e| std::io::Error::other(e.to_string()));
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                result.err()
             } {
                 rinfo.replication_status = ReplicationStatusType::Failed;
                 rinfo.error = Some(err.to_string());
@@ -3748,6 +3807,34 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_should_count_head_proxy_failure_ignores_not_found_and_405() {
+        assert!(
+            !should_count_head_proxy_failure(true, Some("NoSuchKey"), Some(404)),
+            "not-found heads are expected when the object has not reached the target yet"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("MethodNotAllowed"), Some(405)),
+            "405 delete-marker probing responses should not be counted as proxy failures"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("405"), Some(405)),
+            "numeric 405 codes must align with MethodNotAllowed semantics"
+        );
+    }
+
+    #[test]
+    fn test_should_count_head_proxy_failure_counts_unexpected_errors() {
+        assert!(
+            should_count_head_proxy_failure(false, Some("AccessDenied"), Some(403)),
+            "non-NotFound and non-405 service errors should be counted as failures"
+        );
+        assert!(
+            should_count_head_proxy_failure(false, None, Some(500)),
+            "raw 5xx head responses should be counted as proxy failures"
+        );
+    }
+
     #[tokio::test]
     async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
         let oi = ObjectInfo {

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -12,18 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::bucket::replication::get_global_replication_pool;
 use crate::error::Error;
 use crate::global::get_global_bucket_monitor;
 use rustfs_filemeta::{ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::interval;
 
+const ROLLING_WINDOW: Duration = Duration::from_secs(60);
+const FAILURE_LAST_HOUR_WINDOW: Duration = Duration::from_secs(60 * 60);
+
 /// Exponential Moving Average with thread-safe interior mutability
 #[derive(Debug)]
 pub struct ExponentialMovingAverage {
@@ -328,6 +332,13 @@ pub struct InQueueStats {
     pub now_count: AtomicI64,
 }
 
+#[derive(Debug, Clone)]
+struct QueueSample {
+    observed_at: Instant,
+    bytes: i64,
+    count: i64,
+}
+
 impl Clone for InQueueStats {
     fn clone(&self) -> Self {
         Self {
@@ -359,9 +370,60 @@ pub struct InQueueMetric {
     pub curr: InQueueStats,
     pub avg: InQueueStats,
     pub max: InQueueStats,
+    pub last_minute: InQueueStats,
+    #[serde(skip)]
+    samples: VecDeque<QueueSample>,
 }
 
 impl InQueueMetric {
+    fn observe(&mut self, observed_at: Instant) {
+        let bytes = self.curr.now_bytes.load(Ordering::Relaxed);
+        let count = self.curr.now_count.load(Ordering::Relaxed);
+
+        self.curr.bytes = bytes;
+        self.curr.count = count;
+        self.samples.push_back(QueueSample {
+            observed_at,
+            bytes,
+            count,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.avg = InQueueStats::default();
+            self.max = InQueueStats::default();
+            self.last_minute = InQueueStats::default();
+            return;
+        }
+
+        let sample_count = self.samples.len() as i64;
+        let total_bytes = self.samples.iter().map(|sample| sample.bytes).sum::<i64>();
+        let total_count = self.samples.iter().map(|sample| sample.count).sum::<i64>();
+        let max_bytes = self.samples.iter().map(|sample| sample.bytes).max().unwrap_or(0);
+        let max_count = self.samples.iter().map(|sample| sample.count).max().unwrap_or(0);
+
+        self.avg.bytes = total_bytes / sample_count;
+        self.avg.count = total_count / sample_count;
+        self.max.bytes = max_bytes;
+        self.max.count = max_count;
+        self.last_minute.bytes = self.avg.bytes;
+        self.last_minute.count = self.avg.count;
+    }
+
+    fn snapshot(&self) -> Self {
+        let mut snapshot = self.clone();
+        snapshot.curr.bytes = snapshot.curr.now_bytes.load(Ordering::Relaxed);
+        snapshot.curr.count = snapshot.curr.now_count.load(Ordering::Relaxed);
+        snapshot
+    }
+
     pub fn merge(&self, other: &InQueueMetric) -> Self {
         Self {
             curr: InQueueStats {
@@ -384,6 +446,12 @@ impl InQueueMetric {
                 count: self.max.count.max(other.max.count),
                 ..Default::default()
             },
+            last_minute: InQueueStats {
+                bytes: self.last_minute.bytes + other.last_minute.bytes,
+                count: self.last_minute.count + other.last_minute.count,
+                ..Default::default()
+            },
+            samples: VecDeque::new(),
         }
     }
 }
@@ -391,8 +459,8 @@ impl InQueueMetric {
 /// Queue cache
 #[derive(Debug, Default)]
 pub struct QueueCache {
-    pub bucket_stats: HashMap<String, InQueueStats>,
-    pub sr_queue_stats: InQueueStats,
+    pub bucket_stats: HashMap<String, InQueueMetric>,
+    pub sr_queue_stats: InQueueMetric,
 }
 
 impl QueueCache {
@@ -401,36 +469,19 @@ impl QueueCache {
     }
 
     pub fn update(&mut self) {
-        // Update queue statistics cache
-        // In actual implementation, this would get latest statistics from queue system
+        let observed_at = Instant::now();
+        self.sr_queue_stats.observe(observed_at);
+        for stats in self.bucket_stats.values_mut() {
+            stats.observe(observed_at);
+        }
     }
 
     pub fn get_bucket_stats(&self, bucket: &str) -> InQueueMetric {
-        if let Some(bucket_stat) = self.bucket_stats.get(bucket) {
-            InQueueMetric {
-                curr: InQueueStats {
-                    bytes: bucket_stat.now_bytes.load(Ordering::Relaxed),
-                    count: bucket_stat.now_count.load(Ordering::Relaxed),
-                    ..Default::default()
-                },
-                avg: InQueueStats::default(), // simplified implementation
-                max: InQueueStats::default(), // simplified implementation
-            }
-        } else {
-            InQueueMetric::default()
-        }
+        self.bucket_stats.get(bucket).map(InQueueMetric::snapshot).unwrap_or_default()
     }
 
     pub fn get_site_stats(&self) -> InQueueMetric {
-        InQueueMetric {
-            curr: InQueueStats {
-                bytes: self.sr_queue_stats.now_bytes.load(Ordering::Relaxed),
-                count: self.sr_queue_stats.now_count.load(Ordering::Relaxed),
-                ..Default::default()
-            },
-            avg: InQueueStats::default(), // simplified implementation
-            max: InQueueStats::default(), // simplified implementation
-        }
+        self.sr_queue_stats.snapshot()
     }
 }
 
@@ -505,11 +556,19 @@ impl ProxyStatsCache {
     }
 }
 
+#[derive(Debug, Clone)]
+struct FailureSample {
+    observed_at: Instant,
+    size: i64,
+}
+
 /// Failure statistics
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct FailStats {
     pub count: i64,
     pub size: i64,
+    #[serde(skip)]
+    recent: VecDeque<FailureSample>,
 }
 
 impl FailStats {
@@ -518,14 +577,42 @@ impl FailStats {
     }
 
     pub fn add_size(&mut self, size: i64, _err: Option<&Error>) {
+        let observed_at = Instant::now();
         self.count += 1;
         self.size += size;
+        self.recent.push_back(FailureSample { observed_at, size });
+        self.prune(observed_at);
+    }
+
+    fn prune(&mut self, observed_at: Instant) {
+        while self
+            .recent
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > FAILURE_LAST_HOUR_WINDOW)
+        {
+            self.recent.pop_front();
+        }
+    }
+
+    pub fn recent_since(&self, window: Duration) -> FailedMetric {
+        let now = Instant::now();
+        let mut count = 0i64;
+        let mut size = 0i64;
+        for sample in self.recent.iter().rev() {
+            if now.duration_since(sample.observed_at) > window {
+                break;
+            }
+            count += 1;
+            size += sample.size;
+        }
+        FailedMetric { count, size }
     }
 
     pub fn merge(&self, other: &FailStats) -> Self {
         Self {
             count: self.count + other.count,
             size: self.size + other.size,
+            recent: VecDeque::new(),
         }
     }
 
@@ -674,6 +761,14 @@ pub struct ActiveWorkerStat {
     pub curr: i32,
     pub max: i32,
     pub avg: f64,
+    #[serde(skip)]
+    samples: VecDeque<WorkerSample>,
+}
+
+#[derive(Debug, Clone)]
+struct WorkerSample {
+    observed_at: Instant,
+    workers: i32,
 }
 
 impl ActiveWorkerStat {
@@ -685,9 +780,31 @@ impl ActiveWorkerStat {
         self.clone()
     }
 
-    pub fn update(&mut self) {
-        // Simulate worker statistics update logic
-        // In actual implementation, this would get current active count from worker pool
+    pub fn update(&mut self, curr: i32) {
+        let observed_at = Instant::now();
+        self.curr = curr;
+        self.samples.push_back(WorkerSample {
+            observed_at,
+            workers: curr,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.max = curr;
+            self.avg = curr as f64;
+            return;
+        }
+
+        self.max = self.samples.iter().map(|sample| sample.workers).max().unwrap_or(curr);
+        let total = self.samples.iter().map(|sample| sample.workers as i64).sum::<i64>();
+        self.avg = total as f64 / self.samples.len() as f64;
     }
 }
 
@@ -740,8 +857,11 @@ impl ReplicationStats {
             let mut interval = interval(Duration::from_secs(2));
             loop {
                 interval.tick().await;
+                let current = get_global_replication_pool()
+                    .map(|pool| pool.active_workers() + pool.active_lrg_workers() + pool.active_mrf_workers())
+                    .unwrap_or(0);
                 let mut workers = workers_clone.lock().await;
-                workers.update();
+                workers.update(current);
             }
         });
 
@@ -925,14 +1045,26 @@ impl ReplicationStats {
     /// Get replication metrics for all buckets
     pub async fn get_all(&self) -> HashMap<String, BucketReplicationStats> {
         let cache = self.cache.read().await;
-        let mut result = HashMap::new();
+        let mut result = HashMap::with_capacity(cache.len());
 
         for (bucket, stats) in cache.iter() {
-            let mut cloned_stats = stats.clone_stats();
-            // Add queue statistics
+            result.insert(bucket.clone(), stats.clone_stats());
+        }
+        drop(cache);
+
+        {
             let q_cache = self.q_cache.lock().await;
-            cloned_stats.q_stat = q_cache.get_bucket_stats(bucket);
-            result.insert(bucket.clone(), cloned_stats);
+            for (bucket, queue_stats) in &q_cache.bucket_stats {
+                let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+                bucket_stats.q_stat = queue_stats.snapshot();
+            }
+        }
+
+        {
+            let p_cache = self.p_cache.lock().await;
+            for bucket in p_cache.bucket_stats.keys() {
+                result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+            }
         }
 
         result
@@ -1114,12 +1246,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        stats.now_count.fetch_add(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_add(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
     }
 
     /// Decrease queue statistics
@@ -1128,12 +1260,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        stats.now_count.fetch_sub(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_sub(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
     }
 
     /// Increase proxy metrics
@@ -1166,6 +1298,51 @@ mod tests {
         assert_eq!(workers.curr, 0);
     }
 
+    #[test]
+    fn test_in_queue_metric_observe_updates_rolling_stats() {
+        let mut metric = InQueueMetric::default();
+        metric.curr.now_bytes.store(128, Ordering::Relaxed);
+        metric.curr.now_count.store(4, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        metric.curr.now_bytes.store(256, Ordering::Relaxed);
+        metric.curr.now_count.store(6, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        assert_eq!(metric.curr.bytes, 256);
+        assert_eq!(metric.curr.count, 6);
+        assert_eq!(metric.max.bytes, 256);
+        assert_eq!(metric.max.count, 6);
+        assert_eq!(metric.last_minute.bytes, 192);
+        assert_eq!(metric.last_minute.count, 5);
+    }
+
+    #[test]
+    fn test_fail_stats_recent_since_tracks_windows() {
+        let mut stats = FailStats::default();
+        stats.add_size(64, None);
+        stats.add_size(32, None);
+
+        let last_minute = stats.recent_since(Duration::from_secs(60));
+        let last_hour = stats.recent_since(Duration::from_secs(60 * 60));
+        assert_eq!(last_minute.count, 2);
+        assert_eq!(last_minute.size, 96);
+        assert_eq!(last_hour.count, 2);
+        assert_eq!(last_hour.size, 96);
+    }
+
+    #[test]
+    fn test_active_worker_stat_update_tracks_rolling_avg_and_max() {
+        let mut stats = ActiveWorkerStat::default();
+        stats.update(2);
+        stats.update(6);
+        stats.update(4);
+
+        assert_eq!(stats.curr, 4);
+        assert_eq!(stats.max, 6);
+        assert_eq!(stats.avg, 4.0);
+    }
+
     #[tokio::test]
     async fn test_delete_bucket_stats() {
         let stats = ReplicationStats::new();
@@ -1218,6 +1395,15 @@ mod tests {
         assert_eq!(stat.replicated_count, 1);
     }
 
+    #[tokio::test]
+    async fn test_get_all_includes_proxy_only_bucket() {
+        let stats = ReplicationStats::new();
+        stats.inc_proxy("proxy-only-bucket", "HeadObject", false).await;
+
+        let all = stats.get_all().await;
+        assert!(all.contains_key("proxy-only-bucket"));
+    }
+
     #[test]
     fn test_sr_stats() {
         let sr_stats = SRStats::new();

crates/ecstore/src/config/audit.rs

@@ -14,15 +14,16 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
-    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
-    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
-    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
-    PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
-    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
-    WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -337,3 +338,63 @@ pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/com.rs

@@ -19,12 +19,12 @@ use crate::global::is_first_cluster_node_local;
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
 use rustfs_config::audit::{
-    AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
-    AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
+    AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::notify::{
-    NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS,
-    NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+    NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
 use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
@@ -58,7 +58,7 @@ struct TargetConfigDescriptor {
     valid_keys: &'static [&'static str],
 }
 
-fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
+fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
     [
         TargetConfigDescriptor {
             external_key: "webhook",
@@ -66,6 +66,12 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
             default_kvs: &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
             valid_keys: NOTIFY_WEBHOOK_KEYS,
         },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: NOTIFY_KAFKA_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_KAFKA_KVS,
+            valid_keys: NOTIFY_KAFKA_KEYS,
+        },
         TargetConfigDescriptor {
             external_key: "mqtt",
             subsystem_key: NOTIFY_MQTT_SUB_SYS,
@@ -87,7 +93,7 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
     ]
 }
 
-fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
+fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] {
     [
         TargetConfigDescriptor {
             external_key: "webhook",
@@ -95,6 +101,12 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
             default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
             valid_keys: AUDIT_WEBHOOK_KEYS,
         },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: AUDIT_KAFKA_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_KAFKA_KVS,
+            valid_keys: AUDIT_KAFKA_KEYS,
+        },
         TargetConfigDescriptor {
             external_key: "mqtt",
             subsystem_key: AUDIT_MQTT_SUB_SYS,
@@ -335,7 +347,7 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
     applied
 }
 
-fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
+fn parse_target_scalar_value(key: &str, value: &Value) -> Option<String> {
     match value {
         Value::String(v) => Some(v.trim().to_string()),
         Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v {
@@ -350,7 +362,7 @@ fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
     }
 }
 
-fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
+fn decode_target_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
     let mut kvs = KVS::new();
 
     for (key, value) in instance {
@@ -358,7 +370,7 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
             continue;
         }
 
-        if let Some(parsed) = parse_notify_scalar_value(key, value) {
+        if let Some(parsed) = parse_target_scalar_value(key, value) {
             kvs.insert(key.clone(), parsed);
         }
     }
@@ -366,21 +378,21 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
     kvs
 }
 
-fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
+fn decode_target_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
     match value {
-        Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)),
+        Value::Object(instance) => Some(decode_target_instance_object(instance, valid_keys)),
         Value::Array(_) => serde_json::from_value::<KVS>(value.clone()).ok(),
         _ => None,
     }
 }
 
-fn is_notify_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
+fn is_target_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
     section
         .iter()
-        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some())
+        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_target_scalar_value(key, value).is_some())
 }
 
-fn apply_external_notify_section(
+fn apply_external_target_section(
     cfg: &mut Config,
     notify_obj: &Map<String, Value>,
     external_key: &str,
@@ -399,8 +411,8 @@ fn apply_external_notify_section(
     let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default();
     let mut applied = false;
 
-    if is_notify_instance_shorthand(section_obj, valid_keys) {
-        let kvs = decode_notify_instance_object(section_obj, valid_keys);
+    if is_target_instance_shorthand(section_obj, valid_keys) {
+        let kvs = decode_target_instance_object(section_obj, valid_keys);
         if !kvs.is_empty() {
             let mut merged = default_kvs.clone();
             merged.extend(kvs);
@@ -411,7 +423,7 @@ fn apply_external_notify_section(
     }
 
     for (raw_instance, value) in section_obj {
-        let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else {
+        let Some(mut kvs) = decode_target_instance_value(value, valid_keys) else {
             continue;
         };
         if kvs.is_empty() {
@@ -444,7 +456,7 @@ fn apply_external_target_descriptors(
 ) -> bool {
     let mut applied = false;
     for descriptor in descriptors {
-        applied |= apply_external_notify_section(
+        applied |= apply_external_target_section(
             cfg,
             section_obj,
             descriptor.external_key,
@@ -663,11 +675,12 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     oidc_obj
 }
 
-fn is_notify_bool_key(key: &str) -> bool {
+fn is_target_bool_key(key: &str) -> bool {
     matches!(
         key,
         ENABLE_KEY
             | rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+            | rustfs_config::KAFKA_TLS_ENABLE
             | rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
             | rustfs_config::NATS_TLS_REQUIRED
             | rustfs_config::PULSAR_TLS_ALLOW_INSECURE
@@ -675,8 +688,8 @@ fn is_notify_bool_key(key: &str) -> bool {
     )
 }
 
-fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
-    if is_notify_bool_key(key) {
+fn encode_target_scalar_value(key: &str, value: &str) -> Value {
+    if is_target_bool_key(key) {
         if let Ok(state) = value.parse::<EnableState>() {
             return Value::Bool(state.is_enabled());
         }
@@ -697,7 +710,7 @@ fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool {
         .unwrap_or(false)
 }
 
-fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
+fn build_target_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
     let mut instance = Map::new();
 
     for key in valid_keys {
@@ -720,13 +733,13 @@ fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&s
             continue;
         }
 
-        instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value));
+        instance.insert((*key).to_string(), encode_target_scalar_value(key, &effective_value));
     }
 
     instance
 }
 
-fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
+fn merged_target_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
     let mut merged = default_kvs.clone();
     if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) {
         merged.extend(kvs.clone());
@@ -734,7 +747,7 @@ fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS
     merged
 }
 
-fn build_notify_subsystem_object(
+fn build_target_subsystem_object(
     cfg: &Config,
     subsystem_key: &str,
     default_kvs: &KVS,
@@ -744,11 +757,11 @@ fn build_notify_subsystem_object(
         return Map::new();
     };
 
-    let effective_default = merged_notify_default_kvs(subsystem, default_kvs);
+    let effective_default = merged_target_default_kvs(subsystem, default_kvs);
     let mut subsystem_obj = Map::new();
 
     if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) {
-        let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
+        let default_obj = build_target_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
         if !default_obj.is_empty() {
             subsystem_obj.insert("default".to_string(), Value::Object(default_obj));
         }
@@ -761,7 +774,7 @@ fn build_notify_subsystem_object(
     instances.sort_by_key(|(lhs, _)| *lhs);
 
     for (instance_key, kvs) in instances {
-        let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
+        let instance_obj = build_target_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
         if !instance_obj.is_empty() {
             subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj));
         }
@@ -774,7 +787,7 @@ fn build_target_object(cfg: &Config, descriptors: &[TargetConfigDescriptor]) ->
     let mut target_obj = Map::new();
     for descriptor in descriptors {
         let subsystem_obj =
-            build_notify_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
+            build_target_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
         if !subsystem_obj.is_empty() {
             target_obj.insert(descriptor.external_key.to_string(), Value::Object(subsystem_obj));
         }
@@ -1118,8 +1131,8 @@ mod tests {
         ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
     };
     use http::HeaderMap;
-    use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-    use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+    use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
+    use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
     use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
     use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
     use rustfs_filemeta::FileInfo;
@@ -1753,6 +1766,15 @@ mod tests {
                 "topic":"events",
                 "queue_dir":""
               }
+            },
+            "kafka":{
+              "streaming":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092,127.0.0.1:9093",
+                "topic":"events-kafka",
+                "acks":"all",
+                "tls_enable":true
+              }
             }
           }
         }"#;
@@ -1781,6 +1803,14 @@ mod tests {
             .expect("mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), "");
+
+        let kafka = cfg
+            .get_value(NOTIFY_KAFKA_SUB_SYS, "streaming")
+            .expect("kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092,127.0.0.1:9093");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "events-kafka");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true");
     }
 
     #[test]
@@ -1850,6 +1880,14 @@ mod tests {
                 "broker":"tcp://127.0.0.1:1883",
                 "topic":"audit-events"
               }
+            },
+            "kafka":{
+              "auditlog":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092",
+                "topic":"audit-events-kafka",
+                "acks":"1"
+              }
             }
           }
         }"#;
@@ -1873,6 +1911,12 @@ mod tests {
             .get_value(AUDIT_MQTT_SUB_SYS, "analytics")
             .expect("audit mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
+
+        let kafka = cfg
+            .get_value(AUDIT_KAFKA_SUB_SYS, "auditlog")
+            .expect("audit kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "audit-events-kafka");
     }
 
     #[test]
@@ -1993,6 +2037,38 @@ mod tests {
         );
         cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = notify::DEFAULT_NOTIFY_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "streaming".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092,127.0.0.1:9093".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_ACKS.to_string(),
+                    value: "all".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_TLS_ENABLE.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(NOTIFY_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let notify = v
@@ -2028,6 +2104,19 @@ mod tests {
             .expect("mqtt target should be encoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883"));
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some(""));
+
+        let kafka = notify
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("streaming"))
+            .and_then(Value::as_object)
+            .expect("kafka target should be encoded");
+        assert_eq!(
+            kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str),
+            Some("127.0.0.1:9092,127.0.0.1:9093")
+        );
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS).and_then(Value::as_str), Some("all"));
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE).and_then(Value::as_bool), Some(true));
     }
 
     #[test]
@@ -2079,6 +2168,28 @@ mod tests {
         );
         cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = audit::DEFAULT_AUDIT_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "audit-events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "auditlog".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(AUDIT_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let logger = v
@@ -2105,6 +2216,14 @@ mod tests {
             .expect("audit mqtt default should be encoded");
         assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
         assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events"));
+
+        let kafka = logger
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("auditlog"))
+            .and_then(Value::as_object)
+            .expect("audit kafka target should be encoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str), Some("127.0.0.1:9092"));
     }
 
     #[test]

crates/ecstore/src/config/mod.rs

@@ -25,8 +25,12 @@ use crate::store::ECStore;
 use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
 use rustfs_config::COMMENT_KEY;
 use rustfs_config::DEFAULT_DELIMITER;
-use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{
+    AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -245,6 +249,8 @@ pub fn init() {
     kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
     kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
     kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
+    kvs.insert(NOTIFY_KAFKA_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_KAFKA_KVS.clone());
+    kvs.insert(AUDIT_KAFKA_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_KAFKA_KVS.clone());
     kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
 
     // Register all default configurations

crates/ecstore/src/config/notify.rs

@@ -14,14 +14,15 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
-    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
-    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
-    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
-    PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
+    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -314,3 +315,63 @@ pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/disk/local.rs

@@ -176,7 +176,15 @@ impl LocalDisk {
             let root = root_clone.clone();
             Box::pin(async move {
                 match get_disk_info(root.clone()).await {
-                    Ok((info, root)) => {
+                    Ok((info, is_root_disk)) => {
+                        let physical_device_ids = match rustfs_utils::os::get_physical_device_ids(root.to_string_lossy().as_ref())
+                        {
+                            Ok(ids) => ids,
+                            Err(err) => {
+                                warn!(root = ?root, error = ?err, "failed to resolve physical device ids for disk root");
+                                Vec::new()
+                            }
+                        };
                         let disk_info = DiskInfo {
                             total: info.total,
                             free: info.free,
@@ -186,7 +194,8 @@ impl LocalDisk {
                             major: info.major,
                             minor: info.minor,
                             fs_type: info.fstype,
-                            root_disk: root,
+                            root_disk: is_root_disk,
+                            physical_device_ids,
                             id: disk_id,
                             ..Default::default()
                         };
@@ -2072,6 +2081,7 @@ impl DiskAPI for LocalDisk {
 
         let mut objs_returned = 0;
 
+        let mut skip_current_dir_object = false;
         if opts.base_dir.ends_with(SLASH_SEPARATOR) {
             if let Ok(data) = self
                 .read_metadata(
@@ -2098,7 +2108,7 @@ impl DiskAPI for LocalDisk {
                 if let Ok(meta) = tokio::fs::metadata(fpath).await
                     && meta.is_file()
                 {
-                    return Err(DiskError::FileNotFound);
+                    skip_current_dir_object = true;
                 }
             }
         }
@@ -2109,7 +2119,7 @@ impl DiskAPI for LocalDisk {
             &opts,
             &mut out,
             &mut objs_returned,
-            false,
+            skip_current_dir_object,
         )
         .await?;
 

crates/ecstore/src/disk/mod.rs

@@ -596,6 +596,8 @@ pub struct DiskInfo {
     pub scanning: bool,
     pub endpoint: String,
     pub mount_path: String,
+    /// Leaf physical block devices backing this mount path when available.
+    pub physical_device_ids: Vec<String>,
     pub id: Option<Uuid>,
     pub rotational: bool,
     pub metrics: DiskMetrics,

crates/ecstore/src/endpoints.rs

@@ -17,10 +17,11 @@ use crate::{
     disks_layout::DisksLayout,
     global::global_rustfs_port,
 };
+use rustfs_config::{DEFAULT_UNSAFE_BYPASS_DISK_CHECK, ENV_MINIO_CI, ENV_UNSAFE_BYPASS_DISK_CHECK};
 use rustfs_utils::{XHost, check_local_server_addr, get_host_ip, is_local_host};
 use std::{
-    collections::{HashMap, HashSet, hash_map::Entry},
-    io::{Error, Result},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
+    io::{Error, ErrorKind, Result},
     net::IpAddr,
 };
 use tracing::{error, info, instrument, warn};
@@ -348,6 +349,8 @@ impl PoolEndpointList {
             }
         }
 
+        validate_local_physical_disk_independence(pool_endpoint_list.as_ref())?;
+
         let setup_type = match pool_endpoint_list.as_ref()[0].as_ref()[0].get_type() {
             EndpointType::Path => SetupType::Erasure,
             EndpointType::Url => match unique_args.len() {
@@ -645,12 +648,118 @@ impl EndpointServerPools {
     }
 }
 
+fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()> {
+    let mut local_paths = BTreeSet::new();
+    for endpoints in pools {
+        for endpoint in endpoints.as_ref() {
+            if endpoint.is_local {
+                local_paths.insert(endpoint.get_file_path());
+            }
+        }
+    }
+
+    if local_paths.is_empty() {
+        return Ok(());
+    }
+
+    let local_paths = local_paths.into_iter().collect::<Vec<_>>();
+    validate_local_cross_device_mounts(&local_paths)?;
+
+    if local_paths.len() <= 1 {
+        return Ok(());
+    }
+
+    // Compatibility behavior:
+    // - canonical key: RUSTFS_UNSAFE_BYPASS_DISK_CHECK
+    // - legacy CI alias: MINIO_CI
+    // If both are set, `get_env_bool_with_aliases` keeps canonical key precedence.
+    if rustfs_utils::get_env_bool_with_aliases(ENV_UNSAFE_BYPASS_DISK_CHECK, &[ENV_MINIO_CI], DEFAULT_UNSAFE_BYPASS_DISK_CHECK) {
+        warn!(
+            env = ENV_UNSAFE_BYPASS_DISK_CHECK,
+            local_paths = ?local_paths,
+            "Skipping local physical disk independence validation due to explicit environment override",
+        );
+        return Ok(());
+    }
+
+    let mut device_paths = BTreeMap::<String, BTreeSet<String>>::new();
+    let mut missing_paths = Vec::new();
+
+    for path in &local_paths {
+        let canonical = match rustfs_utils::canonicalize(path) {
+            Ok(path) => path,
+            Err(err) if err.kind() == ErrorKind::NotFound => {
+                missing_paths.push(path.clone());
+                continue;
+            }
+            Err(err) => {
+                return Err(Error::other(format!(
+                    "failed to resolve local endpoint path '{path}' for disk validation: {err}"
+                )));
+            }
+        };
+        let canonical_path = canonical.to_string_lossy().into_owned();
+        let device_ids = rustfs_utils::os::get_physical_device_ids(&canonical_path).map_err(|err| {
+            Error::other(format!("failed to inspect physical disk for local endpoint '{canonical_path}': {err}"))
+        })?;
+
+        for device_id in device_ids {
+            device_paths.entry(device_id).or_default().insert(canonical_path.clone());
+        }
+    }
+
+    if !missing_paths.is_empty() {
+        warn!(
+            missing_paths = ?missing_paths,
+            "Excluding non-existent local endpoint paths from physical disk independence validation during endpoint parsing",
+        );
+    }
+
+    let shared_devices = device_paths
+        .into_iter()
+        .filter_map(|(device_id, paths)| {
+            if paths.len() <= 1 {
+                return None;
+            }
+
+            Some((device_id, paths.into_iter().collect::<Vec<_>>()))
+        })
+        .collect::<Vec<_>>();
+
+    if shared_devices.is_empty() {
+        return Ok(());
+    }
+
+    let details = shared_devices
+        .into_iter()
+        .map(|(device_id, paths)| format!("{device_id} => {}", paths.join(", ")))
+        .collect::<Vec<_>>()
+        .join("; ");
+
+    Err(Error::other(format!(
+        "local erasure endpoints must use distinct physical disks; detected shared devices [{details}]. \
+Set {ENV_UNSAFE_BYPASS_DISK_CHECK}=true only for local testing or CI to bypass this safety check"
+    )))
+}
+
+fn validate_local_cross_device_mounts(local_paths: &[String]) -> Result<()> {
+    rustfs_utils::os::check_cross_device_mounts(local_paths)
+        .map_err(|err| Error::other(format!("local endpoint cross-device mount validation failed: {err}")))
+}
+
 #[cfg(test)]
 mod test {
     use rustfs_utils::must_get_local_ips;
 
     use super::*;
+
+    #[cfg(target_os = "linux")]
+    use serial_test::serial;
     use std::path::Path;
+    #[cfg(target_os = "linux")]
+    use temp_env::async_with_vars;
+    #[cfg(target_os = "linux")]
+    use tempfile::tempdir;
 
     #[test]
     fn test_new_endpoints() {
@@ -1412,4 +1521,69 @@ mod test {
             }
         }
     }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn reject_shared_local_physical_disks_by_default() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, None::<&str>)], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let err = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout)
+                .await
+                .unwrap_err();
+
+            let err_text = err.to_string();
+            assert!(err_text.contains("distinct physical disks"), "unexpected error: {err_text}");
+            assert!(err_text.contains(ENV_UNSAFE_BYPASS_DISK_CHECK), "unexpected error: {err_text}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_explicit_env_bypass() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected bypassed disk validation to succeed, got {ret:?}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_minio_ci_alias() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, Some("1"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected MINIO_CI alias to bypass disk validation, got {ret:?}");
+        })
+        .await;
+    }
 }

crates/ecstore/src/rpc/client.rs

@@ -20,6 +20,8 @@ use std::error::Error;
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
 use tracing::debug;
 
+use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
+
 /// 3. Subsequent calls will attempt fresh connections
 /// 4. If node is still down, connection will fail fast (3s timeout)
 pub async fn node_service_time_out_client(
@@ -55,6 +57,8 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
     fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
         let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET);
         req.metadata_mut().as_mut().extend(headers);
+        inject_trace_context_into_metadata(req.metadata_mut());
+        inject_request_id_into_metadata(req.metadata_mut());
         Ok(req)
     }
 }
@@ -84,3 +88,34 @@ impl tonic::service::Interceptor for TonicInterceptor {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tonic::service::Interceptor;
+
+    #[test]
+    fn test_signature_interceptor_keeps_auth_headers() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        assert!(req.metadata().contains_key("x-rustfs-signature"));
+        assert!(req.metadata().contains_key("x-rustfs-timestamp"));
+    }
+
+    #[test]
+    fn test_signature_interceptor_may_inject_request_id() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let span = tracing::info_span!("grpc-rpc-test-span");
+        let _guard = span.enter();
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        if let Some(v) = req.metadata().get("x-request-id") {
+            assert!(!v.as_encoded_bytes().is_empty());
+        }
+    }
+}

crates/ecstore/src/rpc/context_propagation.rs

@@ -0,0 +1,223 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use http::{HeaderMap, HeaderValue};
+use opentelemetry::{global, propagation::Injector, trace::TraceContextExt};
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+pub(crate) const REQUEST_ID_HEADER: &str = "x-request-id";
+
+struct HttpHeaderInjector<'a> {
+    headers: &'a mut HeaderMap,
+}
+
+impl Injector for HttpHeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(name) = http::header::HeaderName::from_bytes(key.as_bytes()) else {
+            return;
+        };
+        let Ok(val) = HeaderValue::from_str(&value) else {
+            return;
+        };
+        self.headers.insert(name, val);
+    }
+}
+
+struct MetadataInjector<'a> {
+    metadata: &'a mut tonic::metadata::MetadataMap,
+}
+
+impl Injector for MetadataInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(meta_key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) else {
+            return;
+        };
+        let Ok(meta_value) = tonic::metadata::MetadataValue::try_from(value.as_str()) else {
+            return;
+        };
+        self.metadata.insert(meta_key, meta_value);
+    }
+}
+
+fn current_trace_id() -> Option<String> {
+    let current_context = Span::current().context();
+    let current_span = current_context.span();
+    let span_context = current_span.span_context();
+    if !span_context.is_valid() {
+        return None;
+    }
+    Some(span_context.trace_id().to_string())
+}
+
+fn fallback_request_id() -> String {
+    format!("req-{}", &uuid::Uuid::new_v4().to_string()[..8])
+}
+
+fn propagated_request_id() -> String {
+    current_trace_id()
+        .map(|trace_id| format!("trace-{trace_id}"))
+        .unwrap_or_else(fallback_request_id)
+}
+
+pub(crate) fn inject_trace_context_into_http_headers(headers: &mut HeaderMap) {
+    let current_context = Span::current().context();
+    global::get_text_map_propagator(|propagator| {
+        let mut injector = HttpHeaderInjector { headers };
+        propagator.inject_context(&current_context, &mut injector);
+    });
+}
+
+pub(crate) fn inject_request_id_into_http_headers(headers: &mut HeaderMap) {
+    if headers.contains_key(REQUEST_ID_HEADER) {
+        return;
+    }
+    let request_id = propagated_request_id();
+    if let Ok(value) = HeaderValue::from_str(&request_id) {
+        headers.insert(REQUEST_ID_HEADER, value);
+    }
+}
+
+pub(crate) fn inject_trace_context_into_metadata(metadata: &mut tonic::metadata::MetadataMap) {
+    let current_context = Span::current().context();
+    global::get_text_map_propagator(|propagator| {
+        let mut injector = MetadataInjector { metadata };
+        propagator.inject_context(&current_context, &mut injector);
+    });
+}
+
+pub(crate) fn inject_request_id_into_metadata(metadata: &mut tonic::metadata::MetadataMap) {
+    let request_id_key = tonic::metadata::MetadataKey::from_static(REQUEST_ID_HEADER);
+    if metadata.contains_key(&request_id_key) {
+        return;
+    }
+    let request_id = propagated_request_id();
+    let Ok(value) = tonic::metadata::MetadataValue::try_from(request_id.as_str()) else {
+        return;
+    };
+    metadata.insert(request_id_key, value);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _};
+    use opentelemetry_sdk::trace::SdkTracerProvider;
+    use tracing_opentelemetry::OpenTelemetrySpanExt;
+    use tracing_subscriber::{Registry, layer::SubscriberExt};
+
+    fn with_trace_parent<F>(trace_id_hex: &str, f: F)
+    where
+        F: FnOnce(),
+    {
+        let provider = SdkTracerProvider::builder().build();
+        let tracer = provider.tracer("context-propagation-tests");
+        let subscriber = Registry::default().with(tracing_opentelemetry::layer().with_tracer(tracer));
+
+        tracing::subscriber::with_default(subscriber, || {
+            let span = tracing::info_span!("context-propagation-test-span");
+
+            let trace_id = TraceId::from_hex(trace_id_hex).expect("trace id should be valid hex");
+            let span_id = opentelemetry::trace::SpanId::from_hex("0102030405060708").expect("span id should be valid hex");
+            let parent = SpanContext::new(trace_id, span_id, TraceFlags::SAMPLED, true, TraceState::default());
+            span.set_parent(opentelemetry::Context::new().with_remote_span_context(parent))
+                .expect("failed to set parent context");
+            let _guard = span.enter();
+
+            f();
+        });
+        let _ = provider.shutdown();
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_preserves_existing_value() {
+        let mut headers = HeaderMap::new();
+        headers.insert(REQUEST_ID_HEADER, HeaderValue::from_static("req-upstream-123"));
+
+        with_trace_parent("0123456789abcdef0123456789abcdef", || {
+            inject_request_id_into_http_headers(&mut headers);
+        });
+
+        assert_eq!(headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-123"));
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_uses_trace_id_when_missing() {
+        let trace_id = "abcdefabcdefabcdefabcdefabcdefab";
+        let mut headers = HeaderMap::new();
+
+        with_trace_parent(trace_id, || {
+            inject_request_id_into_http_headers(&mut headers);
+        });
+
+        assert_eq!(
+            headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()),
+            Some(format!("trace-{trace_id}").as_str())
+        );
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_preserves_existing_value() {
+        let mut metadata = tonic::metadata::MetadataMap::new();
+        metadata.insert(
+            tonic::metadata::MetadataKey::from_static(REQUEST_ID_HEADER),
+            tonic::metadata::MetadataValue::from_static("req-upstream-456"),
+        );
+
+        with_trace_parent("fedcba9876543210fedcba9876543210", || {
+            inject_request_id_into_metadata(&mut metadata);
+        });
+
+        assert_eq!(metadata.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-456"));
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_uses_trace_id_when_missing() {
+        let trace_id = "1234567890abcdef1234567890abcdef";
+        let mut metadata = tonic::metadata::MetadataMap::new();
+
+        with_trace_parent(trace_id, || {
+            inject_request_id_into_metadata(&mut metadata);
+        });
+
+        assert_eq!(
+            metadata.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()),
+            Some(format!("trace-{trace_id}").as_str())
+        );
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_uses_req_fallback_when_trace_missing() {
+        let mut headers = HeaderMap::new();
+        inject_request_id_into_http_headers(&mut headers);
+
+        let request_id = headers
+            .get(REQUEST_ID_HEADER)
+            .and_then(|v| v.to_str().ok())
+            .expect("request id should be injected");
+        assert!(request_id.starts_with("req-"), "expected req- fallback, got: {request_id}");
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_uses_req_fallback_when_trace_missing() {
+        let mut metadata = tonic::metadata::MetadataMap::new();
+        inject_request_id_into_metadata(&mut metadata);
+
+        let request_id = metadata
+            .get(REQUEST_ID_HEADER)
+            .and_then(|v| v.to_str().ok())
+            .expect("request id should be injected");
+        assert!(request_id.starts_with("req-"), "expected req- fallback, got: {request_id}");
+    }
+}

crates/ecstore/src/rpc/http_auth.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers};
 use base64::Engine as _;
 use base64::engine::general_purpose;
 use hmac::{Hmac, KeyInit, Mac};
@@ -62,6 +63,8 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) {
     let auth_headers = gen_signature_headers(url, method);
 
     headers.extend(auth_headers);
+    inject_trace_context_into_http_headers(headers);
+    inject_request_id_into_http_headers(headers);
 }
 
 pub fn gen_signature_headers(url: &str, method: &Method) -> HeaderMap {
@@ -132,6 +135,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::rpc::context_propagation::REQUEST_ID_HEADER;
     use http::{HeaderMap, Method};
     use time::OffsetDateTime;
 
@@ -210,6 +214,33 @@ mod tests {
         assert!((current_time - timestamp).abs() <= 1, "Timestamp should be close to current time");
     }
 
+    #[test]
+    fn test_build_auth_headers_preserves_existing_request_id() {
+        let url = "http://example.com/api/test";
+        let method = Method::GET;
+        let mut headers = HeaderMap::new();
+        headers.insert(REQUEST_ID_HEADER, HeaderValue::from_static("req-upstream-123"));
+
+        build_auth_headers(url, &method, &mut headers);
+
+        assert_eq!(headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-123"));
+    }
+
+    #[test]
+    fn test_build_auth_headers_may_set_request_id_from_trace_id() {
+        let url = "http://example.com/api/test";
+        let method = Method::GET;
+        let mut headers = HeaderMap::new();
+
+        let span = tracing::info_span!("rpc-test-span");
+        let _guard = span.enter();
+        build_auth_headers(url, &method, &mut headers);
+
+        if let Some(value) = headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()) {
+            assert!(!value.is_empty(), "request id should not be empty");
+        }
+    }
+
     #[test]
     fn test_verify_rpc_signature_success() {
         let url = "http://example.com/api/test";

crates/ecstore/src/rpc/mod.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod client;
+mod context_propagation;
 mod http_auth;
 mod peer_rest_client;
 mod peer_s3_client;

crates/ecstore/src/rpc/remote_disk.rs

@@ -886,7 +886,7 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout_for_op(
             "write_metadata",
-            || async {
+            move || async move {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -896,8 +896,8 @@ impl DiskAPI for RemoteDisk {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
-                    file_info: file_info.clone(),
-                    file_info_bin: file_info_bin.clone(),
+                    file_info,
+                    file_info_bin: file_info_bin.into(),
                 });
 
                 let response = client.write_metadata(request).await?.into_inner();
@@ -951,7 +951,7 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout_for_op(
             "update_metadata",
-            || async {
+            move || async move {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -961,10 +961,10 @@ impl DiskAPI for RemoteDisk {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
-                    file_info: file_info.clone(),
-                    opts: opts_str.clone(),
-                    file_info_bin: file_info_bin.clone(),
-                    opts_bin: opts_bin.clone(),
+                    file_info,
+                    opts: opts_str,
+                    file_info_bin: file_info_bin.into(),
+                    opts_bin: opts_bin.into(),
                 });
 
                 let response = client.update_metadata(request).await?.into_inner();
@@ -994,7 +994,7 @@ impl DiskAPI for RemoteDisk {
         let opts_bin = encode_msgpack(opts)?;
 
         self.execute_with_timeout(
-            || async {
+            move || async {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -1005,8 +1005,8 @@ impl DiskAPI for RemoteDisk {
                     volume: volume.to_string(),
                     path: path.to_string(),
                     version_id: version_id.to_string(),
-                    opts: opts_str.clone(),
-                    opts_bin: opts_bin.clone(),
+                    opts: opts_str,
+                    opts_bin: opts_bin.into(),
                 });
 
                 let response = client.read_version(request).await?.into_inner();
@@ -1480,7 +1480,7 @@ impl DiskAPI for RemoteDisk {
                 let request = Request::new(ReadMultipleRequest {
                     disk,
                     read_multiple_req,
-                    read_multiple_req_bin,
+                    read_multiple_req_bin: read_multiple_req_bin.into(),
                 });
 
                 let response = client.read_multiple(request).await?.into_inner();

crates/ecstore/src/rpc/remote_locker.rs

@@ -190,11 +190,6 @@ impl LockClient for RemoteClient {
             Err(err) => return Ok(Self::rpc_failure_response(request, &err)),
         };
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -204,7 +199,8 @@ impl LockClient for RemoteClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/ecstore/src/set_disk.rs

@@ -18,6 +18,7 @@
 use crate::batch_processor::{AsyncBatchProcessor, get_global_processors};
 use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
+use crate::bucket::object_lock::objectlock_sys::check_retention_for_modification;
 use crate::bucket::replication::check_replicate_delete;
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
@@ -1343,6 +1344,22 @@ impl BucketOperations for SetDisks {
     }
 }
 
+fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> {
+    if let Some(retention) = &opts.object_lock_retention
+        && check_retention_for_modification(
+            &obj_info.user_defined,
+            retention.mode.as_deref(),
+            retention.retain_until,
+            retention.bypass_governance,
+        )
+        .is_some()
+    {
+        return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
+    }
+
+    Ok(())
+}
+
 #[async_trait::async_trait]
 impl ObjectOperations for SetDisks {
     #[tracing::instrument(skip(self))]
@@ -1989,6 +2006,8 @@ impl ObjectOperations for SetDisks {
 
         let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
 
+        check_object_lock_retention_update(bucket, object, &obj_info, opts)?;
+
         for (k, v) in obj_info.user_defined {
             fi.metadata.insert(k, v);
         }
@@ -4129,6 +4148,7 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
                         total_space: res.total,
                         used_space: res.used,
                         available_space: res.free,
+                        physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
                         utilization: {
                             if res.total > 0 {
                                 res.used as f64 / res.total as f64 * 100_f64
@@ -5483,6 +5503,74 @@ mod tests {
         assert!(rendered.contains("object"), "{rendered}");
     }
 
+    #[test]
+    fn test_check_object_lock_retention_update_blocks_compliance_shorten() {
+        let now = OffsetDateTime::now_utc();
+        let existing_until = now + Duration::from_secs(60 * 60 * 24 * 60);
+        let requested_until = now + Duration::from_secs(60 * 60 * 24);
+
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
+        );
+
+        let obj_info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions {
+                mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()),
+                retain_until: Some(requested_until),
+                bypass_governance: true,
+            }),
+            ..Default::default()
+        };
+
+        let err = check_object_lock_retention_update("bucket", "object", &obj_info, &opts)
+            .expect_err("COMPLIANCE shortening must be blocked");
+
+        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
+    }
+
+    #[test]
+    fn test_check_object_lock_retention_update_allows_governance_shorten_with_bypass() {
+        let now = OffsetDateTime::now_utc();
+        let existing_until = now + Duration::from_secs(60 * 60 * 24 * 60);
+        let requested_until = now + Duration::from_secs(60 * 60 * 24);
+
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
+        );
+
+        let obj_info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions {
+                mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()),
+                retain_until: Some(requested_until),
+                bypass_governance: true,
+            }),
+            ..Default::default()
+        };
+
+        check_object_lock_retention_update("bucket", "object", &obj_info, &opts)
+            .expect("GOVERNANCE shortening with bypass should remain allowed");
+    }
+
     #[test]
     fn test_should_prevent_write() {
         let oi = ObjectInfo {
@@ -5576,6 +5664,100 @@ mod tests {
         assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string())));
     }
 
+    #[tokio::test]
+    async fn range_reads_use_shard_span_length_for_non_zero_offsets() {
+        use tokio::io::AsyncReadExt;
+        use uuid::Uuid;
+
+        let tempdir = tempfile::tempdir().expect("tempdir should be created");
+        let endpoint =
+            Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("disk should be created");
+
+        let bucket = "bucket";
+        let object = "object";
+        let payload = vec![b'x'; 3 * 1024 * 1024 + 1234];
+        let range_offset = 2 * 1024 * 1024 + 17;
+        let range_length = 512 * 1024;
+
+        disk.make_volume(bucket).await.expect("bucket should be created");
+
+        let mut fi = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
+        let data_dir = Uuid::new_v4();
+        fi.data_dir = Some(data_dir);
+        fi.size = payload.len() as i64;
+        fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None);
+
+        let erasure = erasure_coding::Erasure::new_with_options(
+            fi.erasure.data_blocks,
+            fi.erasure.parity_blocks,
+            fi.erasure.block_size,
+            fi.uses_legacy_checksum,
+        );
+        let shard_path = format!("{object}/{data_dir}/part.1");
+        let checksum_info = fi.erasure.get_checksum_info(1);
+
+        let mut bitrot_writer = create_bitrot_writer(
+            true,
+            None,
+            bucket,
+            &shard_path,
+            payload.len() as i64,
+            erasure.shard_size(),
+            checksum_info.algorithm.clone(),
+        )
+        .await
+        .expect("bitrot writer should be created");
+
+        for chunk in payload.chunks(erasure.shard_size()) {
+            bitrot_writer.write(chunk).await.expect("payload chunk should be written");
+        }
+
+        let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
+        disk.write_all(bucket, &shard_path, Bytes::from(encoded))
+            .await
+            .expect("encoded shard should be stored");
+
+        let files = vec![fi.clone()];
+        let disks = vec![Some(disk.clone())];
+        let (mut reader, mut writer) = tokio::io::duplex(range_length * 2);
+
+        let read_task = tokio::spawn(async move {
+            SetDisks::get_object_with_fileinfo(
+                bucket,
+                object,
+                range_offset,
+                range_length as i64,
+                &mut writer,
+                fi,
+                files,
+                &disks,
+                0,
+                0,
+                true,
+            )
+            .await
+        });
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.expect("range bytes should be readable");
+
+        read_task
+            .await
+            .expect("read task should complete")
+            .expect("range read should succeed");
+
+        assert_eq!(out, payload[range_offset..range_offset + range_length]);
+    }
+
     #[test]
     fn parts_after_marker_uses_marker_position() {
         let part_numbers = (1..=1002).collect::<Vec<_>>();

crates/ecstore/src/store_api/types.rs

@@ -43,6 +43,13 @@ impl HTTPPreconditions {
     }
 }
 
+#[derive(Debug, Default, Clone)]
+pub struct ObjectLockRetentionOptions {
+    pub mode: Option<String>,
+    pub retain_until: Option<OffsetDateTime>,
+    pub bypass_governance: bool,
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -79,6 +86,7 @@ pub struct ObjectOptions {
     pub lifecycle_audit_event: LcAuditEvent,
 
     pub eval_metadata: Option<HashMap<String, String>>,
+    pub object_lock_retention: Option<ObjectLockRetentionOptions>,
 
     pub want_checksum: Option<Checksum>,
     pub skip_verify_bitrot: bool,

crates/ecstore/tests/protobuf_bytes_regression_test.rs

@@ -0,0 +1,36 @@
+// Copyright (c) RustFS contributors
+// SPDX-License-Identifier: Apache-2.0
+
+use bytes::Bytes;
+use rustfs_protos::proto_gen::node_service::{
+    ReadMultipleRequest, ReadMultipleResponse, ReadVersionResponse, ReadXlResponse, UpdateMetadataRequest, WriteMetadataRequest,
+};
+
+fn expect_bytes(_: &Bytes) {}
+
+#[test]
+fn protobuf_bytes_fields_use_bytes_consistently() {
+    let update = UpdateMetadataRequest::default();
+    expect_bytes(&update.file_info_bin);
+    expect_bytes(&update.opts_bin);
+
+    let write = WriteMetadataRequest::default();
+    expect_bytes(&write.file_info_bin);
+
+    let version = ReadVersionResponse::default();
+    expect_bytes(&version.file_info_bin);
+
+    let read_xl = ReadXlResponse::default();
+    expect_bytes(&read_xl.raw_file_info_bin);
+
+    let read_multiple = ReadMultipleRequest::default();
+    expect_bytes(&read_multiple.read_multiple_req_bin);
+
+    let read_multiple_response = ReadMultipleResponse::default();
+    let first = read_multiple_response
+        .read_multiple_resps_bin
+        .first()
+        .cloned()
+        .unwrap_or_default();
+    expect_bytes(&first);
+}

crates/iam/src/oidc.rs

@@ -18,11 +18,11 @@
 //! `openidconnect` crate for standards-compliant discovery, token exchange,
 //! and ID token verification.
 
-use crate::oidc_state::{OidcAuthSession, OidcStateStore};
-use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreProviderMetadata};
+use crate::oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore};
+use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken};
 use openidconnect::{
-    AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce,
-    PkceCodeChallenge, PkceCodeVerifier, RedirectUrl, Scope,
+    AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, LogoutRequest, Nonce,
+    PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, ProviderMetadataWithLogout, RedirectUrl, Scope,
 };
 use reqwest::Client;
 use rustfs_config::oidc::*;
@@ -216,7 +216,7 @@ pub struct OidcClaims {
 /// on-the-fly from metadata when needed.
 #[derive(Clone)]
 struct ProviderState {
-    metadata: CoreProviderMetadata,
+    metadata: ProviderMetadataWithLogout,
     discovered_at: Instant,
 }
 
@@ -364,7 +364,7 @@ impl OidcSys {
         state: &str,
         code: &str,
         redirect_uri: &str,
-    ) -> Result<(OidcClaims, String, OidcAuthSession), String> {
+    ) -> Result<(OidcClaims, String, OidcAuthSession, String), String> {
         // Retrieve and consume the state (single-use)
         let session = self
             .state_store
@@ -449,7 +449,63 @@ impl OidcSys {
             raw,
         };
 
-        Ok((claims, session.provider_id.clone(), session))
+        Ok((claims, session.provider_id.clone(), session, raw_jwt))
+    }
+
+    /// Store a one-time logout session keyed by an opaque token so the console can
+    /// trigger browser logout without persisting the raw ID token.
+    pub async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result<String, String> {
+        if !self.configs.contains_key(provider_id) {
+            return Err(format!("unknown OIDC provider: {provider_id}"));
+        }
+
+        let token = CsrfToken::new_random().secret().clone();
+        self.state_store
+            .insert_logout(
+                token.clone(),
+                OidcLogoutSession {
+                    provider_id: provider_id.to_string(),
+                    id_token: id_token.to_string(),
+                },
+            )
+            .await;
+
+        Ok(token)
+    }
+
+    /// Build the RP-initiated logout URL for a previously issued logout token.
+    /// Returns `Ok(None)` when the provider does not advertise an end-session endpoint.
+    pub async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result<Option<String>, String> {
+        let session = self
+            .state_store
+            .take_logout(logout_token)
+            .await
+            .ok_or_else(|| "invalid or expired OIDC logout token".to_string())?;
+
+        let config = self
+            .configs
+            .get(&session.provider_id)
+            .ok_or_else(|| format!("unknown OIDC provider: {}", session.provider_id))?;
+        let state = self.ensure_provider_state(&session.provider_id, config).await?;
+        let Some(end_session_endpoint) = state.metadata.additional_metadata().end_session_endpoint.clone() else {
+            return Ok(None);
+        };
+
+        let id_token: CoreIdToken = session
+            .id_token
+            .parse()
+            .map_err(|e: serde_json::Error| format!("failed to parse ID token for logout: {e}"))?;
+        let post_logout_redirect_uri = PostLogoutRedirectUrl::new(post_logout_redirect_uri.to_string())
+            .map_err(|e| format!("invalid post logout redirect URI: {e}"))?;
+
+        let logout_url = LogoutRequest::from(end_session_endpoint)
+            .set_id_token_hint(&id_token)
+            .set_client_id(ClientId::new(config.client_id.clone()))
+            .set_post_logout_redirect_uri(post_logout_redirect_uri)
+            .http_get_url()
+            .to_string();
+
+        Ok(Some(logout_url))
     }
 
     /// Map OIDC claims to rustfs policy names.
@@ -930,7 +986,7 @@ impl OidcSys {
             let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?;
 
             for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES {
-                match CoreProviderMetadata::discover_async(issuer_url.clone(), http_client)
+                match ProviderMetadataWithLogout::discover_async(issuer_url.clone(), http_client)
                     .await
                     .map_err(|e| format!("discovery failed: {e}"))
                 {

crates/iam/src/oidc_state.rs

@@ -32,11 +32,20 @@ pub struct OidcAuthSession {
     pub redirect_after: Option<String>,
 }
 
+/// Stores an ID token behind a one-time opaque handle so the console can trigger
+/// RP-initiated logout without persisting the raw token in browser storage.
+#[derive(Debug, Clone)]
+pub struct OidcLogoutSession {
+    pub provider_id: String,
+    pub id_token: String,
+}
+
 /// TTL cache for OIDC auth state (PKCE verifiers + nonces) during the authorization flow.
 /// Entries expire after 5 minutes and are single-use (removed on retrieval).
 #[derive(Clone)]
 pub struct OidcStateStore {
     cache: Cache<String, OidcAuthSession>,
+    logout_cache: Cache<String, OidcLogoutSession>,
     last_capacity_log_at: Arc<AtomicU64>,
 }
 
@@ -46,8 +55,13 @@ impl OidcStateStore {
             .max_capacity(OIDC_STATE_CAPACITY)
             .time_to_live(Duration::from_secs(300)) // 5 minute TTL
             .build();
+        let logout_cache = Cache::builder()
+            .max_capacity(OIDC_STATE_CAPACITY)
+            .time_to_live(Duration::from_secs(3600)) // 1 hour TTL to match console OIDC sessions
+            .build();
         Self {
             cache,
+            logout_cache,
             last_capacity_log_at: Arc::new(AtomicU64::new(0)),
         }
     }
@@ -98,6 +112,21 @@ impl OidcStateStore {
     pub async fn contains(&self, state: &str) -> bool {
         self.cache.get(state).await.is_some()
     }
+
+    /// Store a new one-time logout session keyed by an opaque logout token.
+    pub async fn insert_logout(&self, token: String, session: OidcLogoutSession) {
+        self.logout_cache.insert(token, session).await;
+    }
+
+    /// Retrieve and remove a logout session (single-use). Returns None if expired or not found.
+    pub async fn take_logout(&self, token: &str) -> Option<OidcLogoutSession> {
+        self.logout_cache.remove(token).await
+    }
+
+    /// Check if a logout token exists (without consuming it).
+    pub async fn contains_logout(&self, token: &str) -> bool {
+        self.logout_cache.get(token).await.is_some()
+    }
 }
 
 impl Default for OidcStateStore {
@@ -167,4 +196,24 @@ mod tests {
             assert!(store.take(&format!("state_{i}")).await.is_none());
         }
     }
+
+    #[tokio::test]
+    async fn test_logout_state_store_insert_and_take() {
+        let store = OidcStateStore::new();
+        let session = OidcLogoutSession {
+            provider_id: "okta".to_string(),
+            id_token: "jwt-token".to_string(),
+        };
+
+        store.insert_logout("logout_abc".to_string(), session.clone()).await;
+        assert!(store.contains_logout("logout_abc").await);
+
+        let retrieved = store.take_logout("logout_abc").await;
+        assert!(retrieved.is_some());
+        let retrieved = retrieved.unwrap();
+        assert_eq!(retrieved.provider_id, "okta");
+        assert_eq!(retrieved.id_token, "jwt-token");
+
+        assert!(store.take_logout("logout_abc").await.is_none());
+    }
 }

crates/io-core/src/pool.rs

@@ -84,6 +84,12 @@ struct PoolTier {
     available_buffers: Mutex<Vec<BytesMut>>,
     /// Metrics for tracking this tier
     metrics: Mutex<Option<Arc<BytesPoolMetrics>>>,
+    /// Total acquisitions for this tier
+    tier_total_acquires: AtomicU64,
+    /// Total hits for this tier
+    tier_pool_hits: AtomicU64,
+    /// Current allocated bytes for this tier
+    tier_current_allocated_bytes: AtomicU64,
 }
 
 /// Pool metrics for monitoring and optimization.
@@ -291,6 +297,9 @@ impl PoolTier {
             name,
             available_buffers: Mutex::new(Vec::new()),
             metrics: Mutex::new(None),
+            tier_total_acquires: AtomicU64::new(0),
+            tier_pool_hits: AtomicU64::new(0),
+            tier_current_allocated_bytes: AtomicU64::new(0),
         }
     }
 
@@ -298,6 +307,65 @@ impl PoolTier {
         *self.metrics.lock().unwrap() = Some(metrics);
     }
 
+    fn take_or_allocate_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> (BytesMut, bool) {
+        let buffer_opt = {
+            let mut available = self.available_buffers.lock().unwrap();
+            available.pop()
+        };
+        let was_reused = buffer_opt.is_some();
+
+        let buffer = if let Some(mut buf) = buffer_opt {
+            let previous_capacity = buf.capacity();
+            buf.clear();
+            if previous_capacity < size {
+                buf.reserve(size - previous_capacity);
+            }
+            let current_capacity = buf.capacity();
+            if current_capacity > previous_capacity {
+                let delta = (current_capacity - previous_capacity) as u64;
+                pool_metrics.total_bytes_allocated.fetch_add(delta, Ordering::Relaxed);
+                pool_metrics.current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+                self.tier_current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+            }
+            buf
+        } else {
+            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
+            let allocated_bytes = buf.capacity() as u64;
+            pool_metrics
+                .total_bytes_allocated
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            pool_metrics
+                .current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            self.tier_current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            buf
+        };
+
+        (buffer, was_reused)
+    }
+
+    fn record_acquire_metrics(&self, pool_metrics: &BytesPoolMetrics, buffer_capacity: usize, was_reused: bool) {
+        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
+
+        if was_reused {
+            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
+            self.tier_pool_hits.fetch_add(1, Ordering::Relaxed);
+        } else {
+            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
+        }
+
+        let tier_total_acquires = self.tier_total_acquires.load(Ordering::Relaxed);
+        let tier_pool_hits = self.tier_pool_hits.load(Ordering::Relaxed);
+        let tier_hit_rate = if tier_total_acquires == 0 {
+            0.0
+        } else {
+            tier_pool_hits as f64 / tier_total_acquires as f64
+        };
+        rustfs_io_metrics::record_bytes_pool_hit_rate(self.name, tier_hit_rate);
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
+    }
+
     async fn acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> PooledBuffer {
         // Acquire semaphore permit (owned for storage in PooledBuffer)
         let permit = Arc::clone(&self.semaphore).acquire_owned().await.unwrap();
@@ -308,45 +376,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer - clear and ensure capacity
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -365,45 +399,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         Some(PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -421,8 +421,24 @@ impl PoolTier {
             if let Some(ref metrics) = *self.metrics.lock().unwrap() {
                 metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
             }
+        } else {
+            let released_bytes = buffer.capacity() as u64;
+            self.tier_current_allocated_bytes
+                .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                    Some(current.saturating_sub(released_bytes))
+                })
+                .ok();
+            if let Some(ref metrics) = *self.metrics.lock().unwrap() {
+                metrics
+                    .current_allocated_bytes
+                    .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                        Some(current.saturating_sub(released_bytes))
+                    })
+                    .ok();
+            }
         }
         // If pool is full, buffer is dropped and memory is freed
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 }
 
@@ -617,4 +633,47 @@ mod tests {
         let delta_hits = pool.metrics().pool_hits.load(Ordering::Relaxed) - initial_hits;
         assert_eq!(delta_hits, 1);
     }
+
+    #[tokio::test]
+    async fn test_tier_allocated_bytes_tracks_real_allocations() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire allocates one small-tier buffer.
+        let buf1 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // Return and reuse should not increase allocated bytes.
+        drop(buf1);
+        let buf2 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // A second in-flight buffer forces one more allocation.
+        let _buf3 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 2048);
+
+        drop(buf2);
+    }
+
+    #[tokio::test]
+    async fn test_tier_hit_rate_counters_track_reuse() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire is miss.
+        let buf1 = pool.acquire_buffer(512).await;
+        drop(buf1);
+
+        // Second acquire reuses previous buffer and counts as hit.
+        let _buf2 = pool.acquire_buffer(512).await;
+
+        assert_eq!(pool.small_pool.tier_total_acquires.load(Ordering::Relaxed), 2);
+        assert_eq!(pool.small_pool.tier_pool_hits.load(Ordering::Relaxed), 1);
+    }
 }

crates/lock/src/distributed_lock.rs

@@ -22,13 +22,15 @@ use futures::future::join_all;
 use rustfs_io_metrics::{
     record_read_lock_held_acquire, record_read_lock_held_release, record_write_lock_held_acquire, record_write_lock_held_release,
 };
-use std::sync::{Arc, LazyLock};
+use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::mpsc;
 use tokio::task::JoinSet;
-use tracing::warn;
+use tracing::{debug, warn};
 use uuid::Uuid;
 
+const UNLOCK_RETRY_ATTEMPTS: usize = 3;
+const UNLOCK_RETRY_BACKOFF: Duration = Duration::from_millis(100);
+
 /// Generate a new aggregate lock ID for multiple client locks
 fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
     LockId {
@@ -37,45 +39,6 @@ fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
     }
 }
 
-#[derive(Debug, Clone)]
-struct UnlockJob {
-    /// Entries to release: each (LockId, client) pair will be released independently.
-    entries: Vec<(LockId, Arc<dyn LockClient>)>,
-}
-
-#[derive(Debug)]
-struct UnlockRuntime {
-    tx: mpsc::Sender<UnlockJob>,
-}
-
-// Global unlock runtime with background worker
-static UNLOCK_RUNTIME: LazyLock<UnlockRuntime> = LazyLock::new(|| {
-    // Larger buffer to reduce contention during bursts
-    let (tx, mut rx) = mpsc::channel::<UnlockJob>(8192);
-
-    // Spawn background worker when first used; assumes a Tokio runtime is available
-    tokio::spawn(async move {
-        while let Some(job) = rx.recv().await {
-            // Best-effort release across all (LockId, client) entries.
-            let results = join_all(
-                job.entries
-                    .into_iter()
-                    .map(|(lock_id, client)| async move { client.release(&lock_id).await.unwrap_or(false) }),
-            )
-            .await;
-            let any_ok = results.into_iter().any(|released| released);
-
-            if !any_ok {
-                tracing::warn!("DistributedLockGuard background release failed for one or more entries");
-            } else {
-                tracing::debug!("DistributedLockGuard background released one or more entries");
-            }
-        }
-    });
-
-    UnlockRuntime { tx }
-});
-
 /// A RAII guard for distributed locks that releases the lock asynchronously when dropped.
 #[derive(Debug)]
 pub struct DistributedLockGuard {
@@ -126,47 +89,22 @@ impl DistributedLockGuard {
     }
 
     /// Manually release the lock early.
-    /// This sends a release job to the background worker and then disarms the guard
+    /// This spawns a background release task and then disarms the guard
     /// to prevent double-release on drop.
-    /// Returns true if the lock was released (or was already released), false otherwise.
+    /// Returns true if release was scheduled or the guard was already disarmed.
     pub fn release(&mut self) -> bool {
         if self.disarmed {
             // Lock was already released, return true to indicate lock is in released state
             return true;
         }
 
-        let job = UnlockJob {
-            entries: self.entries.clone(),
-        };
-
-        // Try a non-blocking send to avoid panics
-        let success = if let Err(err) = UNLOCK_RUNTIME.tx.try_send(job) {
-            // Channel full or closed; best-effort fallback: spawn a detached task
-            let entries = self.entries.clone();
-            tracing::warn!(
-                "DistributedLockGuard channel send failed ({}), spawning fallback unlock task for {} entries",
-                err,
-                entries.len()
-            );
-
-            // If runtime is not available, this will panic; but in RustFS we are inside Tokio contexts.
-            let handle = tokio::spawn(async move {
-                let futures_iter = entries
-                    .into_iter()
-                    .map(|(lock_id, client)| async move { client.release(&lock_id).await.unwrap_or(false) });
-                let _ = join_all(futures_iter).await;
-            });
-            // Explicitly drop the JoinHandle to acknowledge detaching the task.
-            drop(handle);
-            true // Consider it successful even if we had to use fallback
-        } else {
-            true
-        };
+        let entries = self.entries.clone();
+        DistributedLock::spawn_release_cleanup(entries, "distributed_lock_guard_release");
 
         // Disarm to prevent double-release on drop
         self.disarmed = true;
         record_lock_held_release(self.lock_type);
-        success
+        true
     }
 }
 
@@ -349,22 +287,64 @@ impl DistributedLock {
         pending
     }
 
-    async fn release_entries(entries: &[(LockId, Arc<dyn LockClient>)], context: &'static str) {
-        let release_results = join_all(
-            entries
-                .iter()
-                .map(|(lock_id, client)| async move { (lock_id, client.release(lock_id).await) }),
-        )
-        .await;
+    async fn release_entries(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
+        let mut pending = entries;
 
-        for (lock_id, result) in release_results {
-            match result {
-                Ok(true) | Ok(false) => {}
-                Err(err) => {
-                    tracing::warn!("{context}: failed to release lock {} on client: {}", lock_id, err);
+        for attempt in 1..=UNLOCK_RETRY_ATTEMPTS {
+            let release_results = join_all(pending.into_iter().map(|(lock_id, client)| async move {
+                match client.release(&lock_id).await {
+                    Ok(true) => None,
+                    Ok(false) => {
+                        warn!(%lock_id, attempt, context, "distributed unlock did not find lock on client");
+                        Some((lock_id, client))
+                    }
+                    Err(err) => {
+                        warn!(%lock_id, attempt, context, "distributed unlock failed on client: {}", err);
+                        Some((lock_id, client))
+                    }
                 }
+            }))
+            .await;
+
+            pending = release_results.into_iter().flatten().collect();
+            if pending.is_empty() {
+                debug!(attempt, context, "distributed unlock completed");
+                return;
+            }
+
+            if attempt < UNLOCK_RETRY_ATTEMPTS {
+                tokio::time::sleep(UNLOCK_RETRY_BACKOFF * attempt as u32).await;
             }
         }
+
+        warn!(
+            remaining = pending.len(),
+            attempts = UNLOCK_RETRY_ATTEMPTS,
+            context,
+            "distributed unlock left unreleased entries after retry"
+        );
+    }
+
+    fn spawn_release_cleanup(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
+        if entries.is_empty() {
+            return;
+        }
+
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            let join_handle = handle.spawn(async move {
+                Self::release_entries(entries, context).await;
+            });
+            drop(join_handle);
+            return;
+        }
+
+        let join_handle = std::thread::spawn(move || match tokio::runtime::Builder::new_current_thread().enable_all().build() {
+            Ok(runtime) => runtime.block_on(async move {
+                Self::release_entries(entries, context).await;
+            }),
+            Err(err) => warn!(context, "failed to create fallback unlock runtime: {}", err),
+        });
+        drop(join_handle);
     }
 
     fn spawn_pending_cleanup(
@@ -387,9 +367,7 @@ impl DistributedLock {
                             continue;
                         };
 
-                        if let Err(err) = client.release(&lock_id).await {
-                            tracing::warn!("{context}: failed to cleanup late lock {} on client {}: {}", lock_id, idx, err);
-                        }
+                        Self::release_entries(vec![(lock_id, client.clone())], context).await;
                     }
                     Ok((idx, Ok(resp))) => {
                         tracing::debug!(
@@ -506,7 +484,7 @@ impl DistributedLock {
 
             if individual_locks.len() + pending.len() < required_quorum {
                 let rollback_count = individual_locks.len();
-                Self::release_entries(&individual_locks, "distributed_lock_quorum_rollback").await;
+                Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
                 if !pending.is_empty() {
                     Self::spawn_pending_cleanup(
                         pending,
@@ -525,7 +503,7 @@ impl DistributedLock {
         }
 
         let rollback_count = individual_locks.len();
-        Self::release_entries(&individual_locks, "distributed_lock_quorum_rollback").await;
+        Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
         let resp = LockResponse::failure(
             format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
             Duration::ZERO,

crates/lock/src/namespace/tests.rs

@@ -16,7 +16,10 @@ use super::*;
 use crate::client::{ClientFactory, local::LocalClient};
 use crate::types::LockType;
 use crate::{GlobalLockManager, LockError, LockInfo, LockResponse, LockStats};
-use std::sync::Arc;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
 use std::time::Duration;
 
 #[derive(Debug, Default)]
@@ -107,6 +110,75 @@ impl crate::client::LockClient for DelayedClient {
     }
 }
 
+#[derive(Debug)]
+struct FlakyReleaseClient {
+    inner: LocalClient,
+    failed_releases_remaining: AtomicUsize,
+    release_attempts: AtomicUsize,
+}
+
+impl FlakyReleaseClient {
+    fn new(manager: Arc<GlobalLockManager>, failed_releases: usize) -> Self {
+        Self {
+            inner: LocalClient::with_manager(manager),
+            failed_releases_remaining: AtomicUsize::new(failed_releases),
+            release_attempts: AtomicUsize::new(0),
+        }
+    }
+
+    fn release_attempts(&self) -> usize {
+        self.release_attempts.load(Ordering::SeqCst)
+    }
+}
+
+#[async_trait::async_trait]
+impl crate::client::LockClient for FlakyReleaseClient {
+    async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+        self.inner.acquire_lock(request).await
+    }
+
+    async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.release_attempts.fetch_add(1, Ordering::SeqCst);
+        if self
+            .failed_releases_remaining
+            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
+            .is_ok()
+        {
+            return Ok(false);
+        }
+
+        self.inner.release(lock_id).await
+    }
+
+    async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.inner.refresh(lock_id).await
+    }
+
+    async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.inner.force_release(lock_id).await
+    }
+
+    async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+        self.inner.check_status(lock_id).await
+    }
+
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        self.inner.get_stats().await
+    }
+
+    async fn close(&self) -> crate::Result<()> {
+        self.inner.close().await
+    }
+
+    async fn is_online(&self) -> bool {
+        true
+    }
+
+    async fn is_local(&self) -> bool {
+        true
+    }
+}
+
 fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
     ObjectKey {
         bucket: Arc::from(bucket),
@@ -115,6 +187,41 @@ fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
     }
 }
 
+async fn wait_until_all_managers_can_write(managers: &[Arc<GlobalLockManager>], resource: ObjectKey) {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
+
+    loop {
+        let mut guards = Vec::with_capacity(managers.len());
+        let mut all_available = true;
+
+        for (idx, manager) in managers.iter().enumerate() {
+            let local_lock = NamespaceLock::with_local_manager(format!("probe-node-{idx}"), manager.clone());
+            match local_lock
+                .get_write_lock(resource.clone(), "probe-owner", Duration::from_millis(20))
+                .await
+            {
+                Ok(guard) => guards.push(guard),
+                Err(_) => {
+                    all_available = false;
+                    break;
+                }
+            }
+        }
+
+        drop(guards);
+
+        if all_available {
+            return;
+        }
+
+        assert!(
+            tokio::time::Instant::now() < deadline,
+            "distributed lock was not released on all simulated nodes"
+        );
+        tokio::time::sleep(Duration::from_millis(20)).await;
+    }
+}
+
 #[tokio::test]
 async fn test_namespace_lock_new() {
     let client = ClientFactory::create_local();
@@ -174,6 +281,24 @@ async fn test_lock_client_default_batch_acquire_and_release() {
     assert_eq!(released, vec![true, true]);
 }
 
+#[tokio::test]
+async fn test_local_client_uses_request_lock_id_for_release() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let client = LocalClient::with_manager(manager);
+    let resource = create_test_object_key("bucket", "object");
+    let request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(1));
+
+    let response = client.acquire_lock(&request).await.unwrap();
+    let lock_info = response.lock_info.expect("successful acquire should return lock info");
+    assert_eq!(lock_info.id, request.lock_id);
+
+    assert!(client.release(&request.lock_id).await.unwrap());
+
+    let second_request = LockRequest::new(resource, LockType::Exclusive, "owner-b").with_acquire_timeout(Duration::from_secs(1));
+    let second_response = client.acquire_lock(&second_request).await.unwrap();
+    assert!(second_response.success);
+}
+
 #[tokio::test]
 async fn test_namespace_lock_get_resource_key() {
     let client = ClientFactory::create_local();
@@ -488,6 +613,97 @@ async fn test_namespace_lock_distributed_with_clients_and_quorum() {
     drop(guard_b);
 }
 
+#[tokio::test]
+async fn test_namespace_lock_distributed_eight_node_write_releases_all_nodes() {
+    let managers = (0..8).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let clients = managers
+        .iter()
+        .map(|manager| Arc::new(LocalClient::with_manager(manager.clone())) as Arc<dyn LockClient>)
+        .collect::<Vec<_>>();
+
+    let lock = NamespaceLock::with_clients_and_quorum("eight-node".to_string(), clients, 5);
+    let resource = create_test_object_key("bucket", "object-eight-node");
+
+    let mut guard = lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
+        .await
+        .expect("owner-a should acquire write lock across eight simulated nodes");
+
+    let err = lock
+        .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
+        .await
+        .expect_err("owner-b should not acquire while owner-a holds all node locks");
+    let err_str = err.to_string();
+    assert!(
+        err_str.contains("required 5") && err_str.contains("achieved"),
+        "expected 8-node quorum failure below required write quorum, got: {err}"
+    );
+
+    assert!(guard.release(), "distributed guard should enqueue release");
+    wait_until_all_managers_can_write(&managers, resource).await;
+}
+
+#[tokio::test]
+async fn test_namespace_lock_distributed_unlock_retries_release_false() {
+    let managers = (0..3).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let flaky_clients = managers
+        .iter()
+        .map(|manager| Arc::new(FlakyReleaseClient::new(manager.clone(), 1)))
+        .collect::<Vec<_>>();
+    let clients = flaky_clients
+        .iter()
+        .map(|client| client.clone() as Arc<dyn LockClient>)
+        .collect::<Vec<_>>();
+
+    let lock = NamespaceLock::with_clients("flaky-release".to_string(), clients);
+    let resource = create_test_object_key("bucket", "object-flaky-release");
+
+    let mut guard = lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
+        .await
+        .expect("owner-a should acquire write lock before flaky release");
+
+    assert!(guard.release(), "distributed guard should enqueue release");
+    wait_until_all_managers_can_write(&managers, resource).await;
+
+    assert!(
+        flaky_clients.iter().all(|client| client.release_attempts() >= 2),
+        "each simulated node should be retried after an initial false release"
+    );
+}
+
+#[test]
+fn test_namespace_lock_distributed_drop_without_runtime_does_not_panic() {
+    let (manager, resource, guard) = {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should be created");
+        runtime.block_on(async {
+            let manager = Arc::new(GlobalLockManager::new());
+            let resource = create_test_object_key("bucket", "object-drop-no-runtime");
+            let lock = NamespaceLock::with_clients(
+                "drop-no-runtime".to_string(),
+                vec![Arc::new(LocalClient::with_manager(manager.clone()))],
+            );
+            let guard = lock
+                .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
+                .await
+                .expect("lock should be acquired");
+            (manager, resource, guard)
+        })
+    };
+
+    let drop_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(guard)));
+    assert!(drop_result.is_ok(), "dropping distributed guard without runtime should not panic");
+
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .expect("test runtime should be created");
+    runtime.block_on(wait_until_all_managers_can_write(&[manager], resource));
+}
+
 #[tokio::test]
 async fn test_namespace_lock_distributed_read_lock_succeeds_with_two_nodes_one_offline() {
     let manager = Arc::new(GlobalLockManager::new());
@@ -568,6 +784,39 @@ async fn test_namespace_lock_distributed_quorum_failure_rolls_back_successful_no
     drop(guard2);
 }
 
+#[tokio::test]
+async fn test_namespace_lock_distributed_quorum_rollback_retries_release_false() {
+    let managers = (0..2).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let flaky_clients = managers
+        .iter()
+        .map(|manager| Arc::new(FlakyReleaseClient::new(manager.clone(), 1)))
+        .collect::<Vec<_>>();
+    let clients = vec![
+        flaky_clients[0].clone() as Arc<dyn LockClient>,
+        flaky_clients[1].clone() as Arc<dyn LockClient>,
+        Arc::new(FailingClient) as Arc<dyn LockClient>,
+    ];
+    let resource = create_test_object_key("bucket", "object-rollback-retry");
+    let lock = NamespaceLock::with_clients_and_quorum("rollback-retry".to_string(), clients, 3);
+
+    let err = lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_millis(100))
+        .await
+        .expect_err("write lock should fail when quorum requires the offline node");
+
+    let err_str = err.to_string().to_lowercase();
+    assert!(
+        err_str.contains("quorum") || err_str.contains("not reached"),
+        "expected quorum error, got: {err}"
+    );
+    wait_until_all_managers_can_write(&managers, resource).await;
+
+    assert!(
+        flaky_clients.iter().all(|client| client.release_attempts() >= 2),
+        "rollback should retry node releases that initially returned false"
+    );
+}
+
 #[tokio::test]
 async fn test_namespace_lock_distributed_even_node_read_write_quorum_split() {
     let manager1 = Arc::new(GlobalLockManager::new());

crates/madmin/src/info_commands.rs

@@ -97,6 +97,9 @@ pub struct Disk {
     pub runtime_state: Option<String>,
     #[serde(rename = "offlineDurationSeconds", default, skip_serializing_if = "Option::is_none")]
     pub offline_duration_seconds: Option<u64>,
+    /// Leaf physical block devices backing this disk path when the platform can resolve them.
+    #[serde(rename = "physicalDeviceIds", default, skip_serializing_if = "Option::is_none")]
+    pub physical_device_ids: Option<Vec<String>>,
 }
 
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -481,6 +484,7 @@ mod tests {
         assert!(disk.heal_info.is_none());
         assert_eq!(disk.used_inodes, 0);
         assert_eq!(disk.free_inodes, 0);
+        assert!(disk.physical_device_ids.is_none());
         assert!(!disk.local);
         assert_eq!(disk.pool_index, 0);
         assert_eq!(disk.set_index, 0);
@@ -514,6 +518,7 @@ mod tests {
             heal_info: None,
             used_inodes: 1000000,
             free_inodes: 9000000,
+            physical_device_ids: Some(vec!["nvme0n1".to_string()]),
             local: true,
             pool_index: 0,
             set_index: 1,
@@ -533,6 +538,7 @@ mod tests {
         assert!(disk.metrics.is_some());
         assert_eq!(disk.runtime_state.as_deref(), Some("online"));
         assert_eq!(disk.offline_duration_seconds, Some(0));
+        assert_eq!(disk.physical_device_ids, Some(vec!["nvme0n1".to_string()]));
         assert!(disk.local);
     }
 
@@ -575,6 +581,7 @@ mod tests {
         assert_eq!(decoded.used_inodes, 11_125);
         assert_eq!(decoded.runtime_state, None);
         assert_eq!(decoded.offline_duration_seconds, None);
+        assert_eq!(decoded.physical_device_ids, None);
     }
 
     #[test]
@@ -606,6 +613,7 @@ mod tests {
             pool_index: 1,
             set_index: 2,
             disk_index: 3,
+            physical_device_ids: Some(vec!["nvme0n1".to_string(), "nvme1n1".to_string()]),
             runtime_state: Some("online".to_string()),
             offline_duration_seconds: Some(0),
         };
@@ -622,6 +630,19 @@ mod tests {
         assert_eq!(decoded.endpoint, "http://current-node:9000");
     }
 
+    #[test]
+    fn test_disk_serializes_physical_device_ids_when_present() {
+        let disk = Disk {
+            physical_device_ids: Some(vec!["nvme0n1".to_string(), "nvme1n1".to_string()]),
+            ..Default::default()
+        };
+
+        let json = serde_json::to_string(&disk).unwrap();
+        assert!(json.contains("physicalDeviceIds"));
+        assert!(json.contains("nvme0n1"));
+        assert!(json.contains("nvme1n1"));
+    }
+
     #[test]
     fn test_healing_disk_default() {
         let healing_disk = HealingDisk::default();

crates/notify/Cargo.toml

@@ -38,7 +38,6 @@ futures = { workspace = true }
 form_urlencoded = { workspace = true }
 hashbrown = { workspace = true }
 rayon = { workspace = true }
-rumqttc = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true }
 starshard = { workspace = true }

crates/notify/src/factory.rs

@@ -15,13 +15,13 @@
 use crate::Event;
 use async_trait::async_trait;
 use rustfs_config::EVENT_DEFAULT_DIR;
-use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_NATS_KEYS, NOTIFY_PULSAR_KEYS, NOTIFY_WEBHOOK_KEYS};
+use rustfs_config::notify::{NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_NATS_KEYS, NOTIFY_PULSAR_KEYS, NOTIFY_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
     config::{
-        build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
-        validate_pulsar_config, validate_webhook_config,
+        build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+        validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
     },
     error::TargetError,
     target::TargetType,
@@ -119,3 +119,22 @@ impl TargetFactory for PulsarTargetFactory {
         NOTIFY_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }
+
+pub struct KafkaTargetFactory;
+
+#[async_trait]
+impl TargetFactory for KafkaTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
+        let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+        let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_kafka_config(config, EVENT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        NOTIFY_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}

crates/notify/src/integration.rs

@@ -15,12 +15,17 @@
 use crate::notification_system_subscriber::NotificationSystemSubscriberView;
 use crate::notifier::TargetList;
 use crate::{
-    Event, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream,
+    Event,
+    error::NotificationError,
+    notifier::EventNotifier,
+    registry::TargetRegistry,
+    rules::{BucketNotificationConfig, ParseConfigError},
+    stream,
 };
 use hashbrown::HashMap;
 use rustfs_config::notify::{
-    DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS,
-    NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+    DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_s3_common::EventName;
@@ -33,13 +38,14 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
 use tokio::sync::{RwLock, Semaphore, broadcast, mpsc};
-use tracing::{debug, error, info, warn};
+use tracing::{debug, info, warn};
 
 const MAX_RECENT_LIVE_EVENTS: usize = 1024;
 
 fn subsystem_target_type(target_type: &str) -> &str {
     match target_type {
         NOTIFY_WEBHOOK_SUB_SYS => "webhook",
+        NOTIFY_KAFKA_SUB_SYS => "kafka",
         NOTIFY_MQTT_SUB_SYS => "mqtt",
         NOTIFY_NATS_SUB_SYS => "nats",
         NOTIFY_PULSAR_SUB_SYS => "pulsar",
@@ -309,7 +315,10 @@ impl NotificationSystem {
 
         let config = {
             let guard = self.config.read().await;
-            debug!("Initializing notification system with config: {:?}", *guard);
+            debug!(
+                subsystem_count = guard.0.len(),
+                "Initializing notification system with configuration summary"
+            );
             guard.clone()
         };
 
@@ -517,7 +526,10 @@ impl NotificationSystem {
                 if !changed {
                     info!("Target {} of type {} not found, no changes made.", target_name, target_type);
                 }
-                debug!("Config after remove: {:?}", config);
+                debug!(
+                    subsystem_count = config.0.len(),
+                    "Target config removal processed and configuration summary updated"
+                );
                 changed
             })
             .await;
@@ -593,7 +605,6 @@ impl NotificationSystem {
         bucket: &str,
         cfg: &BucketNotificationConfig,
     ) -> Result<(), NotificationError> {
-        self.subscriber_view.apply_bucket_config(bucket, cfg);
         let arn_list = self.notifier.get_arn_list(&cfg.region).await;
         if arn_list.is_empty() {
             return Err(NotificationError::Configuration("No targets configured".to_string()));
@@ -602,13 +613,18 @@ impl NotificationSystem {
         // Validate the configuration against the available ARNs
         if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
             debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e);
-            if !e.to_string().contains("ARN not found") {
+            if !matches!(e, ParseConfigError::ArnNotFound(_)) {
                 return Err(NotificationError::BucketNotification(e.to_string()));
-            } else {
-                error!("config validate failed, err: {}", e);
             }
+            warn!(
+                bucket = %bucket,
+                region = %cfg.region,
+                error = %e,
+                "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
+            );
         }
 
+        self.subscriber_view.apply_bucket_config(bucket, cfg);
         let rules_map = cfg.get_rules_map();
         self.notifier.add_rules_map(bucket, rules_map.clone()).await;
         info!("Loaded notification config for bucket: {}", bucket);
@@ -750,6 +766,13 @@ mod tests {
         assert_eq!(target_id.name, "mqtt");
     }
 
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
+        assert_eq!(target_id.id, "eventbus");
+        assert_eq!(target_id.name, "kafka");
+    }
+
     #[test]
     fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
         let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");

crates/notify/src/registry.rs

@@ -13,7 +13,9 @@
 // limitations under the License.
 
 use crate::Event;
-use crate::factory::{MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory};
+use crate::factory::{
+    KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
+};
 use futures::stream::{FuturesUnordered, StreamExt};
 use hashbrown::HashMap;
 use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
@@ -45,6 +47,7 @@ impl TargetRegistry {
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
         registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
         registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
+        registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
 
         registry
     }

crates/obs/Cargo.toml

@@ -53,6 +53,7 @@ opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 opentelemetry-stdout = { workspace = true }
 opentelemetry-otlp = { workspace = true }
 opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
+percent-encoding = { workspace = true }
 serde = { workspace = true }
 tracing = { workspace = true, features = ["std", "attributes"] }
 tracing-appender = { workspace = true }

crates/obs/src/config.rs

@@ -28,11 +28,14 @@ use rustfs_config::observability::{
     DEFAULT_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_MATCH_MODE,
     DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
     DEFAULT_OBS_LOG_PARALLEL_COMPRESS, DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL,
-    DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT,
-    ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS, ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
-    ENV_OBS_LOG_COMPRESSION_ALGORITHM, ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN,
-    ENV_OBS_LOG_ENDPOINT, ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL,
-    ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
+    DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_ENDPOINT, ENV_OBS_ENDPOINT_HEADERS,
+    ENV_OBS_ENDPOINT_LOGS_HEADERS, ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_METRICS_HEADERS,
+    ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, ENV_OBS_ENDPOINT_TRACES_HEADERS,
+    ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, ENV_OBS_ENVIRONMENT, ENV_OBS_LOG_CLEANUP_INTERVAL_SECONDS,
+    ENV_OBS_LOG_COMPRESS_OLD_FILES, ENV_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, ENV_OBS_LOG_COMPRESSION_ALGORITHM,
+    ENV_OBS_LOG_DELETE_EMPTY_FILES, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_DRY_RUN, ENV_OBS_LOG_ENDPOINT,
+    ENV_OBS_LOG_EXCLUDE_PATTERNS, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_GZIP_COMPRESSION_LEVEL, ENV_OBS_LOG_KEEP_FILES,
+    ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
     ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_PARALLEL_COMPRESS, ENV_OBS_LOG_PARALLEL_WORKERS, ENV_OBS_LOG_ROTATION_TIME,
     ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOG_ZSTD_COMPRESSION_LEVEL, ENV_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, ENV_OBS_LOG_ZSTD_WORKERS,
     ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT,
@@ -45,7 +48,7 @@ use rustfs_config::{
     DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO,
     SERVICE_VERSION, USE_STDOUT,
 };
-use rustfs_utils::{get_env_bool, get_env_f64, get_env_opt_str, get_env_str, get_env_u64, get_env_usize};
+use rustfs_utils::{get_env_bool, get_env_f64, get_env_opt_str, get_env_opt_u64, get_env_str, get_env_u64, get_env_usize};
 use serde::{Deserialize, Serialize};
 use std::env;
 
@@ -101,6 +104,28 @@ pub struct OtelConfig {
     pub metric_endpoint: Option<String>,
     /// Dedicated log endpoint; overrides `endpoint` + `/v1/logs` fallback.
     pub log_endpoint: Option<String>,
+    /// Headers applied to all OTLP signals when using HTTP exporter.
+    /// Format: comma-separated `key=value` pairs with URL-encoded values, for
+    /// example: `Authorization=Bearer%20abc%20123,X-Scope-OrgID=my-tenant`.
+    /// URL-encode reserved characters in values such as spaces, commas, and `=`.
+    pub endpoint_headers: Option<String>,
+    /// Additional headers for traces; merged on top of `endpoint_headers`.
+    /// Uses the same comma-separated `key=value` format with URL-encoded values.
+    pub trace_headers: Option<String>,
+    /// Additional headers for metrics; merged on top of `endpoint_headers`.
+    /// Uses the same comma-separated `key=value` format with URL-encoded values.
+    pub metric_headers: Option<String>,
+    /// Additional headers for logs; merged on top of `endpoint_headers`.
+    /// Uses the same comma-separated `key=value` format with URL-encoded values.
+    pub log_headers: Option<String>,
+    /// Timeout (milliseconds) for all OTLP HTTP exports.
+    pub endpoint_timeout_millis: Option<u64>,
+    /// Timeout (milliseconds) for trace OTLP HTTP export.
+    pub trace_timeout_millis: Option<u64>,
+    /// Timeout (milliseconds) for metrics OTLP HTTP export.
+    pub metric_timeout_millis: Option<u64>,
+    /// Timeout (milliseconds) for log OTLP HTTP export.
+    pub log_timeout_millis: Option<u64>,
     /// Dedicated profiling endpoint.
     pub profiling_endpoint: Option<String>,
     /// Whether to export distributed traces (default: `true`).
@@ -245,6 +270,14 @@ impl OtelConfig {
             trace_endpoint: get_env_opt_str(ENV_OBS_TRACE_ENDPOINT),
             metric_endpoint: get_env_opt_str(ENV_OBS_METRIC_ENDPOINT),
             log_endpoint: get_env_opt_str(ENV_OBS_LOG_ENDPOINT),
+            endpoint_headers: get_env_opt_str(ENV_OBS_ENDPOINT_HEADERS),
+            trace_headers: get_env_opt_str(ENV_OBS_ENDPOINT_TRACES_HEADERS),
+            metric_headers: get_env_opt_str(ENV_OBS_ENDPOINT_METRICS_HEADERS),
+            log_headers: get_env_opt_str(ENV_OBS_ENDPOINT_LOGS_HEADERS),
+            endpoint_timeout_millis: get_env_opt_u64(ENV_OBS_ENDPOINT_TIMEOUT_MILLIS),
+            trace_timeout_millis: get_env_opt_u64(ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS),
+            metric_timeout_millis: get_env_opt_u64(ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS),
+            log_timeout_millis: get_env_opt_u64(ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS),
             profiling_endpoint: get_env_opt_str(ENV_OBS_PROFILING_ENDPOINT),
             traces_export_enabled: Some(get_env_bool(ENV_OBS_TRACES_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED)),
             metrics_export_enabled: Some(get_env_bool(ENV_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED)),

crates/obs/src/global.rs

@@ -24,16 +24,16 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
 pub(crate) static OBSERVABILITY_METRIC_ENABLED: OnceCell<bool> = OnceCell::const_new();
 
 /// Namespaced metrics for cleaner and rolling logging.
-pub(crate) const METRIC_LOG_CLEANER_DELETED_FILES_TOTAL: &str = "rustfs.log_cleaner.deleted_files_total";
-pub(crate) const METRIC_LOG_CLEANER_FREED_BYTES_TOTAL: &str = "rustfs.log_cleaner.freed_bytes_total";
-pub(crate) const METRIC_LOG_CLEANER_COMPRESS_DURATION_SECONDS: &str = "rustfs.log_cleaner.compress_duration_seconds";
-pub(crate) const METRIC_LOG_CLEANER_STEAL_SUCCESS_RATE: &str = "rustfs.log_cleaner.steal_success_rate";
-pub(crate) const METRIC_LOG_CLEANER_RUNS_TOTAL: &str = "rustfs.log_cleaner.runs_total";
-pub(crate) const METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL: &str = "rustfs.log_cleaner.run_failures_total";
-pub(crate) const METRIC_LOG_CLEANER_ROTATION_TOTAL: &str = "rustfs.log_cleaner.rotation_total";
-pub(crate) const METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL: &str = "rustfs.log_cleaner.rotation_failures_total";
-pub(crate) const METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS: &str = "rustfs.log_cleaner.rotation_duration_seconds";
-pub(crate) const METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES: &str = "rustfs.log_cleaner.active_file_size_bytes";
+pub(crate) const METRIC_LOG_CLEANER_DELETED_FILES_TOTAL: &str = "rustfs_log_cleaner_deleted_files_total";
+pub(crate) const METRIC_LOG_CLEANER_FREED_BYTES_TOTAL: &str = "rustfs_log_cleaner_freed_bytes_total";
+pub(crate) const METRIC_LOG_CLEANER_COMPRESS_DURATION_SECONDS: &str = "rustfs_log_cleaner_compress_duration_seconds";
+pub(crate) const METRIC_LOG_CLEANER_STEAL_SUCCESS_RATE: &str = "rustfs_log_cleaner_steal_success_rate";
+pub(crate) const METRIC_LOG_CLEANER_RUNS_TOTAL: &str = "rustfs_log_cleaner_runs_total";
+pub(crate) const METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL: &str = "rustfs_log_cleaner_run_failures_total";
+pub(crate) const METRIC_LOG_CLEANER_ROTATION_TOTAL: &str = "rustfs_log_cleaner_rotation_total";
+pub(crate) const METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL: &str = "rustfs_log_cleaner_rotation_failures_total";
+pub(crate) const METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS: &str = "rustfs_log_cleaner_rotation_duration_seconds";
+pub(crate) const METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES: &str = "rustfs_log_cleaner_active_file_size_bytes";
 
 /// Check whether Observability metric is enabled
 pub fn observability_metric_enabled() -> bool {
@@ -198,8 +198,8 @@ mod tests {
 
         for metric in metrics {
             assert!(
-                metric.starts_with("rustfs.log_cleaner."),
-                "metric '{metric}' should use rustfs.log_cleaner.* namespace"
+                metric.starts_with("rustfs_log_cleaner_"),
+                "metric '{metric}' should use rustfs_log_cleaner_* namespace"
             );
         }
     }

crates/obs/src/metrics/collectors/bucket_replication.rs

@@ -12,34 +12,62 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Bucket replication bandwidth metrics collector.
-//!
-//! Collects bandwidth metrics for bucket replication targets.
-//!
-//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication`
-//! to avoid duplication of metric names, types, and help text.
+//! Bucket replication metrics collector.
 
 use crate::metrics::report::PrometheusMetric;
-use crate::metrics::schema::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD};
+use crate::metrics::schema::bucket_replication::{
+    BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
+    BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
+    BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
+    BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
+    BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
+    BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
+    BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
+    BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD,
+    BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
+};
 use std::borrow::Cow;
 
-/// Bucket replication bandwidth statistics for metrics collection.
+#[derive(Debug, Clone, Default)]
+pub struct BucketReplicationTargetStats {
+    pub target_arn: String,
+    pub bandwidth_limit_bytes_per_sec: u64,
+    pub current_bandwidth_bytes_per_sec: f64,
+    pub latency_ms: f64,
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct BucketReplicationBandwidthStats {
-    /// Name of the bucket
     pub bucket: String,
-    /// Target ARN for replication
     pub target_arn: String,
-    /// Configured bandwidth limit in bytes per second
     pub limit_bytes_per_sec: u64,
-    /// Current bandwidth in bytes per second (EWMA)
     pub current_bandwidth_bytes_per_sec: f64,
 }
 
-/// Collects bucket replication bandwidth metrics from the provided statistics.
-///
-/// Uses the metric descriptors from `metrics_type::bucket_replication` module.
-/// Returns a vector of Prometheus metrics for replication bandwidth.
+#[derive(Debug, Clone, Default)]
+pub struct BucketReplicationStats {
+    pub bucket: String,
+    pub total_failed_bytes: u64,
+    pub total_failed_count: u64,
+    pub last_min_failed_bytes: u64,
+    pub last_min_failed_count: u64,
+    pub last_hour_failed_bytes: u64,
+    pub last_hour_failed_count: u64,
+    pub sent_bytes: u64,
+    pub sent_count: u64,
+    pub proxied_get_requests_total: u64,
+    pub proxied_get_requests_failures: u64,
+    pub proxied_head_requests_total: u64,
+    pub proxied_head_requests_failures: u64,
+    pub proxied_put_tagging_requests_total: u64,
+    pub proxied_put_tagging_requests_failures: u64,
+    pub proxied_get_tagging_requests_total: u64,
+    pub proxied_get_tagging_requests_failures: u64,
+    pub proxied_delete_tagging_requests_total: u64,
+    pub proxied_delete_tagging_requests_failures: u64,
+    pub targets: Vec<BucketReplicationTargetStats>,
+}
+
 pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
     if stats.is_empty() {
         return Vec::new();
@@ -52,24 +80,206 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
 
         metrics.push(
             PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64)
-                .with_label("bucket", bucket_label.clone())
-                .with_label("targetArn", target_arn_label.clone()),
+                .with_label(BUCKET_L, bucket_label.clone())
+                .with_label(TARGET_ARN_L, target_arn_label.clone()),
         );
 
         metrics.push(
             PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec)
-                .with_label("bucket", bucket_label)
-                .with_label("targetArn", target_arn_label),
+                .with_label(BUCKET_L, bucket_label)
+                .with_label(TARGET_ARN_L, target_arn_label),
         );
     }
 
     metrics
 }
 
+pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::new();
+    for stat in stats {
+        let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_TOTAL_FAILED_BYTES_MD, stat.total_failed_bytes as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_TOTAL_FAILED_COUNT_MD, stat.total_failed_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, stat.last_min_failed_bytes as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, stat.last_min_failed_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, stat.last_hour_failed_bytes as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, stat.last_hour_failed_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_SENT_BYTES_MD, stat.sent_bytes as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_SENT_COUNT_MD, stat.sent_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, stat.proxied_get_requests_total as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
+                stat.proxied_get_requests_failures as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
+                stat.proxied_head_requests_total as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
+                stat.proxied_head_requests_failures as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
+                stat.proxied_put_tagging_requests_total as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
+                stat.proxied_put_tagging_requests_failures as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
+                stat.proxied_get_tagging_requests_total as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
+                stat.proxied_get_tagging_requests_failures as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
+                stat.proxied_delete_tagging_requests_total as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
+                stat.proxied_delete_tagging_requests_failures as f64,
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+
+        for target in &stat.targets {
+            let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
+            metrics.push(
+                PrometheusMetric::from_descriptor(&BUCKET_REPL_LATENCY_MS_MD, target.latency_ms)
+                    .with_label(BUCKET_L, bucket_label.clone())
+                    .with_label(OPERATION_L, Cow::Borrowed("object_replication"))
+                    .with_label(RANGE_L, Cow::Borrowed("all"))
+                    .with_label(TARGET_ARN_L, target_label),
+            );
+        }
+    }
+
+    metrics
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_collect_bucket_replication_metrics() {
+        let stats = vec![BucketReplicationStats {
+            bucket: "b1".to_string(),
+            total_failed_bytes: 64,
+            total_failed_count: 2,
+            last_min_failed_bytes: 32,
+            last_min_failed_count: 1,
+            last_hour_failed_bytes: 64,
+            last_hour_failed_count: 2,
+            sent_bytes: 1024,
+            sent_count: 8,
+            proxied_get_requests_total: 5,
+            proxied_get_requests_failures: 1,
+            proxied_head_requests_total: 4,
+            proxied_head_requests_failures: 0,
+            proxied_put_tagging_requests_total: 3,
+            proxied_put_tagging_requests_failures: 1,
+            proxied_get_tagging_requests_total: 2,
+            proxied_get_tagging_requests_failures: 0,
+            proxied_delete_tagging_requests_total: 1,
+            proxied_delete_tagging_requests_failures: 0,
+            targets: vec![BucketReplicationTargetStats {
+                target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
+                bandwidth_limit_bytes_per_sec: 2048,
+                current_bandwidth_bytes_per_sec: 1024.0,
+                latency_ms: 15.0,
+            }],
+        }];
+
+        let metrics = collect_bucket_replication_metrics(&stats);
+        assert_eq!(metrics.len(), 19);
+
+        let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == sent_name
+                && metric.value == 8.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let latency_name = BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == latency_name
+                && metric.value == 15.0
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
+        }));
+    }
+
+    #[test]
+    fn test_collect_bucket_replication_metrics_empty() {
+        let stats: Vec<BucketReplicationStats> = Vec::new();
+        let metrics = collect_bucket_replication_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+
     #[test]
     fn test_collect_bucket_replication_bandwidth_metrics() {
         let stats = vec![BucketReplicationBandwidthStats {
@@ -83,24 +293,29 @@ mod tests {
         assert_eq!(metrics.len(), 2);
 
         let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name();
-        let limit_metric = metrics.iter().find(|m| {
-            m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
+        let limit_metric = metrics.iter().find(|metric| {
+            metric.name == limit_metric_name
+                && metric.value == 1_048_576.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
         });
         assert!(limit_metric.is_some());
         assert!(
             limit_metric
-                .and_then(|m| {
-                    m.labels
+                .and_then(|metric| {
+                    metric
+                        .labels
                         .iter()
-                        .find(|(k, _)| *k == "targetArn")
-                        .map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
+                        .find(|(key, _)| *key == TARGET_ARN_L)
+                        .map(|(_, value)| value.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
                 })
                 .unwrap_or(false)
         );
 
         let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name();
-        let current_metric = metrics.iter().find(|m| {
-            m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
+        let current_metric = metrics.iter().find(|metric| {
+            metric.name == current_metric_name
+                && metric.value == 204_800.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
         });
         assert!(current_metric.is_some());
     }

crates/obs/src/metrics/collectors/mod.rs

@@ -41,7 +41,10 @@ pub mod system_process;
 
 pub use audit::{AuditTargetStats, collect_audit_metrics};
 pub use bucket::{BucketStats, collect_bucket_metrics};
-pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
+pub use bucket_replication::{
+    BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
+    collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
+};
 pub use cluster::{ClusterStats, collect_cluster_metrics};
 pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics};
 pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};

crates/obs/src/metrics/collectors/replication.rs

@@ -25,6 +25,14 @@ use crate::metrics::schema::replication::*;
 /// Replication statistics.
 #[derive(Debug, Clone, Default)]
 pub struct ReplicationStats {
+    /// Average number of active replication workers
+    pub average_active_workers: f64,
+    /// Average queued bytes since server start
+    pub average_queued_bytes: i64,
+    /// Average queued objects since server start
+    pub average_queued_count: i64,
+    /// Average data transfer rate in bytes/sec
+    pub average_data_transfer_rate: f64,
     /// Number of active replication workers
     pub active_workers: u64,
     /// Current data transfer rate in bytes/sec
@@ -50,6 +58,10 @@ pub struct ReplicationStats {
 /// Returns a vector of Prometheus metrics for replication statistics.
 pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
     vec![
+        PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
+        PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_COUNT_MD, stats.average_queued_count as f64),
+        PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD, stats.average_data_transfer_rate),
         PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64),
         PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate),
         PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64),
@@ -70,6 +82,10 @@ mod tests {
     #[test]
     fn test_collect_replication_metrics() {
         let stats = ReplicationStats {
+            average_active_workers: 8.5,
+            average_queued_bytes: 1024 * 1024 * 40,
+            average_queued_count: 240,
+            average_data_transfer_rate: 1024.0 * 1024.0 * 3.0,
             active_workers: 10,
             current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s
             last_minute_queued_bytes: 1024 * 1024 * 100,       // 100 MB
@@ -84,13 +100,17 @@ mod tests {
         let metrics = collect_replication_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 9);
+        assert_eq!(metrics.len(), 13);
 
         // Verify active workers
         let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
         let active = metrics.iter().find(|m| m.name == active_name);
         assert!(active.is_some());
         assert_eq!(active.map(|m| m.value), Some(10.0));
+
+        let avg_active_name = REPLICATION_AVERAGE_ACTIVE_WORKERS_MD.get_full_metric_name();
+        let avg_active = metrics.iter().find(|m| m.name == avg_active_name);
+        assert_eq!(avg_active.map(|m| m.value), Some(8.5));
     }
 
     #[test]
@@ -98,7 +118,7 @@ mod tests {
         let stats = ReplicationStats::default();
         let metrics = collect_replication_metrics(&stats);
 
-        assert_eq!(metrics.len(), 9);
+        assert_eq!(metrics.len(), 13);
         for metric in &metrics {
             assert_eq!(metric.value, 0.0);
             assert!(metric.labels.is_empty());

crates/obs/src/metrics/scheduler.rs

@@ -35,8 +35,14 @@ use crate::metrics::collectors::{
     collect_audit_metrics,
     collect_bucket_metrics,
     collect_bucket_replication_bandwidth_metrics,
+    collect_bucket_replication_metrics,
+    collect_cluster_health_metrics,
     collect_cluster_metrics,
+    collect_cpu_metrics,
+    collect_drive_count_metrics,
+    collect_drive_detailed_metrics,
     collect_host_network_metrics,
+    collect_memory_metrics,
     collect_node_metrics,
     collect_notification_metrics,
     collect_notification_target_metrics,
@@ -45,6 +51,7 @@ use crate::metrics::collectors::{
     collect_process_disk_metrics,
     collect_process_memory_metrics,
     collect_process_metrics,
+    collect_replication_metrics,
     collect_resource_metrics,
 };
 use crate::metrics::config::{
@@ -56,14 +63,16 @@ use crate::metrics::config::{
 };
 use crate::metrics::report::{PrometheusMetric, report_metrics};
 use crate::metrics::stats_collector::{
-    ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats,
-    collect_disk_stats, collect_host_network_stats, collect_process_metric_bundle,
+    ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
+    collect_bucket_stats, collect_cluster_and_health_stats, collect_disk_and_system_drive_stats, collect_host_network_stats,
+    collect_process_metric_bundle, collect_replication_stats, collect_system_cpu_and_memory_stats_with,
 };
 use rustfs_audit::audit_target_metrics;
 use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
 use rustfs_utils::get_env_opt_u64;
 use std::borrow::Cow;
 use std::time::Duration;
+use sysinfo::System;
 use tokio::time::Instant;
 use tokio_util::sync::CancellationToken;
 use tracing::warn;
@@ -154,8 +163,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         loop {
             tokio::select! {
                 _ = interval.tick() => {
-                    let stats = collect_cluster_stats().await;
-                    let metrics = collect_cluster_metrics(&stats);
+                    let (stats, cluster_health) = collect_cluster_and_health_stats().await;
+                    let mut metrics = collect_cluster_metrics(&stats);
+                    metrics.extend(collect_cluster_health_metrics(&cluster_health));
                     report_metrics(&metrics);
                 }
                 _ = token_clone.cancelled() => {
@@ -192,8 +202,10 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         loop {
             tokio::select! {
                 _ = interval.tick() => {
-                    let stats = collect_disk_stats().await;
-                    let metrics = collect_node_metrics(&stats);
+                    let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_stats().await;
+                    let mut metrics = collect_node_metrics(&disk_stats);
+                    metrics.extend(collect_drive_detailed_metrics(&drive_stats));
+                    metrics.extend(collect_drive_count_metrics(&drive_counts));
                     report_metrics(&metrics);
                 }
                 _ = token_clone.cancelled() => {
@@ -212,7 +224,11 @@ pub fn init_metrics_runtime(token: CancellationToken) {
             tokio::select! {
                 _ = interval.tick() => {
                     let stats = collect_bucket_replication_bandwidth_stats();
-                    let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+                    let mut metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+                    let bucket_replication = collect_bucket_replication_detail_stats().await;
+                    metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
+                    let replication = collect_replication_stats().await;
+                    metrics.extend(collect_replication_metrics(&replication));
                     report_metrics(&metrics);
                 }
                 _ = token_clone.cancelled() => {
@@ -297,6 +313,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
     let token_clone = token;
     tokio::spawn(async move {
         let labels = current_process_metric_labels();
+        let mut host_system = System::new_all();
         let process_interval = resource_interval.min(system_interval);
         let mut interval = tokio::time::interval(process_interval);
         let now = Instant::now();
@@ -327,9 +344,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
 
                     if now >= next_system_run {
                         #[cfg(feature = "gpu")]
-                        let mut metrics = collect_system_monitoring_metrics(&bundle, &labels);
+                        let mut metrics = collect_system_monitoring_metrics(&bundle, &labels, &mut host_system);
                         #[cfg(not(feature = "gpu"))]
-                        let metrics = collect_system_monitoring_metrics(&bundle, &labels);
+                        let metrics = collect_system_monitoring_metrics(&bundle, &labels, &mut host_system);
 
                         #[cfg(feature = "gpu")]
                         if let Some(pid) = current_pid {
@@ -418,6 +435,7 @@ fn fallback_process_metric_labels(err: ProcessAttributeError) -> Vec<(&'static s
 fn collect_system_monitoring_metrics(
     bundle: &ProcessMetricBundle,
     labels: &[(&'static str, Cow<'static, str>)],
+    host_system: &mut System,
 ) -> Vec<PrometheusMetric> {
     let cpu_stats = ProcessCpuStats {
         usage: bundle.resource.cpu_percent,
@@ -432,8 +450,11 @@ fn collect_system_monitoring_metrics(
         written_bytes: bundle.disk_write_bytes,
     };
     let network_stats = collect_host_network_stats();
+    let (system_cpu_stats, system_memory_stats) = collect_system_cpu_and_memory_stats_with(host_system);
 
     let mut metrics = Vec::new();
+    metrics.extend(collect_cpu_metrics(&system_cpu_stats));
+    metrics.extend(collect_memory_metrics(&system_memory_stats));
     metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(labels)));
     metrics.extend(collect_process_memory_metrics(&memory_stats, Some(labels)));
     metrics.extend(collect_process_disk_metrics(&disk_stats, Some(labels)));

crates/obs/src/metrics/schema/bucket_replication.rs

@@ -22,7 +22,7 @@ pub const BUCKET_L: &str = "bucket";
 /// Replication operation
 pub const OPERATION_L: &str = "operation";
 /// Replication target ARN
-pub const TARGET_ARN_L: &str = "targetArn";
+pub const TARGET_ARN_L: &str = "target_arn";
 /// Replication range
 pub const RANGE_L: &str = "range";
 

crates/obs/src/metrics/stats_collector.rs

@@ -21,19 +21,41 @@
 //! and convert them to the Stats structs used by collectors.
 
 use crate::metrics::collectors::{
-    BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, HostNetworkStats, ProcessStats, ProcessStatusType,
-    ResourceStats,
+    BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, ClusterHealthStats,
+    ClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, HostNetworkStats, MemoryStats, ProcessStats,
+    ProcessStatusType, ReplicationStats, ResourceStats,
 };
 use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
+use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
 use rustfs_ecstore::data_usage::load_data_usage_from_backend;
 use rustfs_ecstore::global::get_global_bucket_monitor;
 use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
 use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
 use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
 use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
-use sysinfo::Networks;
+use std::time::Duration;
+use sysinfo::{Networks, System};
 use tracing::{instrument, warn};
 
+const DRIVE_STATE_OK: &str = "ok";
+const DRIVE_STATE_ONLINE: &str = "online";
+const DRIVE_STATE_UNFORMATTED: &str = "unformatted";
+const DRIVE_RUNTIME_STATE_RETURNING: &str = "returning";
+
+fn disk_is_online_for_metrics(state: &str, runtime_state: Option<&str>) -> bool {
+    let state_is_acceptable = state.eq_ignore_ascii_case(DRIVE_STATE_OK)
+        || state.eq_ignore_ascii_case(DRIVE_STATE_ONLINE)
+        || state.eq_ignore_ascii_case(DRIVE_STATE_UNFORMATTED);
+
+    if let Some(runtime_state) = runtime_state {
+        let runtime_state_is_acceptable = runtime_state.eq_ignore_ascii_case(DRIVE_STATE_ONLINE)
+            || runtime_state.eq_ignore_ascii_case(DRIVE_RUNTIME_STATE_RETURNING);
+        return runtime_state_is_acceptable && state_is_acceptable;
+    }
+
+    state_is_acceptable
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ProcessMetricBundle {
     pub resource: ResourceStats,
@@ -42,48 +64,76 @@ pub struct ProcessMetricBundle {
     pub disk_write_bytes: u64,
 }
 
-/// Collect cluster statistics from the storage layer.
-#[instrument]
-pub async fn collect_cluster_stats() -> ClusterStats {
+/// Collect cluster and cluster-health statistics from a single storage snapshot.
+pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthStats) {
     let Some(store) = new_object_layer_fn() else {
-        return ClusterStats::default();
+        return (ClusterStats::default(), ClusterHealthStats::default());
     };
 
     let storage_info = store.storage_info().await;
-
     let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
     let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
     let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
     let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
 
-    // Get bucket and object counts from data usage info
+    // Get bucket and object counts from data usage info.
     let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
         Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
         Err(e) => {
             warn!("Failed to load data usage from backend: {}", e);
-            // Fall back to bucket list for buckets_count, objects_count stays 0
+            // Fall back to bucket list for buckets_count, objects_count stays 0.
             let buckets = store
                 .list_bucket(&BucketOptions {
                     cached: true,
                     ..Default::default()
                 })
                 .await
-                .unwrap_or_else(|e| {
-                    warn!("Failed to list buckets for cluster metrics: {}", e);
+                .unwrap_or_else(|err| {
+                    warn!("Failed to list buckets for cluster metrics: {}", err);
                     Vec::new()
                 });
             (buckets.len() as u64, 0)
         }
     };
 
-    ClusterStats {
-        raw_capacity_bytes: raw_capacity,
-        usable_capacity_bytes: usable_capacity,
-        used_bytes: used,
-        free_bytes: free,
-        objects_count,
-        buckets_count,
+    let mut online = 0u64;
+    let mut offline = 0u64;
+    for disk in &storage_info.disks {
+        if disk_is_online_for_metrics(disk.state.as_str(), disk.runtime_state.as_deref()) {
+            online += 1;
+        } else {
+            offline += 1;
+        }
     }
+
+    (
+        ClusterStats {
+            raw_capacity_bytes: raw_capacity,
+            usable_capacity_bytes: usable_capacity,
+            used_bytes: used,
+            free_bytes: free,
+            objects_count,
+            buckets_count,
+        },
+        ClusterHealthStats {
+            drives_offline_count: offline,
+            drives_online_count: online,
+            drives_count: storage_info.disks.len() as u64,
+        },
+    )
+}
+
+/// Collect cluster statistics from the storage layer.
+#[instrument]
+pub async fn collect_cluster_stats() -> ClusterStats {
+    let (cluster_stats, _) = collect_cluster_and_health_stats().await;
+    cluster_stats
+}
+
+/// Collect cluster health statistics from the storage layer.
+pub async fn collect_cluster_health_stats() -> ClusterHealthStats {
+    let (_, cluster_health_stats) = collect_cluster_and_health_stats().await;
+    cluster_health_stats
 }
 
 /// Collect bucket statistics from the storage layer.
@@ -177,15 +227,207 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
         .collect()
 }
 
-/// Collect disk statistics from the storage layer.
-pub async fn collect_disk_stats() -> Vec<DiskStats> {
-    let Some(store) = new_object_layer_fn() else {
+/// Collect bucket and target level replication stats from the global replication runtime.
+pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
+    let Some(stats) = GLOBAL_REPLICATION_STATS.get() else {
         return Vec::new();
     };
 
-    let storage_info = store.storage_info().await;
+    let all_bucket_stats = stats.get_all().await;
+    let mut buckets = Vec::with_capacity(all_bucket_stats.len());
 
-    storage_info
+    for (bucket, bucket_stats) in all_bucket_stats {
+        let proxy = stats.get_proxy_stats(&bucket).await;
+        let mut total_failed_bytes = 0u64;
+        let mut total_failed_count = 0u64;
+        let mut last_min_failed_bytes = 0u64;
+        let mut last_min_failed_count = 0u64;
+        let mut last_hour_failed_bytes = 0u64;
+        let mut last_hour_failed_count = 0u64;
+        let mut sent_bytes = 0u64;
+        let mut sent_count = 0u64;
+        let mut targets = Vec::with_capacity(bucket_stats.stats.len());
+
+        for (target_arn, target_stats) in bucket_stats.stats {
+            total_failed_bytes += target_stats.fail_stats.size.max(0) as u64;
+            total_failed_count += target_stats.fail_stats.count.max(0) as u64;
+
+            let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
+            last_min_failed_bytes += last_min.size.max(0) as u64;
+            last_min_failed_count += last_min.count.max(0) as u64;
+
+            let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
+            last_hour_failed_bytes += last_hour.size.max(0) as u64;
+            last_hour_failed_count += last_hour.count.max(0) as u64;
+
+            sent_bytes += target_stats.replicated_size.max(0) as u64;
+            sent_count += target_stats.replicated_count.max(0) as u64;
+
+            targets.push(BucketReplicationTargetStats {
+                target_arn,
+                bandwidth_limit_bytes_per_sec: target_stats.bandwidth_limit_bytes_per_sec.max(0) as u64,
+                current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
+                latency_ms: target_stats.latency.curr,
+            });
+        }
+
+        buckets.push(BucketReplicationStats {
+            bucket,
+            total_failed_bytes,
+            total_failed_count,
+            last_min_failed_bytes,
+            last_min_failed_count,
+            last_hour_failed_bytes,
+            last_hour_failed_count,
+            sent_bytes,
+            sent_count,
+            proxied_get_requests_total: proxy.get_total.max(0) as u64,
+            proxied_get_requests_failures: proxy.get_failed.max(0) as u64,
+            proxied_head_requests_total: proxy.head_total.max(0) as u64,
+            proxied_head_requests_failures: proxy.head_failed.max(0) as u64,
+            // Proxy cache currently tracks generic PutObject requests, not tagging-specific APIs.
+            // Keep tagging counters zero until PutObjectTagging stats are tracked separately.
+            proxied_put_tagging_requests_total: 0,
+            proxied_put_tagging_requests_failures: 0,
+            proxied_get_tagging_requests_total: 0,
+            proxied_get_tagging_requests_failures: 0,
+            proxied_delete_tagging_requests_total: 0,
+            proxied_delete_tagging_requests_failures: 0,
+            targets,
+        });
+    }
+
+    buckets
+}
+
+/// Collect site-level replication stats from the global replication runtime.
+pub async fn collect_replication_stats() -> ReplicationStats {
+    let Some(stats) = GLOBAL_REPLICATION_STATS.get() else {
+        return ReplicationStats::default();
+    };
+
+    let site_metrics = stats.get_sr_metrics_for_node().await;
+    let current_active_workers = u64::try_from(site_metrics.active_workers.curr).unwrap_or(0);
+
+    let bandwidth_stats = collect_bucket_replication_bandwidth_stats();
+    let current_data_transfer_rate = bandwidth_stats
+        .iter()
+        .map(|stat| stat.current_bandwidth_bytes_per_sec)
+        .sum::<f64>();
+
+    let all_bucket_stats = stats.get_all().await;
+    let average_data_transfer_rate = all_bucket_stats
+        .values()
+        .flat_map(|bucket| bucket.stats.values())
+        .map(|stat| stat.xfer_rate_lrg.avg + stat.xfer_rate_sml.avg)
+        .sum::<f64>();
+    let max_data_transfer_rate = all_bucket_stats
+        .values()
+        .flat_map(|bucket| bucket.stats.values())
+        .map(|stat| stat.xfer_rate_lrg.peak + stat.xfer_rate_sml.peak)
+        .sum::<f64>();
+    let recent_backlog_count = stats
+        .mrf_stats
+        .values()
+        .copied()
+        .filter(|value| *value > 0)
+        .sum::<i64>()
+        .try_into()
+        .unwrap_or(0);
+
+    ReplicationStats {
+        average_active_workers: site_metrics.active_workers.avg,
+        average_queued_bytes: site_metrics.queued.avg.bytes,
+        average_queued_count: site_metrics.queued.avg.count,
+        average_data_transfer_rate,
+        active_workers: current_active_workers,
+        current_data_transfer_rate,
+        last_minute_queued_bytes: site_metrics.queued.last_minute.bytes.max(0) as u64,
+        last_minute_queued_count: site_metrics.queued.last_minute.count.max(0) as u64,
+        max_active_workers: u64::try_from(site_metrics.active_workers.max).unwrap_or(0),
+        max_queued_bytes: site_metrics.queued.max.bytes.max(0) as u64,
+        max_queued_count: site_metrics.queued.max.count.max(0) as u64,
+        max_data_transfer_rate,
+        recent_backlog_count,
+    }
+}
+
+/// Collect disk statistics from the storage layer.
+pub async fn collect_disk_stats() -> Vec<DiskStats> {
+    let (disk_stats, _, _) = collect_disk_and_system_drive_stats().await;
+    disk_stats
+}
+
+fn build_system_cpu_stats(system: &System) -> CpuStats {
+    let cpu_usage = system.global_cpu_usage() as f64;
+    let cpu_count = system.cpus().len().max(1) as f64;
+    let load_avg = System::load_average().one;
+
+    CpuStats {
+        avg_idle: (100.0 - cpu_usage).max(0.0),
+        avg_iowait: 0.0,
+        load_avg,
+        load_avg_perc: (load_avg / cpu_count) * 100.0,
+        nice: 0.0,
+        steal: 0.0,
+        system: cpu_usage,
+        user: 0.0,
+    }
+}
+
+fn build_system_memory_stats(system: &System) -> MemoryStats {
+    let total = system.total_memory();
+    let used = system.used_memory();
+
+    MemoryStats {
+        total,
+        used,
+        used_perc: if total > 0 {
+            (used as f64 / total as f64) * 100.0
+        } else {
+            0.0
+        },
+        free: system.free_memory(),
+        buffers: 0,
+        cache: 0,
+        shared: 0,
+        available: system.available_memory(),
+    }
+}
+
+/// Collect system CPU and memory statistics from a shared sysinfo snapshot.
+pub fn collect_system_cpu_and_memory_stats() -> (CpuStats, MemoryStats) {
+    let mut system = System::new_all();
+    collect_system_cpu_and_memory_stats_with(&mut system)
+}
+
+/// Collect system CPU and memory statistics by refreshing a reusable sysinfo instance.
+pub fn collect_system_cpu_and_memory_stats_with(system: &mut System) -> (CpuStats, MemoryStats) {
+    system.refresh_cpu_all();
+    system.refresh_memory();
+    (build_system_cpu_stats(system), build_system_memory_stats(system))
+}
+
+/// Collect system CPU statistics from the current host.
+pub fn collect_system_cpu_stats() -> CpuStats {
+    let (cpu_stats, _) = collect_system_cpu_and_memory_stats();
+    cpu_stats
+}
+
+/// Collect system memory statistics from the current host.
+pub fn collect_system_memory_stats() -> MemoryStats {
+    let (_, memory_stats) = collect_system_cpu_and_memory_stats();
+    memory_stats
+}
+
+/// Collect node disk stats and drive stats from a single storage snapshot.
+pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<DriveDetailedStats>, DriveCountStats) {
+    let Some(store) = new_object_layer_fn() else {
+        return (Vec::new(), Vec::new(), DriveCountStats::default());
+    };
+
+    let storage_info = store.storage_info().await;
+    let disk_stats = storage_info
         .disks
         .iter()
         .map(|disk| DiskStats {
@@ -195,7 +437,63 @@ pub async fn collect_disk_stats() -> Vec<DiskStats> {
             used_bytes: disk.used_space,
             free_bytes: disk.available_space,
         })
-        .collect()
+        .collect();
+
+    let mut online_count = 0u64;
+    let mut offline_count = 0u64;
+    let drive_stats = storage_info
+        .disks
+        .iter()
+        .map(|disk| {
+            let is_online = disk_is_online_for_metrics(disk.state.as_str(), disk.runtime_state.as_deref());
+            if is_online {
+                online_count += 1;
+            } else {
+                offline_count += 1;
+            }
+
+            DriveDetailedStats {
+                server: disk.endpoint.clone(),
+                drive: disk.drive_path.clone(),
+                total_bytes: disk.total_space,
+                used_bytes: disk.used_space,
+                free_bytes: disk.available_space,
+                used_inodes: 0,
+                free_inodes: 0,
+                total_inodes: 0,
+                timeout_errors_total: 0,
+                io_errors_total: 0,
+                availability_errors_total: 0,
+                waiting_io: 0,
+                api_latency_micros: 0,
+                health: if is_online { 1 } else { 0 },
+                reads_per_sec: 0.0,
+                reads_kb_per_sec: 0.0,
+                reads_await: 0.0,
+                writes_per_sec: 0.0,
+                writes_kb_per_sec: 0.0,
+                writes_await: 0.0,
+                perc_util: if disk.total_space > 0 {
+                    (disk.used_space as f64 / disk.total_space as f64) * 100.0
+                } else {
+                    0.0
+                },
+            }
+        })
+        .collect();
+
+    let drive_count_stats = DriveCountStats {
+        offline_count,
+        online_count,
+        total_count: online_count + offline_count,
+    };
+    (disk_stats, drive_stats, drive_count_stats)
+}
+
+/// Collect system drive statistics using the storage layer snapshot.
+pub async fn collect_system_drive_stats() -> (Vec<DriveDetailedStats>, DriveCountStats) {
+    let (_, drive_stats, drive_count_stats) = collect_disk_and_system_drive_stats().await;
+    (drive_stats, drive_count_stats)
 }
 
 /// Collect resource and process statistics for the current process in one sysinfo refresh.
@@ -286,3 +584,18 @@ pub fn collect_host_network_stats() -> HostNetworkStats {
         per_interface,
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn disk_is_online_for_metrics_accepts_online_state_case_insensitive() {
+        assert!(disk_is_online_for_metrics("OnLiNe", Some("online")));
+    }
+
+    #[test]
+    fn disk_is_online_for_metrics_rejects_offline_runtime_state() {
+        assert!(!disk_is_online_for_metrics(DRIVE_STATE_OK, Some("offline")));
+    }
+}

crates/obs/src/telemetry/otel.rs

@@ -57,11 +57,13 @@ use opentelemetry_sdk::{
     metrics::{PeriodicReader, SdkMeterProvider},
     trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
 };
+use percent_encoding::percent_decode_str;
 use rustfs_config::observability::{DEFAULT_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES};
 use rustfs_config::{
     APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED,
     DEFAULT_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
 };
+use std::collections::HashMap;
 use std::{fs, io::IsTerminal, time::Duration};
 use tracing::info;
 use tracing_error::ErrorLayer;
@@ -94,7 +96,7 @@ use tracing_subscriber::{
 ///
 /// # Note
 /// This function is intentionally kept unchanged from the pre-refactor
-/// implementation to preserve existing OTLP behaviour.
+/// implementation to preserve existing OTLP behavior.
 pub(super) fn init_observability_http(
     config: &OtelConfig,
     logger_level: &str,
@@ -339,11 +341,19 @@ fn build_tracer_provider(
         return Ok(None);
     }
 
-    let exporter = opentelemetry_otlp::SpanExporter::builder()
+    let mut exporter_builder = opentelemetry_otlp::SpanExporter::builder()
         .with_http()
         .with_endpoint(trace_ep)
         .with_protocol(Protocol::HttpBinary)
-        .with_compression(Compression::Gzip)
+        .with_compression(Compression::Gzip);
+    let trace_headers = resolve_signal_headers(config.endpoint_headers.as_deref(), config.trace_headers.as_deref());
+    if !trace_headers.is_empty() {
+        exporter_builder = exporter_builder.with_headers(trace_headers);
+    }
+    if let Some(timeout) = resolve_signal_timeout(config.endpoint_timeout_millis, config.trace_timeout_millis) {
+        exporter_builder = exporter_builder.with_timeout(timeout);
+    }
+    let exporter = exporter_builder
         .build()
         .map_err(|e| TelemetryError::BuildSpanExporter(e.to_string()))?;
 
@@ -398,12 +408,20 @@ fn build_meter_provider(
         return Ok(None);
     }
 
-    let exporter = opentelemetry_otlp::MetricExporter::builder()
+    let mut exporter_builder = opentelemetry_otlp::MetricExporter::builder()
         .with_http()
         .with_endpoint(metric_ep)
         .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
         .with_protocol(Protocol::HttpBinary)
-        .with_compression(Compression::Gzip)
+        .with_compression(Compression::Gzip);
+    let metric_headers = resolve_signal_headers(config.endpoint_headers.as_deref(), config.metric_headers.as_deref());
+    if !metric_headers.is_empty() {
+        exporter_builder = exporter_builder.with_headers(metric_headers);
+    }
+    if let Some(timeout) = resolve_signal_timeout(config.endpoint_timeout_millis, config.metric_timeout_millis) {
+        exporter_builder = exporter_builder.with_timeout(timeout);
+    }
+    let exporter = exporter_builder
         .build()
         .map_err(|e| TelemetryError::BuildMetricExporter(e.to_string()))?;
 
@@ -444,11 +462,19 @@ fn build_logger_provider(
         return Ok(None);
     }
 
-    let exporter = opentelemetry_otlp::LogExporter::builder()
+    let mut exporter_builder = opentelemetry_otlp::LogExporter::builder()
         .with_http()
         .with_endpoint(log_ep)
         .with_protocol(Protocol::HttpBinary)
-        .with_compression(Compression::Gzip)
+        .with_compression(Compression::Gzip);
+    let log_headers = resolve_signal_headers(config.endpoint_headers.as_deref(), config.log_headers.as_deref());
+    if !log_headers.is_empty() {
+        exporter_builder = exporter_builder.with_headers(log_headers);
+    }
+    if let Some(timeout) = resolve_signal_timeout(config.endpoint_timeout_millis, config.log_timeout_millis) {
+        exporter_builder = exporter_builder.with_timeout(timeout);
+    }
+    let exporter = exporter_builder
         .build()
         .map_err(|e| TelemetryError::BuildLogExporter(e.to_string()))?;
 
@@ -513,6 +539,39 @@ fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout:
         .build()
 }
 
+fn resolve_signal_headers(common_headers: Option<&str>, signal_headers: Option<&str>) -> HashMap<String, String> {
+    let mut headers = HashMap::new();
+    if let Some(raw_headers) = common_headers {
+        headers.extend(parse_otlp_headers(raw_headers));
+    }
+    if let Some(raw_headers) = signal_headers {
+        headers.extend(parse_otlp_headers(raw_headers));
+    }
+    headers
+}
+
+fn parse_otlp_headers(raw_headers: &str) -> HashMap<String, String> {
+    raw_headers
+        .split(',')
+        .filter_map(|entry| {
+            let (key, value) = entry.split_once('=')?;
+            let key = key.trim();
+            if key.is_empty() {
+                return None;
+            }
+            let value = percent_decode_str(value.trim()).decode_utf8().ok()?;
+            Some((key.to_string(), value.into_owned()))
+        })
+        .collect()
+}
+
+fn resolve_signal_timeout(common_timeout_millis: Option<u64>, signal_timeout_millis: Option<u64>) -> Option<Duration> {
+    signal_timeout_millis
+        .or(common_timeout_millis)
+        .filter(|timeout_millis| *timeout_millis > 0)
+        .map(Duration::from_millis)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -539,4 +598,34 @@ mod tests {
         let sampler = build_tracer_sampler(1.2);
         assert!(format!("{sampler:?}").contains("AlwaysOn"));
     }
+
+    #[test]
+    fn test_parse_otlp_headers_ignores_invalid_entries() {
+        let headers = parse_otlp_headers("Authorization=Bearer%20abc,empty=,missing, =ignored,key=value,bad=%FF");
+        assert_eq!(headers.len(), 3);
+        assert_eq!(headers.get("Authorization"), Some(&"Bearer abc".to_string()));
+        assert_eq!(headers.get("empty"), Some(&"".to_string()));
+        assert_eq!(headers.get("key"), Some(&"value".to_string()));
+    }
+
+    #[test]
+    fn test_resolve_signal_headers_signal_overrides_common() {
+        let headers = resolve_signal_headers(Some("k1=v1,k2=common"), Some("k2=signal,k3=v3"));
+        assert_eq!(headers.get("k1"), Some(&"v1".to_string()));
+        assert_eq!(headers.get("k2"), Some(&"signal".to_string()));
+        assert_eq!(headers.get("k3"), Some(&"v3".to_string()));
+    }
+
+    #[test]
+    fn test_resolve_signal_timeout_prefers_signal_value() {
+        assert_eq!(resolve_signal_timeout(Some(2_000), Some(5_000)), Some(Duration::from_millis(5_000)));
+    }
+
+    #[test]
+    fn test_resolve_signal_timeout_falls_back_to_common() {
+        assert_eq!(resolve_signal_timeout(Some(3_000), None), Some(Duration::from_millis(3_000)));
+        assert_eq!(resolve_signal_timeout(None, None), None);
+        assert_eq!(resolve_signal_timeout(Some(0), None), None);
+        assert_eq!(resolve_signal_timeout(None, Some(0)), None);
+    }
 }

crates/policy/src/policy/action.rs

@@ -599,15 +599,13 @@ impl AdminAction {
     }
 }
 
-#[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, IntoStaticStr, Debug, Copy)]
+#[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, IntoStaticStr, Debug, Copy, EnumString)]
 #[serde(try_from = "&str", into = "&str")]
-pub enum StsAction {}
-
-impl TryFrom<&str> for StsAction {
-    type Error = strum::ParseError;
-    fn try_from(_value: &str) -> std::result::Result<Self, Self::Error> {
-        Err(strum::ParseError::VariantNotFound)
-    }
+pub enum StsAction {
+    #[strum(serialize = "sts:*")]
+    AllActions,
+    #[strum(serialize = "sts:AssumeRole")]
+    AssumeRoleAction,
 }
 
 #[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, IntoStaticStr, Debug, Copy, EnumString)]
@@ -629,6 +627,16 @@ mod tests {
         assert!(matches!(action, Action::S3Action(S3Action::AllActions)));
     }
 
+    #[test]
+    fn test_sts_action_parsing() {
+        let action = Action::try_from("sts:AssumeRole").expect("Should parse STS AssumeRole action");
+        assert!(matches!(action, Action::StsAction(StsAction::AssumeRoleAction)));
+
+        let wildcard = Action::try_from("sts:*").expect("Should parse STS wildcard action");
+        assert!(matches!(wildcard, Action::StsAction(StsAction::AllActions)));
+        assert!(wildcard.is_match(&action));
+    }
+
     #[test]
     fn test_actionset_serialize_single_element() {
         // Single element should serialize as array for S3 specification compliance

crates/policy/src/policy/effect.rs

@@ -19,7 +19,7 @@ use strum::{EnumString, IntoStaticStr};
 use super::Validator;
 
 #[derive(Serialize, Clone, Deserialize, EnumString, IntoStaticStr, Default, Debug, PartialEq)]
-#[serde(try_from = "&str", into = "&str")]
+#[serde(try_from = "String", into = "&str")]
 pub enum Effect {
     #[default]
     #[strum(serialize = "Allow")]
@@ -28,6 +28,16 @@ pub enum Effect {
     Deny,
 }
 
+impl TryFrom<String> for Effect {
+    type Error = Error;
+
+    fn try_from(value: String) -> std::result::Result<Self, Self::Error> {
+        value
+            .parse::<Self>()
+            .map_err(|e: strum::ParseError| Error::StringError(e.to_string()))
+    }
+}
+
 impl Effect {
     pub fn is_allowed(&self, allowed: bool) -> bool {
         if matches!(self, Self::Allow) {

crates/policy/src/policy/function/binary.rs

@@ -14,19 +14,394 @@
 
 use std::collections::HashMap;
 
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
 
 use super::func::InnerFunc;
 
 pub type BinaryFunc = InnerFunc<BinaryFuncValue>;
 
-// todo implement it
-#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)]
-#[serde(transparent)]
-pub struct BinaryFuncValue(String);
+#[derive(thiserror::Error, Clone, Debug, Eq, PartialEq)]
+pub enum BinaryFuncValueError {
+    #[error("invalid base64 for BinaryEquals")]
+    InvalidBase64,
+}
 
-impl BinaryFunc {
-    pub fn evaluate(&self, _values: &HashMap<String, Vec<String>>) -> bool {
-        todo!()
+/// Policy value for the AWS IAM `BinaryEquals` condition.
+///
+/// Policies store the value as a base64-encoded string or array of strings.
+/// During deserialization the values are validated and the raw bytes are
+/// cached, so evaluation is a plain byte comparison and malformed policies
+/// are rejected at parse time.
+#[derive(Clone, Debug)]
+pub struct BinaryFuncValue {
+    /// Original base64 forms, preserved for serialization round-trips.
+    encoded: Vec<String>,
+    /// Decoded bytes used for comparison during `evaluate`.
+    decoded: Vec<Vec<u8>>,
+}
+
+impl BinaryFuncValue {
+    /// Construct from a base64-encoded string, validating the encoding.
+    pub fn new(encoded: impl Into<String>) -> Result<Self, BinaryFuncValueError> {
+        Self::from_encoded_values(vec![encoded.into()])
+    }
+
+    fn from_encoded_values(encoded: Vec<String>) -> Result<Self, BinaryFuncValueError> {
+        let decoded = encoded
+            .iter()
+            .map(|value| {
+                base64_simd::STANDARD
+                    .decode_to_vec(value.as_bytes())
+                    .map_err(|_| BinaryFuncValueError::InvalidBase64)
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        Ok(Self { encoded, decoded })
+    }
+}
+
+impl TryFrom<String> for BinaryFuncValue {
+    type Error = BinaryFuncValueError;
+
+    fn try_from(encoded: String) -> Result<Self, Self::Error> {
+        Self::new(encoded)
+    }
+}
+
+impl TryFrom<&str> for BinaryFuncValue {
+    type Error = BinaryFuncValueError;
+
+    fn try_from(encoded: &str) -> Result<Self, Self::Error> {
+        Self::new(encoded)
+    }
+}
+
+// Equality is defined over decoded bytes so that semantically equal values
+// compare equal regardless of incidental base64 formatting differences.
+impl PartialEq for BinaryFuncValue {
+    fn eq(&self, other: &Self) -> bool {
+        self.decoded == other.decoded
+    }
+}
+
+impl Eq for BinaryFuncValue {}
+
+impl Serialize for BinaryFuncValue {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        if self.encoded.len() == 1 {
+            serializer.serialize_str(&self.encoded[0])
+        } else {
+            self.encoded.serialize(serializer)
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for BinaryFuncValue {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        struct StringOrVecVisitor;
+
+        impl<'de> de::Visitor<'de> for StringOrVecVisitor {
+            type Value = BinaryFuncValue;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                formatter.write_str("a base64 string or an array of base64 strings")
+            }
+
+            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+            where
+                E: de::Error,
+            {
+                BinaryFuncValue::new(value).map_err(E::custom)
+            }
+
+            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+            where
+                A: de::SeqAccess<'de>,
+            {
+                let mut values = Vec::with_capacity(seq.size_hint().unwrap_or(0));
+                while let Some(value) = seq.next_element::<String>()? {
+                    values.push(value);
+                }
+                if values.is_empty() {
+                    return Err(de::Error::custom("empty"));
+                }
+
+                BinaryFuncValue::from_encoded_values(values).map_err(de::Error::custom)
+            }
+        }
+
+        deserializer.deserialize_any(StringOrVecVisitor)
+    }
+}
+
+impl BinaryFunc {
+    /// Evaluate an AWS IAM `BinaryEquals` condition.
+    ///
+    /// AWS semantics compare the base64-decoded bytes of the policy value
+    /// against the base64-decoded bytes of the request context value. In this
+    /// codebase request context values come directly from HTTP header strings
+    /// (see `rustfs::auth::get_condition_values_with_query`), so for real
+    /// binary condition keys (e.g. SSE-C customer-key headers) the request
+    /// value is itself base64. Decoding both sides is therefore required for
+    /// the comparison to ever succeed.
+    ///
+    /// All key/value pairs in the function must match (logical AND); for a
+    /// given key, any decoded request value that equals any expected decoded
+    /// policy value satisfies that pair (OR across request values and policy
+    /// values). A missing request key, or *any* request value that is not
+    /// valid base64, causes the condition to evaluate to false (fail-closed).
+    pub fn evaluate(&self, values: &HashMap<String, Vec<String>>) -> bool {
+        for inner in self.0.iter() {
+            let Some(rvalues) = values.get(inner.key.name().as_str()) else {
+                return false;
+            };
+
+            let mut matched = false;
+            for v in rvalues {
+                let Ok(decoded) = base64_simd::STANDARD.decode_to_vec(v.as_bytes()) else {
+                    return false;
+                };
+                if inner
+                    .values
+                    .decoded
+                    .iter()
+                    .any(|expected| decoded.as_slice() == expected.as_slice())
+                {
+                    matched = true;
+                }
+            }
+            if !matched {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{BinaryFunc, BinaryFuncValue, BinaryFuncValueError};
+    use crate::policy::function::func::FuncKeyValue;
+    use crate::policy::function::{
+        key::Key,
+        key_name::AwsKeyName::*,
+        key_name::KeyName::{self, *},
+    };
+    use std::collections::HashMap;
+
+    fn new_func(name: KeyName, variable: Option<String>, value: &str) -> BinaryFunc {
+        BinaryFunc {
+            0: vec![FuncKeyValue {
+                key: Key { name, variable },
+                values: BinaryFuncValue::new(value).expect("valid base64 in test"),
+            }],
+        }
+    }
+
+    fn new_multi_func(name: KeyName, variable: Option<String>, values: &[&str]) -> BinaryFunc {
+        BinaryFunc {
+            0: vec![FuncKeyValue {
+                key: Key { name, variable },
+                values: BinaryFuncValue::from_encoded_values(values.iter().map(|value| (*value).to_string()).collect())
+                    .expect("valid binary array in test"),
+            }],
+        }
+    }
+
+    #[test]
+    fn evaluate_matches_decoded_bytes() {
+        // base64("hello") = "aGVsbG8="
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8=");
+        let mut ctx = HashMap::new();
+        // Request value is itself base64 — BinaryEquals decodes both sides.
+        ctx.insert("username".to_string(), vec!["aGVsbG8=".to_string()]);
+        assert!(f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_rejects_non_matching_value() {
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8="); // "hello"
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["d29ybGQ=".to_string()]); // "world"
+        assert!(!f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_matches_any_request_value() {
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8="); // "hello"
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["d29ybGQ=".to_string(), "aGVsbG8=".to_string()]);
+        assert!(f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_missing_key_is_false() {
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8=");
+        let ctx = HashMap::new();
+        assert!(!f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_empty_request_values_is_false() {
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8=");
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec![]);
+        assert!(!f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_matches_multibyte_utf8() {
+        // base64("café") = "Y2Fmw6k=" — exercises multi-byte UTF-8 round trip.
+        let f = new_func(Aws(AWSUsername), None, "Y2Fmw6k=");
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["Y2Fmw6k=".to_string()]);
+        assert!(f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_matches_any_policy_value() {
+        let f = new_multi_func(Aws(AWSUsername), None, &["aGVsbG8=", "d29ybGQ="]);
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["d29ybGQ=".to_string()]);
+        assert!(f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_invalid_base64_request_value_fails_closed() {
+        // Malformed base64 in the request must never match, regardless of policy value.
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8=");
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["!!!not-base64!!!".to_string()]);
+        assert!(!f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn evaluate_mixed_valid_and_invalid_request_values_fails_closed() {
+        // A valid matching value alongside an invalid base64 value must still
+        // fail closed — for BinaryEquals, any unparsable request value causes
+        // evaluation to return false even if another request value matches.
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8="); // "hello"
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["aGVsbG8=".to_string(), "!!!not-base64!!!".to_string()]);
+        assert!(!f.evaluate(&ctx));
+
+        // Order-independent: invalid first, valid second — still false.
+        let mut ctx2 = HashMap::new();
+        ctx2.insert("username".to_string(), vec!["!!!not-base64!!!".to_string(), "aGVsbG8=".to_string()]);
+        assert!(!f.evaluate(&ctx2));
+    }
+
+    #[test]
+    fn evaluate_raw_request_value_does_not_match() {
+        // A raw (non-base64) request value that happens to equal the decoded
+        // policy bytes must NOT match — both sides are decoded first. "hello"
+        // is not valid standard base64 (length 5, not a multiple of 4), so
+        // decoding fails and the evaluation fails closed.
+        let f = new_func(Aws(AWSUsername), None, "aGVsbG8="); // decodes to "hello"
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["hello".to_string()]);
+        assert!(!f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn try_from_constructs_binary_func_value() {
+        // Ergonomic alternatives to BinaryFuncValue::new — parity with the
+        // prior public-struct API and idiomatic Rust conversion.
+        let from_str: BinaryFuncValue = "aGVsbG8=".try_into().unwrap();
+        let from_string: BinaryFuncValue = String::from("aGVsbG8=").try_into().unwrap();
+        assert_eq!(from_str, from_string);
+        assert_eq!(BinaryFuncValue::try_from("!!!bad!!!").unwrap_err(), BinaryFuncValueError::InvalidBase64,);
+    }
+
+    #[test]
+    fn evaluate_all_key_values_must_match() {
+        // Two key/value pairs — both must be satisfied.
+        let f = BinaryFunc {
+            0: vec![
+                FuncKeyValue {
+                    key: Key {
+                        name: Aws(AWSUsername),
+                        variable: None,
+                    },
+                    values: BinaryFuncValue::new("aGVsbG8=").unwrap(), // "hello"
+                },
+                FuncKeyValue {
+                    key: Key {
+                        name: Aws(AWSPrincipalType),
+                        variable: None,
+                    },
+                    values: BinaryFuncValue::new("d29ybGQ=").unwrap(), // "world"
+                },
+            ],
+        };
+
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["aGVsbG8=".to_string()]);
+        ctx.insert("principaltype".to_string(), vec!["d29ybGQ=".to_string()]);
+        assert!(f.evaluate(&ctx));
+
+        // Second key missing — must fail.
+        let mut ctx2 = HashMap::new();
+        ctx2.insert("username".to_string(), vec!["aGVsbG8=".to_string()]);
+        assert!(!f.evaluate(&ctx2));
+    }
+
+    #[test]
+    fn deserializes_from_policy_json() {
+        let json = r#"{"aws:username": "aGVsbG8="}"#;
+        let f: BinaryFunc = serde_json::from_str(json).unwrap();
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["aGVsbG8=".to_string()]);
+        assert!(f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn deserializes_array_from_policy_json() {
+        let json = r#"{"aws:username": ["aGVsbG8=", "d29ybGQ="]}"#;
+        let f: BinaryFunc = serde_json::from_str(json).unwrap();
+        let mut ctx = HashMap::new();
+        ctx.insert("username".to_string(), vec!["d29ybGQ=".to_string()]);
+        assert!(f.evaluate(&ctx));
+    }
+
+    #[test]
+    fn deserialize_rejects_invalid_base64_at_parse_time() {
+        // Malformed policies must be rejected eagerly, not silently fail at eval.
+        let json = r#"{"aws:username": "!!!not-base64!!!"}"#;
+        let err = serde_json::from_str::<BinaryFunc>(json).unwrap_err();
+        let msg = err.to_string();
+        assert!(msg.contains("invalid base64"), "unexpected error message: {msg}");
+    }
+
+    #[test]
+    fn deserialize_rejects_invalid_base64_in_array_at_parse_time() {
+        let json = r#"{"aws:username": ["aGVsbG8=", "!!!not-base64!!!"]}"#;
+        let err = serde_json::from_str::<BinaryFunc>(json).unwrap_err();
+        let msg = err.to_string();
+        assert!(msg.contains("invalid base64"), "unexpected error message: {msg}");
+    }
+
+    #[test]
+    fn deserialize_rejects_empty_array() {
+        let json = r#"{"aws:username": []}"#;
+        let err = serde_json::from_str::<BinaryFunc>(json).unwrap_err();
+        let msg = err.to_string();
+        assert!(msg.contains("empty"), "unexpected error message: {msg}");
+    }
+
+    #[test]
+    fn serialize_round_trip_preserves_encoded_form() {
+        let json = r#"{"aws:username":"aGVsbG8="}"#;
+        let f: BinaryFunc = serde_json::from_str(json).unwrap();
+        let out = serde_json::to_string(&f).unwrap();
+        assert_eq!(out, json);
+    }
+
+    #[test]
+    fn serialize_round_trip_preserves_encoded_array_form() {
+        let json = r#"{"aws:username":["aGVsbG8=","d29ybGQ="]}"#;
+        let f: BinaryFunc = serde_json::from_str(json).unwrap();
+        let out = serde_json::to_string(&f).unwrap();
+        assert_eq!(out, json);
     }
 }

crates/policy/src/policy/policy.rs

@@ -1761,4 +1761,28 @@ mod test {
         assert!(!found);
         assert!(policies.is_empty());
     }
+
+    #[test]
+    fn test_policy_round_trips_through_json_value() {
+        let policy = Policy::parse_config(
+            br#"{
+  "Version":"2012-10-17",
+  "Statement":[
+    {
+      "Effect":"Allow",
+      "Action":["s3:GetObject"],
+      "Resource":["arn:aws:s3:::bucket/*"]
+    }
+  ]
+}"#,
+        )
+        .expect("policy should parse");
+
+        let value = serde_json::to_value(&policy).expect("policy should serialize");
+        let round_trip: Policy = serde_json::from_value(value).expect("policy should deserialize from serde_json::Value");
+
+        assert_eq!(round_trip.version, policy.version);
+        assert_eq!(round_trip.statements.len(), policy.statements.len());
+        assert_eq!(round_trip.statements[0].effect, policy.statements[0].effect);
+    }
 }

crates/protos/src/generated/flatbuffers_generated/models.rs

@@ -1,55 +1,46 @@
 // automatically generated by the FlatBuffers compiler, do not modify
-
 // @generated
 
-use core::cmp::Ordering;
-use core::mem;
-
-extern crate flatbuffers;
-use self::flatbuffers::{EndianScalar, Follow};
+extern crate alloc;
 
 #[allow(unused_imports, dead_code)]
 pub mod models {
 
-    use core::cmp::Ordering;
-    use core::mem;
-
-    extern crate flatbuffers;
-    use self::flatbuffers::{EndianScalar, Follow};
+    extern crate alloc;
 
     pub enum PingBodyOffset {}
     #[derive(Copy, Clone, PartialEq)]
 
     pub struct PingBody<'a> {
-        pub _tab: flatbuffers::Table<'a>,
+        pub _tab: ::flatbuffers::Table<'a>,
     }
 
-    impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
+    impl<'a> ::flatbuffers::Follow<'a> for PingBody<'a> {
         type Inner = PingBody<'a>;
         #[inline]
         unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
             Self {
-                _tab: unsafe { flatbuffers::Table::new(buf, loc) },
+                _tab: unsafe { ::flatbuffers::Table::new(buf, loc) },
             }
         }
     }
 
     impl<'a> PingBody<'a> {
-        pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
+        pub const VT_PAYLOAD: ::flatbuffers::VOffsetT = 4;
 
         pub const fn get_fully_qualified_name() -> &'static str {
             "models.PingBody"
         }
 
         #[inline]
-        pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+        pub unsafe fn init_from_table(table: ::flatbuffers::Table<'a>) -> Self {
             PingBody { _tab: table }
         }
         #[allow(unused_mut)]
-        pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
-            _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+        pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: ::flatbuffers::Allocator + 'bldr>(
+            _fbb: &'mut_bldr mut ::flatbuffers::FlatBufferBuilder<'bldr, A>,
             args: &'args PingBodyArgs<'args>,
-        ) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
+        ) -> ::flatbuffers::WIPOffset<PingBody<'bldr>> {
             let mut builder = PingBodyBuilder::new(_fbb);
             if let Some(x) = args.payload {
                 builder.add_payload(x);
@@ -58,29 +49,28 @@ pub mod models {
         }
 
         #[inline]
-        pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+        pub fn payload(&self) -> Option<::flatbuffers::Vector<'a, u8>> {
             // Safety:
             // Created from valid Table for this object
             // which contains a valid value in this slot
             unsafe {
                 self._tab
-                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)
+                    .get::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)
             }
         }
     }
 
-    impl flatbuffers::Verifiable for PingBody<'_> {
+    impl ::flatbuffers::Verifiable for PingBody<'_> {
         #[inline]
-        fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> {
-            use self::flatbuffers::Verifiable;
+        fn run_verifier(v: &mut ::flatbuffers::Verifier, pos: usize) -> Result<(), ::flatbuffers::InvalidFlatbuffer> {
             v.visit_table(pos)?
-                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
+                .visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
                 .finish();
             Ok(())
         }
     }
     pub struct PingBodyArgs<'a> {
-        pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+        pub payload: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, u8>>>,
     }
     impl<'a> Default for PingBodyArgs<'a> {
         #[inline]
@@ -89,18 +79,18 @@ pub mod models {
         }
     }
 
-    pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
-        fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
-        start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+    pub struct PingBodyBuilder<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> {
+        fbb_: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>,
+        start_: ::flatbuffers::WIPOffset<::flatbuffers::TableUnfinishedWIPOffset>,
     }
-    impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
+    impl<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
         #[inline]
-        pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b, u8>>) {
+        pub fn add_payload(&mut self, payload: ::flatbuffers::WIPOffset<::flatbuffers::Vector<'b, u8>>) {
             self.fbb_
-                .push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
+                .push_slot_always::<::flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
         }
         #[inline]
-        pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
+        pub fn new(_fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
             let start = _fbb.start_table();
             PingBodyBuilder {
                 fbb_: _fbb,
@@ -108,14 +98,14 @@ pub mod models {
             }
         }
         #[inline]
-        pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
+        pub fn finish(self) -> ::flatbuffers::WIPOffset<PingBody<'a>> {
             let o = self.fbb_.end_table(self.start_);
-            flatbuffers::WIPOffset::new(o.value())
+            ::flatbuffers::WIPOffset::new(o.value())
         }
     }
 
-    impl core::fmt::Debug for PingBody<'_> {
-        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    impl ::core::fmt::Debug for PingBody<'_> {
+        fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
             let mut ds = f.debug_struct("PingBody");
             ds.field("payload", &self.payload());
             ds.finish()

crates/protos/src/generated/proto_gen/node_service.rs

@@ -467,10 +467,10 @@ pub struct UpdateMetadataRequest {
     pub file_info: ::prost::alloc::string::String,
     #[prost(string, tag = "5")]
     pub opts: ::prost::alloc::string::String,
-    #[prost(bytes = "vec", tag = "6")]
-    pub file_info_bin: ::prost::alloc::vec::Vec<u8>,
-    #[prost(bytes = "vec", tag = "7")]
-    pub opts_bin: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes = "bytes", tag = "6")]
+    pub file_info_bin: ::prost::bytes::Bytes,
+    #[prost(bytes = "bytes", tag = "7")]
+    pub opts_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct UpdateMetadataResponse {
@@ -490,8 +490,8 @@ pub struct WriteMetadataRequest {
     pub path: ::prost::alloc::string::String,
     #[prost(string, tag = "4")]
     pub file_info: ::prost::alloc::string::String,
-    #[prost(bytes = "vec", tag = "5")]
-    pub file_info_bin: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes = "bytes", tag = "5")]
+    pub file_info_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct WriteMetadataResponse {
@@ -512,8 +512,8 @@ pub struct ReadVersionRequest {
     pub version_id: ::prost::alloc::string::String,
     #[prost(string, tag = "5")]
     pub opts: ::prost::alloc::string::String,
-    #[prost(bytes = "vec", tag = "6")]
-    pub opts_bin: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes = "bytes", tag = "6")]
+    pub opts_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReadVersionResponse {
@@ -523,8 +523,8 @@ pub struct ReadVersionResponse {
     pub file_info: ::prost::alloc::string::String,
     #[prost(message, optional, tag = "3")]
     pub error: ::core::option::Option<Error>,
-    #[prost(bytes = "vec", tag = "4")]
-    pub file_info_bin: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes = "bytes", tag = "4")]
+    pub file_info_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReadXlRequest {
@@ -545,8 +545,8 @@ pub struct ReadXlResponse {
     pub raw_file_info: ::prost::alloc::string::String,
     #[prost(message, optional, tag = "3")]
     pub error: ::core::option::Option<Error>,
-    #[prost(bytes = "vec", tag = "4")]
-    pub raw_file_info_bin: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes = "bytes", tag = "4")]
+    pub raw_file_info_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct DeleteVersionRequest {
@@ -598,8 +598,8 @@ pub struct ReadMultipleRequest {
     pub disk: ::prost::alloc::string::String,
     #[prost(string, tag = "2")]
     pub read_multiple_req: ::prost::alloc::string::String,
-    #[prost(bytes = "vec", tag = "3")]
-    pub read_multiple_req_bin: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes = "bytes", tag = "3")]
+    pub read_multiple_req_bin: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ReadMultipleResponse {
@@ -609,8 +609,8 @@ pub struct ReadMultipleResponse {
     pub read_multiple_resps: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
     #[prost(message, optional, tag = "3")]
     pub error: ::core::option::Option<Error>,
-    #[prost(bytes = "vec", repeated, tag = "4")]
-    pub read_multiple_resps_bin: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
+    #[prost(bytes = "bytes", repeated, tag = "4")]
+    pub read_multiple_resps_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct DeleteVolumeRequest {
@@ -673,7 +673,7 @@ pub struct GenerallyLockResult {
     #[prost(string, optional, tag = "3")]
     pub lock_info: ::core::option::Option<::prost::alloc::string::String>,
 }
-#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct BatchGenerallyLockResponse {
     #[prost(message, repeated, tag = "1")]
     pub results: ::prost::alloc::vec::Vec<GenerallyLockResult>,
@@ -816,26 +816,6 @@ pub struct GetMetricsResponse {
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
-pub struct GetLiveEventsRequest {
-    #[prost(uint64, tag = "1")]
-    pub after_sequence: u64,
-    #[prost(uint32, tag = "2")]
-    pub limit: u32,
-}
-#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
-pub struct GetLiveEventsResponse {
-    #[prost(bool, tag = "1")]
-    pub success: bool,
-    #[prost(bytes = "bytes", tag = "2")]
-    pub events: ::prost::bytes::Bytes,
-    #[prost(uint64, tag = "3")]
-    pub next_sequence: u64,
-    #[prost(bool, tag = "4")]
-    pub truncated: bool,
-    #[prost(string, optional, tag = "5")]
-    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
-}
-#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct GetProcInfoRequest {}
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct GetProcInfoResponse {
@@ -1130,6 +1110,26 @@ pub struct LoadTransitionTierConfigResponse {
     #[prost(string, optional, tag = "2")]
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct GetLiveEventsRequest {
+    #[prost(uint64, tag = "1")]
+    pub after_sequence: u64,
+    #[prost(uint32, tag = "2")]
+    pub limit: u32,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct GetLiveEventsResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(bytes = "bytes", tag = "2")]
+    pub events: ::prost::bytes::Bytes,
+    #[prost(uint64, tag = "3")]
+    pub next_sequence: u64,
+    #[prost(bool, tag = "4")]
+    pub truncated: bool,
+    #[prost(string, optional, tag = "5")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
 /// Generated client implementations.
 pub mod node_service_client {
     #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
@@ -1991,21 +1991,6 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "GetMetrics"));
             self.inner.unary(req, path, codec).await
         }
-        pub async fn get_live_events(
-            &mut self,
-            request: impl tonic::IntoRequest<super::GetLiveEventsRequest>,
-        ) -> std::result::Result<tonic::Response<super::GetLiveEventsResponse>, tonic::Status> {
-            self.inner
-                .ready()
-                .await
-                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
-            let codec = tonic_prost::ProstCodec::default();
-            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetLiveEvents");
-            let mut req = request.into_request();
-            req.extensions_mut()
-                .insert(GrpcMethod::new("node_service.NodeService", "GetLiveEvents"));
-            self.inner.unary(req, path, codec).await
-        }
         pub async fn get_proc_info(
             &mut self,
             request: impl tonic::IntoRequest<super::GetProcInfoRequest>,
@@ -2383,6 +2368,21 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "LoadTransitionTierConfig"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn get_live_events(
+            &mut self,
+            request: impl tonic::IntoRequest<super::GetLiveEventsRequest>,
+        ) -> std::result::Result<tonic::Response<super::GetLiveEventsResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetLiveEvents");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "GetLiveEvents"));
+            self.inner.unary(req, path, codec).await
+        }
     }
 }
 /// Generated server implementations.
@@ -2614,10 +2614,6 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::GetMetricsRequest>,
         ) -> std::result::Result<tonic::Response<super::GetMetricsResponse>, tonic::Status>;
-        async fn get_live_events(
-            &self,
-            request: tonic::Request<super::GetLiveEventsRequest>,
-        ) -> std::result::Result<tonic::Response<super::GetLiveEventsResponse>, tonic::Status>;
         async fn get_proc_info(
             &self,
             request: tonic::Request<super::GetProcInfoRequest>,
@@ -2720,6 +2716,10 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::LoadTransitionTierConfigRequest>,
         ) -> std::result::Result<tonic::Response<super::LoadTransitionTierConfigResponse>, tonic::Status>;
+        async fn get_live_events(
+            &self,
+            request: tonic::Request<super::GetLiveEventsRequest>,
+        ) -> std::result::Result<tonic::Response<super::GetLiveEventsResponse>, tonic::Status>;
     }
     #[derive(Debug)]
     pub struct NodeServiceServer<T> {
@@ -4250,34 +4250,6 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
-                "/node_service.NodeService/GetLiveEvents" => {
-                    #[allow(non_camel_case_types)]
-                    struct GetLiveEventsSvc<T: NodeService>(pub Arc<T>);
-                    impl<T: NodeService> tonic::server::UnaryService<super::GetLiveEventsRequest> for GetLiveEventsSvc<T> {
-                        type Response = super::GetLiveEventsResponse;
-                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
-                        fn call(&mut self, request: tonic::Request<super::GetLiveEventsRequest>) -> Self::Future {
-                            let inner = Arc::clone(&self.0);
-                            let fut = async move { <T as NodeService>::get_live_events(&inner, request).await };
-                            Box::pin(fut)
-                        }
-                    }
-                    let accept_compression_encodings = self.accept_compression_encodings;
-                    let send_compression_encodings = self.send_compression_encodings;
-                    let max_decoding_message_size = self.max_decoding_message_size;
-                    let max_encoding_message_size = self.max_encoding_message_size;
-                    let inner = self.inner.clone();
-                    let fut = async move {
-                        let method = GetLiveEventsSvc(inner);
-                        let codec = tonic_prost::ProstCodec::default();
-                        let mut grpc = tonic::server::Grpc::new(codec)
-                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
-                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
-                        let res = grpc.unary(method, req).await;
-                        Ok(res)
-                    };
-                    Box::pin(fut)
-                }
                 "/node_service.NodeService/GetProcInfo" => {
                     #[allow(non_camel_case_types)]
                     struct GetProcInfoSvc<T: NodeService>(pub Arc<T>);
@@ -4980,6 +4952,34 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/GetLiveEvents" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetLiveEventsSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::GetLiveEventsRequest> for GetLiveEventsSvc<T> {
+                        type Response = super::GetLiveEventsResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::GetLiveEventsRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::get_live_events(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = GetLiveEventsSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 _ => Box::pin(async move {
                     let mut response = http::Response::new(tonic::body::Body::default());
                     let headers = response.headers_mut();

crates/targets/Cargo.toml

@@ -33,6 +33,7 @@ url = { workspace = true }
 urlencoding = { workspace = true }
 uuid = { workspace = true, features = ["v4", "serde"] }
 sysinfo = { workspace = true, features = ["multithread"] }
+rustfs-kafka-async = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true }

crates/targets/src/check.rs

@@ -128,3 +128,50 @@ pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarA
         Err(_) => Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())),
     }
 }
+
+pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs) -> Result<(), crate::TargetError> {
+    use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
+    use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, RequiredAcks, SecurityConfig};
+    use std::time::Duration;
+
+    let map_kafka_error = |err: KafkaError, context: &str| match err {
+        KafkaError::Connection(ConnectionError::NoHostReachable) => crate::TargetError::NotConnected,
+        KafkaError::Connection(ConnectionError::Timeout(_)) => crate::TargetError::Timeout(format!("{context}: {err}")),
+        KafkaError::Connection(_) => crate::TargetError::Network(format!("{context}: {err}")),
+        KafkaError::Config(_) => crate::TargetError::Configuration(format!("{context}: {err}")),
+        _ => crate::TargetError::Request(format!("{context}: {err}")),
+    };
+
+    let acks = match args.acks {
+        0 => RequiredAcks::None,
+        1 => RequiredAcks::One,
+        _ => RequiredAcks::All,
+    };
+
+    let mut config = AsyncProducerConfig::new()
+        .with_ack_timeout(Duration::from_secs(5))
+        .with_required_acks(acks);
+
+    if args.tls_enable {
+        let mut security = SecurityConfig::new();
+        if !args.tls_ca.is_empty() {
+            security = security.with_ca_cert(args.tls_ca.clone());
+        }
+        if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
+            security = security.with_client_cert(args.tls_client_cert.clone(), args.tls_client_key.clone());
+        }
+        config = config.with_security(security);
+    }
+
+    match tokio::time::timeout(Duration::from_secs(5), async {
+        let _ = AsyncProducer::from_hosts_with_config(args.brokers.clone(), config)
+            .await
+            .map_err(|err| map_kafka_error(err, "Kafka broker check failed to create producer"))?;
+        Ok(())
+    })
+    .await
+    {
+        Ok(result) => result,
+        Err(_) => Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())),
+    }
+}

crates/targets/src/config/loader.rs

@@ -28,6 +28,33 @@ pub fn collect_target_configs(
     collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
 }
 
+fn is_sensitive_target_field(field_name: &str) -> bool {
+    let field_name = field_name.to_ascii_lowercase();
+    field_name.contains("password")
+        || field_name.contains("secret")
+        || field_name.contains("token")
+        || field_name.contains("credential")
+        || field_name.contains("private_key")
+        || field_name.contains("client_key")
+        || field_name.contains("access_key")
+        || field_name.contains("auth")
+}
+
+fn redact_target_field_value(field_name: &str, value: &str) -> String {
+    if is_sensitive_target_field(field_name) && !value.is_empty() {
+        return "***redacted***".to_string();
+    }
+    value.to_string()
+}
+
+fn redacted_target_config(config: &KVS) -> Vec<(String, String)> {
+    config
+        .0
+        .iter()
+        .map(|kv| (kv.key.clone(), redact_target_field_value(&kv.key, &kv.value)))
+        .collect()
+}
+
 pub fn collect_env_target_instance_ids(route_prefix: &str, target_type: &str, valid_fields: &HashSet<String>) -> HashSet<String> {
     collect_env_target_instance_ids_from_env(route_prefix, target_type, valid_fields, std::env::vars())
 }
@@ -104,7 +131,7 @@ where
         debug!(
             instance_id = %if instance_id == DEFAULT_DELIMITER { DEFAULT_DELIMITER } else { &instance_id },
             %field_name,
-            %value,
+            value = %redact_target_field_value(&field_name, value),
             "Parsed target environment override"
         );
         env_overrides
@@ -137,7 +164,10 @@ where
             merged_config.extend(env_instance_cfg.clone());
         }
 
-        debug!(instance_id = %id, ?merged_config, "Merged target configuration");
+        if tracing::enabled!(tracing::Level::DEBUG) {
+            let redacted_config = redacted_target_config(&merged_config);
+            debug!(instance_id = %id, ?redacted_config, "Merged target configuration");
+        }
         if is_target_enabled(&merged_config) {
             merged_configs.push((id, merged_config));
         }
@@ -148,7 +178,10 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::{collect_env_target_instance_ids_from_env, collect_target_configs_from_env};
+    use super::{
+        collect_env_target_instance_ids_from_env, collect_target_configs_from_env, redact_target_field_value,
+        redacted_target_config,
+    };
     use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
     use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_LIMIT};
     use rustfs_ecstore::config::{Config, KVS};
@@ -236,4 +269,40 @@ mod tests {
 
         assert_eq!(ids, HashSet::from(["primary".to_string()]));
     }
+
+    #[test]
+    fn redact_target_field_value_redacts_sensitive_fields() {
+        assert_eq!(redact_target_field_value("password", "secret"), "***redacted***");
+        assert_eq!(redact_target_field_value("auth_token", "token"), "***redacted***");
+        assert_eq!(redact_target_field_value("credentials_file", "/tmp/creds"), "***redacted***");
+    }
+
+    #[test]
+    fn redact_target_field_value_keeps_non_sensitive_fields() {
+        assert_eq!(redact_target_field_value("endpoint", "https://example.com"), "https://example.com");
+        assert_eq!(redact_target_field_value("queue_limit", "1000"), "1000");
+    }
+
+    #[test]
+    fn redacted_target_config_masks_sensitive_values_without_mutating_shape() {
+        let mut config = KVS::new();
+        config.insert("endpoint".to_string(), "https://example.com/hook".to_string());
+        config.insert("password".to_string(), "super-secret".to_string());
+        config.insert("client_key".to_string(), "private-key".to_string());
+        config.insert("auth_token".to_string(), "bearer-token".to_string());
+        config.insert("empty_secret".to_string(), String::new());
+
+        let redacted = redacted_target_config(&config);
+
+        assert_eq!(
+            redacted,
+            vec![
+                ("endpoint".to_string(), "https://example.com/hook".to_string()),
+                ("password".to_string(), "***redacted***".to_string()),
+                ("client_key".to_string(), "***redacted***".to_string()),
+                ("auth_token".to_string(), "***redacted***".to_string()),
+                ("empty_secret".to_string(), String::new()),
+            ]
+        );
+    }
 }

crates/targets/src/config/mod.rs

@@ -21,6 +21,6 @@ pub use loader::{
     collect_target_configs_from_env,
 };
 pub use target_args::{
-    build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
-    validate_pulsar_config, validate_webhook_config,
+    build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+    validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
 };

crates/targets/src/config/target_args.rs

@@ -16,6 +16,7 @@ use super::common::{parse_target_bool, parse_url, validate_nats_server_config, v
 use crate::error::TargetError;
 use crate::target::{
     TargetType,
+    kafka::KafkaArgs,
     mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
     nats::{NATSArgs, validate_nats_address},
     pulsar::{PulsarArgs, validate_pulsar_broker},
@@ -23,19 +24,38 @@ use crate::target::{
 };
 use rumqttc::QoS;
 use rustfs_config::{
-    DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT,
-    MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA,
-    MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
-    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
-    NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
-    PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME,
-    RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
+    DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT,
+    KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS,
+    MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
+    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
+    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
+    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
+    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
+    PULSAR_TOPIC, PULSAR_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use rustfs_ecstore::config::KVS;
 use std::path::Path;
 use std::time::Duration;
 
+fn parse_kafka_acks_value(value: Option<&str>) -> Result<i16, TargetError> {
+    let Some(value) = value else {
+        return Ok(1);
+    };
+
+    let normalized = value.trim();
+    if normalized.is_empty() {
+        return Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string()));
+    }
+
+    match normalized.to_ascii_lowercase().as_str() {
+        "0" => Ok(0),
+        "1" => Ok(1),
+        "-1" | "all" => Ok(-1),
+        _ => Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string())),
+    }
+}
+
 pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<WebhookArgs, TargetError> {
     let endpoint = config
         .lookup(WEBHOOK_ENDPOINT)
@@ -265,3 +285,114 @@ pub fn validate_pulsar_config(config: &KVS, default_queue_dir: &str) -> Result<(
         .ok_or_else(|| TargetError::Configuration("Missing Pulsar broker".to_string()))?;
     validate_pulsar_broker_config(&broker, config, default_queue_dir)
 }
+
+pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<KafkaArgs, TargetError> {
+    let brokers_raw = config
+        .lookup(KAFKA_BROKERS)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
+    if brokers_raw.split(',').all(|s| s.trim().is_empty()) {
+        return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
+    }
+    let brokers: Vec<String> = brokers_raw
+        .split(',')
+        .map(|s| s.trim().to_string())
+        .filter(|s| !s.is_empty())
+        .collect();
+
+    let topic = config
+        .lookup(KAFKA_TOPIC)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka topic".to_string()))?;
+
+    Ok(KafkaArgs {
+        enable: true,
+        brokers,
+        topic,
+        acks: parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?,
+        tls_enable: parse_target_bool(config.lookup(KAFKA_TLS_ENABLE).as_deref()).unwrap_or(false),
+        tls_ca: config.lookup(KAFKA_TLS_CA).unwrap_or_default(),
+        tls_client_cert: config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default(),
+        tls_client_key: config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default(),
+        queue_dir: config
+            .lookup(KAFKA_QUEUE_DIR)
+            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_limit: config
+            .lookup(KAFKA_QUEUE_LIMIT)
+            .and_then(|v| v.parse::<u64>().ok())
+            .unwrap_or(DEFAULT_LIMIT),
+        target_type,
+    })
+}
+
+pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
+    let brokers_raw = config
+        .lookup(KAFKA_BROKERS)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
+    if brokers_raw.split(',').map(|s| s.trim()).all(|s| s.is_empty()) {
+        return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
+    }
+
+    if config.lookup(KAFKA_TOPIC).is_none() {
+        return Err(TargetError::Configuration("Missing Kafka topic".to_string()));
+    }
+
+    parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?;
+
+    let tls_client_cert = config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default();
+    let tls_client_key = config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default();
+    if tls_client_cert.is_empty() != tls_client_key.is_empty() {
+        return Err(TargetError::Configuration(
+            "Kafka tls_client_cert and tls_client_key must be specified together".to_string(),
+        ));
+    }
+
+    let queue_dir = config
+        .lookup(KAFKA_QUEUE_DIR)
+        .unwrap_or_else(|| default_queue_dir.to_string());
+    if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
+        return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string()));
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_kafka_args, validate_kafka_config};
+    use crate::target::TargetType;
+    use rustfs_config::{KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC};
+    use rustfs_ecstore::config::KVS;
+
+    fn kafka_base_config() -> KVS {
+        let mut config = KVS::new();
+        config.insert(KAFKA_BROKERS.to_string(), "127.0.0.1:9092".to_string());
+        config.insert(KAFKA_TOPIC.to_string(), "events".to_string());
+        config
+    }
+
+    #[test]
+    fn build_kafka_args_accepts_all_ack_alias() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "all".to_string());
+
+        let args = build_kafka_args(&config, "", TargetType::NotifyEvent).expect("valid kafka args");
+        assert_eq!(args.acks, -1);
+    }
+
+    #[test]
+    fn build_kafka_args_rejects_invalid_acks() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "leader".to_string());
+
+        let err = build_kafka_args(&config, "", TargetType::NotifyEvent).expect_err("invalid acks should fail");
+        assert!(err.to_string().contains("Kafka acks must be one of"));
+    }
+
+    #[test]
+    fn validate_kafka_config_rejects_invalid_acks() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "2".to_string());
+
+        let err = validate_kafka_config(&config, "").expect_err("invalid acks should fail");
+        assert!(err.to_string().contains("Kafka acks must be one of"));
+    }
+}

crates/targets/src/lib.rs

@@ -21,7 +21,8 @@ pub mod sys;
 pub mod target;
 
 pub use check::{
-    check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available, check_pulsar_broker_available,
+    check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
+    check_pulsar_broker_available,
 };
 pub use error::{StoreError, TargetError};
 pub use rustfs_s3_common::EventName;

crates/targets/src/target/kafka.rs

@@ -0,0 +1,437 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    StoreError, Target, TargetLog,
+    arn::TargetID,
+    error::TargetError,
+    store::{Key, QueueStore, Store},
+    target::{
+        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
+        TargetType,
+    },
+};
+use async_trait::async_trait;
+use rustfs_config::audit::AUDIT_STORE_EXTENSION;
+use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
+use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
+use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use std::{marker::PhantomData, path::PathBuf, sync::Arc, time::Duration};
+use tokio::sync::Mutex;
+use tracing::{debug, error, info, instrument, warn};
+
+/// Arguments for configuring a Kafka target
+#[derive(Debug, Clone)]
+pub struct KafkaArgs {
+    /// Whether the target is enabled
+    pub enable: bool,
+    /// Comma-separated list of broker addresses (e.g. "localhost:9092,broker2:9092")
+    pub brokers: Vec<String>,
+    /// The topic to publish events to
+    pub topic: String,
+    /// Required acks: 0 = none, 1 = leader, -1 = all
+    pub acks: i16,
+    /// Whether to enable TLS for Kafka transport
+    pub tls_enable: bool,
+    /// Optional path to CA cert used for broker verification
+    pub tls_ca: String,
+    /// Optional path to client certificate for mTLS
+    pub tls_client_cert: String,
+    /// Optional path to client private key for mTLS
+    pub tls_client_key: String,
+    /// The directory to store events in case of failure
+    pub queue_dir: String,
+    /// The maximum number of events to store
+    pub queue_limit: u64,
+    /// The target type (audit or notify)
+    pub target_type: TargetType,
+}
+
+impl KafkaArgs {
+    /// Validates the KafkaArgs configuration
+    pub fn validate(&self) -> Result<(), TargetError> {
+        if !self.enable {
+            return Ok(());
+        }
+
+        if self.brokers.is_empty() {
+            return Err(TargetError::Configuration("kafka brokers cannot be empty".to_string()));
+        }
+
+        if self.topic.is_empty() {
+            return Err(TargetError::Configuration("kafka topic cannot be empty".to_string()));
+        }
+
+        if !matches!(self.acks, -1..=1) {
+            return Err(TargetError::Configuration("kafka acks must be one of: 0, 1, -1".to_string()));
+        }
+
+        if self.tls_client_cert.is_empty() != self.tls_client_key.is_empty() {
+            return Err(TargetError::Configuration(
+                "kafka tls_client_cert and tls_client_key must be specified together".to_string(),
+            ));
+        }
+
+        if !self.queue_dir.is_empty() {
+            let path = std::path::Path::new(&self.queue_dir);
+            if !path.is_absolute() {
+                return Err(TargetError::Configuration("kafka queueDir path should be absolute".to_string()));
+            }
+        }
+
+        Ok(())
+    }
+}
+
+/// A target that sends events to an Apache Kafka topic
+pub struct KafkaTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    id: TargetID,
+    args: KafkaArgs,
+    store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
+    producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    delivery_counters: Arc<TargetDeliveryCounters>,
+    _phantom: PhantomData<E>,
+}
+
+impl<E> KafkaTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn map_kafka_error(err: KafkaError, context: &str) -> TargetError {
+        match err {
+            KafkaError::Connection(ConnectionError::NoHostReachable) => TargetError::NotConnected,
+            KafkaError::Connection(ConnectionError::Timeout(_)) => TargetError::Timeout(format!("{context}: {err}")),
+            KafkaError::Connection(_) => TargetError::Network(format!("{context}: {err}")),
+            KafkaError::Config(_) => TargetError::Configuration(format!("{context}: {err}")),
+            _ => TargetError::Request(format!("{context}: {err}")),
+        }
+    }
+
+    fn is_connection_error(err: &TargetError) -> bool {
+        matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
+    }
+
+    /// Creates a new KafkaTarget
+    #[instrument(skip(args), fields(target_id = %id))]
+    pub fn new(id: String, args: KafkaArgs) -> Result<Self, TargetError> {
+        args.validate()?;
+
+        let target_id = TargetID::new(id, ChannelTargetType::Kafka.as_str().to_string());
+
+        let queue_store = if !args.queue_dir.is_empty() {
+            let queue_dir =
+                PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Kafka.as_str(), target_id.id));
+
+            let extension = match args.target_type {
+                TargetType::AuditLog => AUDIT_STORE_EXTENSION,
+                TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
+            };
+
+            let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
+            if let Err(e) = store.open() {
+                error!("Failed to open store for Kafka target {}: {}", target_id.id, e);
+                return Err(TargetError::Storage(format!("{e}")));
+            }
+
+            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
+        } else {
+            None
+        };
+
+        info!(target_id = %target_id.id, "Kafka target created");
+        Ok(KafkaTarget {
+            id: target_id,
+            args,
+            store: queue_store,
+            producer: Arc::new(Mutex::new(None)),
+            delivery_counters: Arc::new(TargetDeliveryCounters::default()),
+            _phantom: PhantomData,
+        })
+    }
+
+    /// Builds a Kafka producer from the current args
+    async fn build_producer(&self) -> Result<AsyncProducer, TargetError> {
+        let acks = match self.args.acks {
+            0 => RequiredAcks::None,
+            1 => RequiredAcks::One,
+            _ => RequiredAcks::All,
+        };
+
+        let mut config = AsyncProducerConfig::new()
+            .with_ack_timeout(Duration::from_secs(30))
+            .with_required_acks(acks);
+
+        if self.args.tls_enable {
+            let mut security = SecurityConfig::new();
+            if !self.args.tls_ca.is_empty() {
+                security = security.with_ca_cert(self.args.tls_ca.clone());
+            }
+            if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
+                security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone());
+            }
+            config = config.with_security(security);
+        }
+
+        AsyncProducer::from_hosts_with_config(self.args.brokers.clone(), config)
+            .await
+            .map_err(|e| Self::map_kafka_error(e, "Failed to create Kafka producer"))
+    }
+
+    async fn get_or_build_producer(&self) -> Result<Arc<AsyncProducer>, TargetError> {
+        let mut cached = self.producer.lock().await;
+        if let Some(producer) = cached.as_ref() {
+            return Ok(Arc::clone(producer));
+        }
+
+        let producer = Arc::new(self.build_producer().await?);
+        *cached = Some(Arc::clone(&producer));
+        Ok(producer)
+    }
+
+    async fn invalidate_cached_producer(&self) {
+        let mut cached = self.producer.lock().await;
+        *cached = None;
+    }
+
+    /// Serializes the event and builds a QueuedPayload
+    fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
+        let object_name = crate::target::decode_object_name(&event.object_name)?;
+        let key = format!("{}/{}", event.bucket_name, object_name);
+
+        let log = TargetLog {
+            event_name: event.event_name,
+            key,
+            records: vec![event.data.clone()],
+        };
+
+        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
+
+        let meta = QueuedPayloadMeta::new(
+            event.event_name,
+            event.bucket_name.clone(),
+            event.object_name.clone(),
+            "application/json",
+            body.len(),
+        );
+
+        Ok(QueuedPayload::new(meta, body))
+    }
+
+    /// Sends the raw body to Kafka
+    #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
+    async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
+        debug!(
+            target = %self.id,
+            bucket = %meta.bucket_name,
+            object = %meta.object_name,
+            event = %meta.event_name,
+            payload_len = body.len(),
+            "Sending Kafka payload"
+        );
+
+        let producer = self.get_or_build_producer().await?;
+
+        if let Err(err) = producer.send(&Record::from_value(&self.args.topic, body.as_slice())).await {
+            let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
+            if Self::is_connection_error(&mapped) {
+                self.invalidate_cached_producer().await;
+            }
+            return Err(mapped);
+        }
+
+        debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
+        self.delivery_counters.record_success();
+        Ok(())
+    }
+
+    /// Clones this target into a boxed trait object
+    pub fn clone_box(&self) -> Box<dyn Target<E> + Send + Sync> {
+        Box::new(KafkaTarget::<E> {
+            id: self.id.clone(),
+            args: self.args.clone(),
+            store: self.store.as_ref().map(|s| s.boxed_clone()),
+            producer: Arc::clone(&self.producer),
+            delivery_counters: Arc::clone(&self.delivery_counters),
+            _phantom: PhantomData,
+        })
+    }
+}
+
+#[async_trait]
+impl<E> Target<E> for KafkaTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn id(&self) -> TargetID {
+        self.id.clone()
+    }
+
+    async fn is_active(&self) -> Result<bool, TargetError> {
+        let _ = self.get_or_build_producer().await?;
+        Ok(true)
+    }
+
+    async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+        let queued = match self.build_queued_payload(&event) {
+            Ok(queued) => queued,
+            Err(err) => {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+        };
+
+        if let Some(store) = &self.store {
+            let encoded = match queued.encode() {
+                Ok(encoded) => encoded,
+                Err(err) => {
+                    self.delivery_counters.record_final_failure();
+                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
+                }
+            };
+            if let Err(e) = store.put_raw(&encoded) {
+                self.delivery_counters.record_final_failure();
+                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+            }
+            debug!("Event saved to store for Kafka target: {}", self.id);
+            Ok(())
+        } else {
+            if let Err(err) = self.send_body(queued.body, &queued.meta).await {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+            Ok(())
+        }
+    }
+
+    async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+        debug!("Sending queued payload from store for Kafka target: {}, key: {}", self.id, key);
+
+        if let Err(e) = self.send_body(body, &meta).await {
+            if matches!(e, TargetError::NotConnected) {
+                warn!(target_id = %self.id, "Kafka not reachable, event remains in store.");
+                return Err(TargetError::NotConnected);
+            }
+            error!(target_id = %self.id, error = %e, "Failed to send event from store.");
+            return Err(e);
+        }
+
+        debug!("Event sent from store for Kafka target: {}", self.id);
+        Ok(())
+    }
+
+    async fn close(&self) -> Result<(), TargetError> {
+        info!("Kafka target closed: {}", self.id);
+        Ok(())
+    }
+
+    fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+        self.store.as_deref()
+    }
+
+    fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+        self.clone_box()
+    }
+
+    fn is_enabled(&self) -> bool {
+        self.args.enable
+    }
+
+    fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
+        self.delivery_counters
+            .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
+    }
+
+    fn record_final_failure(&self) {
+        self.delivery_counters.record_final_failure();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn base_args() -> KafkaArgs {
+        KafkaArgs {
+            enable: true,
+            brokers: vec!["localhost:9092".to_string()],
+            topic: "rustfs-events".to_string(),
+            acks: 1,
+            tls_enable: false,
+            tls_ca: String::new(),
+            tls_client_cert: String::new(),
+            tls_client_key: String::new(),
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
+    #[test]
+    fn test_validate_empty_brokers() {
+        let args = KafkaArgs {
+            brokers: vec![],
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn test_validate_empty_topic() {
+        let args = KafkaArgs {
+            topic: String::new(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn test_validate_relative_queue_dir() {
+        let args = KafkaArgs {
+            queue_dir: "relative/path".to_string(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn test_validate_valid_args() {
+        assert!(base_args().validate().is_ok());
+    }
+
+    #[test]
+    fn test_validate_disabled_target_skips_validation() {
+        let args = KafkaArgs {
+            enable: false,
+            brokers: vec![],
+            topic: String::new(),
+            ..base_args()
+        };
+        assert!(args.validate().is_ok());
+    }
+
+    #[test]
+    fn test_validate_tls_client_cert_and_key_must_be_paired() {
+        let args = KafkaArgs {
+            tls_client_cert: "/tmp/client.crt".to_string(),
+            tls_client_key: String::new(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+}

crates/targets/src/target/mod.rs

@@ -25,6 +25,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 use tracing::warn;
 
+pub mod kafka;
 pub mod mqtt;
 pub mod nats;
 pub mod pulsar;

crates/targets/src/target/mqtt.rs

@@ -672,7 +672,7 @@ where
             bucket = %meta.bucket_name,
             object = %meta.object_name,
             event = %meta.event_name,
-            preview = %meta.best_effort_preview(&body, 256),
+            payload_len = body.len(),
             "Sending MQTT payload"
         );
 

crates/targets/src/target/webhook.rs

@@ -288,7 +288,7 @@ where
             bucket = %meta.bucket_name,
             object = %meta.object_name,
             event = %meta.event_name,
-            preview = %meta.best_effort_preview(&body, 256),
+            payload_len = body.len(),
             "Sending webhook payload"
         );
 

crates/trusted-proxies/README.md

@@ -4,6 +4,13 @@ The `rustfs-trusted-proxies` module provides secure and efficient management of
 ecosystem. It is designed to handle multi-layer proxy architectures, ensuring accurate client IP identification while
 maintaining a zero-trust security model.
 
+## Modes
+
+- **Simple default**: only trusts forwarding headers when the direct peer IP is
+  internal.
+- **Legacy full mode**: keeps the original proxy-chain validation, available
+  via `legacy_*` helpers.
+
 ## Features
 
 - **Multi-Layer Proxy Validation**: Supports `Strict`, `Lenient`, and `HopByHop` validation modes to accurately identify
@@ -23,6 +30,7 @@ The module is configured primarily through environment variables:
 | Variable                                      | Default             | Description                                             |
 |-----------------------------------------------|---------------------|---------------------------------------------------------|
 | `RUSTFS_TRUSTED_PROXY_ENABLED`                | `true`              | Enable the trusted proxy middleware                     |
+| `RUSTFS_TRUSTED_PROXY_IMPLEMENTATION`         | `simple`            | Select `simple` or `legacy` implementation              |
 | `RUSTFS_TRUSTED_PROXY_VALIDATION_MODE`        | `hop_by_hop`        | Validation strategy (`strict`, `lenient`, `hop_by_hop`) |
 | `RUSTFS_TRUSTED_PROXY_NETWORKS`               | `127.0.0.1,::1,...` | Comma-separated list of trusted CIDR ranges             |
 | `RUSTFS_TRUSTED_PROXY_MAX_HOPS`               | `10`                | Maximum allowed proxy hops                              |
@@ -58,6 +66,29 @@ let app = Router::new()
     });
 ```
 
+### Simple default mode
+
+The default mode only trusts forwarding headers from internal IPs.
+
+```bash
+RUSTFS_TRUSTED_PROXY_IMPLEMENTATION=simple
+```
+
+### Legacy mode
+
+The original implementation is still available:
+
+```rust
+rustfs_trusted_proxies::legacy_init();
+let layer = rustfs_trusted_proxies::LegacyTrustedProxyLayer::enabled(config, None);
+```
+
+Or switch the global default path:
+
+```bash
+RUSTFS_TRUSTED_PROXY_IMPLEMENTATION=legacy
+```
+
 ### Accessing Client Info
 
 Retrieve the verified client information in your handlers or other middleware:

crates/trusted-proxies/src/config/env.rs

@@ -19,8 +19,8 @@ use ipnetwork::IpNetwork;
 use rustfs_config::{
     ENV_TRUSTED_PROXY_CHAIN_CONTINUITY_CHECK, ENV_TRUSTED_PROXY_CLOUD_METADATA_ENABLED, ENV_TRUSTED_PROXY_CLOUD_METADATA_TIMEOUT,
     ENV_TRUSTED_PROXY_CLOUDFLARE_IPS_ENABLED, ENV_TRUSTED_PROXY_ENABLE_RFC7239, ENV_TRUSTED_PROXY_ENABLED,
-    ENV_TRUSTED_PROXY_EXTRA_PROXIES, ENV_TRUSTED_PROXY_IPS, ENV_TRUSTED_PROXY_MAX_HOPS, ENV_TRUSTED_PROXY_PROXIES,
-    ENV_TRUSTED_PROXY_VALIDATION_MODE,
+    ENV_TRUSTED_PROXY_EXTRA_PROXIES, ENV_TRUSTED_PROXY_IMPLEMENTATION, ENV_TRUSTED_PROXY_IPS, ENV_TRUSTED_PROXY_MAX_HOPS,
+    ENV_TRUSTED_PROXY_PROXIES, ENV_TRUSTED_PROXY_VALIDATION_MODE,
 };
 use std::str::FromStr;
 // ==================== Helper Functions ====================
@@ -72,6 +72,7 @@ pub fn is_env_set(key: &str) -> bool {
 pub fn get_all_proxy_env_vars() -> Vec<(String, String)> {
     let vars = [
         ENV_TRUSTED_PROXY_ENABLED,
+        ENV_TRUSTED_PROXY_IMPLEMENTATION,
         ENV_TRUSTED_PROXY_VALIDATION_MODE,
         ENV_TRUSTED_PROXY_ENABLE_RFC7239,
         ENV_TRUSTED_PROXY_MAX_HOPS,

crates/trusted-proxies/src/global.rs

@@ -17,7 +17,7 @@
 //! This module provides a unified interface for initializing and using the
 //! trusted proxy functionality within the RustFS server.
 
-use crate::{AppConfig, ConfigLoader, ProxyMetrics, TrustedProxyLayer, default_proxy_metrics};
+use crate::{AppConfig, ConfigLoader, LegacyTrustedProxyLayer, ProxyMetrics, default_proxy_metrics};
 use rustfs_config::{DEFAULT_TRUSTED_PROXY_ENABLED, ENV_TRUSTED_PROXY_ENABLED};
 use std::sync::Arc;
 use std::sync::OnceLock;
@@ -29,7 +29,7 @@ static CONFIG: OnceLock<Arc<AppConfig>> = OnceLock::new();
 static METRICS: OnceLock<Option<ProxyMetrics>> = OnceLock::new();
 
 /// Global instance of the trusted proxy layer.
-static PROXY_LAYER: OnceLock<TrustedProxyLayer> = OnceLock::new();
+static PROXY_LAYER: OnceLock<LegacyTrustedProxyLayer> = OnceLock::new();
 
 /// Global flag indicating if the trusted proxy middleware is enabled.
 static ENABLED: OnceLock<bool> = OnceLock::new();
@@ -39,33 +39,32 @@ static ENABLED: OnceLock<bool> = OnceLock::new();
 /// This function should be called once at the start of the application.
 /// It loads the configuration, initializes metrics, and sets up the proxy layer.
 pub fn init() {
-    // Check if the trusted proxy system is enabled via environment variable.
-    let enabled = rustfs_utils::get_env_bool(ENV_TRUSTED_PROXY_ENABLED, DEFAULT_TRUSTED_PROXY_ENABLED);
-    ENABLED.set(enabled).expect("Trusted proxy enabled flag already initialized");
+    let enabled = is_enabled();
+    ENABLED.get_or_init(|| enabled);
 
     if !enabled {
         tracing::info!("Trusted Proxies module is disabled via configuration");
         return;
     }
 
-    // Load configuration from environment variables.
-    let config = Arc::new(ConfigLoader::from_env_or_default());
-    CONFIG.set(config.clone()).expect("Trusted proxy config already initialized");
+    let config = CONFIG.get_or_init(|| Arc::new(ConfigLoader::from_env_or_default())).clone();
 
-    // Initialize metrics if enabled.
-    let metrics = if config.monitoring.metrics_enabled {
-        let m = default_proxy_metrics(enabled);
-        Some(m)
-    } else {
-        None
-    };
-    METRICS
-        .set(metrics.clone())
-        .expect("Trusted proxy metrics already initialized");
+    METRICS.get_or_init(|| {
+        if config.monitoring.metrics_enabled {
+            Some(default_proxy_metrics(enabled))
+        } else {
+            None
+        }
+    });
 
-    // Initialize the trusted proxy layer.
-    let layer = TrustedProxyLayer::with_cache_config(config.proxy.clone(), config.cache.clone(), metrics, enabled);
-    PROXY_LAYER.set(layer).expect("Trusted proxy layer already initialized");
+    PROXY_LAYER.get_or_init(|| {
+        LegacyTrustedProxyLayer::with_cache_config(
+            config.proxy.clone(),
+            config.cache.clone(),
+            METRICS.get().and_then(|m| m.clone()),
+            enabled,
+        )
+    });
 
     tracing::info!("Trusted Proxies module initialized");
     ConfigLoader::print_summary(&config);
@@ -78,7 +77,7 @@ pub fn init() {
 /// # Panics
 ///
 /// Panics if `init()` has not been called.
-pub fn layer() -> &'static TrustedProxyLayer {
+pub fn layer() -> &'static LegacyTrustedProxyLayer {
     PROXY_LAYER
         .get()
         .expect("Trusted proxy system not initialized. Call init() first.")
@@ -102,5 +101,5 @@ pub fn metrics() -> Option<&'static ProxyMetrics> {
 
 /// Returns true if the trusted proxy system is enabled.
 pub fn is_enabled() -> bool {
-    *ENABLED.get().unwrap_or(&false)
+    *ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_TRUSTED_PROXY_ENABLED, DEFAULT_TRUSTED_PROXY_ENABLED))
 }

crates/trusted-proxies/src/lib.rs

@@ -18,12 +18,19 @@ mod error;
 mod global;
 mod middleware;
 mod proxy;
+mod simple;
 mod utils;
 
 pub use cloud::*;
 pub use config::*;
 pub use error::*;
-pub use global::{config as global_config, init, is_enabled, layer, metrics};
-pub use middleware::{TrustedProxyLayer, TrustedProxyMiddleware};
+pub use global::{
+    config as legacy_global_config, init as legacy_init, is_enabled as legacy_is_enabled, layer as legacy_layer,
+    metrics as legacy_metrics,
+};
+pub use middleware::{TrustedProxyLayer as LegacyTrustedProxyLayer, TrustedProxyMiddleware as LegacyTrustedProxyMiddleware};
 pub use proxy::*;
+pub use simple::{
+    TrustedProxyImplementation, TrustedProxyLayer, TrustedProxyMiddleware, implementation, init, is_enabled, layer,
+};
 pub use utils::*;

crates/trusted-proxies/src/middleware/layer.rs

@@ -17,9 +17,9 @@
 use std::sync::Arc;
 use tower::Layer;
 
+use crate::LegacyTrustedProxyMiddleware;
 use crate::ProxyValidator;
 use crate::TrustedProxyConfig;
-use crate::TrustedProxyMiddleware;
 use crate::{CacheConfig, ProxyMetrics};
 
 /// Tower Layer for the trusted proxy middleware.
@@ -73,10 +73,10 @@ impl TrustedProxyLayer {
 }
 
 impl<S> Layer<S> for TrustedProxyLayer {
-    type Service = TrustedProxyMiddleware<S>;
+    type Service = LegacyTrustedProxyMiddleware<S>;
 
     fn layer(&self, inner: S) -> Self::Service {
-        TrustedProxyMiddleware {
+        LegacyTrustedProxyMiddleware {
             inner,
             validator: self.validator.clone(),
             enabled: self.enabled,

crates/trusted-proxies/src/middleware/service.rs

@@ -14,7 +14,7 @@
 
 //! Tower service implementation for the trusted proxy middleware.
 
-use crate::{ClientInfo, ProxyValidator, TrustedProxyLayer};
+use crate::{ClientInfo, ProxyValidator};
 use http::Request;
 use std::sync::Arc;
 use std::task::{Context, Poll};
@@ -43,7 +43,7 @@ impl<S> TrustedProxyMiddleware<S> {
     }
 
     /// Creates a new `TrustedProxyMiddleware` from a `TrustedProxyLayer`.
-    pub fn from_layer(inner: S, layer: &TrustedProxyLayer) -> Self {
+    pub fn from_layer(inner: S, layer: &super::layer::TrustedProxyLayer) -> Self {
         Self::new(inner, layer.validator.clone(), layer.enabled)
     }
 }

crates/trusted-proxies/src/simple.rs

@@ -0,0 +1,463 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Simplified trusted proxy mode.
+//!
+//! The crate keeps both the simplified and legacy implementations. The default
+//! runtime path uses the simplified rule set, and an environment variable can
+//! switch the global entrypoints to the legacy chain validator.
+
+use crate::{ClientInfo, LegacyTrustedProxyLayer, LegacyTrustedProxyMiddleware, ValidationMode, global};
+use axum::http::{HeaderMap, Request};
+use rustfs_config::{
+    DEFAULT_TRUSTED_PROXY_ENABLED, DEFAULT_TRUSTED_PROXY_IMPLEMENTATION, ENV_TRUSTED_PROXY_ENABLED,
+    ENV_TRUSTED_PROXY_IMPLEMENTATION,
+};
+use std::fmt;
+use std::net::{IpAddr, SocketAddr};
+use std::str::FromStr;
+use std::sync::OnceLock;
+use std::task::{Context, Poll};
+use tower::{Layer, Service};
+use tracing::debug;
+
+/// Constant switch for the crate's default integration path.
+pub const SIMPLE_INTERNAL_ONLY_DEFAULT: bool = true;
+
+const HEADER_FORWARDED: &str = "forwarded";
+const HEADER_X_FORWARDED_FOR: &str = "x-forwarded-for";
+const HEADER_X_REAL_IP: &str = "x-real-ip";
+
+static ENABLED: OnceLock<bool> = OnceLock::new();
+static IMPLEMENTATION: OnceLock<TrustedProxyImplementation> = OnceLock::new();
+static LAYER: OnceLock<TrustedProxyLayer> = OnceLock::new();
+
+/// Selects which implementation is used by the global entrypoints.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
+pub enum TrustedProxyImplementation {
+    #[default]
+    Simple,
+    Legacy,
+}
+
+impl TrustedProxyImplementation {
+    fn from_env() -> Self {
+        parse_implementation(std::env::var(ENV_TRUSTED_PROXY_IMPLEMENTATION).ok().as_deref())
+    }
+}
+
+impl fmt::Display for TrustedProxyImplementation {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(match self {
+            Self::Simple => "simple",
+            Self::Legacy => "legacy",
+        })
+    }
+}
+
+/// Initializes the default trusted proxy implementation.
+pub fn init() {
+    let enabled = is_enabled();
+    let implementation = implementation();
+    let _ = layer();
+
+    tracing::info!(
+        enabled,
+        implementation = %implementation,
+        simple_internal_only = SIMPLE_INTERNAL_ONLY_DEFAULT,
+        "Trusted proxy middleware initialized"
+    );
+}
+
+/// Returns whether the default trusted proxy implementation is enabled.
+pub fn is_enabled() -> bool {
+    *ENABLED.get_or_init(|| rustfs_utils::get_env_bool(ENV_TRUSTED_PROXY_ENABLED, DEFAULT_TRUSTED_PROXY_ENABLED))
+}
+
+/// Returns the selected implementation.
+pub fn implementation() -> TrustedProxyImplementation {
+    *IMPLEMENTATION.get_or_init(TrustedProxyImplementation::from_env)
+}
+
+/// Returns the default trusted proxy layer.
+pub fn layer() -> &'static TrustedProxyLayer {
+    LAYER.get_or_init(build_layer)
+}
+
+fn build_layer() -> TrustedProxyLayer {
+    if !is_enabled() {
+        return TrustedProxyLayer::disabled();
+    }
+
+    match implementation() {
+        TrustedProxyImplementation::Simple => TrustedProxyLayer::enabled(),
+        TrustedProxyImplementation::Legacy => {
+            global::init();
+            TrustedProxyLayer::legacy(global::layer().clone())
+        }
+    }
+}
+
+/// Public layer wrapper for both implementations.
+#[derive(Clone, Debug)]
+pub enum TrustedProxyLayer {
+    Simple(SimpleTrustedProxyLayer),
+    Legacy(LegacyTrustedProxyLayer),
+}
+
+impl TrustedProxyLayer {
+    pub fn enabled() -> Self {
+        Self::Simple(SimpleTrustedProxyLayer::enabled())
+    }
+
+    pub fn disabled() -> Self {
+        Self::Simple(SimpleTrustedProxyLayer::disabled())
+    }
+
+    pub fn legacy(layer: LegacyTrustedProxyLayer) -> Self {
+        Self::Legacy(layer)
+    }
+
+    pub fn is_enabled(&self) -> bool {
+        match self {
+            Self::Simple(layer) => layer.is_enabled(),
+            Self::Legacy(layer) => layer.is_enabled(),
+        }
+    }
+
+    pub fn is_legacy(&self) -> bool {
+        matches!(self, Self::Legacy(_))
+    }
+}
+
+impl<S> Layer<S> for TrustedProxyLayer {
+    type Service = TrustedProxyMiddleware<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        match self {
+            Self::Simple(layer) => TrustedProxyMiddleware::Simple(layer.layer(inner)),
+            Self::Legacy(layer) => TrustedProxyMiddleware::Legacy(layer.layer(inner)),
+        }
+    }
+}
+
+/// Public middleware wrapper for both implementations.
+#[derive(Clone)]
+pub enum TrustedProxyMiddleware<S> {
+    Simple(SimpleTrustedProxyMiddleware<S>),
+    Legacy(LegacyTrustedProxyMiddleware<S>),
+}
+
+impl<S, ReqBody> Service<Request<ReqBody>> for TrustedProxyMiddleware<S>
+where
+    S: Service<Request<ReqBody>> + Clone + Send + 'static,
+    S::Future: Send,
+{
+    type Response = S::Response;
+    type Error = S::Error;
+    type Future = S::Future;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        match self {
+            Self::Simple(service) => service.poll_ready(cx),
+            Self::Legacy(service) => service.poll_ready(cx),
+        }
+    }
+
+    fn call(&mut self, req: Request<ReqBody>) -> Self::Future {
+        match self {
+            Self::Simple(service) => service.call(req),
+            Self::Legacy(service) => service.call(req),
+        }
+    }
+}
+
+/// Minimal layer used by RustFS by default.
+#[derive(Clone, Debug, Default)]
+pub struct SimpleTrustedProxyLayer {
+    enabled: bool,
+}
+
+impl SimpleTrustedProxyLayer {
+    pub fn enabled() -> Self {
+        Self { enabled: true }
+    }
+
+    pub fn disabled() -> Self {
+        Self { enabled: false }
+    }
+
+    pub fn is_enabled(&self) -> bool {
+        self.enabled
+    }
+}
+
+impl<S> Layer<S> for SimpleTrustedProxyLayer {
+    type Service = SimpleTrustedProxyMiddleware<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        SimpleTrustedProxyMiddleware {
+            inner,
+            enabled: self.enabled,
+        }
+    }
+}
+
+/// Minimal middleware used by RustFS by default.
+#[derive(Clone)]
+pub struct SimpleTrustedProxyMiddleware<S> {
+    inner: S,
+    enabled: bool,
+}
+
+impl<S, ReqBody> Service<Request<ReqBody>> for SimpleTrustedProxyMiddleware<S>
+where
+    S: Service<Request<ReqBody>> + Clone + Send + 'static,
+    S::Future: Send,
+{
+    type Response = S::Response;
+    type Error = S::Error;
+    type Future = S::Future;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut req: Request<ReqBody>) -> Self::Future {
+        if self.enabled {
+            let peer_addr = req.extensions().get::<SocketAddr>().copied();
+            let client_info = resolve_client_info(peer_addr, req.headers());
+            req.extensions_mut().insert(client_info);
+        } else {
+            debug!("Simple trusted proxy middleware is disabled");
+        }
+
+        self.inner.call(req)
+    }
+}
+
+fn resolve_client_info(peer_addr: Option<SocketAddr>, headers: &HeaderMap) -> ClientInfo {
+    let Some(peer_addr) = peer_addr else {
+        return ClientInfo::direct(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0));
+    };
+
+    if !is_internal_ip(peer_addr.ip()) {
+        return ClientInfo::direct(peer_addr);
+    }
+
+    match forwarded_client_ip(headers) {
+        Some(real_ip) if is_usable_ip(real_ip) && real_ip != peer_addr.ip() => {
+            ClientInfo::from_trusted_proxy(real_ip, None, None, peer_addr.ip(), 1, ValidationMode::Lenient, Vec::new())
+        }
+        _ => ClientInfo::direct(peer_addr),
+    }
+}
+
+fn forwarded_client_ip(headers: &HeaderMap) -> Option<IpAddr> {
+    parse_x_forwarded_for(headers)
+        .or_else(|| parse_single_ip_header(headers, HEADER_X_REAL_IP))
+        .or_else(|| parse_forwarded_header(headers))
+}
+
+fn parse_x_forwarded_for(headers: &HeaderMap) -> Option<IpAddr> {
+    let value = headers.get(HEADER_X_FORWARDED_FOR)?.to_str().ok()?;
+    let first = value.split(',').next()?.trim();
+    parse_ip_token(first)
+}
+
+fn parse_single_ip_header(headers: &HeaderMap, name: &str) -> Option<IpAddr> {
+    let value = headers.get(name)?.to_str().ok()?;
+    parse_ip_token(value)
+}
+
+fn parse_forwarded_header(headers: &HeaderMap) -> Option<IpAddr> {
+    let value = headers.get(HEADER_FORWARDED)?.to_str().ok()?;
+    let first_entry = value.split(',').next()?.trim();
+
+    for part in first_entry.split(';') {
+        let Some((key, raw_value)) = part.split_once('=') else {
+            continue;
+        };
+        if key.trim().eq_ignore_ascii_case("for") {
+            return parse_ip_token(raw_value.trim());
+        }
+    }
+
+    None
+}
+
+fn parse_ip_token(value: &str) -> Option<IpAddr> {
+    let value = value.trim().trim_matches('"');
+    if value.is_empty() || value.eq_ignore_ascii_case("unknown") || value.starts_with('_') {
+        return None;
+    }
+
+    if let Some(bracketed) = value.strip_prefix('[')
+        && let Some(end) = bracketed.find(']')
+    {
+        return IpAddr::from_str(&bracketed[..end]).ok();
+    }
+
+    if let Ok(ip) = IpAddr::from_str(value) {
+        return Some(ip);
+    }
+
+    if let Ok(socket_addr) = SocketAddr::from_str(value) {
+        return Some(socket_addr.ip());
+    }
+
+    None
+}
+
+fn is_internal_ip(ip: IpAddr) -> bool {
+    match ip {
+        IpAddr::V4(ip) => ip.is_private() || ip.is_loopback() || ip.is_link_local(),
+        IpAddr::V6(ip) => ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local(),
+    }
+}
+
+fn is_usable_ip(ip: IpAddr) -> bool {
+    !ip.is_unspecified() && !ip.is_multicast()
+}
+
+fn parse_implementation(value: Option<&str>) -> TrustedProxyImplementation {
+    match value.map(|v| v.trim().to_ascii_lowercase()) {
+        Some(mode) if mode == "legacy" || mode == "full" || mode == "full_legacy" => TrustedProxyImplementation::Legacy,
+        Some(mode) if mode == "simple" || mode == "internal_only" || mode == "internal-only" => {
+            TrustedProxyImplementation::Simple
+        }
+        Some(mode) if mode == DEFAULT_TRUSTED_PROXY_IMPLEMENTATION => TrustedProxyImplementation::Simple,
+        _ => TrustedProxyImplementation::Simple,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        ENV_TRUSTED_PROXY_IMPLEMENTATION, HEADER_FORWARDED, HEADER_X_FORWARDED_FOR, HEADER_X_REAL_IP, TrustedProxyImplementation,
+        TrustedProxyLayer, forwarded_client_ip, is_internal_ip, parse_implementation, parse_ip_token, resolve_client_info,
+    };
+    use crate::ClientInfo;
+    use axum::http::{HeaderMap, HeaderValue};
+    use serial_test::serial;
+    use std::net::{IpAddr, SocketAddr};
+
+    #[test]
+    fn test_simple_mode_is_default() {
+        assert!(TrustedProxyLayer::enabled().is_enabled());
+        assert!(!TrustedProxyLayer::disabled().is_enabled());
+    }
+
+    #[test]
+    fn test_parse_implementation() {
+        assert_eq!(parse_implementation(Some("simple")), TrustedProxyImplementation::Simple);
+        assert_eq!(parse_implementation(Some("legacy")), TrustedProxyImplementation::Legacy);
+        assert_eq!(parse_implementation(Some("full")), TrustedProxyImplementation::Legacy);
+        assert_eq!(parse_implementation(Some("internal-only")), TrustedProxyImplementation::Simple);
+        assert_eq!(parse_implementation(Some("unknown")), TrustedProxyImplementation::Simple);
+    }
+
+    #[test]
+    fn test_parse_ip_token() {
+        assert_eq!(parse_ip_token("203.0.113.10"), Some(IpAddr::from([203, 0, 113, 10])));
+        assert_eq!(parse_ip_token("203.0.113.10:9000"), Some(IpAddr::from([203, 0, 113, 10])));
+        assert_eq!(parse_ip_token("[2001:db8::10]:9000"), Some("2001:db8::10".parse().unwrap()));
+        assert_eq!(parse_ip_token("unknown"), None);
+    }
+
+    #[test]
+    fn test_forwarded_header_priority() {
+        let mut headers = HeaderMap::new();
+        headers.insert(HEADER_X_FORWARDED_FOR, HeaderValue::from_static("203.0.113.10, 10.0.0.5"));
+        headers.insert(HEADER_X_REAL_IP, HeaderValue::from_static("198.51.100.10"));
+        headers.insert(HEADER_FORWARDED, HeaderValue::from_static("for=192.0.2.60;proto=https"));
+        assert_eq!(forwarded_client_ip(&headers), Some(IpAddr::from([203, 0, 113, 10])));
+    }
+
+    #[test]
+    fn test_forwarded_header_fallback() {
+        let mut headers = HeaderMap::new();
+        headers.insert(HEADER_FORWARDED, HeaderValue::from_static("for=203.0.113.10;proto=https"));
+        assert_eq!(forwarded_client_ip(&headers), Some(IpAddr::from([203, 0, 113, 10])));
+    }
+
+    #[test]
+    fn test_internal_peer_can_override_real_ip() {
+        let mut headers = HeaderMap::new();
+        headers.insert(HEADER_X_FORWARDED_FOR, HeaderValue::from_static("203.0.113.10"));
+
+        let client_info = resolve_client_info(Some(SocketAddr::from(([10, 0, 0, 5], 9000))), &headers);
+        assert_eq!(client_info.real_ip, IpAddr::from([203, 0, 113, 10]));
+        assert!(client_info.is_from_trusted_proxy);
+        assert_eq!(client_info.proxy_ip, Some(IpAddr::from([10, 0, 0, 5])));
+    }
+
+    #[test]
+    fn test_public_peer_keeps_direct_ip() {
+        let mut headers = HeaderMap::new();
+        headers.insert(HEADER_X_FORWARDED_FOR, HeaderValue::from_static("203.0.113.10"));
+
+        let peer_addr = SocketAddr::from(([8, 8, 8, 8], 9000));
+        let client_info = resolve_client_info(Some(peer_addr), &headers);
+        assert_eq!(client_info.real_ip, peer_addr.ip());
+        assert!(!client_info.is_from_trusted_proxy);
+    }
+
+    #[test]
+    fn test_missing_headers_keep_direct_ip() {
+        let peer_addr = SocketAddr::from(([192, 168, 1, 20], 9000));
+        let client_info = resolve_client_info(Some(peer_addr), &HeaderMap::new());
+        assert_eq!(client_info.real_ip, peer_addr.ip());
+        assert!(!client_info.is_from_trusted_proxy);
+    }
+
+    #[test]
+    fn test_missing_peer_addr_uses_direct_placeholder() {
+        let client_info = resolve_client_info(None, &HeaderMap::new());
+        assert_eq!(
+            client_info.real_ip,
+            ClientInfo::direct(SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0)).real_ip
+        );
+    }
+
+    #[test]
+    fn test_forwarded_header_segment_without_equals() {
+        // A segment without '=' before 'for=' must not abort parsing.
+        let mut headers = HeaderMap::new();
+        headers.insert(HEADER_FORWARDED, HeaderValue::from_static("proto;for=203.0.113.10"));
+        assert_eq!(forwarded_client_ip(&headers), Some(IpAddr::from([203, 0, 113, 10])));
+    }
+
+    #[test]
+    fn test_parse_ip_token_invalid_port_rejected() {
+        // A bare "ip:notaport" token must not be accepted as a valid IP.
+        assert_eq!(parse_ip_token("203.0.113.10:notaport"), None);
+    }
+
+    #[test]
+    fn test_internal_ip_detection() {
+        assert!(is_internal_ip(IpAddr::from([10, 0, 0, 1])));
+        assert!(is_internal_ip(IpAddr::from([127, 0, 0, 1])));
+        assert!(is_internal_ip("fd00::1".parse().unwrap()));
+        assert!(!is_internal_ip(IpAddr::from([203, 0, 113, 10])));
+    }
+
+    #[test]
+    #[serial]
+    fn test_implementation_from_env() {
+        temp_env::with_vars(vec![(ENV_TRUSTED_PROXY_IMPLEMENTATION, Some("legacy"))], || {
+            assert_eq!(TrustedProxyImplementation::from_env(), TrustedProxyImplementation::Legacy);
+        });
+    }
+}

crates/trusted-proxies/tests/integration/proxy_tests.rs

@@ -14,14 +14,14 @@
 
 use axum::body::Body;
 use axum::{Router, routing::get};
-use rustfs_trusted_proxies::{TrustedProxy, TrustedProxyConfig, TrustedProxyLayer, ValidationMode};
+use rustfs_trusted_proxies::{LegacyTrustedProxyLayer, TrustedProxy, TrustedProxyConfig, ValidationMode};
 use tower::ServiceExt;
 
 #[tokio::test]
 async fn test_proxy_validation_flow() {
     let proxies = vec![TrustedProxy::Single("127.0.0.1".parse().unwrap())];
     let config = TrustedProxyConfig::new(proxies, ValidationMode::HopByHop, true, 10, true, vec![]);
-    let proxy_layer = TrustedProxyLayer::enabled(config, None);
+    let proxy_layer = LegacyTrustedProxyLayer::enabled(config, None);
 
     let app = Router::new().route("/test", get(|| async { "OK" })).layer(proxy_layer);
 

crates/trusted-proxies/tests/proxy_layer.rs

@@ -0,0 +1,66 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use axum::{
+    body::{self, Body},
+    http::{Request, Response},
+};
+use rustfs_trusted_proxies::{ClientInfo, TrustedProxyLayer};
+use std::convert::Infallible;
+use std::net::SocketAddr;
+use tower::{Layer, ServiceExt, service_fn};
+
+#[tokio::test]
+async fn test_layer_inserts_client_info_for_internal_proxy() {
+    let peer_addr = SocketAddr::from(([10, 0, 0, 5], 9000));
+    let service = service_fn(|request: Request<Body>| async move {
+        let client_info = request.extensions().get::<ClientInfo>().cloned().unwrap();
+        Ok::<_, Infallible>(Response::new(Body::from(client_info.real_ip.to_string())))
+    });
+    let service = TrustedProxyLayer::enabled().layer(service);
+
+    let mut request = Request::builder()
+        .uri("/")
+        .header("x-forwarded-for", "203.0.113.10")
+        .body(Body::empty())
+        .unwrap();
+    request.extensions_mut().insert(peer_addr);
+
+    let response = service.oneshot(request).await.unwrap();
+
+    let body = body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    assert_eq!(std::str::from_utf8(&body).unwrap(), "203.0.113.10");
+}
+
+#[tokio::test]
+async fn test_layer_ignores_forwarded_header_for_public_peer() {
+    let peer_addr = SocketAddr::from(([8, 8, 8, 8], 9000));
+    let service = service_fn(|request: Request<Body>| async move {
+        let client_info = request.extensions().get::<ClientInfo>().cloned().unwrap();
+        Ok::<_, Infallible>(Response::new(Body::from(client_info.real_ip.to_string())))
+    });
+    let service = TrustedProxyLayer::enabled().layer(service);
+
+    let mut request = Request::builder()
+        .uri("/")
+        .header("x-forwarded-for", "203.0.113.10")
+        .body(Body::empty())
+        .unwrap();
+    request.extensions_mut().insert(peer_addr);
+
+    let response = service.oneshot(request).await.unwrap();
+
+    let body = body::to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    assert_eq!(std::str::from_utf8(&body).unwrap(), "8.8.8.8");
+}