test(get): reject range with part number (#2725)

Co-authored-by: houseme <housemecn@gmail.com>

.docker/compose/README.md

@@ -36,6 +36,24 @@ To start a 4-node cluster for distributed testing:
 docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```
 
+### Script-Based 4-Node Validation (Recommended)
+
+Use the local validation script when you need local-source image build, failover checks,
+and benchmark workflow in one command:
+
+```bash
+# Default mode: WAIT_PROBE_MODE=service
+# This avoids false negatives where /health/ready remains 503 locally
+# while the service path is already available.
+./scripts/run_four_node_cluster_failover_bench.sh
+```
+
+Strict mode is available when you explicitly want `/health/ready == 200` as the gate:
+
+```bash
+WAIT_PROBE_MODE=ready ./scripts/run_four_node_cluster_failover_bench.sh
+```
+
 ### (Deprecated) Minimal Observability
 
 ```bash

.docker/compose/docker-compose.cluster.local-build.yml

@@ -0,0 +1,144 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  node1:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node1
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node1_data_0:/data/rustfs0
+      - node1_data_1:/data/rustfs1
+      - node1_data_2:/data/rustfs2
+      - node1_data_3:/data/rustfs3
+    ports:
+      - "9000:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node2:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node2
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node2_data_0:/data/rustfs0
+      - node2_data_1:/data/rustfs1
+      - node2_data_2:/data/rustfs2
+      - node2_data_3:/data/rustfs3
+    ports:
+      - "9001:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node3:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node3
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node3_data_0:/data/rustfs0
+      - node3_data_1:/data/rustfs1
+      - node3_data_2:/data/rustfs2
+      - node3_data_3:/data/rustfs3
+    ports:
+      - "9002:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node4:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node4
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node4_data_0:/data/rustfs0
+      - node4_data_1:/data/rustfs1
+      - node4_data_2:/data/rustfs2
+      - node4_data_3:/data/rustfs3
+    ports:
+      - "9003:9000"
+    networks:
+      - rustfs-cluster-net
+
+volumes:
+  node1_data_0:
+  node1_data_1:
+  node1_data_2:
+  node1_data_3:
+  node2_data_0:
+  node2_data_1:
+  node2_data_2:
+  node2_data_3:
+  node3_data_0:
+  node3_data_1:
+  node3_data_2:
+  node3_data_3:
+  node4_data_0:
+  node4_data_1:
+  node4_data_2:
+  node4_data_3:
+
+networks:
+  rustfs-cluster-net:
+    driver: bridge

.docker/compose/docker-compose.observability.yaml

@@ -26,7 +26,7 @@ services:
     restart: "no"
 
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.10.5
     user: "10001"
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
@@ -36,9 +36,19 @@ services:
       - "3200:3200" # tempo
       - "4317"      # otlp grpc
       - "4318"      # otlp http
+      - "7946"      # memberlist
     restart: unless-stopped
     networks:
       - rustfs-network
+    depends_on:
+      tempo-init:
+        condition: service_completed_successfully
+    healthcheck:
+      test: [ "CMD", "/tempo", "-version" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
 
   otel-collector:
     image: otel/opentelemetry-collector-contrib:latest
@@ -61,6 +71,12 @@ services:
       - jaeger
       - prometheus
       - loki
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   jaeger:
     image: jaegertracing/jaeger:latest
@@ -72,12 +88,23 @@ services:
       - BADGER_DIRECTORY_KEY=/badger/key
       - COLLECTOR_OTLP_ENABLED=true
     volumes:
+      - ../../.docker/observability/jaeger.yaml:/etc/jaeger/config.yml:ro
       - jaeger-data:/badger
     ports:
       - "16686:16686" # Web UI
       - "14269:14269" # Admin/Metrics
+      - "4317"        # otlp grpc
+      - "4318"        # otlp http
+    command: [ "--config", "/etc/jaeger/config.yml" ]
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
 
   prometheus:
     image: prom/prometheus:latest
@@ -85,6 +112,7 @@ services:
       - TZ=Asia/Shanghai
     volumes:
       - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ../../.docker/observability/prometheus-rules:/etc/prometheus/rules:ro
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"
@@ -94,23 +122,45 @@ services:
       - '--web.enable-remote-write-receiver'
       - '--enable-feature=promql-experimental-functions'
       - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - '--storage.tsdb.retention.time=30d'
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   loki:
     image: grafana/loki:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/loki.yaml:/etc/loki/local-config.yaml:ro
+      - ../../.docker/observability/loki.yaml:/etc/loki/loki.yaml:ro
       - loki-data:/loki
     ports:
       - "3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/loki.yaml
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  pyroscope:
+    image: grafana/pyroscope:latest
+    ports:
+      - "4040:4040"
+    command:
+      - -self-profiling.disable-push=true
+    networks:
+      - rustfs-network
+    restart: unless-stopped
 
   grafana:
     image: grafana/grafana:latest
@@ -127,10 +177,17 @@ services:
     volumes:
       - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
       - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
     depends_on:
       - prometheus
       - tempo
       - loki
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   # --- RustFS Cluster ---
 
@@ -215,6 +272,7 @@ volumes:
   tempo-data:
   loki-data:
   jaeger-data:
+  grafana-data:
 
 networks:
   rustfs-network:

.docker/observability/README.md

@@ -13,6 +13,9 @@ The stack is composed of the following best-in-class open-source components:
 - **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
 - **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
 
+By default, this stack uses Tempo in single-binary mode and does not require Kafka/Redpanda.
+If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rustfs.yml` together with `docker-compose-tempo-ha-override.yml`.
+
 ## Architecture
 
 1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
@@ -46,6 +49,15 @@ Run the following command to start the entire stack:
 docker compose up -d
 ```
 
+### High Availability Tempo
+
+The default `docker-compose.yml` is the single-node stack.
+If you need the Kafka-backed HA Tempo configuration, start it with:
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### Access Dashboards
 
 | Service        | URL                                              | Credentials       | Description                    |

.docker/observability/README_ZH.md

@@ -13,6 +13,9 @@
 - **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
 - **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
 
+默认情况下，这套技术栈使用 Tempo 单二进制模式，不依赖 Kafka/Redpanda。
+如果需要基于 Kafka 的 HA Tempo 路径，请使用 `docker-compose-example-for-rustfs.yml` 配合 `docker-compose-tempo-ha-override.yml`。
+
 ## 架构
 
 1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
@@ -46,6 +49,15 @@
 docker compose up -d
 ```
 
+### Tempo 高可用模式
+
+默认的 `docker-compose.yml` 对应单机栈。
+如果需要基于 Kafka 的 HA Tempo 配置，请使用：
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### 访问仪表盘
 
 | 服务 | URL | 凭据 | 描述 |

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -85,10 +85,8 @@ services:
     networks:
       - otel-network
     restart: unless-stopped
-    depends_on:
-      - redpanda
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
@@ -205,7 +203,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/docker-compose.yml

@@ -16,10 +16,8 @@ services:
 
   # --- Tracing ---
   tempo:
-    image: grafana/tempo:2.10.3
+    image: grafana/tempo:2.10.5
     container_name: tempo
-    depends_on:
-      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -33,31 +31,11 @@ services:
       - otel-network
     restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
-  redpanda:
-    image: redpandadata/redpanda:latest
-    ports:
-      - "9092:9092" # Kafka API for clients
-    command: >
-      redpanda start --overprovisioned
-      --mode=dev-container
-      --kafka-addr=PLAINTEXT://0.0.0.0:9092
-      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
-
-  redpanda-console:
-    image: docker.redpanda.com/redpandadata/console:latest
-    environment:
-      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
-    volumes:
-      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
-    ports:
-      - "8080:8080"
-    depends_on:
-      - redpanda
 
   vulture:
     image: grafana/tempo-vulture:latest
@@ -106,6 +84,7 @@ services:
     container_name: prometheus
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus-rules:/etc/prometheus/rules:ro
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"
@@ -168,7 +147,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/grafana/provisioning/datasources.yaml

@@ -1,97 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    uid: prometheus
-    access: proxy
-    orgId: 1
-    url: http://prometheus:9090
-    isDefault: true
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      exemplarTraceIdDestinations:
-        - name: trace_id
-          datasourceUid: tempo
-
-  - name: Tempo
-    type: tempo
-    uid: tempo
-    access: proxy
-    orgId: 1
-    url: http://tempo:3200
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      serviceMap:
-        datasourceUid: prometheus
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false
-      tracesToMetrics:
-        datasourceUid: prometheus
-        tags: [ { key: 'service.name' }, { key: 'job' } ]
-        queries:
-          - name: 'Service-Level Latency'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
-          - name: 'Service-Level Calls'
-            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
-          - name: 'Service-Level Errors'
-            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
-      nodeGraph:
-        enabled: true
-
-  - name: Loki
-    type: loki
-    uid: loki
-    orgId: 1
-    url: http://loki:3100
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      derivedFields:
-        - datasourceUid: tempo
-          matcherRegex: 'trace_id=(\w+)'
-          name: 'TraceID'
-          url: '$${__value.raw}'
-
-  - name: Jaeger
-    type: jaeger
-    uid: jaeger
-    url: http://jaeger:16686
-    access: proxy
-    isDefault: false
-    editable: false
-    jsonData:
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false

.docker/observability/prometheus-rules/rustfs-dashboard.yml

@@ -0,0 +1,40 @@
+groups:
+  - name: rustfs-dashboard
+    interval: 30s
+    rules:
+      - record: rustfs:http_server_requests:rate5m
+        expr: sum by (job) (rate(rustfs_http_server_requests_total[5m]))
+
+      - record: rustfs:http_server_request_duration_seconds:p50_5m
+        expr: histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+      - record: rustfs:http_server_request_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+      - record: rustfs:http_server_request_duration_seconds:p99_5m
+        expr: histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+
+      - record: rustfs:http_server_response_body_size_bytes:p50_5m
+        expr: histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+      - record: rustfs:http_server_response_body_size_bytes:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+      - record: rustfs:http_server_response_body_size_bytes:p99_5m
+        expr: histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+
+      - record: rustfs:log_cleaner_runs:rate15m
+        expr: sum by (job) (rate(rustfs_log_cleaner_runs_total[15m]))
+      - record: rustfs:log_cleaner_failure_ratio:rate5m
+        expr: sum by (job) (rate(rustfs_log_cleaner_run_failures_total[5m])) / clamp_min(sum by (job) (rate(rustfs_log_cleaner_runs_total[5m])), 1e-9)
+      - record: rustfs:log_cleaner_rotation_failure_ratio:rate5m
+        expr: sum by (job) (rate(rustfs_log_cleaner_rotation_failures_total[5m])) / clamp_min(sum by (job) (rate(rustfs_log_cleaner_rotation_total[5m])), 1e-9)
+      - record: rustfs:log_cleaner_rotation_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_log_cleaner_rotation_duration_seconds_bucket[5m])))
+      - record: rustfs:log_cleaner_compress_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_log_cleaner_compress_duration_seconds_bucket[5m])))
+
+      - record: rustfs:scanner_objects_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_objects_scanned_total[5m]))
+      - record: rustfs:scanner_directories_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_directories_scanned_total[5m]))
+      - record: rustfs:scanner_buckets_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_buckets_scanned_total[5m]))
+      - record: rustfs:scanner_cycles_success:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_cycles_total{result="success"}[5m]))

.docker/observability/prometheus.yml

@@ -19,6 +19,9 @@ global:
     cluster: 'rustfs-dev'  # Label to identify the cluster
     replica: '1'  # Replica identifier
 
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
 scrape_configs:
   - job_name: 'otel-collector'
     static_configs:

.docker/observability/tempo.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-partition_ring_live_store: true
 stream_over_http_enabled: true
 
 server:
@@ -33,33 +32,17 @@ distributor:
           endpoint: "0.0.0.0:4317"
         http:
           endpoint: "0.0.0.0:4318"
-  #log_received_spans:
-  #    enabled: true
-  #  log_discarded_spans:
-  #    enabled: true
 
-backend_scheduler:
-  provider:
-    compaction:
-      compaction:
-        block_retention: 1h
-
-backend_worker:
-  backend_scheduler_addr: localhost:3200
-  compaction:
-    block_retention: 1h
-  ring:
-    kvstore:
-      store: memberlist
-
-querier:
-  query_live_store: true
+ingester:
+  max_block_duration: 5m
 
 metrics_generator:
   registry:
     external_labels:
       source: tempo
       cluster: docker-compose
+  traces_storage:
+    path: /var/tempo/generator/traces
   storage:
     path: /var/tempo/generator/wal
     remote_write:
@@ -85,15 +68,5 @@ overrides:
       processors: [ "span-metrics", "service-graphs", "local-blocks" ]
       generate_native_histograms: both
 
-ingest:
-  enabled: true
-  kafka:
-    address: redpanda:9092
-    topic: tempo-ingest
-
-block_builder:
-  consume_cycle_duration: 30s
-
 usage_report:
   reporting_enabled: false
-

.github/dependabot.yml

@@ -27,9 +27,8 @@ updates:
       timezone: "Asia/Shanghai"
       time: "08:00"
     assignees:
-      - "heihutu"
-    reviewers:
       - "houseme"
+    reviewers:
       - "overtrue"
       - "majinghe"
     ignore:
@@ -39,6 +38,8 @@ updates:
         versions: [ "0.23.x" ]
       - dependency-name: "ratelimit"
         versions: [ "1.x" ]
+      - dependency-name: "ratelimit"
+        versions: [ "2.x" ]
     groups:
       s3s:
         update-types:

Cargo.toml

@@ -59,7 +59,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.93.0"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -76,46 +76,46 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "0.0.5" }
-rustfs-heal = { path = "crates/heal", version = "0.0.5" }
-rustfs-appauth = { path = "crates/appauth", version = "0.0.5" }
-rustfs-audit = { path = "crates/audit", version = "0.0.5" }
-rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
-rustfs-common = { path = "crates/common", version = "0.0.5" }
-rustfs-config = { path = "./crates/config", version = "0.0.5" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "0.0.5" }
-rustfs-credentials = { path = "crates/credentials", version = "0.0.5" }
-rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
-rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
-rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
-rustfs-iam = { path = "crates/iam", version = "0.0.5" }
-rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
-rustfs-kms = { path = "crates/kms", version = "0.0.5" }
-rustfs-lock = { path = "crates/lock", version = "0.0.5" }
-rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
-rustfs-notify = { path = "crates/notify", version = "0.0.5" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
-rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
-rustfs-obs = { path = "crates/obs", version = "0.0.5" }
-rustfs-policy = { path = "crates/policy", version = "0.0.5" }
-rustfs-protos = { path = "crates/protos", version = "0.0.5" }
-rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
-rustfs-rio = { path = "crates/rio", version = "0.0.5" }
-rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
-rustfs-scanner = { path = "crates/scanner", version = "0.0.5" }
-rustfs-signer = { path = "crates/signer", version = "0.0.5" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "0.0.5" }
-rustfs-targets = { path = "crates/targets", version = "0.0.5" }
-rustfs-utils = { path = "crates/utils", version = "0.0.5" }
-rustfs-workers = { path = "crates/workers", version = "0.0.5" }
-rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
+rustfs = { path = "./rustfs", version = "1.0.0-beta.1" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.1" }
+rustfs-appauth = { path = "crates/appauth", version = "1.0.0-beta.1" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.1" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.1" }
+rustfs-common = { path = "crates/common", version = "1.0.0-beta.1" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-beta.1" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.1" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.1" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.1" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.1" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.1" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.1" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.1" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.1" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.1" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.1" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.1" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.1" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.1" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.1" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.1" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.1" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.1" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.1" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.1" }
+rustfs-s3-common = { path = "crates/s3-common", version = "1.0.0-beta.1" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.1" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.1" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.1" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.1" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.1" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.1" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.1" }
+rustfs-workers = { path = "crates/workers", version = "1.0.0-beta.1" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.1" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.41" }
+async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
 async-nats = "0.47.0"
@@ -131,7 +131,8 @@ hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-g
 http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
-reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+rustfs-kafka-async = { version = "1.2.0" }
 socket2 = { version = "0.6.3", features = ["all"] }
 tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
@@ -167,10 +168,10 @@ crc-fast = "1.9.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
-pbkdf2 = "0.13.0-rc.10"
-rsa = { version = "0.10.0-rc.17" }
-rustls = { version = "0.23.38", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pki-types = "1.14.0"
+pbkdf2 = "0.13.0"
+rsa = { version = "0.10.0-rc.18" }
+rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls-pki-types = "1.14.1"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"
@@ -179,18 +180,18 @@ zeroize = { version = "1.8.2", features = ["derive"] }
 # Time and Date
 chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.23", features = ["serde"] }
+jiff = { version = "0.2.24", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
 anyhow = "1.0.102"
 arc-swap = "1.9.1"
-astral-tokio-tar = "0.6.0"
+astral-tokio-tar = "0.6.1"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.15" }
+aws-config = { version = "1.8.16" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.131.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
@@ -220,9 +221,9 @@ hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.185"
+libc = "0.2.186"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.11"
+local-ip-address = "0.6.12"
 memmap2 = "0.9.10"
 lz4 = "1.28.1"
 matchit = "0.9.2"
@@ -252,7 +253,7 @@ rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
 s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
 serial_test = "3.4.0"
-shadow-rs = { version = "1.7.1", default-features = false }
+shadow-rs = { version = "2.0.0", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
@@ -279,7 +280,7 @@ walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "8.5.1"
+zip = "8.6.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
@@ -291,12 +292,12 @@ opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rus
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }
-pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.0.3", features = ["backend-pprof-rs"] }
 
 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }
 unftp-core = "0.1.0"
-suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"
 
 # WebDAV

Dockerfile.decommission-local

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.91-trixie
+FROM rust:1.93-trixie
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \

Dockerfile.source

@@ -30,7 +30,7 @@ ARG BUILDPLATFORM
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.91-trixie AS builder
+FROM rust:1.93-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
@@ -60,7 +60,15 @@ RUN set -eux; \
 
 # Optional: cross toolchain for aarch64 (only when targeting linux/arm64)
 RUN set -eux; \
-    if [ "${TARGETPLATFORM:-linux/amd64}" = "linux/arm64" ]; then \
+    target_platform="${TARGETPLATFORM:-}"; \
+    if [ -z "${target_platform}" ]; then \
+    case "$(uname -m)" in \
+    x86_64) target_platform="linux/amd64" ;; \
+    aarch64|arm64) target_platform="linux/arm64" ;; \
+    *) target_platform="linux/amd64" ;; \
+    esac; \
+    fi; \
+    if [ "${target_platform}" = "linux/arm64" ]; then \
     export DEBIAN_FRONTEND=noninteractive; \
     apt-get update; \
     apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu; \
@@ -120,7 +128,16 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
     --mount=type=cache,target=/usr/src/rustfs/target \
     set -eux; \
-    case "${TARGETPLATFORM:-linux/amd64}" in \
+    rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; \
+    target_platform="${TARGETPLATFORM:-}"; \
+    if [ -z "${target_platform}" ]; then \
+    case "$(uname -m)" in \
+    x86_64) target_platform="linux/amd64" ;; \
+    aarch64|arm64) target_platform="linux/arm64" ;; \
+    *) target_platform="linux/amd64" ;; \
+    esac; \
+    fi; \
+    case "${target_platform}" in \
     linux/amd64) \
     echo "Building for x86_64-unknown-linux-gnu"; \
     cargo build --release --locked --target x86_64-unknown-linux-gnu --bin rustfs -j "$(nproc)"; \
@@ -132,7 +149,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     install -m 0755 target/aarch64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \
     ;; \
     *) \
-    echo "Unsupported TARGETPLATFORM=${TARGETPLATFORM}" >&2; exit 1 \
+    echo "Unsupported target platform=${target_platform}" >&2; exit 1 \
     ;; \
     esac
 
@@ -178,7 +195,7 @@ CMD ["cargo", "run", "--bin", "rustfs", "--"]
 # -----------------------------
 # Runtime stage (Ubuntu minimal)
 # -----------------------------
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ARG BUILD_DATE
 ARG VCS_REF

README.md

@@ -115,7 +115,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:v1.0.0-beta.1
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command

README_ZH.md

@@ -113,7 +113,7 @@ RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用
  docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
  # 使用指定版本运行
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.68
+ docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:v1.0.0-beta.1
 ```
 
 您也可以使用 Docker Compose。使用根目录下的 `docker-compose.yml` 文件：

crates/audit/Cargo.toml

@@ -41,11 +41,10 @@ serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
-url = { workspace = true }
-rumqttc = { workspace = true }
 
 [dev-dependencies]
 temp-env = { workspace = true }
+url = { workspace = true }
 
 [lints]
 workspace = true

crates/audit/src/factory.rs

@@ -15,13 +15,13 @@
 use crate::AuditEntry;
 use async_trait::async_trait;
 use rustfs_config::AUDIT_DEFAULT_DIR;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
+use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
     config::{
-        build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
-        validate_pulsar_config, validate_webhook_config,
+        build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+        validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
     },
     error::TargetError,
     target::TargetType,
@@ -119,3 +119,22 @@ impl TargetFactory for PulsarTargetFactory {
         AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }
+
+pub struct KafkaTargetFactory;
+
+#[async_trait]
+impl TargetFactory for KafkaTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_kafka_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}

crates/audit/src/registry.rs

@@ -14,7 +14,9 @@
 
 use crate::{
     AuditEntry, AuditError, AuditResult,
-    factory::{MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory},
+    factory::{
+        KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
+    },
 };
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
@@ -53,6 +55,7 @@ impl AuditRegistry {
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
         registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
         registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
+        registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
 
         registry
     }
@@ -197,8 +200,8 @@ impl AuditRegistry {
             }
         }
 
-        if !errors.is_empty() {
-            return Err(AuditError::Target(errors.into_iter().next().unwrap()));
+        if let Some(error) = errors.into_iter().next() {
+            return Err(AuditError::Target(error));
         }
 
         Ok(())

crates/common/src/internode_metrics.rs

@@ -51,7 +51,7 @@ impl InternodeMetrics {
             return;
         }
         self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-        counter!("rustfs.internode.sent.bytes.total").increment(bytes);
+        counter!("rustfs_system_network_internode_sent_bytes_total").increment(bytes);
     }
 
     pub fn record_recv_bytes(&self, bytes: usize) {
@@ -60,22 +60,22 @@ impl InternodeMetrics {
             return;
         }
         self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-        counter!("rustfs.internode.recv.bytes.total").increment(bytes);
+        counter!("rustfs_system_network_internode_recv_bytes_total").increment(bytes);
     }
 
     pub fn record_outgoing_request(&self) {
         self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.requests.outgoing.total").increment(1);
+        counter!("rustfs_system_network_internode_requests_outgoing_total").increment(1);
     }
 
     pub fn record_incoming_request(&self) {
         self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.requests.incoming.total").increment(1);
+        counter!("rustfs_system_network_internode_requests_incoming_total").increment(1);
     }
 
     pub fn record_error(&self) {
         self.errors_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.errors.total").increment(1);
+        counter!("rustfs_system_network_internode_errors_total").increment(1);
     }
 
     pub fn record_dial_result(&self, duration: Duration, success: bool) {
@@ -83,11 +83,11 @@ impl InternodeMetrics {
         self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
         let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
         let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
-        gauge!("rustfs.internode.dial.avg_time.nanos").set(total as f64 / samples as f64);
+        gauge!("rustfs_system_network_internode_dial_avg_time_nanos").set(total as f64 / samples as f64);
 
         if !success {
             self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
-            counter!("rustfs.internode.dial.errors.total").increment(1);
+            counter!("rustfs_system_network_internode_dial_errors_total").increment(1);
         }
 
         let now_ms = SystemTime::now()

crates/config/src/audit/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Kafka Environment Variables
+pub const ENV_AUDIT_KAFKA_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_ENABLE";
+pub const ENV_AUDIT_KAFKA_BROKERS: &str = "RUSTFS_AUDIT_KAFKA_BROKERS";
+pub const ENV_AUDIT_KAFKA_TOPIC: &str = "RUSTFS_AUDIT_KAFKA_TOPIC";
+pub const ENV_AUDIT_KAFKA_ACKS: &str = "RUSTFS_AUDIT_KAFKA_ACKS";
+pub const ENV_AUDIT_KAFKA_TLS_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_TLS_ENABLE";
+pub const ENV_AUDIT_KAFKA_TLS_CA: &str = "RUSTFS_AUDIT_KAFKA_TLS_CA";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_KAFKA_QUEUE_DIR: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_DIR";
+pub const ENV_AUDIT_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_AUDIT_KAFKA_ENABLE,
+    ENV_AUDIT_KAFKA_BROKERS,
+    ENV_AUDIT_KAFKA_TOPIC,
+    ENV_AUDIT_KAFKA_ACKS,
+    ENV_AUDIT_KAFKA_TLS_ENABLE,
+    ENV_AUDIT_KAFKA_TLS_CA,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_CERT,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_KEY,
+    ENV_AUDIT_KAFKA_QUEUE_DIR,
+    ENV_AUDIT_KAFKA_QUEUE_LIMIT,
+];
+
+/// A list of all valid configuration keys for a Kafka audit target.
+pub const AUDIT_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/mod.rs

@@ -16,11 +16,13 @@
 //! This module defines the configuration for audit systems, including
 //! webhook and MQTT audit-related settings.
 
+mod kafka;
 mod mqtt;
 mod nats;
 mod pulsar;
 mod webhook;
 
+pub use kafka::*;
 pub use mqtt::*;
 pub use nats::*;
 pub use pulsar::*;
@@ -33,6 +35,7 @@ pub const AUDIT_PREFIX: &str = "audit";
 pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
 
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
+pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
 pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
 pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
 pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
@@ -40,6 +43,7 @@ pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
 #[allow(dead_code)]
 pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
+    AUDIT_KAFKA_SUB_SYS,
     AUDIT_MQTT_SUB_SYS,
     AUDIT_NATS_SUB_SYS,
     AUDIT_PULSAR_SUB_SYS,

crates/config/src/constants/app.rs

@@ -131,6 +131,18 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable to explicitly bypass local physical disk independence checks.
+pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
+
+/// Compatibility alias used by legacy MinIO CI pipelines.
+///
+/// RustFS keeps this alias for backward compatibility only. Prefer
+/// `ENV_UNSAFE_BYPASS_DISK_CHECK` for explicit bypass control.
+pub const ENV_MINIO_CI: &str = "MINIO_CI";
+
+/// Default flag value for bypassing local physical disk independence checks.
+pub const DEFAULT_UNSAFE_BYPASS_DISK_CHECK: bool = false;
+
 /// Environment variable for server access key.
 pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
 
@@ -284,9 +296,9 @@ pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true;
 
 /// Default profiling export enabled
 /// It is used to enable or disable exporting profiles
-/// Default value: true
+/// Default value: false
 /// Environment variable: RUSTFS_OBS_PROFILING_EXPORT_ENABLED
-pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = true;
+pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = false;
 
 /// Default log local logging enabled for rustfs
 /// This is the default log local logging enabled for rustfs.

crates/config/src/constants/drive.rs

@@ -39,6 +39,14 @@ pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
 pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
 pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
 
+/// Interval in seconds between active health probes for local and remote drives.
+pub const ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_INTERVAL_SECS";
+pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
+
+/// Timeout in seconds for a single active health probe.
+pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5;
+
 /// Number of consecutive failures before a suspect drive is classified as offline.
 pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
 pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;

crates/config/src/constants/env.rs

@@ -26,6 +26,15 @@ pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";
 
+/// Global switch for enabling the audit module.
+pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
+/// Global switch for enabling the notify module.
+pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
+/// Default global audit switch (disabled by default).
+pub const DEFAULT_AUDIT_ENABLE: bool = false;
+/// Default global notify switch (disabled by default).
+pub const DEFAULT_NOTIFY_ENABLE: bool = false;
+
 /// Medium-drawn lines separator
 /// This is used to separate words in environment variable names.
 pub const ENV_WORD_DELIMITER_DASH: &str = "-";
@@ -290,4 +299,10 @@ mod tests {
             assert!(state.is_disabled());
         }
     }
+
+    #[test]
+    fn test_global_audit_notify_switch_constants() {
+        assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
+        assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
+    }
 }

crates/config/src/constants/health.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Enable or disable public `/health` and `/health/ready` endpoints.
+/// When disabled, the routes are not registered and return 404.
+pub const ENV_HEALTH_ENDPOINT_ENABLE: &str = "RUSTFS_HEALTH_ENDPOINT_ENABLE";
+pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
+
+/// Cache TTL for storage readiness runtime-state evaluation (milliseconds).
+/// This reduces storage-layer pressure when probes are called at high frequency.
+pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
+pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
+
+/// Enable minimal health payload mode for GET `/health*` responses.
+/// When enabled, only `status` and `ready` fields are returned.
+pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
+pub const DEFAULT_HEALTH_MINIMAL_RESPONSE_ENABLE: bool = false;

crates/config/src/constants/internode.rs

@@ -0,0 +1,62 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Timeout for establishing a new internode gRPC connection.
+pub const ENV_INTERNODE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS: u64 = 3;
+
+/// TCP keepalive interval for internode gRPC channels.
+pub const ENV_INTERNODE_TCP_KEEPALIVE_SECS: &str = "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS";
+pub const DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS: u64 = 10;
+
+/// HTTP/2 keepalive interval for internode gRPC channels.
+pub const ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS";
+pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5;
+
+/// HTTP/2 keepalive timeout for internode gRPC channels.
+pub const ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
+
+/// Overall timeout for a single internode gRPC request.
+pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn internode_timeout_defaults_stay_in_expected_bounds() {
+        assert_eq!(DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, 10);
+        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
+        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
+    }
+
+    #[test]
+    fn internode_timeout_env_names_are_stable() {
+        assert_eq!(ENV_INTERNODE_CONNECT_TIMEOUT_SECS, "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS");
+        assert_eq!(ENV_INTERNODE_TCP_KEEPALIVE_SECS, "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS");
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
+            "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS"
+        );
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
+            "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
+        );
+        assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
+    }
+}

crates/config/src/constants/mod.rs

@@ -20,6 +20,8 @@ pub(crate) mod console;
 pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
+pub(crate) mod health;
+pub(crate) mod internode;
 pub(crate) mod object;
 pub(crate) mod oidc;
 pub(crate) mod profiler;

crates/config/src/constants/proxy.rs

@@ -20,6 +20,11 @@ pub const ENV_TRUSTED_PROXY_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_ENABLED";
 /// Trusted proxy middleware is enabled by default.
 pub const DEFAULT_TRUSTED_PROXY_ENABLED: bool = true;
 
+/// Environment variable to select the trusted proxy implementation.
+pub const ENV_TRUSTED_PROXY_IMPLEMENTATION: &str = "RUSTFS_TRUSTED_PROXY_IMPLEMENTATION";
+/// The simplified implementation is used by default.
+pub const DEFAULT_TRUSTED_PROXY_IMPLEMENTATION: &str = "simple";
+
 /// Environment variable for the proxy validation mode.
 pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDATION_MODE";
 /// Default validation mode is "hop_by_hop".

crates/config/src/constants/runtime.rs

@@ -59,9 +59,28 @@ pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10;
 pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling
 // Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option<String>)
 
+// Allocator reclaim configuration
+pub const ENV_ALLOCATOR_RECLAIM_ENABLED: &str = "RUSTFS_ALLOCATOR_RECLAIM_ENABLED";
+pub const ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS: &str = "RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS";
+pub const ENV_ALLOCATOR_RECLAIM_FORCE: &str = "RUSTFS_ALLOCATOR_RECLAIM_FORCE";
+pub const ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS: &str = "RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS";
+pub const DEFAULT_ALLOCATOR_RECLAIM_ENABLED: bool = false;
+pub const DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS: u64 = 30;
+pub const DEFAULT_ALLOCATOR_RECLAIM_FORCE: bool = true;
+pub const DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS: u64 = 3;
+
+// File page-cache reclaim configuration
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE";
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE";
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD";
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false;
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = false;
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024;
+
 /// Threshold for small object seek support in megabytes.
 ///
 /// When an object is smaller than this size, rustfs will provide seek support.
 ///
 /// Default is set to 10MB.
+pub const ENV_OBJECT_SEEK_SUPPORT_THRESHOLD: &str = "RUSTFS_OBJECT_SEEK_SUPPORT_THRESHOLD";
 pub const DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD: usize = 10 * 1024 * 1024;

crates/config/src/constants/targets.rs

@@ -40,6 +40,15 @@ pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
 pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
 pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
+pub const KAFKA_BROKERS: &str = "brokers";
+pub const KAFKA_TOPIC: &str = "topic";
+pub const KAFKA_ACKS: &str = "acks";
+pub const KAFKA_QUEUE_DIR: &str = "queue_dir";
+pub const KAFKA_QUEUE_LIMIT: &str = "queue_limit";
+pub const KAFKA_TLS_ENABLE: &str = "tls_enable";
+pub const KAFKA_TLS_CA: &str = "tls_ca";
+pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key";
 
 pub const NATS_ADDRESS: &str = "address";
 pub const NATS_SUBJECT: &str = "subject";

crates/config/src/lib.rs

@@ -31,6 +31,10 @@ pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::heal::*;
 #[cfg(feature = "constants")]
+pub use constants::health::*;
+#[cfg(feature = "constants")]
+pub use constants::internode::*;
+#[cfg(feature = "constants")]
 pub use constants::object::*;
 #[cfg(feature = "constants")]
 pub use constants::profiler::*;

crates/config/src/notify/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// A list of all valid configuration keys for a Kafka target.
+pub const NOTIFY_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+// Kafka Environment Variables
+pub const ENV_NOTIFY_KAFKA_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_ENABLE";
+pub const ENV_NOTIFY_KAFKA_BROKERS: &str = "RUSTFS_NOTIFY_KAFKA_BROKERS";
+pub const ENV_NOTIFY_KAFKA_TOPIC: &str = "RUSTFS_NOTIFY_KAFKA_TOPIC";
+pub const ENV_NOTIFY_KAFKA_ACKS: &str = "RUSTFS_NOTIFY_KAFKA_ACKS";
+pub const ENV_NOTIFY_KAFKA_TLS_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_TLS_ENABLE";
+pub const ENV_NOTIFY_KAFKA_TLS_CA: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CA";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_KAFKA_QUEUE_DIR: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_DIR";
+pub const ENV_NOTIFY_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_NOTIFY_KAFKA_ENABLE,
+    ENV_NOTIFY_KAFKA_BROKERS,
+    ENV_NOTIFY_KAFKA_TOPIC,
+    ENV_NOTIFY_KAFKA_ACKS,
+    ENV_NOTIFY_KAFKA_TLS_ENABLE,
+    ENV_NOTIFY_KAFKA_TLS_CA,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY,
+    ENV_NOTIFY_KAFKA_QUEUE_DIR,
+    ENV_NOTIFY_KAFKA_QUEUE_LIMIT,
+];

crates/config/src/notify/mod.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod arn;
+mod kafka;
 mod mqtt;
 mod nats;
 mod pulsar;
@@ -20,6 +21,7 @@ mod store;
 mod webhook;
 
 pub use arn::*;
+pub use kafka::*;
 pub use mqtt::*;
 pub use nats::*;
 pub use pulsar::*;
@@ -69,13 +71,13 @@ pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
 
 #[allow(dead_code)]
 pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
+    NOTIFY_KAFKA_SUB_SYS,
     NOTIFY_MQTT_SUB_SYS,
     NOTIFY_NATS_SUB_SYS,
     NOTIFY_PULSAR_SUB_SYS,
     NOTIFY_WEBHOOK_SUB_SYS,
 ];
 
-#[allow(dead_code)]
 pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
 pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
 #[allow(dead_code)]

crates/config/src/observability/mod.rs

@@ -22,6 +22,14 @@ pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
 pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT";
 pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT";
 pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT";
+pub const ENV_OBS_ENDPOINT_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_HEADERS";
+pub const ENV_OBS_ENDPOINT_TRACES_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS";
+pub const ENV_OBS_ENDPOINT_METRICS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS";
+pub const ENV_OBS_ENDPOINT_LOGS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS";
+pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
 pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
@@ -108,6 +116,14 @@ mod tests {
         assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT");
         assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT");
         assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT");
+        assert_eq!(ENV_OBS_ENDPOINT_HEADERS, "RUSTFS_OBS_ENDPOINT_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_HEADERS, "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_HEADERS, "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_HEADERS, "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS");
         assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT");
         assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
         assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");

crates/e2e_test/Cargo.toml

@@ -73,3 +73,4 @@ rcgen.workspace = true
 anyhow.workspace = true
 rustls.workspace = true
 zip.workspace = true
+clap.workspace = true

crates/e2e_test/src/bin/tls_gen.rs

@@ -0,0 +1,22 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Parser;
+use e2e_test::tls_gen::{Args, run};
+
+fn main() -> anyhow::Result<()> {
+    let out_dir = run(Args::parse())?;
+    println!("Generated RustFS TLS bundle in {}", out_dir.display());
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -125,3 +125,5 @@ mod replication_extension_test;
 
 #[cfg(test)]
 mod snowball_auto_extract_test;
+
+pub mod tls_gen;

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -19,6 +19,7 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
     use serial_test::serial;
     use tracing::info;
@@ -179,4 +180,84 @@ mod tests {
             "Delete marker should no longer be latest after the second put"
         );
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_prefix_with_marker_object_returns_children() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns prefix children when a marker object also exists");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-versions-prefix-marker";
+        let marker_key = "data01";
+        let child_keys = [
+            "data01/meta/dump-2026-04-08-053205.json.gz",
+            "data01/meta/dump-2026-04-08-063209.json.gz",
+        ];
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to suspend versioning");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(marker_key)
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to put marker object");
+
+        for key in child_keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(ByteStream::from_static(b"payload"))
+                .send()
+                .await
+                .expect("Failed to put child object");
+        }
+
+        let listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix("data01/")
+            .send()
+            .await
+            .expect("Failed to list object versions by prefix");
+
+        let version_keys: Vec<_> = listing.versions().iter().filter_map(|version| version.key()).collect();
+
+        assert_eq!(
+            version_keys.len(),
+            child_keys.len(),
+            "ListObjectVersions with a trailing slash prefix should include child objects even when the marker object exists"
+        );
+
+        for key in child_keys {
+            assert!(
+                version_keys.contains(&key),
+                "ListObjectVersions(prefix=data01/) should include child object {key}"
+            );
+        }
+    }
 }

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -120,11 +120,6 @@ impl LockClient for GrpcLockClient {
             .map_err(|e| LockError::internal(e.to_string()))?
             .into_inner();
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -134,7 +129,8 @@ impl LockClient for GrpcLockClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -50,6 +50,18 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
     }
 }
 
+fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> GenerallyLockResult {
+    if success {
+        GenerallyLockResult {
+            success: true,
+            error_info: None,
+            lock_info: None,
+        }
+    } else {
+        lock_result_from_error(format!("lock not found for release: {lock_id}"))
+    }
+}
+
 /// Minimal NodeService implementation that only supports Lock RPCs
 /// Used for testing distributed lock scenarios with real gRPC
 #[derive(Debug)]
@@ -102,7 +114,7 @@ impl NodeService for MinimalLockNodeService {
                 let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
                 Ok(Response::new(GenerallyLockResponse {
                     success: result.success,
-                    error_info: None,
+                    error_info: result.error,
                     lock_info: lock_info_json,
                 }))
             }
@@ -131,11 +143,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -161,11 +176,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.force_release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -271,10 +289,9 @@ impl NodeService for MinimalLockNodeService {
                 Ok(batch_results) => {
                     for (result_idx, success) in batch_results.into_iter().enumerate() {
                         if let Some(request_idx) = valid_indices.get(result_idx) {
-                            results[*request_idx] = GenerallyLockResult {
-                                success,
-                                error_info: None,
-                                lock_info: None,
+                            results[*request_idx] = match lock_ids.get(result_idx) {
+                                Some(lock_id) => lock_result_from_release(lock_id, success),
+                                None => lock_result_from_error(format!("unlock response index out of range: {result_idx}")),
                             };
                         }
                     }

crates/e2e_test/src/reliant/head_tls_bodyless_test.rs

@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for TLS/HTTP2 `HEAD` responses on missing objects.
+//!
+//! Before the fix, RustFS returned `404` for a missing object but still wrote
+//! the XML error payload on a `HEAD` request. Under HTTP/2 this emitted DATA
+//! frames after the response headers, which clients surfaced as a protocol
+//! error. This test keeps the request at the raw HTTPS layer so it can validate
+//! the final wire-facing behavior rather than SDK-level error mapping.
+
+#![cfg(test)]
+
+use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use http::Version;
+use http::header::HOST;
+use rcgen::generate_simple_self_signed;
+use reqwest::{Certificate, Client, Response, StatusCode};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+use std::path::Path;
+use std::process::Command;
+use tokio::fs;
+use tokio::time::{Duration, sleep};
+use tracing::info;
+
+const ACCESS_KEY: &str = "rustfsadmin";
+const SECRET_KEY: &str = "rustfsadmin";
+const BUCKET: &str = "test-head-tls-bodyless-bucket";
+
+async fn generate_tls_bundle(tls_dir: &Path) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    fs::create_dir_all(tls_dir).await?;
+    let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
+    let cert_pem = cert.cert.pem();
+    let key_pem = cert.signing_key.serialize_pem();
+
+    fs::write(tls_dir.join("rustfs_cert.pem"), cert_pem.as_bytes()).await?;
+    fs::write(tls_dir.join("rustfs_key.pem"), key_pem.as_bytes()).await?;
+
+    Ok(cert_pem.into_bytes())
+}
+
+fn local_https_h2_client(ca_pem: &[u8]) -> Result<Client, Box<dyn Error + Send + Sync>> {
+    let _ca_cert = Certificate::from_pem(ca_pem)?;
+    Ok(Client::builder()
+        .no_proxy()
+        .no_gzip()
+        .no_brotli()
+        .no_zstd()
+        .no_deflate()
+        .danger_accept_invalid_certs(true)
+        .build()?)
+}
+
+async fn signed_empty_request(
+    client: &Client,
+    method: http::Method,
+    url: &str,
+) -> Result<Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let request = http::Request::builder()
+        .method(method.as_str())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+
+    let signed = sign_v4(request, 0, ACCESS_KEY, SECRET_KEY, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        builder = builder.header(name, value);
+    }
+
+    Ok(builder.send().await?)
+}
+
+async fn ensure_bucket_exists(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let bucket_url = format!("{endpoint}/{BUCKET}/");
+    let response = signed_empty_request(client, http::Method::HEAD, &bucket_url).await?;
+
+    if response.status() == StatusCode::OK {
+        return Ok(());
+    }
+
+    let response = signed_empty_request(client, http::Method::PUT, &bucket_url).await?;
+    match response.status() {
+        StatusCode::OK => Ok(()),
+        StatusCode::CONFLICT => Ok(()),
+        status => Err(format!("unexpected bucket setup status: {status}").into()),
+    }
+}
+
+async fn wait_for_tls_server_ready(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let ready_url = format!("{endpoint}/");
+    for _attempt in 0..60 {
+        match signed_empty_request(client, http::Method::GET, &ready_url).await {
+            Ok(response) if response.status().is_success() => return Ok(()),
+            Ok(_) | Err(_) => sleep(Duration::from_millis(500)).await,
+        }
+    }
+
+    Err("RustFS TLS server failed to become ready within 30 seconds".into())
+}
+
+async fn start_tls_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let binary_path = rustfs_binary_path();
+    let mut command = Command::new(&binary_path);
+    command
+        .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .current_dir(&env.temp_dir);
+
+    for key in [
+        "RUSTFS_ADDRESS",
+        "RUSTFS_VOLUMES",
+        "RUSTFS_ACCESS_KEY",
+        "RUSTFS_SECRET_KEY",
+        "RUSTFS_TLS_PATH",
+        "RUSTFS_OBS_LOG_DIRECTORY",
+    ] {
+        command.env_remove(key);
+    }
+
+    let process = command
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .env("RUSTFS_CONSOLE_ENABLE", "false")
+        .args([
+            "--address",
+            &env.address,
+            "--access-key",
+            &env.access_key,
+            "--secret-key",
+            &env.secret_key,
+            &env.temp_dir,
+        ])
+        .spawn()?;
+
+    env.process = Some(process);
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_head_missing_object_over_tls_http2_is_bodyless() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let tls_dir = std::path::PathBuf::from(&env.temp_dir).join("tls");
+    let ca_pem = generate_tls_bundle(&tls_dir).await?;
+    start_tls_rustfs_server(&mut env, &tls_dir).await?;
+
+    let endpoint = format!("https://{}", env.address);
+    let client = local_https_h2_client(&ca_pem)?;
+    wait_for_tls_server_ready(&client, &endpoint).await?;
+    ensure_bucket_exists(&client, &endpoint).await?;
+
+    let missing_key = "head-does-not-exist.txt";
+    let object_url = format!("{endpoint}/{BUCKET}/{missing_key}");
+
+    let get_response = signed_empty_request(&client, http::Method::GET, &object_url).await?;
+    assert_eq!(get_response.status(), StatusCode::NOT_FOUND);
+    let get_version = get_response.version();
+    let get_body = get_response.bytes().await?;
+    let get_body_text = String::from_utf8_lossy(&get_body);
+    assert!(
+        get_body_text.contains("<Code>NoSuchKey</Code>") || get_body_text.contains("<Code>NoSuchObject</Code>"),
+        "GET missing-object error body should expose NoSuchKey/NoSuchObject, got: {}",
+        get_body_text
+    );
+    info!("GET missing object over TLS used {:?} and returned {} bytes", get_version, get_body.len());
+
+    let head_response = signed_empty_request(&client, http::Method::HEAD, &object_url).await?;
+    assert_eq!(head_response.status(), StatusCode::NOT_FOUND);
+    assert_eq!(head_response.version(), Version::HTTP_2, "HEAD regression test must exercise HTTP/2");
+    let head_body = head_response.bytes().await?;
+    assert!(
+        head_body.is_empty(),
+        "HEAD missing-object response must not send body bytes over TLS/HTTP2, got {} bytes: {:?}",
+        head_body.len(),
+        head_body
+    );
+
+    Ok(())
+}

crates/e2e_test/src/reliant/lock.rs

@@ -275,6 +275,41 @@ async fn test_grpc_lock_client_batch_acquire_and_release() {
     handle.abort();
 }
 
+#[tokio::test]
+async fn test_grpc_lock_client_uses_request_lock_id_and_reports_missing_unlock() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+    let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client = GrpcLockClient::new(addr);
+    let request = LockRequest::new(test_resource(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(2));
+
+    let response = grpc_client.acquire_lock(&request).await.expect("gRPC acquire should succeed");
+    let lock_info = response.lock_info.expect("gRPC acquire should include lock info");
+    assert_eq!(lock_info.id, request.lock_id);
+
+    assert!(
+        grpc_client
+            .release(&request.lock_id)
+            .await
+            .expect("gRPC release should succeed"),
+        "release should find the request lock id"
+    );
+
+    let missing_release = grpc_client
+        .release(&request.lock_id)
+        .await
+        .expect_err("second release should report missing lock");
+    assert!(
+        missing_release.to_string().contains("lock not found for release"),
+        "missing release should preserve server error, got: {missing_release}"
+    );
+
+    handle.abort();
+}
+
 #[tokio::test]
 async fn test_distributed_lock_4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes() {
     let manager1 = Arc::new(GlobalLockManager::new());

crates/e2e_test/src/reliant/mod.rs

@@ -17,6 +17,7 @@ mod get_deleted_object_test;
 mod grpc_lock_client;
 mod grpc_lock_server;
 mod head_deleted_object_versioning_test;
+mod head_tls_bodyless_test;
 mod lifecycle;
 mod lock;
 mod node_interact_test;

crates/e2e_test/src/tls_gen.rs

@@ -0,0 +1,259 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::{Context, Result, bail};
+use clap::Parser;
+use rcgen::{
+    BasicConstraints, CertificateParams, CertifiedIssuer, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose,
+    SanType,
+};
+use std::fs;
+use std::net::IpAddr;
+use std::path::{Path, PathBuf};
+use time::{Duration, OffsetDateTime};
+
+pub const DEFAULT_OUT_DIR: &str = "target/tls";
+pub const OUTPUT_FILES: [&str; 7] = [
+    "rustfs_cert.pem",
+    "rustfs_key.pem",
+    "ca.crt",
+    "public.crt",
+    "client_ca.crt",
+    "client_cert.pem",
+    "client_key.pem",
+];
+
+#[derive(Debug, Parser)]
+#[command(name = "tls_gen", about = "Generate a full RustFS TLS bundle for local TLS and mTLS tests.")]
+pub struct Args {
+    #[arg(long, default_value = DEFAULT_OUT_DIR)]
+    pub out_dir: PathBuf,
+    #[arg(long, default_value_t = 365)]
+    pub days: i64,
+    #[arg(long)]
+    pub force: bool,
+}
+
+pub fn run(args: Args) -> Result<PathBuf> {
+    if args.days <= 0 {
+        bail!("--days must be a positive integer");
+    }
+
+    write_bundle(&args.out_dir, args.force, args.days)?;
+    Ok(args.out_dir)
+}
+
+pub fn ensure_writable(out_dir: &Path, force: bool) -> Result<()> {
+    if force {
+        return Ok(());
+    }
+
+    let existing: Vec<_> = OUTPUT_FILES
+        .iter()
+        .map(|name| out_dir.join(name))
+        .filter(|path| path.exists())
+        .collect();
+
+    if existing.is_empty() {
+        return Ok(());
+    }
+
+    let existing_list = existing
+        .iter()
+        .map(|path| path.file_name().and_then(|name| name.to_str()).unwrap_or("<unknown>"))
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    bail!(
+        "Refusing to overwrite existing files in {}: {}. Re-run with --force to replace them.",
+        out_dir.display(),
+        existing_list
+    )
+}
+
+fn write_bundle(out_dir: &Path, force: bool, days: i64) -> Result<()> {
+    fs::create_dir_all(out_dir).with_context(|| format!("failed to create output directory {}", out_dir.display()))?;
+    ensure_writable(out_dir, force)?;
+
+    let ca_key = generate_private_key()?;
+    let ca = build_ca_certificate(ca_key, days)?;
+
+    let server_key = generate_private_key()?;
+    let server_cert = build_leaf_certificate(
+        &server_key,
+        "localhost",
+        &[SanType::DnsName("localhost".try_into()?)],
+        &[
+            SanType::IpAddress(IpAddr::V4("127.0.0.1".parse()?)),
+            SanType::IpAddress(IpAddr::V6("::1".parse()?)),
+        ],
+        ExtendedKeyUsagePurpose::ServerAuth,
+        &ca,
+        days,
+    )?;
+
+    let client_key = generate_private_key()?;
+    let client_cert = build_leaf_certificate(
+        &client_key,
+        "rustfs-test-client",
+        &[SanType::DnsName("rustfs-test-client".try_into()?)],
+        &[],
+        ExtendedKeyUsagePurpose::ClientAuth,
+        &ca,
+        days,
+    )?;
+
+    let ca_pem = ca.pem();
+    let bundle = [
+        ("rustfs_cert.pem", server_cert.pem()),
+        ("rustfs_key.pem", server_key.serialize_pem()),
+        ("ca.crt", ca_pem.clone()),
+        ("public.crt", ca_pem.clone()),
+        ("client_ca.crt", ca_pem),
+        ("client_cert.pem", client_cert.pem()),
+        ("client_key.pem", client_key.serialize_pem()),
+    ];
+
+    for (name, content) in bundle {
+        fs::write(out_dir.join(name), content).with_context(|| format!("failed to write {}", out_dir.join(name).display()))?;
+    }
+
+    Ok(())
+}
+
+fn build_ca_certificate(signing_key: KeyPair, days: i64) -> Result<CertifiedIssuer<'static, KeyPair>> {
+    let mut params = base_params("RustFS Test CA", days)?;
+    params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+    params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
+
+    CertifiedIssuer::self_signed(params, signing_key).context("failed to create CA certificate")
+}
+
+fn build_leaf_certificate(
+    signing_key: &KeyPair,
+    common_name: &str,
+    dns_names: &[SanType],
+    ip_addresses: &[SanType],
+    usage: ExtendedKeyUsagePurpose,
+    issuer: &CertifiedIssuer<'_, KeyPair>,
+    days: i64,
+) -> Result<rcgen::Certificate> {
+    let mut params = base_params(common_name, days)?;
+    params.is_ca = IsCa::ExplicitNoCa;
+    params.key_usages = vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment];
+    params.extended_key_usages = vec![usage];
+    params.use_authority_key_identifier_extension = true;
+    params.subject_alt_names.extend_from_slice(dns_names);
+    params.subject_alt_names.extend_from_slice(ip_addresses);
+
+    params
+        .signed_by(signing_key, issuer)
+        .with_context(|| format!("failed to create leaf certificate for {common_name}"))
+}
+
+fn base_params(common_name: &str, days: i64) -> Result<CertificateParams> {
+    let mut params = CertificateParams::default();
+    let issued_at = OffsetDateTime::now_utc() - Duration::minutes(5);
+    params.not_before = issued_at;
+    params.not_after = issued_at + Duration::days(days);
+    params.distinguished_name.push(DnType::CountryName, "US");
+    params.distinguished_name.push(DnType::OrganizationName, "RustFS");
+    params.distinguished_name.push(DnType::CommonName, common_name);
+    Ok(params)
+}
+
+fn generate_private_key() -> Result<KeyPair> {
+    KeyPair::generate().context("failed to generate private key")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Args, OUTPUT_FILES, ensure_writable, run};
+    use std::fs;
+    use std::path::{Path, PathBuf};
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn unique_temp_dir() -> PathBuf {
+        let suffix = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("system time must be after unix epoch")
+            .as_nanos();
+        std::env::temp_dir().join(format!("rustfs-tls-gen-{suffix}"))
+    }
+
+    struct TempDir {
+        path: PathBuf,
+    }
+
+    impl TempDir {
+        fn new() -> Self {
+            let path = unique_temp_dir();
+            fs::create_dir_all(&path).expect("temporary directory should be created");
+            Self { path }
+        }
+
+        fn path(&self) -> &Path {
+            &self.path
+        }
+    }
+
+    impl Drop for TempDir {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.path);
+        }
+    }
+
+    #[test]
+    fn run_writes_full_bundle() {
+        let temp_dir = TempDir::new();
+
+        let out_dir = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 365,
+            force: false,
+        })
+        .expect("bundle generation should succeed");
+
+        for name in OUTPUT_FILES {
+            let content = fs::read(out_dir.join(name)).unwrap_or_else(|error| panic!("{name} should exist: {error}"));
+            assert!(!content.is_empty(), "{name} should not be empty");
+        }
+    }
+
+    #[test]
+    fn ensure_writable_rejects_existing_files_without_force() {
+        let temp_dir = TempDir::new();
+        let existing = temp_dir.path().join(OUTPUT_FILES[0]);
+        fs::write(&existing, "existing").expect("existing file should be created");
+
+        let error = ensure_writable(temp_dir.path(), false).expect_err("existing files must be rejected");
+        let message = format!("{error:#}");
+
+        assert!(message.contains("Refusing to overwrite existing files"));
+        assert!(message.contains(OUTPUT_FILES[0]));
+    }
+
+    #[test]
+    fn run_rejects_non_positive_days() {
+        let temp_dir = TempDir::new();
+        let error = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 0,
+            force: false,
+        })
+        .expect_err("non-positive days must fail");
+
+        assert_eq!(format!("{error:#}"), "--days must be a positive integer");
+    }
+}

crates/ecstore/Cargo.toml

@@ -55,6 +55,7 @@ flatbuffers.workspace = true
 futures.workspace = true
 futures-util.workspace = true
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
 serde.workspace = true
 time.workspace = true
 bytesize.workspace = true
@@ -62,6 +63,7 @@ serde_json.workspace = true
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
 s3s.workspace = true
 http.workspace = true
+opentelemetry.workspace = true
 http-body = { workspace = true }
 http-body-util.workspace = true
 url.workspace = true
@@ -98,6 +100,7 @@ pin-project-lite.workspace = true
 md-5.workspace = true
 memmap2 = { workspace = true }
 libc.workspace = true
+rustix = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-workers.workspace = true
 reqwest = { workspace = true }
@@ -124,9 +127,10 @@ metrics = { workspace = true }
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true }
 serial_test = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 
 [build-dependencies]
 shadow-rs = { workspace = true, features = ["build", "metadata"] }

crates/ecstore/src/batch_processor.rs

@@ -87,25 +87,70 @@ impl AsyncBatchProcessor {
         T: Send + 'static,
         F: Future<Output = Result<T>> + Send + 'static,
     {
-        let results = self.execute_batch(tasks).await;
-        let mut successes = Vec::new();
+        if required_successes == 0 {
+            return Ok(Vec::new());
+        }
 
-        for value in results.into_iter().flatten() {
-            successes.push(value);
-            if successes.len() >= required_successes {
-                return Ok(successes);
+        if tasks.is_empty() {
+            return Err(Error::other(format!(
+                "Insufficient successful results: got 0, needed {required_successes}"
+            )));
+        }
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
+        let mut join_set = JoinSet::new();
+        let mut successes = Vec::new();
+        let mut pending_tasks = tasks.len();
+        let mut first_error = None;
+
+        for task in tasks {
+            let sem = semaphore.clone();
+            join_set.spawn(async move {
+                let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
+                task.await
+            });
+        }
+
+        while let Some(join_result) = join_set.join_next().await {
+            pending_tasks = pending_tasks.saturating_sub(1);
+
+            match join_result {
+                Ok(Ok(value)) => {
+                    successes.push(value);
+                    if successes.len() >= required_successes {
+                        return Ok(successes);
+                    }
+                }
+                Ok(Err(err)) => {
+                    if first_error.is_none() {
+                        first_error = Some(err);
+                    }
+                }
+                Err(join_error) => {
+                    if first_error.is_none() {
+                        first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
+                    }
+                }
+            }
+
+            if successes.len() + pending_tasks < required_successes {
+                return Err(first_error.unwrap_or_else(|| {
+                    Error::other(format!(
+                        "Insufficient successful results: got {}, needed {}",
+                        successes.len(),
+                        required_successes
+                    ))
+                }));
             }
         }
 
-        if successes.len() >= required_successes {
-            Ok(successes)
-        } else {
-            Err(Error::other(format!(
+        Err(first_error.unwrap_or_else(|| {
+            Error::other(format!(
                 "Insufficient successful results: got {}, needed {}",
                 successes.len(),
                 required_successes
-            )))
-        }
+            ))
+        }))
     }
 }
 
@@ -228,4 +273,52 @@ mod tests {
         let successes = results.unwrap();
         assert!(successes.len() >= 2);
     }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_returns_before_slow_tail() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))]
+            .into_iter()
+            .map(|(delay_ms, outcome)| async move {
+                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                outcome
+            })
+            .collect();
+
+        let results = processor
+            .execute_batch_with_quorum(tasks, 2)
+            .await
+            .expect("quorum should succeed");
+        assert_eq!(results.len(), 2);
+        assert!(started.elapsed() < Duration::from_millis(100));
+    }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok(1_i32)),
+            (15, Err(Error::other("first failure"))),
+            (20, Err(Error::other("second failure"))),
+            (250, Ok(4)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let err = processor
+            .execute_batch_with_quorum(tasks, 3)
+            .await
+            .expect_err("quorum should fail once it becomes impossible");
+
+        assert!(err.to_string().contains("first failure"));
+        assert!(started.elapsed() < Duration::from_millis(120));
+    }
 }

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -395,8 +395,27 @@ impl BucketTargetSys {
         }
     }
 
-    pub async fn set_target(&self, bucket: &str, target: &BucketTarget, update: bool) -> Result<(), BucketTargetError> {
-        if !target.target_type.is_valid() && !update {
+    pub async fn set_target(
+        &self,
+        bucket: &str,
+        target: &BucketTarget,
+        update: bool,
+    ) -> Result<BucketTargets, BucketTargetError> {
+        self.validate_target(bucket, target).await?;
+
+        let mut bucket_targets = match self.list_bucket_targets(bucket).await {
+            Ok(targets) => targets,
+            Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
+            Err(err) => return Err(err),
+        };
+
+        Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
+
+        Ok(bucket_targets)
+    }
+
+    pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
+        if !target.target_type.is_valid() {
             return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
                 bucket: bucket.to_string(),
             });
@@ -450,52 +469,44 @@ impl BucketTargetSys {
             }
         }
 
-        {
-            let mut targets_map = self.targets_map.write().await;
-            let bucket_targets = targets_map.entry(bucket.to_string()).or_insert_with(Vec::new);
-            let mut found = false;
+        Ok(())
+    }
 
-            for (idx, existing_target) in bucket_targets.iter().enumerate() {
-                if existing_target.target_type.to_string() == target.target_type.to_string() {
-                    if existing_target.arn == target.arn {
-                        if !update {
-                            return Err(BucketTargetError::BucketRemoteAlreadyExists {
-                                bucket: existing_target.target_bucket.clone(),
-                            });
-                        }
-                        bucket_targets[idx] = target.clone();
-                        found = true;
-                        break;
-                    }
-                    if existing_target.endpoint == target.endpoint {
+    fn upsert_target_entry(
+        bucket_targets: &mut Vec<BucketTarget>,
+        target: &BucketTarget,
+        update: bool,
+    ) -> Result<(), BucketTargetError> {
+        let mut found = false;
+
+        for (idx, existing_target) in bucket_targets.iter().enumerate() {
+            if existing_target.target_type.to_string() == target.target_type.to_string() {
+                if existing_target.arn == target.arn {
+                    if !update {
                         return Err(BucketTargetError::BucketRemoteAlreadyExists {
                             bucket: existing_target.target_bucket.clone(),
                         });
                     }
+                    bucket_targets[idx] = target.clone();
+                    found = true;
+                    break;
+                }
+                if existing_target.endpoint == target.endpoint {
+                    return Err(BucketTargetError::BucketRemoteAlreadyExists {
+                        bucket: existing_target.target_bucket.clone(),
+                    });
                 }
             }
-
-            if !found && !update {
-                bucket_targets.push(target.clone());
-            }
         }
 
-        {
-            let mut arn_remotes_map = self.arn_remotes_map.write().await;
-            arn_remotes_map.insert(
-                target.arn.clone(),
-                ArnTarget {
-                    client: Some(Arc::new(target_client)),
-                    last_refresh: OffsetDateTime::now_utc(),
-                },
-            );
+        if !found && !update {
+            bucket_targets.push(target.clone());
         }
 
-        self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
         Ok(())
     }
 
-    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<(), BucketTargetError> {
+    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<BucketTargets, BucketTargetError> {
         if arn_str.is_empty() {
             return Err(BucketTargetError::BucketRemoteArnInvalid {
                 bucket: bucket.to_string(),
@@ -524,33 +535,16 @@ impl BucketTargetSys {
             }
         }
 
-        {
-            let mut targets_map = self.targets_map.write().await;
+        let targets = self.list_bucket_targets(bucket).await?;
+        let new_targets: Vec<BucketTarget> = targets.targets.iter().filter(|t| t.arn != arn_str).cloned().collect();
 
-            let Some(targets) = targets_map.get(bucket) else {
-                return Err(BucketTargetError::BucketRemoteTargetNotFound {
-                    bucket: bucket.to_string(),
-                });
-            };
-
-            let new_targets: Vec<BucketTarget> = targets.iter().filter(|t| t.arn != arn_str).cloned().collect();
-
-            if new_targets.len() == targets.len() {
-                return Err(BucketTargetError::BucketRemoteTargetNotFound {
-                    bucket: bucket.to_string(),
-                });
-            }
-
-            targets_map.insert(bucket.to_string(), new_targets);
+        if new_targets.len() == targets.targets.len() {
+            return Err(BucketTargetError::BucketRemoteTargetNotFound {
+                bucket: bucket.to_string(),
+            });
         }
 
-        {
-            self.arn_remotes_map.write().await.remove(arn_str);
-        }
-
-        self.update_bandwidth_limit(bucket, arn_str, 0);
-
-        Ok(())
+        Ok(BucketTargets { targets: new_targets })
     }
 
     pub async fn mark_refresh_in_progress(&self, bucket: &str, arn: &str) {
@@ -603,7 +597,7 @@ impl BucketTargetSys {
 
         if let Some(last_refresh) = last_refresh {
             let now = OffsetDateTime::now_utc();
-            if now - last_refresh > Duration::from_secs(60 * 5) {
+            if now - last_refresh < Duration::from_secs(60 * 5) {
                 return None;
             }
         }
@@ -619,6 +613,16 @@ impl BucketTargetSys {
             }
         };
 
+        let cli = self
+            .arn_remotes_map
+            .read()
+            .await
+            .get(arn)
+            .and_then(|target| target.client.clone());
+        if cli.is_some() {
+            return cli;
+        }
+
         self.inc_arn_errs(bucket, arn).await;
         None
     }

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -54,8 +54,8 @@ use rustfs_filemeta::{
 use rustfs_s3_common::EventName;
 use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
 use s3s::dto::{
-    BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
-    Timestamp,
+    BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
+    RestoreRequestType, RestoreStatus, Timestamp,
 };
 use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
 use sha2::{Digest, Sha256};
@@ -92,6 +92,7 @@ const ENV_STALE_UPLOADS_EXPIRY: &str = "RUSTFS_API_STALE_UPLOADS_EXPIRY";
 const ENV_STALE_UPLOADS_CLEANUP_INTERVAL: &str = "RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL";
 const DEFAULT_STALE_UPLOADS_EXPIRY: StdDuration = StdDuration::from_secs(24 * 60 * 60);
 const DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL: StdDuration = StdDuration::from_secs(6 * 60 * 60);
+const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
 
 lazy_static! {
     pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
@@ -1241,6 +1242,29 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket:
     }
 }
 
+fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
+    lc.rules.iter().any(|rule| {
+        rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
+            && rule.id.as_deref() == Some(rule_id)
+            && rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
+    })
+}
+
+fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
+    lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
+        let updated_at = OffsetDateTime::from(updated_at.clone());
+        now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
+    })
+}
+
+async fn apply_existing_object_expiry(api: Arc<ECStore>, object: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
+    if object.is_remote() {
+        apply_expiry_on_transitioned_object(api, object, event, src).await;
+    } else {
+        apply_expiry_on_non_transitioned_objects(api, object, event, src).await;
+    }
+}
+
 pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
     let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
         return Ok(());
@@ -1253,6 +1277,8 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
     let mut marker = None;
     let mut version_marker = None;
     let src = LcEventSrc::Scanner;
+    let defer_date_expiry_once = should_defer_date_expiry_for_recent_config_update(&lc, OffsetDateTime::now_utc());
+    let mut date_expiry_deferred_once = false;
 
     loop {
         let page = api
@@ -1269,15 +1295,16 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
                 | IlmAction::DeleteRestoredVersionAction
                 | IlmAction::DeleteAllVersionsAction
                 | IlmAction::DelMarkerDeleteAllVersionsAction => {
-                    if event
-                        .due
-                        .is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp())
-                    {
-                        if object.is_remote() {
-                            apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await;
-                        } else {
-                            apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await;
+                    let now = OffsetDateTime::now_utc();
+                    if event.due.is_some_and(|due| due.unix_timestamp() <= now.unix_timestamp()) {
+                        if defer_date_expiry_once
+                            && !date_expiry_deferred_once
+                            && lifecycle_rule_has_date_expiration(&lc, &event.rule_id)
+                        {
+                            tokio::time::sleep(StdDuration::from_secs(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS as u64)).await;
+                            date_expiry_deferred_once = true;
                         }
+                        apply_existing_object_expiry(api.clone(), object, &event, &src).await;
                     } else {
                         apply_expiry_rule(&event, &src, object).await;
                     }
@@ -1977,9 +2004,10 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
 #[cfg(test)]
 mod tests {
     use super::{
-        StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
-        lifecycle_deleted_object, lifecycle_version_purge_state_from_completed_targets,
-        mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete,
+        DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks,
+        cleanup_stale_multipart_uploads_once_at, lifecycle_deleted_object, lifecycle_rule_has_date_expiration,
+        lifecycle_version_purge_state_from_completed_targets, mark_delete_opts_skip_decommissioned_on_remote_success,
+        merge_stale_multipart_candidate, replication_state_for_delete, should_defer_date_expiry_for_recent_config_update,
         should_reuse_lifecycle_delete_replication_state,
     };
     use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
@@ -1994,6 +2022,7 @@ mod tests {
         BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader,
     };
     use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
+    use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
     use std::collections::HashMap;
@@ -2014,6 +2043,49 @@ mod tests {
         assert!(opts.skip_decommissioned);
     }
 
+    #[test]
+    fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
+        let lc = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    date: Some(Timestamp::from(OffsetDateTime::now_utc())),
+                    ..Default::default()
+                }),
+                id: Some("rule-date".to_string()),
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        assert!(lifecycle_rule_has_date_expiration(&lc, "rule-date"));
+        assert!(!lifecycle_rule_has_date_expiration(&lc, "missing-rule"));
+    }
+
+    #[test]
+    fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
+        let now = OffsetDateTime::now_utc();
+        let recent = BucketLifecycleConfiguration {
+            expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
+            rules: Vec::new(),
+        };
+        let stale = BucketLifecycleConfiguration {
+            expiry_updated_at: Some(Timestamp::from(
+                now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
+            )),
+            rules: Vec::new(),
+        };
+
+        assert!(should_defer_date_expiry_for_recent_config_update(&recent, now));
+        assert!(!should_defer_date_expiry_for_recent_config_update(&stale, now));
+    }
+
     #[test]
     fn mark_delete_opts_skip_decommissioned_on_remote_success_preserves_false_on_failure() {
         let mut opts = ObjectOptions::default();

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -20,15 +20,144 @@
 
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
 use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts};
+use crate::client::signer_error::error_chain_contains_signer_header_marker;
 use crate::global::GLOBAL_TierConfigMgr;
+use rustfs_utils::get_env_usize;
 use sha2::{Digest, Sha256};
 use std::any::Any;
+use std::collections::VecDeque;
 use std::io::Write;
+use std::sync::LazyLock;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::{Duration, Instant};
+use tokio::sync::{Mutex, Semaphore};
+use tracing::warn;
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
 static XXHASH_SEED: u64 = 0;
 
+const ENV_REMOTE_DELETE_MAX_CONCURRENCY: &str = "RUSTFS_REMOTE_DELETE_MAX_CONCURRENCY";
+const ENV_REMOTE_DELETE_BREAKER_THRESHOLD: &str = "RUSTFS_REMOTE_DELETE_BREAKER_THRESHOLD";
+const ENV_REMOTE_DELETE_BREAKER_WINDOW_SECS: &str = "RUSTFS_REMOTE_DELETE_BREAKER_WINDOW_SECS";
+const DEFAULT_REMOTE_DELETE_BREAKER_THRESHOLD: usize = 50;
+const DEFAULT_REMOTE_DELETE_BREAKER_WINDOW_SECS: usize = 30;
+const METRIC_DELETE_REMOTE_FAILED_TOTAL: &str = "rustfs_delete_remote_failed_total";
+const METRIC_DELETE_REMOTE_BREAKER_TOTAL: &str = "rustfs_delete_remote_breaker_total";
+const METRIC_DELETE_REMOTE_INFLIGHT: &str = "rustfs_delete_remote_inflight";
+
+static REMOTE_DELETE_INFLIGHT: AtomicUsize = AtomicUsize::new(0);
+
+static REMOTE_DELETE_LIMITER: LazyLock<Semaphore> = LazyLock::new(|| {
+    let default_limit = std::cmp::min(num_cpus::get(), 16).max(1);
+    let concurrency = get_env_usize(ENV_REMOTE_DELETE_MAX_CONCURRENCY, default_limit).max(1);
+    Semaphore::new(concurrency)
+});
+
+static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::new(|| {
+    Mutex::new(RemoteDeleteBreaker::new(
+        get_env_usize(ENV_REMOTE_DELETE_BREAKER_THRESHOLD, DEFAULT_REMOTE_DELETE_BREAKER_THRESHOLD).max(1),
+        Duration::from_secs(
+            get_env_usize(ENV_REMOTE_DELETE_BREAKER_WINDOW_SECS, DEFAULT_REMOTE_DELETE_BREAKER_WINDOW_SECS) as u64,
+        ),
+    ))
+});
+
+#[derive(Debug)]
+struct RemoteDeleteBreaker {
+    threshold: usize,
+    window: Duration,
+    failures: VecDeque<Instant>,
+}
+
+impl RemoteDeleteBreaker {
+    fn new(threshold: usize, window: Duration) -> Self {
+        Self {
+            threshold: threshold.max(1),
+            window: window.max(Duration::from_secs(1)),
+            failures: VecDeque::new(),
+        }
+    }
+
+    fn should_short_circuit(&mut self, now: Instant) -> bool {
+        self.prune(now);
+        self.failures.len() >= self.threshold
+    }
+
+    fn record_signer_failure(&mut self, now: Instant) -> bool {
+        self.prune(now);
+        let was_open = self.failures.len() >= self.threshold;
+        self.failures.push_back(now);
+        !was_open && self.failures.len() >= self.threshold
+    }
+
+    fn prune(&mut self, now: Instant) {
+        while let Some(ts) = self.failures.front().copied() {
+            if now.duration_since(ts) > self.window {
+                self.failures.pop_front();
+            } else {
+                break;
+            }
+        }
+    }
+}
+
+struct RemoteDeleteInflightGuard;
+
+impl RemoteDeleteInflightGuard {
+    fn new() -> Self {
+        let inflight = REMOTE_DELETE_INFLIGHT.fetch_add(1, Ordering::Relaxed) + 1;
+        metrics::gauge!(METRIC_DELETE_REMOTE_INFLIGHT).set(inflight as f64);
+        Self
+    }
+}
+
+impl Drop for RemoteDeleteInflightGuard {
+    fn drop(&mut self) {
+        let inflight = REMOTE_DELETE_INFLIGHT.fetch_sub(1, Ordering::Relaxed) - 1;
+        metrics::gauge!(METRIC_DELETE_REMOTE_INFLIGHT).set(inflight as f64);
+    }
+}
+
+fn is_signer_header_error(err: &std::io::Error) -> bool {
+    if err.kind() != std::io::ErrorKind::InvalidInput {
+        return false;
+    }
+
+    if let Some(source) = err.get_ref() {
+        if error_chain_contains_signer_header_marker(source) {
+            return true;
+        }
+    }
+
+    let message = err.to_string().to_ascii_lowercase();
+    message.contains("invalid utf-8 header value")
+        || message.contains("invalidheadervalue")
+        || (message.contains("sign v4") && message.contains("header value"))
+}
+
+async fn remote_delete_breaker_is_open(now: Instant) -> bool {
+    let mut breaker = REMOTE_DELETE_BREAKER.lock().await;
+    breaker.should_short_circuit(now)
+}
+
+async fn record_remote_delete_failure(err: &std::io::Error, now: Instant) {
+    metrics::counter!(METRIC_DELETE_REMOTE_FAILED_TOTAL).increment(1);
+
+    if !is_signer_header_error(err) {
+        return;
+    }
+
+    let mut breaker = REMOTE_DELETE_BREAKER.lock().await;
+    if breaker.record_signer_failure(now) {
+        warn!(
+            threshold = breaker.threshold,
+            window_secs = breaker.window.as_secs(),
+            "remote tier delete breaker opened by signer/header failures"
+        );
+    }
+}
+
 #[derive(Default)]
 #[allow(dead_code)]
 struct ObjSweeper {
@@ -148,12 +277,31 @@ impl ExpiryOp for Jentry {
 }
 
 pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
+    if remote_delete_breaker_is_open(Instant::now()).await {
+        metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
+        return Err(std::io::Error::other("remote tier delete breaker is open due to signer/header failures"));
+    }
+
+    let _permit = REMOTE_DELETE_LIMITER
+        .acquire()
+        .await
+        .map_err(|_| std::io::Error::other("remote tier delete limiter is closed"))?;
+    let _inflight = RemoteDeleteInflightGuard::new();
+
     let mut config_mgr = GLOBAL_TierConfigMgr.write().await;
     let w = match config_mgr.get_driver(tier_name).await {
         Ok(w) => w,
-        Err(e) => return Err(std::io::Error::other(e)),
+        Err(e) => {
+            let err = std::io::Error::other(e);
+            record_remote_delete_failure(&err, Instant::now()).await;
+            return Err(err);
+        }
     };
-    w.remove(obj_name, rv_id).await
+    let result = w.remove(obj_name, rv_id).await;
+    if let Err(err) = &result {
+        record_remote_delete_failure(err, Instant::now()).await;
+    }
+    result
 }
 
 pub fn transitioned_delete_journal_entry(
@@ -189,4 +337,44 @@ pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject
 }
 
 #[cfg(test)]
-mod test {}
+mod test {
+    use crate::client::signer_error::invalid_utf8_header_error;
+
+    use super::{RemoteDeleteBreaker, is_signer_header_error};
+    use std::io::{Error, ErrorKind};
+    use std::time::{Duration, Instant};
+
+    #[test]
+    fn signer_header_error_detection_matches_utf8_failures() {
+        let err = Error::new(
+            ErrorKind::InvalidInput,
+            "failed to sign v4 request: invalid UTF-8 header value for `x-amz-meta-invalid`",
+        );
+        assert!(is_signer_header_error(&err));
+    }
+
+    #[test]
+    fn signer_header_error_detection_rejects_unrelated_errors() {
+        let err = Error::other("dial tcp: i/o timeout");
+        assert!(!is_signer_header_error(&err));
+    }
+
+    #[test]
+    fn signer_header_error_detection_matches_structured_marker() {
+        let err = invalid_utf8_header_error("failed to sign v4 request", "x-amz-meta-invalid");
+        assert!(is_signer_header_error(&err));
+    }
+
+    #[test]
+    fn breaker_opens_at_threshold_and_recovers_after_window() {
+        let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
+        let start = Instant::now();
+
+        assert!(!breaker.should_short_circuit(start));
+        assert!(!breaker.record_signer_failure(start));
+        assert!(!breaker.record_signer_failure(start + Duration::from_secs(1)));
+        assert!(breaker.record_signer_failure(start + Duration::from_secs(2)));
+        assert!(breaker.should_short_circuit(start + Duration::from_secs(3)));
+        assert!(!breaker.should_short_circuit(start + Duration::from_secs(40)));
+    }
+}

crates/ecstore/src/bucket/replication/config.rs

@@ -224,19 +224,98 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
                 continue;
             }
 
-            if !self.role.is_empty() {
-                arns.push(self.role.clone()); // Use the legacy RoleArn when present
-                return arns;
-            }
-
-            if !targets_map.contains(&rule.destination.bucket) {
+            if !rule.destination.bucket.is_empty() && !targets_map.contains(&rule.destination.bucket) {
                 targets_map.insert(rule.destination.bucket.clone());
             }
         }
 
+        if targets_map.is_empty() && !self.role.is_empty() {
+            arns.push(self.role.clone());
+            return arns;
+        }
+
         for arn in targets_map {
             arns.push(arn);
         }
         arns
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use s3s::dto::{DeleteMarkerReplication, Destination, ExistingObjectReplication, ReplicationRule};
+
+    fn replication_rule(id: &str, arn: &str) -> ReplicationRule {
+        ReplicationRule {
+            delete_marker_replication: Some(DeleteMarkerReplication::default()),
+            delete_replication: None,
+            destination: Destination {
+                bucket: arn.to_string(),
+                ..Default::default()
+            },
+            existing_object_replication: Some(ExistingObjectReplication {
+                status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
+            }),
+            filter: None,
+            id: Some(id.to_string()),
+            prefix: Some(String::new()),
+            priority: Some(1),
+            source_selection_criteria: None,
+            status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+        }
+    }
+
+    #[test]
+    fn filter_target_arns_keeps_multiple_destinations_when_role_is_present() {
+        let config = ReplicationConfiguration {
+            role: "arn:legacy:target".to_string(),
+            rules: vec![
+                replication_rule("rule-1", "arn:target:a"),
+                replication_rule("rule-2", "arn:target:b"),
+            ],
+        };
+
+        let arns = config.filter_target_arns(&ObjectOpts {
+            name: "object".to_string(),
+            op_type: ReplicationType::Object,
+            ..Default::default()
+        });
+
+        assert_eq!(arns.len(), 2);
+        assert!(arns.iter().any(|arn| arn == "arn:target:a"));
+        assert!(arns.iter().any(|arn| arn == "arn:target:b"));
+    }
+
+    #[test]
+    fn filter_target_arns_falls_back_to_role_when_destination_is_empty() {
+        let config = ReplicationConfiguration {
+            role: "arn:legacy:target".to_string(),
+            rules: vec![ReplicationRule {
+                delete_marker_replication: Some(DeleteMarkerReplication::default()),
+                delete_replication: None,
+                destination: Destination {
+                    bucket: String::new(),
+                    ..Default::default()
+                },
+                existing_object_replication: Some(ExistingObjectReplication {
+                    status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
+                }),
+                filter: None,
+                id: Some("rule-1".to_string()),
+                prefix: Some(String::new()),
+                priority: Some(1),
+                source_selection_criteria: None,
+                status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+            }],
+        };
+
+        let arns = config.filter_target_arns(&ObjectOpts {
+            name: "object".to_string(),
+            op_type: ReplicationType::Object,
+            ..Default::default()
+        });
+
+        assert_eq!(arns, vec!["arn:legacy:target".to_string()]);
+    }
+}

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -956,6 +956,9 @@ pub type DynReplicationPool = dyn ReplicationPoolTrait + Send + Sync;
 /// Trait that abstracts the replication pool operations
 #[async_trait::async_trait]
 pub trait ReplicationPoolTrait: std::fmt::Debug {
+    fn active_workers(&self) -> i32;
+    fn active_mrf_workers(&self) -> i32;
+    fn active_lrg_workers(&self) -> i32;
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo);
     async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo);
     async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
@@ -972,6 +975,18 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
 // Implement the trait for ReplicationPool
 #[async_trait::async_trait]
 impl<S: StorageAPI> ReplicationPoolTrait for ReplicationPool<S> {
+    fn active_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_workers(self)
+    }
+
+    fn active_mrf_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_mrf_workers(self)
+    }
+
+    fn active_lrg_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_lrg_workers(self)
+    }
+
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo) {
         self.queue_replica_task(ri).await;
     }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -35,7 +35,7 @@ use crate::set_disk::get_lock_acquire_timeout;
 use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
-use aws_sdk_s3::operation::head_object::HeadObjectOutput;
+use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
 use aws_smithy_types::body::SdkBody;
@@ -115,6 +115,41 @@ fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &Res
     state.resync_id.is_empty() || opts.resync_id.is_empty() || state.resync_id == opts.resync_id
 }
 
+fn should_count_head_proxy_failure(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
+    if is_not_found || matches!(code, Some("MethodNotAllowed" | "405")) {
+        return false;
+    }
+    !matches!(raw_status, Some(404 | 405))
+}
+
+fn is_head_proxy_failure(err: &SdkError<HeadObjectError>) -> bool {
+    let (is_not_found, code) = err
+        .as_service_error()
+        .map(|service_err| (service_err.is_not_found(), service_err.code()))
+        .unwrap_or((false, None));
+    let raw_status = err.raw_response().map(|resp| resp.status().as_u16());
+    should_count_head_proxy_failure(is_not_found, code, raw_status)
+}
+
+async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) {
+    if let Some(stats) = GLOBAL_REPLICATION_STATS.get() {
+        stats.inc_proxy(bucket, api, is_err).await;
+    }
+}
+
+async fn head_object_with_proxy_stats(
+    source_bucket: &str,
+    target_client: &TargetClient,
+    target_bucket: &str,
+    object: &str,
+    version_id: Option<String>,
+) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    let result = target_client.head_object(target_bucket, object, version_id).await;
+    let is_err = result.as_ref().err().is_some_and(is_head_proxy_failure);
+    record_proxy_request(source_bucket, "HeadObject", is_err).await;
+    result
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ResyncOpts {
     pub bucket: String,
@@ -748,10 +783,15 @@ impl ReplicationResyncer {
 
                     let reset_id = target_client.reset_id.clone();
 
-                    let (size, err) = if let Err(err) = target_client
-                        .head_object(&target_client.bucket, &roi.name, roi.version_id.map(|v| v.to_string()))
-                        .await
-                    {
+                    let head_result = head_object_with_proxy_stats(
+                        &bucket_name,
+                        target_client.as_ref(),
+                        &target_client.bucket,
+                        &roi.name,
+                        roi.version_id.map(|v| v.to_string()),
+                    )
+                    .await;
+                    let (size, err) = if let Err(err) = head_result {
                         if roi.delete_marker {
                             st.replicated_count += 1;
                         } else {
@@ -2120,9 +2160,14 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
     };
 
     if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
-        match tgt_client
-            .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
-            .await
+        match head_object_with_proxy_stats(
+            &dobj.bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &dobj.delete_object.object_name,
+            version_id.clone(),
+        )
+        .await
         {
             Ok(_) => {}
             Err(e) => {
@@ -2471,9 +2516,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         }
 
         let mut replication_action = replication_action;
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2528,9 +2578,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             }
         };
 
+        let has_tagging_replication = !put_opts.user_tags.is_empty();
         if let Some(err) = if is_multipart {
             drop(gr);
-            replicate_object_with_multipart(MultipartReplicationContext {
+            let result = replicate_object_with_multipart(MultipartReplicationContext {
                 storage: storage.clone(),
                 cli: tgt_client.clone(),
                 src_bucket: &bucket,
@@ -2541,16 +2592,24 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 arn: &rinfo.arn,
                 put_opts,
             })
-            .await
-            .err()
+            .await;
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            if has_tagging_replication {
+                record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+            }
+            result.err()
         } else {
             gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
             let byte_stream = async_read_to_bytestream(gr.stream);
-            tgt_client
+            let result = tgt_client
                 .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                 .await
-                .map_err(|e| std::io::Error::other(e.to_string()))
-                .err()
+                .map_err(|e| std::io::Error::other(e.to_string()));
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            if has_tagging_replication {
+                record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+            }
+            result.err()
         } {
             rinfo.replication_status = ReplicationStatusType::Failed;
             rinfo.error = Some(err.to_string());
@@ -2690,9 +2749,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             warn!("failed to set replication tagging directive header: {err}");
         }
 
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2810,9 +2874,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 }
             };
 
+            let has_tagging_replication = !put_opts.user_tags.is_empty();
             if let Some(err) = if is_multipart {
                 drop(gr);
-                replicate_object_with_multipart(MultipartReplicationContext {
+                let result = replicate_object_with_multipart(MultipartReplicationContext {
                     storage: storage.clone(),
                     cli: tgt_client.clone(),
                     src_bucket: &bucket,
@@ -2823,16 +2888,24 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     arn: &rinfo.arn,
                     put_opts,
                 })
-                .await
-                .err()
+                .await;
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                if has_tagging_replication {
+                    record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+                }
+                result.err()
             } else {
                 gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
                 let byte_stream = async_read_to_bytestream(gr.stream);
-                tgt_client
+                let result = tgt_client
                     .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                     .await
-                    .map_err(|e| std::io::Error::other(e.to_string()))
-                    .err()
+                    .map_err(|e| std::io::Error::other(e.to_string()));
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                if has_tagging_replication {
+                    record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+                }
+                result.err()
             } {
                 rinfo.replication_status = ReplicationStatusType::Failed;
                 rinfo.error = Some(err.to_string());
@@ -3748,6 +3821,34 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_should_count_head_proxy_failure_ignores_not_found_and_405() {
+        assert!(
+            !should_count_head_proxy_failure(true, Some("NoSuchKey"), Some(404)),
+            "not-found heads are expected when the object has not reached the target yet"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("MethodNotAllowed"), Some(405)),
+            "405 delete-marker probing responses should not be counted as proxy failures"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("405"), Some(405)),
+            "numeric 405 codes must align with MethodNotAllowed semantics"
+        );
+    }
+
+    #[test]
+    fn test_should_count_head_proxy_failure_counts_unexpected_errors() {
+        assert!(
+            should_count_head_proxy_failure(false, Some("AccessDenied"), Some(403)),
+            "non-NotFound and non-405 service errors should be counted as failures"
+        );
+        assert!(
+            should_count_head_proxy_failure(false, None, Some(500)),
+            "raw 5xx head responses should be counted as proxy failures"
+        );
+    }
+
     #[tokio::test]
     async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
         let oi = ObjectInfo {

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -12,18 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::bucket::replication::get_global_replication_pool;
 use crate::error::Error;
 use crate::global::get_global_bucket_monitor;
 use rustfs_filemeta::{ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::interval;
 
+const ROLLING_WINDOW: Duration = Duration::from_secs(60);
+const FAILURE_LAST_HOUR_WINDOW: Duration = Duration::from_secs(60 * 60);
+
 /// Exponential Moving Average with thread-safe interior mutability
 #[derive(Debug)]
 pub struct ExponentialMovingAverage {
@@ -328,6 +332,13 @@ pub struct InQueueStats {
     pub now_count: AtomicI64,
 }
 
+#[derive(Debug, Clone)]
+struct QueueSample {
+    observed_at: Instant,
+    bytes: i64,
+    count: i64,
+}
+
 impl Clone for InQueueStats {
     fn clone(&self) -> Self {
         Self {
@@ -359,9 +370,60 @@ pub struct InQueueMetric {
     pub curr: InQueueStats,
     pub avg: InQueueStats,
     pub max: InQueueStats,
+    pub last_minute: InQueueStats,
+    #[serde(skip)]
+    samples: VecDeque<QueueSample>,
 }
 
 impl InQueueMetric {
+    fn observe(&mut self, observed_at: Instant) {
+        let bytes = self.curr.now_bytes.load(Ordering::Relaxed);
+        let count = self.curr.now_count.load(Ordering::Relaxed);
+
+        self.curr.bytes = bytes;
+        self.curr.count = count;
+        self.samples.push_back(QueueSample {
+            observed_at,
+            bytes,
+            count,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.avg = InQueueStats::default();
+            self.max = InQueueStats::default();
+            self.last_minute = InQueueStats::default();
+            return;
+        }
+
+        let sample_count = self.samples.len() as i64;
+        let total_bytes = self.samples.iter().map(|sample| sample.bytes).sum::<i64>();
+        let total_count = self.samples.iter().map(|sample| sample.count).sum::<i64>();
+        let max_bytes = self.samples.iter().map(|sample| sample.bytes).max().unwrap_or(0);
+        let max_count = self.samples.iter().map(|sample| sample.count).max().unwrap_or(0);
+
+        self.avg.bytes = total_bytes / sample_count;
+        self.avg.count = total_count / sample_count;
+        self.max.bytes = max_bytes;
+        self.max.count = max_count;
+        self.last_minute.bytes = self.avg.bytes;
+        self.last_minute.count = self.avg.count;
+    }
+
+    fn snapshot(&self) -> Self {
+        let mut snapshot = self.clone();
+        snapshot.curr.bytes = snapshot.curr.now_bytes.load(Ordering::Relaxed);
+        snapshot.curr.count = snapshot.curr.now_count.load(Ordering::Relaxed);
+        snapshot
+    }
+
     pub fn merge(&self, other: &InQueueMetric) -> Self {
         Self {
             curr: InQueueStats {
@@ -384,6 +446,12 @@ impl InQueueMetric {
                 count: self.max.count.max(other.max.count),
                 ..Default::default()
             },
+            last_minute: InQueueStats {
+                bytes: self.last_minute.bytes + other.last_minute.bytes,
+                count: self.last_minute.count + other.last_minute.count,
+                ..Default::default()
+            },
+            samples: VecDeque::new(),
         }
     }
 }
@@ -391,8 +459,8 @@ impl InQueueMetric {
 /// Queue cache
 #[derive(Debug, Default)]
 pub struct QueueCache {
-    pub bucket_stats: HashMap<String, InQueueStats>,
-    pub sr_queue_stats: InQueueStats,
+    pub bucket_stats: HashMap<String, InQueueMetric>,
+    pub sr_queue_stats: InQueueMetric,
 }
 
 impl QueueCache {
@@ -401,36 +469,19 @@ impl QueueCache {
     }
 
     pub fn update(&mut self) {
-        // Update queue statistics cache
-        // In actual implementation, this would get latest statistics from queue system
+        let observed_at = Instant::now();
+        self.sr_queue_stats.observe(observed_at);
+        for stats in self.bucket_stats.values_mut() {
+            stats.observe(observed_at);
+        }
     }
 
     pub fn get_bucket_stats(&self, bucket: &str) -> InQueueMetric {
-        if let Some(bucket_stat) = self.bucket_stats.get(bucket) {
-            InQueueMetric {
-                curr: InQueueStats {
-                    bytes: bucket_stat.now_bytes.load(Ordering::Relaxed),
-                    count: bucket_stat.now_count.load(Ordering::Relaxed),
-                    ..Default::default()
-                },
-                avg: InQueueStats::default(), // simplified implementation
-                max: InQueueStats::default(), // simplified implementation
-            }
-        } else {
-            InQueueMetric::default()
-        }
+        self.bucket_stats.get(bucket).map(InQueueMetric::snapshot).unwrap_or_default()
     }
 
     pub fn get_site_stats(&self) -> InQueueMetric {
-        InQueueMetric {
-            curr: InQueueStats {
-                bytes: self.sr_queue_stats.now_bytes.load(Ordering::Relaxed),
-                count: self.sr_queue_stats.now_count.load(Ordering::Relaxed),
-                ..Default::default()
-            },
-            avg: InQueueStats::default(), // simplified implementation
-            max: InQueueStats::default(), // simplified implementation
-        }
+        self.sr_queue_stats.snapshot()
     }
 }
 
@@ -438,8 +489,14 @@ impl QueueCache {
 pub struct ProxyMetric {
     pub get_total: i64,
     pub get_failed: i64,
+    pub get_tag_total: i64,
+    pub get_tag_failed: i64,
     pub put_total: i64,
     pub put_failed: i64,
+    pub put_tag_total: i64,
+    pub put_tag_failed: i64,
+    pub delete_tag_total: i64,
+    pub delete_tag_failed: i64,
     pub head_total: i64,
     pub head_failed: i64,
 }
@@ -448,8 +505,14 @@ impl ProxyMetric {
     pub fn add(&mut self, other: &ProxyMetric) {
         self.get_total += other.get_total;
         self.get_failed += other.get_failed;
+        self.get_tag_total += other.get_tag_total;
+        self.get_tag_failed += other.get_tag_failed;
         self.put_total += other.put_total;
         self.put_failed += other.put_failed;
+        self.put_tag_total += other.put_tag_total;
+        self.put_tag_failed += other.put_tag_failed;
+        self.delete_tag_total += other.delete_tag_total;
+        self.delete_tag_failed += other.delete_tag_failed;
         self.head_total += other.head_total;
         self.head_failed += other.head_failed;
     }
@@ -476,18 +539,36 @@ impl ProxyStatsCache {
                     metric.get_failed += 1;
                 }
             }
+            "GetObjectTagging" => {
+                metric.get_tag_total += 1;
+                if is_err {
+                    metric.get_tag_failed += 1;
+                }
+            }
             "PutObject" => {
                 metric.put_total += 1;
                 if is_err {
                     metric.put_failed += 1;
                 }
             }
+            "PutObjectTagging" => {
+                metric.put_tag_total += 1;
+                if is_err {
+                    metric.put_tag_failed += 1;
+                }
+            }
             "HeadObject" => {
                 metric.head_total += 1;
                 if is_err {
                     metric.head_failed += 1;
                 }
             }
+            "DeleteObjectTagging" => {
+                metric.delete_tag_total += 1;
+                if is_err {
+                    metric.delete_tag_failed += 1;
+                }
+            }
             _ => {}
         }
     }
@@ -505,11 +586,19 @@ impl ProxyStatsCache {
     }
 }
 
+#[derive(Debug, Clone)]
+struct FailureSample {
+    observed_at: Instant,
+    size: i64,
+}
+
 /// Failure statistics
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct FailStats {
     pub count: i64,
     pub size: i64,
+    #[serde(skip)]
+    recent: VecDeque<FailureSample>,
 }
 
 impl FailStats {
@@ -518,14 +607,42 @@ impl FailStats {
     }
 
     pub fn add_size(&mut self, size: i64, _err: Option<&Error>) {
+        let observed_at = Instant::now();
         self.count += 1;
         self.size += size;
+        self.recent.push_back(FailureSample { observed_at, size });
+        self.prune(observed_at);
+    }
+
+    fn prune(&mut self, observed_at: Instant) {
+        while self
+            .recent
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > FAILURE_LAST_HOUR_WINDOW)
+        {
+            self.recent.pop_front();
+        }
+    }
+
+    pub fn recent_since(&self, window: Duration) -> FailedMetric {
+        let now = Instant::now();
+        let mut count = 0i64;
+        let mut size = 0i64;
+        for sample in self.recent.iter().rev() {
+            if now.duration_since(sample.observed_at) > window {
+                break;
+            }
+            count += 1;
+            size += sample.size;
+        }
+        FailedMetric { count, size }
     }
 
     pub fn merge(&self, other: &FailStats) -> Self {
         Self {
             count: self.count + other.count,
             size: self.size + other.size,
+            recent: VecDeque::new(),
         }
     }
 
@@ -674,6 +791,14 @@ pub struct ActiveWorkerStat {
     pub curr: i32,
     pub max: i32,
     pub avg: f64,
+    #[serde(skip)]
+    samples: VecDeque<WorkerSample>,
+}
+
+#[derive(Debug, Clone)]
+struct WorkerSample {
+    observed_at: Instant,
+    workers: i32,
 }
 
 impl ActiveWorkerStat {
@@ -685,9 +810,31 @@ impl ActiveWorkerStat {
         self.clone()
     }
 
-    pub fn update(&mut self) {
-        // Simulate worker statistics update logic
-        // In actual implementation, this would get current active count from worker pool
+    pub fn update(&mut self, curr: i32) {
+        let observed_at = Instant::now();
+        self.curr = curr;
+        self.samples.push_back(WorkerSample {
+            observed_at,
+            workers: curr,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.max = curr;
+            self.avg = curr as f64;
+            return;
+        }
+
+        self.max = self.samples.iter().map(|sample| sample.workers).max().unwrap_or(curr);
+        let total = self.samples.iter().map(|sample| sample.workers as i64).sum::<i64>();
+        self.avg = total as f64 / self.samples.len() as f64;
     }
 }
 
@@ -740,8 +887,11 @@ impl ReplicationStats {
             let mut interval = interval(Duration::from_secs(2));
             loop {
                 interval.tick().await;
+                let current = get_global_replication_pool()
+                    .map(|pool| pool.active_workers() + pool.active_lrg_workers() + pool.active_mrf_workers())
+                    .unwrap_or(0);
                 let mut workers = workers_clone.lock().await;
-                workers.update();
+                workers.update(current);
             }
         });
 
@@ -925,14 +1075,26 @@ impl ReplicationStats {
     /// Get replication metrics for all buckets
     pub async fn get_all(&self) -> HashMap<String, BucketReplicationStats> {
         let cache = self.cache.read().await;
-        let mut result = HashMap::new();
+        let mut result = HashMap::with_capacity(cache.len());
 
         for (bucket, stats) in cache.iter() {
-            let mut cloned_stats = stats.clone_stats();
-            // Add queue statistics
+            result.insert(bucket.clone(), stats.clone_stats());
+        }
+        drop(cache);
+
+        {
             let q_cache = self.q_cache.lock().await;
-            cloned_stats.q_stat = q_cache.get_bucket_stats(bucket);
-            result.insert(bucket.clone(), cloned_stats);
+            for (bucket, queue_stats) in &q_cache.bucket_stats {
+                let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+                bucket_stats.q_stat = queue_stats.snapshot();
+            }
+        }
+
+        {
+            let p_cache = self.p_cache.lock().await;
+            for bucket in p_cache.bucket_stats.keys() {
+                result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+            }
         }
 
         result
@@ -1114,12 +1276,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        stats.now_count.fetch_add(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_add(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
     }
 
     /// Decrease queue statistics
@@ -1128,12 +1290,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        stats.now_count.fetch_sub(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_sub(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
     }
 
     /// Increase proxy metrics
@@ -1166,6 +1328,51 @@ mod tests {
         assert_eq!(workers.curr, 0);
     }
 
+    #[test]
+    fn test_in_queue_metric_observe_updates_rolling_stats() {
+        let mut metric = InQueueMetric::default();
+        metric.curr.now_bytes.store(128, Ordering::Relaxed);
+        metric.curr.now_count.store(4, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        metric.curr.now_bytes.store(256, Ordering::Relaxed);
+        metric.curr.now_count.store(6, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        assert_eq!(metric.curr.bytes, 256);
+        assert_eq!(metric.curr.count, 6);
+        assert_eq!(metric.max.bytes, 256);
+        assert_eq!(metric.max.count, 6);
+        assert_eq!(metric.last_minute.bytes, 192);
+        assert_eq!(metric.last_minute.count, 5);
+    }
+
+    #[test]
+    fn test_fail_stats_recent_since_tracks_windows() {
+        let mut stats = FailStats::default();
+        stats.add_size(64, None);
+        stats.add_size(32, None);
+
+        let last_minute = stats.recent_since(Duration::from_secs(60));
+        let last_hour = stats.recent_since(Duration::from_secs(60 * 60));
+        assert_eq!(last_minute.count, 2);
+        assert_eq!(last_minute.size, 96);
+        assert_eq!(last_hour.count, 2);
+        assert_eq!(last_hour.size, 96);
+    }
+
+    #[test]
+    fn test_active_worker_stat_update_tracks_rolling_avg_and_max() {
+        let mut stats = ActiveWorkerStat::default();
+        stats.update(2);
+        stats.update(6);
+        stats.update(4);
+
+        assert_eq!(stats.curr, 4);
+        assert_eq!(stats.max, 6);
+        assert_eq!(stats.avg, 4.0);
+    }
+
     #[tokio::test]
     async fn test_delete_bucket_stats() {
         let stats = ReplicationStats::new();
@@ -1218,6 +1425,15 @@ mod tests {
         assert_eq!(stat.replicated_count, 1);
     }
 
+    #[tokio::test]
+    async fn test_get_all_includes_proxy_only_bucket() {
+        let stats = ReplicationStats::new();
+        stats.inc_proxy("proxy-only-bucket", "HeadObject", false).await;
+
+        let all = stats.get_all().await;
+        assert!(all.contains_key("proxy-only-bucket"));
+    }
+
     #[test]
     fn test_sr_stats() {
         let sr_stats = SRStats::new();

crates/ecstore/src/client/bucket_cache.rs

@@ -22,6 +22,7 @@ use super::constants::UNSIGNED_PAYLOAD;
 use super::credentials::SignatureType;
 use crate::client::{
     api_error_response::http_resp_to_error_response,
+    signer_error,
     transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
 };
 use http::Request;
@@ -35,6 +36,10 @@ use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
 use s3s::S3ErrorCode;
 use std::collections::HashMap;
 
+fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> std::io::Error {
+    signer_error::signer_error_to_io_error(scope, error)
+}
+
 #[derive(Debug, Clone)]
 pub struct BucketLocationCache {
     items: HashMap<String, String>,
@@ -179,10 +184,15 @@ impl TransitionClient {
             content_sha256 = UNSIGNED_PAYLOAD.to_string();
         }
 
-        if let Ok(content_sha256_value) = content_sha256.parse() {
-            req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
-        }
-        let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
+        let content_sha256_value = content_sha256.parse().map_err(|err| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                format!("invalid X-Amz-Content-Sha256 header value: {err}"),
+            )
+        })?;
+        req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
+        let req = rustfs_signer::try_sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1")
+            .map_err(|err| signer_error_to_io_error("failed to sign bucket location request", err))?;
         Ok(req)
     }
 }

crates/ecstore/src/client/mod.rs

@@ -35,5 +35,6 @@ pub mod constants;
 pub mod credentials;
 pub mod object_api_utils;
 pub mod object_handlers_common;
+pub mod signer_error;
 pub mod transition_api;
 pub mod utils;

crates/ecstore/src/client/signer_error.rs

@@ -0,0 +1,105 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::error::Error as StdError;
+use std::fmt::{Display, Formatter};
+use std::io::{Error, ErrorKind};
+
+pub(crate) const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error";
+
+#[derive(Debug)]
+struct SignerHeaderError {
+    scope: String,
+    header_name: String,
+}
+
+impl SignerHeaderError {
+    fn new(scope: &str, header_name: &str) -> Self {
+        Self {
+            scope: scope.to_string(),
+            header_name: header_name.to_string(),
+        }
+    }
+}
+
+impl Display for SignerHeaderError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}: invalid UTF-8 header value for `{}` [{}]",
+            self.scope, self.header_name, SIGNER_HEADER_ERROR_MARKER
+        )
+    }
+}
+
+impl StdError for SignerHeaderError {}
+
+pub(crate) fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error {
+    Error::new(ErrorKind::InvalidInput, SignerHeaderError::new(scope, header_name))
+}
+
+pub(crate) fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error {
+    match error {
+        rustfs_signer::SignV4Error::InvalidHeaderValue { name } => invalid_utf8_header_error(scope, &name),
+        other => Error::other(format!("{scope}: {other}")),
+    }
+}
+
+pub(crate) fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool {
+    let mut current = Some(err);
+    while let Some(source) = current {
+        if source.downcast_ref::<SignerHeaderError>().is_some() {
+            return true;
+        }
+
+        if source.to_string().contains(SIGNER_HEADER_ERROR_MARKER) {
+            return true;
+        }
+
+        current = source.source();
+    }
+
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{error_chain_contains_signer_header_marker, invalid_utf8_header_error, signer_error_to_io_error};
+
+    #[test]
+    fn invalid_utf8_header_error_is_detected_through_error_chain() {
+        let err = invalid_utf8_header_error("failed to sign request", "x-amz-meta-invalid");
+
+        assert!(error_chain_contains_signer_header_marker(&err));
+    }
+
+    #[test]
+    fn mapped_signer_header_error_is_detected_through_error_chain() {
+        let err = signer_error_to_io_error(
+            "failed to sign request",
+            rustfs_signer::SignV4Error::InvalidHeaderValue {
+                name: "x-amz-meta-invalid".to_string(),
+            },
+        );
+
+        assert!(error_chain_contains_signer_header_marker(&err));
+    }
+
+    #[test]
+    fn generic_io_errors_do_not_match_signer_header_marker() {
+        let err = std::io::Error::other("unrelated failure");
+
+        assert!(!error_chain_contains_signer_header_marker(&err));
+    }
+}

crates/ecstore/src/client/transition_api.rs

@@ -31,6 +31,7 @@ use crate::client::{
     },
     constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
     credentials::{CredContext, Credentials, SignatureType, Static},
+    signer_error,
 };
 use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
 use futures::{Future, StreamExt};
@@ -85,6 +86,21 @@ const C_UNKNOWN: i32 = -1;
 const C_OFFLINE: i32 = 0;
 const C_ONLINE: i32 = 1;
 
+fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
+    signer_error::invalid_utf8_header_error(scope, header_name)
+}
+
+fn validate_header_values(headers: &HeaderMap, scope: &str) -> Result<(), std::io::Error> {
+    for (name, value) in headers {
+        value.to_str().map_err(|_| invalid_utf8_header_error(scope, name.as_str()))?;
+    }
+    Ok(())
+}
+
+fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> std::io::Error {
+    signer_error::signer_error_to_io_error(scope, error)
+}
+
 //pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>;
 pub enum ReaderImpl {
     Body(Bytes),
@@ -560,8 +576,9 @@ impl TransitionClient {
                         "extra signed headers for presign with signature v2 is not supported.",
                     )));
                 }
-                let headers = req.headers_mut();
                 if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
+                    validate_header_values(extra_headers, "presign extra header")?;
+                    let headers = req.headers_mut();
                     for (k, v) in extra_headers {
                         headers.insert(k, v.clone());
                     }
@@ -570,7 +587,7 @@ impl TransitionClient {
             if signer_type == SignatureType::SignatureV2 {
                 req = rustfs_signer::pre_sign_v2(req, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
             } else if signer_type == SignatureType::SignatureV4 {
-                req = rustfs_signer::pre_sign_v4(
+                req = rustfs_signer::try_pre_sign_v4(
                     req,
                     &access_key_id,
                     &secret_access_key,
@@ -578,12 +595,14 @@ impl TransitionClient {
                     &location,
                     metadata.expires,
                     OffsetDateTime::now_utc(),
-                );
+                )
+                .map_err(|err| signer_error_to_io_error("failed to presign v4 request", err))?;
             }
             return Ok(req);
         }
 
         self.set_user_agent(&mut req);
+        validate_header_values(&metadata.custom_header, "request custom header")?;
 
         for (k, v) in metadata.custom_header.clone() {
             if let Some(key) = k {
@@ -593,15 +612,15 @@ impl TransitionClient {
 
         //req.content_length = metadata.content_length;
         if metadata.content_length <= -1 {
-            if let Ok(chunked_value) = HeaderValue::from_str(&vec!["chunked"].join(",")) {
-                req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
-            }
+            req.headers_mut()
+                .insert(http::header::TRANSFER_ENCODING, HeaderValue::from_static("chunked"));
         }
 
-        if metadata.content_md5_base64.len() > 0 {
-            if let Ok(md5_value) = HeaderValue::from_str(&metadata.content_md5_base64) {
-                req.headers_mut().insert("Content-Md5", md5_value);
-            }
+        if !metadata.content_md5_base64.is_empty() {
+            let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).map_err(|err| {
+                std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("invalid Content-Md5 header value: {err}"))
+            })?;
+            req.headers_mut().insert("Content-Md5", md5_value);
         }
 
         if signer_type == SignatureType::SignatureAnonymous {
@@ -634,14 +653,15 @@ impl TransitionClient {
                 .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
             req.headers_mut().insert(header_name, header_value);
 
-            req = rustfs_signer::sign_v4_trailer(
+            req = rustfs_signer::try_sign_v4_trailer(
                 req,
                 &access_key_id,
                 &secret_access_key,
                 &session_token,
                 &location,
                 metadata.trailer.clone(),
-            );
+            )
+            .map_err(|err| signer_error_to_io_error("failed to sign v4 request", err))?;
         }
 
         if metadata.content_length > 0 {
@@ -1354,7 +1374,10 @@ pub struct CreateBucketConfiguration {
 
 #[cfg(test)]
 mod tests {
-    use super::{build_tls_config, load_root_store_from_tls_path, with_rustls_init_guard};
+    use super::{
+        build_tls_config, load_root_store_from_tls_path, signer_error_to_io_error, validate_header_values, with_rustls_init_guard,
+    };
+    use http::{HeaderMap, HeaderValue};
 
     #[test]
     fn rustls_guard_converts_panics_to_io_errors() {
@@ -1404,4 +1427,29 @@ mod tests {
         });
         assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
     }
+
+    #[test]
+    fn validate_header_values_returns_header_name_for_non_utf8_values() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            "x-amz-meta-invalid",
+            HeaderValue::from_bytes(&[0xFF]).expect("invalid utf8 bytes should be accepted by HeaderValue"),
+        );
+
+        let err =
+            validate_header_values(&headers, "request custom header").expect_err("invalid header value should fail validation");
+        assert!(err.to_string().contains("x-amz-meta-invalid"));
+    }
+
+    #[test]
+    fn signer_error_mapping_preserves_header_name() {
+        let err = signer_error_to_io_error(
+            "failed to sign v4 request",
+            rustfs_signer::SignV4Error::InvalidHeaderValue {
+                name: "x-amz-meta-invalid".to_string(),
+            },
+        );
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(err.to_string().contains("x-amz-meta-invalid"));
+    }
 }

crates/ecstore/src/config/audit.rs

@@ -14,15 +14,16 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
-    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
-    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
-    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
-    PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
-    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
-    WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -337,3 +338,63 @@ pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/com.rs

@@ -19,12 +19,12 @@ use crate::global::is_first_cluster_node_local;
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
 use rustfs_config::audit::{
-    AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
-    AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
+    AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::notify::{
-    NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS,
-    NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+    NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
 use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
@@ -58,7 +58,7 @@ struct TargetConfigDescriptor {
     valid_keys: &'static [&'static str],
 }
 
-fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
+fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
     [
         TargetConfigDescriptor {
             external_key: "webhook",
@@ -66,6 +66,12 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
             default_kvs: &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
             valid_keys: NOTIFY_WEBHOOK_KEYS,
         },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: NOTIFY_KAFKA_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_KAFKA_KVS,
+            valid_keys: NOTIFY_KAFKA_KEYS,
+        },
         TargetConfigDescriptor {
             external_key: "mqtt",
             subsystem_key: NOTIFY_MQTT_SUB_SYS,
@@ -87,7 +93,7 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
     ]
 }
 
-fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
+fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] {
     [
         TargetConfigDescriptor {
             external_key: "webhook",
@@ -95,6 +101,12 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
             default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
             valid_keys: AUDIT_WEBHOOK_KEYS,
         },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: AUDIT_KAFKA_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_KAFKA_KVS,
+            valid_keys: AUDIT_KAFKA_KEYS,
+        },
         TargetConfigDescriptor {
             external_key: "mqtt",
             subsystem_key: AUDIT_MQTT_SUB_SYS,
@@ -335,7 +347,7 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
     applied
 }
 
-fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
+fn parse_target_scalar_value(key: &str, value: &Value) -> Option<String> {
     match value {
         Value::String(v) => Some(v.trim().to_string()),
         Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v {
@@ -350,7 +362,7 @@ fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
     }
 }
 
-fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
+fn decode_target_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
     let mut kvs = KVS::new();
 
     for (key, value) in instance {
@@ -358,7 +370,7 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
             continue;
         }
 
-        if let Some(parsed) = parse_notify_scalar_value(key, value) {
+        if let Some(parsed) = parse_target_scalar_value(key, value) {
             kvs.insert(key.clone(), parsed);
         }
     }
@@ -366,21 +378,21 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
     kvs
 }
 
-fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
+fn decode_target_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
     match value {
-        Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)),
+        Value::Object(instance) => Some(decode_target_instance_object(instance, valid_keys)),
         Value::Array(_) => serde_json::from_value::<KVS>(value.clone()).ok(),
         _ => None,
     }
 }
 
-fn is_notify_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
+fn is_target_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
     section
         .iter()
-        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some())
+        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_target_scalar_value(key, value).is_some())
 }
 
-fn apply_external_notify_section(
+fn apply_external_target_section(
     cfg: &mut Config,
     notify_obj: &Map<String, Value>,
     external_key: &str,
@@ -399,8 +411,8 @@ fn apply_external_notify_section(
     let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default();
     let mut applied = false;
 
-    if is_notify_instance_shorthand(section_obj, valid_keys) {
-        let kvs = decode_notify_instance_object(section_obj, valid_keys);
+    if is_target_instance_shorthand(section_obj, valid_keys) {
+        let kvs = decode_target_instance_object(section_obj, valid_keys);
         if !kvs.is_empty() {
             let mut merged = default_kvs.clone();
             merged.extend(kvs);
@@ -411,7 +423,7 @@ fn apply_external_notify_section(
     }
 
     for (raw_instance, value) in section_obj {
-        let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else {
+        let Some(mut kvs) = decode_target_instance_value(value, valid_keys) else {
             continue;
         };
         if kvs.is_empty() {
@@ -444,7 +456,7 @@ fn apply_external_target_descriptors(
 ) -> bool {
     let mut applied = false;
     for descriptor in descriptors {
-        applied |= apply_external_notify_section(
+        applied |= apply_external_target_section(
             cfg,
             section_obj,
             descriptor.external_key,
@@ -663,11 +675,12 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     oidc_obj
 }
 
-fn is_notify_bool_key(key: &str) -> bool {
+fn is_target_bool_key(key: &str) -> bool {
     matches!(
         key,
         ENABLE_KEY
             | rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+            | rustfs_config::KAFKA_TLS_ENABLE
             | rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
             | rustfs_config::NATS_TLS_REQUIRED
             | rustfs_config::PULSAR_TLS_ALLOW_INSECURE
@@ -675,8 +688,8 @@ fn is_notify_bool_key(key: &str) -> bool {
     )
 }
 
-fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
-    if is_notify_bool_key(key) {
+fn encode_target_scalar_value(key: &str, value: &str) -> Value {
+    if is_target_bool_key(key) {
         if let Ok(state) = value.parse::<EnableState>() {
             return Value::Bool(state.is_enabled());
         }
@@ -697,7 +710,7 @@ fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool {
         .unwrap_or(false)
 }
 
-fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
+fn build_target_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
     let mut instance = Map::new();
 
     for key in valid_keys {
@@ -720,13 +733,13 @@ fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&s
             continue;
         }
 
-        instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value));
+        instance.insert((*key).to_string(), encode_target_scalar_value(key, &effective_value));
     }
 
     instance
 }
 
-fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
+fn merged_target_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
     let mut merged = default_kvs.clone();
     if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) {
         merged.extend(kvs.clone());
@@ -734,7 +747,7 @@ fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS
     merged
 }
 
-fn build_notify_subsystem_object(
+fn build_target_subsystem_object(
     cfg: &Config,
     subsystem_key: &str,
     default_kvs: &KVS,
@@ -744,11 +757,11 @@ fn build_notify_subsystem_object(
         return Map::new();
     };
 
-    let effective_default = merged_notify_default_kvs(subsystem, default_kvs);
+    let effective_default = merged_target_default_kvs(subsystem, default_kvs);
     let mut subsystem_obj = Map::new();
 
     if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) {
-        let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
+        let default_obj = build_target_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
         if !default_obj.is_empty() {
             subsystem_obj.insert("default".to_string(), Value::Object(default_obj));
         }
@@ -761,7 +774,7 @@ fn build_notify_subsystem_object(
     instances.sort_by_key(|(lhs, _)| *lhs);
 
     for (instance_key, kvs) in instances {
-        let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
+        let instance_obj = build_target_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
         if !instance_obj.is_empty() {
             subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj));
         }
@@ -774,7 +787,7 @@ fn build_target_object(cfg: &Config, descriptors: &[TargetConfigDescriptor]) ->
     let mut target_obj = Map::new();
     for descriptor in descriptors {
         let subsystem_obj =
-            build_notify_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
+            build_target_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
         if !subsystem_obj.is_empty() {
             target_obj.insert(descriptor.external_key.to_string(), Value::Object(subsystem_obj));
         }
@@ -1118,8 +1131,8 @@ mod tests {
         ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
     };
     use http::HeaderMap;
-    use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-    use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+    use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
+    use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
     use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
     use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
     use rustfs_filemeta::FileInfo;
@@ -1753,6 +1766,15 @@ mod tests {
                 "topic":"events",
                 "queue_dir":""
               }
+            },
+            "kafka":{
+              "streaming":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092,127.0.0.1:9093",
+                "topic":"events-kafka",
+                "acks":"all",
+                "tls_enable":true
+              }
             }
           }
         }"#;
@@ -1781,6 +1803,14 @@ mod tests {
             .expect("mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), "");
+
+        let kafka = cfg
+            .get_value(NOTIFY_KAFKA_SUB_SYS, "streaming")
+            .expect("kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092,127.0.0.1:9093");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "events-kafka");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true");
     }
 
     #[test]
@@ -1850,6 +1880,14 @@ mod tests {
                 "broker":"tcp://127.0.0.1:1883",
                 "topic":"audit-events"
               }
+            },
+            "kafka":{
+              "auditlog":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092",
+                "topic":"audit-events-kafka",
+                "acks":"1"
+              }
             }
           }
         }"#;
@@ -1873,6 +1911,12 @@ mod tests {
             .get_value(AUDIT_MQTT_SUB_SYS, "analytics")
             .expect("audit mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
+
+        let kafka = cfg
+            .get_value(AUDIT_KAFKA_SUB_SYS, "auditlog")
+            .expect("audit kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "audit-events-kafka");
     }
 
     #[test]
@@ -1993,6 +2037,38 @@ mod tests {
         );
         cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = notify::DEFAULT_NOTIFY_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "streaming".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092,127.0.0.1:9093".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_ACKS.to_string(),
+                    value: "all".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_TLS_ENABLE.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(NOTIFY_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let notify = v
@@ -2028,6 +2104,19 @@ mod tests {
             .expect("mqtt target should be encoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883"));
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some(""));
+
+        let kafka = notify
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("streaming"))
+            .and_then(Value::as_object)
+            .expect("kafka target should be encoded");
+        assert_eq!(
+            kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str),
+            Some("127.0.0.1:9092,127.0.0.1:9093")
+        );
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS).and_then(Value::as_str), Some("all"));
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE).and_then(Value::as_bool), Some(true));
     }
 
     #[test]
@@ -2079,6 +2168,28 @@ mod tests {
         );
         cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = audit::DEFAULT_AUDIT_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "audit-events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "auditlog".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(AUDIT_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let logger = v
@@ -2105,6 +2216,14 @@ mod tests {
             .expect("audit mqtt default should be encoded");
         assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
         assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events"));
+
+        let kafka = logger
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("auditlog"))
+            .and_then(Value::as_object)
+            .expect("audit kafka target should be encoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str), Some("127.0.0.1:9092"));
     }
 
     #[test]

crates/ecstore/src/config/mod.rs

@@ -25,8 +25,12 @@ use crate::store::ECStore;
 use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
 use rustfs_config::COMMENT_KEY;
 use rustfs_config::DEFAULT_DELIMITER;
-use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{
+    AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -245,6 +249,8 @@ pub fn init() {
     kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
     kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
     kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
+    kvs.insert(NOTIFY_KAFKA_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_KAFKA_KVS.clone());
+    kvs.insert(AUDIT_KAFKA_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_KAFKA_KVS.clone());
     kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
 
     // Register all default configurations

crates/ecstore/src/config/notify.rs

@@ -14,14 +14,15 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
-    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
-    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
-    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
-    PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
+    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -314,3 +315,63 @@ pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/disk/disk_store.rs

@@ -46,9 +46,7 @@ const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
-pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
 
 lazy_static::lazy_static! {
     static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]);
@@ -104,6 +102,20 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
     )
 }
 
+pub fn get_drive_active_check_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_active_check_timeout() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
+    ))
+}
+
 /// DiskHealthTracker tracks the health status of a disk.
 /// Similar to Go's diskHealthTracker.
 #[derive(Debug)]
@@ -231,9 +243,26 @@ impl DiskHealthTracker {
             RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
         };
 
-        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        let became_offline = next == RuntimeDriveHealthState::Offline && current != RuntimeDriveHealthState::Offline;
+        if next == RuntimeDriveHealthState::Offline {
+            self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        } else {
+            self.status.store(DISK_HEALTH_OK, Ordering::Release);
+        }
         self.transition_state(endpoint, current, next, reason);
-        current == RuntimeDriveHealthState::Online
+        became_offline
+    }
+
+    pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        if current == RuntimeDriveHealthState::Offline {
+            return false;
+        }
+
+        self.consecutive_successes.store(0, Ordering::Release);
+        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        self.transition_state(endpoint, current, RuntimeDriveHealthState::Offline, reason);
+        true
     }
 
     pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
@@ -268,6 +297,14 @@ impl DiskHealthTracker {
         became_online
     }
 
+    pub fn record_operation_success(&self, endpoint: &Endpoint, reason: &'static str) {
+        if self.runtime_state() == RuntimeDriveHealthState::Online {
+            self.log_success();
+        } else {
+            self.mark_recovery_success(endpoint, reason);
+        }
+    }
+
     fn transition_state(
         &self,
         endpoint: &Endpoint,
@@ -449,7 +486,7 @@ impl LocalDiskWrapper {
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         // TODO: config interval
 
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());
 
         loop {
             tokio::select! {
@@ -482,7 +519,16 @@ impl LocalDiskWrapper {
 
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                    if Self::perform_health_check(
+                        disk.clone(),
+                        &TEST_BUCKET,
+                        &test_obj,
+                        &TEST_DATA,
+                        true,
+                        get_drive_active_check_timeout(),
+                    )
+                    .await
+                    .is_err()
                         && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
                     {
                         // Health check failed, disk is considered faulty
@@ -588,7 +634,16 @@ impl LocalDiskWrapper {
                     }
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
+                    match Self::perform_health_check(
+                        disk.clone(),
+                        &TEST_BUCKET,
+                        &test_obj,
+                        &TEST_DATA,
+                        false,
+                        get_drive_active_check_timeout(),
+                    )
+                    .await
+                    {
                         Ok(_) => {
                             let state_before = health.runtime_state();
                             let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
@@ -707,7 +762,7 @@ impl LocalDiskWrapper {
             let result = operation().await;
             self.health.decrement_waiting();
             if result.is_ok() {
-                self.health.log_success();
+                self.health.record_operation_success(&self.endpoint(), "operation_success");
             }
             return result;
         }
@@ -718,7 +773,7 @@ impl LocalDiskWrapper {
             Ok(operation_result) => {
                 // Log success and decrement waiting counter
                 if operation_result.is_ok() {
-                    self.health.log_success();
+                    self.health.record_operation_success(&self.endpoint(), "operation_success");
                 }
                 self.health.decrement_waiting();
                 operation_result
@@ -919,7 +974,7 @@ impl DiskAPI for LocalDiskWrapper {
         let has_err = result.iter().any(|e| e.is_some());
         if !has_err {
             // Log success and decrement waiting counter
-            self.health.log_success();
+            self.health.record_operation_success(&self.endpoint(), "operation_success");
         }
 
         result
@@ -1112,37 +1167,92 @@ mod tests {
         });
     }
 
+    #[test]
+    fn drive_active_check_interval_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, || {
+            assert_eq!(
+                get_drive_active_check_interval(),
+                Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS)
+            );
+        });
+    }
+
+    #[test]
+    fn drive_active_check_interval_reads_env_override() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, Some("3"), || {
+            assert_eq!(get_drive_active_check_interval(), Duration::from_secs(3));
+        });
+    }
+
+    #[test]
+    fn drive_active_check_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, || {
+            assert_eq!(
+                get_drive_active_check_timeout(),
+                Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS)
+            );
+        });
+    }
+
+    #[test]
+    fn drive_active_check_timeout_reads_env_override() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1"), || {
+            assert_eq!(get_drive_active_check_timeout(), Duration::from_secs(1));
+        });
+    }
+
     #[test]
     fn runtime_state_transitions_from_online_to_suspect_then_offline() {
-        let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
-        let health = DiskHealthTracker::new();
+        temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || {
+            let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
+            let health = DiskHealthTracker::new();
 
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
-        assert!(health.mark_failure(&endpoint, "timeout"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+            assert!(!health.mark_failure(&endpoint, "timeout"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+            assert!(!health.is_faulty());
 
-        assert!(!health.mark_failure(&endpoint, "timeout"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
-        assert!(health.offline_duration().is_some());
+            assert!(health.mark_failure(&endpoint, "timeout"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+            assert!(health.is_faulty());
+            assert!(health.offline_duration().is_some());
+        });
     }
 
     #[test]
     fn runtime_state_transitions_back_online_after_recovery_threshold() {
-        let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+        temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || {
+            let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+            let health = DiskHealthTracker::new();
+
+            health.mark_failure(&endpoint, "timeout");
+            health.mark_failure(&endpoint, "timeout");
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+            assert!(!health.mark_recovery_success(&endpoint, "probe"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+            assert!(!health.mark_recovery_success(&endpoint, "probe"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+            assert!(health.mark_recovery_success(&endpoint, "probe"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+            assert!(health.offline_duration().is_none());
+        });
+    }
+
+    #[test]
+    fn operation_success_recovers_suspect_drive_without_faulting() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-suspect-success").expect("endpoint should parse");
         let health = DiskHealthTracker::new();
 
-        health.mark_failure(&endpoint, "timeout");
-        health.mark_failure(&endpoint, "timeout");
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+        assert!(!health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+        assert!(!health.is_faulty());
 
-        assert!(!health.mark_recovery_success(&endpoint, "probe"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
-
-        assert!(!health.mark_recovery_success(&endpoint, "probe"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
-
-        assert!(health.mark_recovery_success(&endpoint, "probe"));
+        health.record_operation_success(&endpoint, "operation_success");
         assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(!health.is_faulty());
         assert!(health.offline_duration().is_none());
     }
 }

crates/ecstore/src/disk/local.rs

@@ -31,6 +31,7 @@ use crate::disk::{
 use crate::erasure_coding::bitrot_verify;
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
 use bytes::Bytes;
+use metrics::counter;
 use parking_lot::RwLock as ParkingLotRwLock;
 use rustfs_filemeta::{
     Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
@@ -79,6 +80,238 @@ pub enum InternalBuf<'a> {
     Owned(Bytes),
 }
 
+struct FileCacheReclaimWriter {
+    inner: File,
+    reclaim_len: usize,
+    reclaim_on_shutdown: bool,
+    reclaimed: bool,
+}
+
+struct FileCacheReclaimReader {
+    inner: File,
+    reclaim_offset: u64,
+    reclaim_len: usize,
+    reclaim_on_drop: bool,
+    reclaimed: bool,
+}
+
+fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
+    counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "ok".to_string()).increment(1);
+    counter!("rustfs_page_cache_reclaim_bytes_total", "kind" => kind.to_string()).increment(reclaim_len as u64);
+    metrics::histogram!("rustfs_page_cache_reclaim_duration_seconds", "kind" => kind.to_string())
+        .record(started.elapsed().as_secs_f64());
+}
+
+fn record_file_cache_reclaim_error(kind: &'static str) {
+    counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "err".to_string()).increment(1);
+}
+
+impl FileCacheReclaimReader {
+    fn new(inner: File, reclaim_offset: u64, reclaim_len: usize, reclaim_on_drop: bool) -> Self {
+        #[cfg(target_os = "macos")]
+        if reclaim_on_drop {
+            let _ = set_fd_nocache(&inner);
+        }
+
+        Self {
+            inner,
+            reclaim_offset,
+            reclaim_len,
+            reclaim_on_drop,
+            reclaimed: false,
+        }
+    }
+
+    #[cfg(target_os = "linux")]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        use core::num::NonZeroU64;
+        use rustix::fs::{Advice, fadvise};
+
+        if !self.reclaim_on_drop || self.reclaimed || self.reclaim_len == 0 {
+            return Ok(());
+        }
+
+        let started = std::time::Instant::now();
+        let reclaim_len =
+            NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return");
+        fadvise(&self.inner, self.reclaim_offset, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?;
+
+        self.reclaimed = true;
+        record_file_cache_reclaim_success("read", self.reclaim_len, started);
+        Ok(())
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg(target_os = "macos")]
+#[allow(unsafe_code)]
+fn set_fd_nocache(file: &File) -> std::io::Result<()> {
+    use std::os::fd::AsRawFd;
+
+    // SAFETY: `fcntl` is called on a valid file descriptor owned by `file`.
+    let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) };
+    if ret == -1 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(())
+}
+
+#[cfg(target_os = "macos")]
+#[allow(unsafe_code)]
+fn set_std_fd_nocache(file: &std::fs::File) -> std::io::Result<()> {
+    use std::os::fd::AsRawFd;
+
+    // SAFETY: `fcntl` is called on a valid file descriptor owned by `file`.
+    let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) };
+    if ret == -1 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(())
+}
+
+impl Drop for FileCacheReclaimReader {
+    fn drop(&mut self) {
+        if let Err(err) = self.reclaim_file_cache() {
+            record_file_cache_reclaim_error("read");
+            debug!(error = ?err, reclaim_offset = self.reclaim_offset, reclaim_len = self.reclaim_len, "failed to reclaim file cache after read");
+        }
+    }
+}
+
+impl tokio::io::AsyncRead for FileCacheReclaimReader {
+    fn poll_read(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
+    }
+}
+
+impl FileCacheReclaimWriter {
+    fn new(inner: File, reclaim_len: usize, reclaim_on_shutdown: bool) -> Self {
+        #[cfg(target_os = "macos")]
+        if reclaim_on_shutdown {
+            let _ = set_fd_nocache(&inner);
+        }
+
+        Self {
+            inner,
+            reclaim_len,
+            reclaim_on_shutdown,
+            reclaimed: false,
+        }
+    }
+
+    #[cfg(target_os = "linux")]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        use core::num::NonZeroU64;
+        use rustix::fs::{Advice, fadvise};
+
+        if !self.reclaim_on_shutdown || self.reclaimed || self.reclaim_len == 0 {
+            return Ok(());
+        }
+
+        let started = std::time::Instant::now();
+        let reclaim_len =
+            NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return");
+        fadvise(&self.inner, 0, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?;
+
+        self.reclaimed = true;
+        record_file_cache_reclaim_success("write", self.reclaim_len, started);
+        Ok(())
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+impl AsyncWrite for FileCacheReclaimWriter {
+    fn poll_write(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        std::pin::Pin::new(&mut self.inner).poll_write(cx, buf)
+    }
+
+    fn poll_flush(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
+        std::pin::Pin::new(&mut self.inner).poll_flush(cx)
+    }
+
+    fn poll_shutdown(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        match std::pin::Pin::new(&mut self.inner).poll_shutdown(cx) {
+            std::task::Poll::Ready(Ok(())) => {
+                if let Err(err) = self.reclaim_file_cache() {
+                    record_file_cache_reclaim_error("write");
+                    debug!(error = ?err, reclaim_len = self.reclaim_len, "failed to reclaim file cache after write");
+                }
+                std::task::Poll::Ready(Ok(()))
+            }
+            other => other,
+        }
+    }
+
+    fn poll_write_vectored(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        bufs: &[std::io::IoSlice<'_>],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        std::pin::Pin::new(&mut self.inner).poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.is_write_vectored()
+    }
+}
+
+fn should_reclaim_file_cache_after_write(file_size: i64) -> bool {
+    if file_size <= 0 {
+        return false;
+    }
+
+    if !rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE,
+    ) {
+        return false;
+    }
+
+    let threshold = rustfs_utils::get_env_usize(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+    );
+    file_size as usize >= threshold
+}
+
+fn should_reclaim_file_cache_after_read(length: usize) -> bool {
+    if length == 0 {
+        return false;
+    }
+
+    if !rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE,
+    ) {
+        return false;
+    }
+
+    let threshold = rustfs_utils::get_env_usize(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+    );
+    length >= threshold
+}
+
 pub struct LocalDisk {
     pub root: PathBuf,
     pub format_path: PathBuf,
@@ -176,7 +409,15 @@ impl LocalDisk {
             let root = root_clone.clone();
             Box::pin(async move {
                 match get_disk_info(root.clone()).await {
-                    Ok((info, root)) => {
+                    Ok((info, is_root_disk)) => {
+                        let physical_device_ids = match rustfs_utils::os::get_physical_device_ids(root.to_string_lossy().as_ref())
+                        {
+                            Ok(ids) => ids,
+                            Err(err) => {
+                                warn!(root = ?root, error = ?err, "failed to resolve physical device ids for disk root");
+                                Vec::new()
+                            }
+                        };
                         let disk_info = DiskInfo {
                             total: info.total,
                             free: info.free,
@@ -186,7 +427,8 @@ impl LocalDisk {
                             major: info.major,
                             minor: info.minor,
                             fs_type: info.fstype,
-                            root_disk: root,
+                            root_disk: is_root_disk,
+                            physical_device_ids,
                             id: disk_id,
                             ..Default::default()
                         };
@@ -1838,8 +2080,9 @@ impl DiskAPI for LocalDisk {
         let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY)
             .await
             .map_err(to_file_error)?;
+        let reclaim_on_shutdown = should_reclaim_file_cache_after_write(_file_size);
 
-        Ok(Box::new(f))
+        Ok(Box::new(FileCacheReclaimWriter::new(f, _file_size.max(0) as usize, reclaim_on_shutdown)))
 
         // Ok(())
     }
@@ -1911,7 +2154,8 @@ impl DiskAPI for LocalDisk {
             f.seek(SeekFrom::Start(offset as u64)).await?;
         }
 
-        Ok(Box::new(f))
+        let reclaim_on_drop = should_reclaim_file_cache_after_read(length);
+        Ok(Box::new(FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop)))
     }
 
     /// Zero-copy file read using memory mapping (Unix) or efficient read (non-Unix).
@@ -1956,9 +2200,15 @@ impl DiskAPI for LocalDisk {
             use memmap2::MmapOptions;
             let file_path_clone = file_path.clone();
 
+            let should_reclaim_after_read = should_reclaim_file_cache_after_read(length);
             let bytes = tokio::task::spawn_blocking(move || {
                 let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?;
 
+                #[cfg(target_os = "macos")]
+                if should_reclaim_after_read {
+                    let _ = set_std_fd_nocache(&file);
+                }
+
                 // mmap offsets on Unix must be page-size aligned. Align the
                 // mapping down to the nearest page boundary, then slice out the
                 // originally requested logical range.
@@ -1986,7 +2236,21 @@ impl DiskAPI for LocalDisk {
                 let end = logical_offset
                     .checked_add(length)
                     .ok_or_else(|| DiskError::other("mmap slice length overflow"))?;
-                Ok::<Bytes, DiskError>(Bytes::copy_from_slice(&mmap[logical_offset..end]))
+                let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]);
+
+                #[cfg(target_os = "linux")]
+                if should_reclaim_after_read {
+                    use core::num::NonZeroU64;
+                    use rustix::fs::{Advice, fadvise};
+
+                    let reclaim_len =
+                        NonZeroU64::new(map_len as u64).ok_or_else(|| DiskError::other("mmap reclaim length overflow"))?;
+                    fadvise(&file, aligned_offset, Some(reclaim_len), Advice::DontNeed)
+                        .map_err(std::io::Error::from)
+                        .map_err(DiskError::from)?;
+                }
+
+                Ok::<Bytes, DiskError>(bytes)
             })
             .await
             .map_err(DiskError::from)??;
@@ -2072,6 +2336,7 @@ impl DiskAPI for LocalDisk {
 
         let mut objs_returned = 0;
 
+        let mut skip_current_dir_object = false;
         if opts.base_dir.ends_with(SLASH_SEPARATOR) {
             if let Ok(data) = self
                 .read_metadata(
@@ -2098,7 +2363,7 @@ impl DiskAPI for LocalDisk {
                 if let Ok(meta) = tokio::fs::metadata(fpath).await
                     && meta.is_file()
                 {
-                    return Err(DiskError::FileNotFound);
+                    skip_current_dir_object = true;
                 }
             }
         }
@@ -2109,7 +2374,7 @@ impl DiskAPI for LocalDisk {
             &opts,
             &mut out,
             &mut objs_returned,
-            false,
+            skip_current_dir_object,
         )
         .await?;
 
@@ -3362,4 +3627,32 @@ mod test {
             assert_eq!(normalize_path_components("C:\\a\\..\\b"), PathBuf::from("C:\\b"));
         }
     }
+
+    #[test]
+    fn should_reclaim_file_cache_after_write_respects_env_and_threshold() {
+        temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, || {
+            assert!(!should_reclaim_file_cache_after_write(8 * 1024 * 1024));
+        });
+
+        temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, Some("true"), || {
+            temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || {
+                assert!(should_reclaim_file_cache_after_write(8 * 1024 * 1024));
+                assert!(!should_reclaim_file_cache_after_write(1024));
+            });
+        });
+    }
+
+    #[test]
+    fn should_reclaim_file_cache_after_read_respects_env_and_threshold() {
+        temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, || {
+            assert!(!should_reclaim_file_cache_after_read(8 * 1024 * 1024));
+        });
+
+        temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, Some("true"), || {
+            temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || {
+                assert!(should_reclaim_file_cache_after_read(8 * 1024 * 1024));
+                assert!(!should_reclaim_file_cache_after_read(1024));
+            });
+        });
+    }
 }

crates/ecstore/src/disk/mod.rs

@@ -596,6 +596,8 @@ pub struct DiskInfo {
     pub scanning: bool,
     pub endpoint: String,
     pub mount_path: String,
+    /// Leaf physical block devices backing this mount path when available.
+    pub physical_device_ids: Vec<String>,
     pub id: Option<Uuid>,
     pub rotational: bool,
     pub metrics: DiskMetrics,

crates/ecstore/src/endpoints.rs

@@ -17,10 +17,11 @@ use crate::{
     disks_layout::DisksLayout,
     global::global_rustfs_port,
 };
+use rustfs_config::{DEFAULT_UNSAFE_BYPASS_DISK_CHECK, ENV_MINIO_CI, ENV_UNSAFE_BYPASS_DISK_CHECK};
 use rustfs_utils::{XHost, check_local_server_addr, get_host_ip, is_local_host};
 use std::{
-    collections::{HashMap, HashSet, hash_map::Entry},
-    io::{Error, Result},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
+    io::{Error, ErrorKind, Result},
     net::IpAddr,
 };
 use tracing::{error, info, instrument, warn};
@@ -348,6 +349,8 @@ impl PoolEndpointList {
             }
         }
 
+        validate_local_physical_disk_independence(pool_endpoint_list.as_ref())?;
+
         let setup_type = match pool_endpoint_list.as_ref()[0].as_ref()[0].get_type() {
             EndpointType::Path => SetupType::Erasure,
             EndpointType::Url => match unique_args.len() {
@@ -645,12 +648,118 @@ impl EndpointServerPools {
     }
 }
 
+fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()> {
+    let mut local_paths = BTreeSet::new();
+    for endpoints in pools {
+        for endpoint in endpoints.as_ref() {
+            if endpoint.is_local {
+                local_paths.insert(endpoint.get_file_path());
+            }
+        }
+    }
+
+    if local_paths.is_empty() {
+        return Ok(());
+    }
+
+    let local_paths = local_paths.into_iter().collect::<Vec<_>>();
+    validate_local_cross_device_mounts(&local_paths)?;
+
+    if local_paths.len() <= 1 {
+        return Ok(());
+    }
+
+    // Compatibility behavior:
+    // - canonical key: RUSTFS_UNSAFE_BYPASS_DISK_CHECK
+    // - legacy CI alias: MINIO_CI
+    // If both are set, `get_env_bool_with_aliases` keeps canonical key precedence.
+    if rustfs_utils::get_env_bool_with_aliases(ENV_UNSAFE_BYPASS_DISK_CHECK, &[ENV_MINIO_CI], DEFAULT_UNSAFE_BYPASS_DISK_CHECK) {
+        warn!(
+            env = ENV_UNSAFE_BYPASS_DISK_CHECK,
+            local_paths = ?local_paths,
+            "Skipping local physical disk independence validation due to explicit environment override",
+        );
+        return Ok(());
+    }
+
+    let mut device_paths = BTreeMap::<String, BTreeSet<String>>::new();
+    let mut missing_paths = Vec::new();
+
+    for path in &local_paths {
+        let canonical = match rustfs_utils::canonicalize(path) {
+            Ok(path) => path,
+            Err(err) if err.kind() == ErrorKind::NotFound => {
+                missing_paths.push(path.clone());
+                continue;
+            }
+            Err(err) => {
+                return Err(Error::other(format!(
+                    "failed to resolve local endpoint path '{path}' for disk validation: {err}"
+                )));
+            }
+        };
+        let canonical_path = canonical.to_string_lossy().into_owned();
+        let device_ids = rustfs_utils::os::get_physical_device_ids(&canonical_path).map_err(|err| {
+            Error::other(format!("failed to inspect physical disk for local endpoint '{canonical_path}': {err}"))
+        })?;
+
+        for device_id in device_ids {
+            device_paths.entry(device_id).or_default().insert(canonical_path.clone());
+        }
+    }
+
+    if !missing_paths.is_empty() {
+        warn!(
+            missing_paths = ?missing_paths,
+            "Excluding non-existent local endpoint paths from physical disk independence validation during endpoint parsing",
+        );
+    }
+
+    let shared_devices = device_paths
+        .into_iter()
+        .filter_map(|(device_id, paths)| {
+            if paths.len() <= 1 {
+                return None;
+            }
+
+            Some((device_id, paths.into_iter().collect::<Vec<_>>()))
+        })
+        .collect::<Vec<_>>();
+
+    if shared_devices.is_empty() {
+        return Ok(());
+    }
+
+    let details = shared_devices
+        .into_iter()
+        .map(|(device_id, paths)| format!("{device_id} => {}", paths.join(", ")))
+        .collect::<Vec<_>>()
+        .join("; ");
+
+    Err(Error::other(format!(
+        "local erasure endpoints must use distinct physical disks; detected shared devices [{details}]. \
+Set {ENV_UNSAFE_BYPASS_DISK_CHECK}=true only for local testing or CI to bypass this safety check"
+    )))
+}
+
+fn validate_local_cross_device_mounts(local_paths: &[String]) -> Result<()> {
+    rustfs_utils::os::check_cross_device_mounts(local_paths)
+        .map_err(|err| Error::other(format!("local endpoint cross-device mount validation failed: {err}")))
+}
+
 #[cfg(test)]
 mod test {
     use rustfs_utils::must_get_local_ips;
 
     use super::*;
+
+    #[cfg(target_os = "linux")]
+    use serial_test::serial;
     use std::path::Path;
+    #[cfg(target_os = "linux")]
+    use temp_env::async_with_vars;
+    #[cfg(target_os = "linux")]
+    use tempfile::tempdir;
 
     #[test]
     fn test_new_endpoints() {
@@ -1412,4 +1521,69 @@ mod test {
             }
         }
     }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn reject_shared_local_physical_disks_by_default() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, None::<&str>)], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let err = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout)
+                .await
+                .unwrap_err();
+
+            let err_text = err.to_string();
+            assert!(err_text.contains("distinct physical disks"), "unexpected error: {err_text}");
+            assert!(err_text.contains(ENV_UNSAFE_BYPASS_DISK_CHECK), "unexpected error: {err_text}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_explicit_env_bypass() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected bypassed disk validation to succeed, got {ret:?}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_minio_ci_alias() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, Some("1"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected MINIO_CI alias to bypass disk validation, got {ret:?}");
+        })
+        .await;
+    }
 }

crates/ecstore/src/erasure_coding/encode.rs

@@ -26,6 +26,30 @@ use tokio::io::AsyncRead;
 use tokio::sync::mpsc;
 use tracing::error;
 
+const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
+const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 8 * 1024 * 1024;
+const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 8;
+
+fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
+    if expanded_block_bytes == 0 {
+        return 1;
+    }
+
+    max_inflight_bytes
+        .saturating_div(expanded_block_bytes)
+        .clamp(1, DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS)
+}
+
+fn queued_block_bytes(block: &[Bytes]) -> usize {
+    block.iter().map(Bytes::len).sum()
+}
+
+async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
+    while let Some(block) = rx.recv().await {
+        rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
+    }
+}
+
 pub(crate) struct MultiWriter<'a> {
     writers: &'a mut [Option<BitrotWriterWrapper>],
     write_quorum: usize,
@@ -185,7 +209,14 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
-        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
+        // Bound queued encoded blocks by memory budget to avoid per-request spikes.
+        let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
+        let max_inflight_bytes = rustfs_utils::get_env_usize(
+            ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
+            DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
+        );
+        let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
+        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
 
         let task = tokio::spawn(async move {
             let block_size = self.block_size;
@@ -196,7 +227,10 @@ impl Erasure {
                     Ok(n) if n > 0 => {
                         total += n;
                         let res = self.encode_data(&buf[..n])?;
+                        let queued_bytes = queued_block_bytes(&res);
+                        rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
                         if let Err(err) = tx.send(res).await {
+                            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
                             return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                         }
                     }
@@ -229,6 +263,8 @@ impl Erasure {
             if block.is_empty() {
                 break;
             }
+            let queued_bytes = queued_block_bytes(&block);
+            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
             if let Err(err) = writers.write(block).await {
                 write_err = Some(err);
                 break;
@@ -238,6 +274,7 @@ impl Erasure {
         if let Some(err) = write_err {
             task.abort();
             let _ = task.await;
+            drain_queued_inflight_bytes(&mut rx).await;
             if let Err(shutdown_err) = writers.shutdown().await {
                 error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
             }
@@ -310,4 +347,18 @@ mod tests {
         assert_eq!(written, b"small payload".len());
         assert!(!committed.lock().unwrap().is_empty());
     }
+
+    #[test]
+    fn encode_channel_capacity_never_returns_zero() {
+        assert_eq!(encode_channel_capacity(0, 1024), 1);
+        assert_eq!(encode_channel_capacity(4096, 0), 1);
+        assert_eq!(encode_channel_capacity(4096, 1024), 1);
+    }
+
+    #[test]
+    fn encode_channel_capacity_respects_budget_and_hard_cap() {
+        assert_eq!(encode_channel_capacity(4 * 1024 * 1024, 32 * 1024 * 1024), 8);
+        assert_eq!(encode_channel_capacity(16 * 1024 * 1024, 32 * 1024 * 1024), 2);
+        assert_eq!(encode_channel_capacity(1, usize::MAX), DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS);
+    }
 }

crates/ecstore/src/pools.rs

@@ -52,7 +52,7 @@ use rmp_serde::Serializer;
 use rustfs_common::defer;
 use rustfs_common::heal_channel::HealOpts;
 use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
-use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join};
+use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
 use rustfs_workers::workers::Workers;
 use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
 use serde::{Deserialize, Serialize};
@@ -986,25 +986,11 @@ impl PoolMeta {
 }
 
 pub fn path2_bucket_object(name: &str) -> (String, String) {
-    path2_bucket_object_with_base_path("", name)
+    path_to_bucket_object(name)
 }
 
 pub fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
-    // Trim the base path and leading slash
-    let trimmed_path = path
-        .strip_prefix(base_path)
-        .unwrap_or(path)
-        .strip_prefix(SLASH_SEPARATOR)
-        .unwrap_or(path);
-    // Find the position of the first '/'
-    let Some(pos) = trimmed_path.find(SLASH_SEPARATOR) else {
-        return (trimmed_path.to_string(), "".to_string());
-    };
-    // Split into bucket and prefix
-    let bucket = &trimmed_path[0..pos];
-    let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists
-
-    (bucket.to_string(), prefix.to_string())
+    path_to_bucket_object_with_base_path(base_path, path)
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -3746,4 +3732,13 @@ mod pools_tests {
                 .contains("failed to start decommission routines: scheduled 1 of 2 expected workers")
         );
     }
+
+    #[test]
+    #[cfg(windows)]
+    fn test_path2_bucket_object_with_base_path_supports_windows_separators() {
+        let (bucket, object) = super::path2_bucket_object_with_base_path("C:\\data", "C:\\data\\my-bucket\\nested\\object.txt");
+
+        assert_eq!(bucket, "my-bucket");
+        assert_eq!(object, "nested/object.txt");
+    }
 }

crates/ecstore/src/rpc/client.rs

@@ -12,14 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::error::{DiskError, Error as DiskErrorType};
 use crate::rpc::{TONIC_RPC_PREFIX, gen_signature_headers};
 use http::Method;
 use rustfs_common::GLOBAL_CONN_MAP;
 use rustfs_protos::{create_new_channel, proto_gen::node_service::node_service_client::NodeServiceClient};
-use std::error::Error;
+use std::{error::Error, io::ErrorKind};
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
 use tracing::debug;
 
+use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
+
 /// 3. Subsequent calls will attempt fresh connections
 /// 4. If node is still down, connection will fail fast (3s timeout)
 pub async fn node_service_time_out_client(
@@ -49,12 +52,54 @@ pub async fn node_service_time_out_client_no_auth(
     node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
 }
 
+pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
+    match err {
+        DiskError::Timeout => true,
+        DiskError::Io(io_err) => {
+            if matches!(
+                io_err.kind(),
+                ErrorKind::TimedOut
+                    | ErrorKind::ConnectionRefused
+                    | ErrorKind::ConnectionReset
+                    | ErrorKind::BrokenPipe
+                    | ErrorKind::NotConnected
+                    | ErrorKind::ConnectionAborted
+                    | ErrorKind::UnexpectedEof
+            ) {
+                return true;
+            }
+
+            let message = io_err.to_string().to_ascii_lowercase();
+            [
+                "transport error",
+                "unavailable",
+                "error trying to connect",
+                "connection refused",
+                "connection reset",
+                "broken pipe",
+                "not connected",
+                "unexpected eof",
+                "timed out",
+                "deadline has elapsed",
+                "connection closed",
+                "connection aborted",
+                "tcp connect error",
+            ]
+            .iter()
+            .any(|needle| message.contains(needle))
+        }
+        _ => false,
+    }
+}
+
 pub struct TonicSignatureInterceptor;
 
 impl tonic::service::Interceptor for TonicSignatureInterceptor {
     fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
         let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET);
         req.metadata_mut().as_mut().extend(headers);
+        inject_trace_context_into_metadata(req.metadata_mut());
+        inject_request_id_into_metadata(req.metadata_mut());
         Ok(req)
     }
 }
@@ -84,3 +129,34 @@ impl tonic::service::Interceptor for TonicInterceptor {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tonic::service::Interceptor;
+
+    #[test]
+    fn test_signature_interceptor_keeps_auth_headers() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        assert!(req.metadata().contains_key("x-rustfs-signature"));
+        assert!(req.metadata().contains_key("x-rustfs-timestamp"));
+    }
+
+    #[test]
+    fn test_signature_interceptor_may_inject_request_id() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let span = tracing::info_span!("grpc-rpc-test-span");
+        let _guard = span.enter();
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        if let Some(v) = req.metadata().get("x-request-id") {
+            assert!(!v.as_encoded_bytes().is_empty());
+        }
+    }
+}

crates/ecstore/src/rpc/context_propagation.rs

@@ -0,0 +1,223 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use http::{HeaderMap, HeaderValue};
+use opentelemetry::{global, propagation::Injector, trace::TraceContextExt};
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+pub(crate) const REQUEST_ID_HEADER: &str = "x-request-id";
+
+struct HttpHeaderInjector<'a> {
+    headers: &'a mut HeaderMap,
+}
+
+impl Injector for HttpHeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(name) = http::header::HeaderName::from_bytes(key.as_bytes()) else {
+            return;
+        };
+        let Ok(val) = HeaderValue::from_str(&value) else {
+            return;
+        };
+        self.headers.insert(name, val);
+    }
+}
+
+struct MetadataInjector<'a> {
+    metadata: &'a mut tonic::metadata::MetadataMap,
+}
+
+impl Injector for MetadataInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(meta_key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) else {
+            return;
+        };
+        let Ok(meta_value) = tonic::metadata::MetadataValue::try_from(value.as_str()) else {
+            return;
+        };
+        self.metadata.insert(meta_key, meta_value);
+    }
+}
+
+fn current_trace_id() -> Option<String> {
+    let current_context = Span::current().context();
+    let current_span = current_context.span();
+    let span_context = current_span.span_context();
+    if !span_context.is_valid() {
+        return None;
+    }
+    Some(span_context.trace_id().to_string())
+}
+
+fn fallback_request_id() -> String {
+    format!("req-{}", &uuid::Uuid::new_v4().to_string()[..8])
+}
+
+fn propagated_request_id() -> String {
+    current_trace_id()
+        .map(|trace_id| format!("trace-{trace_id}"))
+        .unwrap_or_else(fallback_request_id)
+}
+
+pub(crate) fn inject_trace_context_into_http_headers(headers: &mut HeaderMap) {
+    let current_context = Span::current().context();
+    global::get_text_map_propagator(|propagator| {
+        let mut injector = HttpHeaderInjector { headers };
+        propagator.inject_context(&current_context, &mut injector);
+    });
+}
+
+pub(crate) fn inject_request_id_into_http_headers(headers: &mut HeaderMap) {
+    if headers.contains_key(REQUEST_ID_HEADER) {
+        return;
+    }
+    let request_id = propagated_request_id();
+    if let Ok(value) = HeaderValue::from_str(&request_id) {
+        headers.insert(REQUEST_ID_HEADER, value);
+    }
+}
+
+pub(crate) fn inject_trace_context_into_metadata(metadata: &mut tonic::metadata::MetadataMap) {
+    let current_context = Span::current().context();
+    global::get_text_map_propagator(|propagator| {
+        let mut injector = MetadataInjector { metadata };
+        propagator.inject_context(&current_context, &mut injector);
+    });
+}
+
+pub(crate) fn inject_request_id_into_metadata(metadata: &mut tonic::metadata::MetadataMap) {
+    let request_id_key = tonic::metadata::MetadataKey::from_static(REQUEST_ID_HEADER);
+    if metadata.contains_key(&request_id_key) {
+        return;
+    }
+    let request_id = propagated_request_id();
+    let Ok(value) = tonic::metadata::MetadataValue::try_from(request_id.as_str()) else {
+        return;
+    };
+    metadata.insert(request_id_key, value);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _};
+    use opentelemetry_sdk::trace::SdkTracerProvider;
+    use tracing_opentelemetry::OpenTelemetrySpanExt;
+    use tracing_subscriber::{Registry, layer::SubscriberExt};
+
+    fn with_trace_parent<F>(trace_id_hex: &str, f: F)
+    where
+        F: FnOnce(),
+    {
+        let provider = SdkTracerProvider::builder().build();
+        let tracer = provider.tracer("context-propagation-tests");
+        let subscriber = Registry::default().with(tracing_opentelemetry::layer().with_tracer(tracer));
+
+        tracing::subscriber::with_default(subscriber, || {
+            let span = tracing::info_span!("context-propagation-test-span");
+
+            let trace_id = TraceId::from_hex(trace_id_hex).expect("trace id should be valid hex");
+            let span_id = opentelemetry::trace::SpanId::from_hex("0102030405060708").expect("span id should be valid hex");
+            let parent = SpanContext::new(trace_id, span_id, TraceFlags::SAMPLED, true, TraceState::default());
+            span.set_parent(opentelemetry::Context::new().with_remote_span_context(parent))
+                .expect("failed to set parent context");
+            let _guard = span.enter();
+
+            f();
+        });
+        let _ = provider.shutdown();
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_preserves_existing_value() {
+        let mut headers = HeaderMap::new();
+        headers.insert(REQUEST_ID_HEADER, HeaderValue::from_static("req-upstream-123"));
+
+        with_trace_parent("0123456789abcdef0123456789abcdef", || {
+            inject_request_id_into_http_headers(&mut headers);
+        });
+
+        assert_eq!(headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-123"));
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_uses_trace_id_when_missing() {
+        let trace_id = "abcdefabcdefabcdefabcdefabcdefab";
+        let mut headers = HeaderMap::new();
+
+        with_trace_parent(trace_id, || {
+            inject_request_id_into_http_headers(&mut headers);
+        });
+
+        assert_eq!(
+            headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()),
+            Some(format!("trace-{trace_id}").as_str())
+        );
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_preserves_existing_value() {
+        let mut metadata = tonic::metadata::MetadataMap::new();
+        metadata.insert(
+            tonic::metadata::MetadataKey::from_static(REQUEST_ID_HEADER),
+            tonic::metadata::MetadataValue::from_static("req-upstream-456"),
+        );
+
+        with_trace_parent("fedcba9876543210fedcba9876543210", || {
+            inject_request_id_into_metadata(&mut metadata);
+        });
+
+        assert_eq!(metadata.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-456"));
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_uses_trace_id_when_missing() {
+        let trace_id = "1234567890abcdef1234567890abcdef";
+        let mut metadata = tonic::metadata::MetadataMap::new();
+
+        with_trace_parent(trace_id, || {
+            inject_request_id_into_metadata(&mut metadata);
+        });
+
+        assert_eq!(
+            metadata.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()),
+            Some(format!("trace-{trace_id}").as_str())
+        );
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_uses_req_fallback_when_trace_missing() {
+        let mut headers = HeaderMap::new();
+        inject_request_id_into_http_headers(&mut headers);
+
+        let request_id = headers
+            .get(REQUEST_ID_HEADER)
+            .and_then(|v| v.to_str().ok())
+            .expect("request id should be injected");
+        assert!(request_id.starts_with("req-"), "expected req- fallback, got: {request_id}");
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_uses_req_fallback_when_trace_missing() {
+        let mut metadata = tonic::metadata::MetadataMap::new();
+        inject_request_id_into_metadata(&mut metadata);
+
+        let request_id = metadata
+            .get(REQUEST_ID_HEADER)
+            .and_then(|v| v.to_str().ok())
+            .expect("request id should be injected");
+        assert!(request_id.starts_with("req-"), "expected req- fallback, got: {request_id}");
+    }
+}

crates/ecstore/src/rpc/http_auth.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers};
 use base64::Engine as _;
 use base64::engine::general_purpose;
 use hmac::{Hmac, KeyInit, Mac};
@@ -62,6 +63,8 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) {
     let auth_headers = gen_signature_headers(url, method);
 
     headers.extend(auth_headers);
+    inject_trace_context_into_http_headers(headers);
+    inject_request_id_into_http_headers(headers);
 }
 
 pub fn gen_signature_headers(url: &str, method: &Method) -> HeaderMap {
@@ -132,6 +135,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::rpc::context_propagation::REQUEST_ID_HEADER;
     use http::{HeaderMap, Method};
     use time::OffsetDateTime;
 
@@ -210,6 +214,33 @@ mod tests {
         assert!((current_time - timestamp).abs() <= 1, "Timestamp should be close to current time");
     }
 
+    #[test]
+    fn test_build_auth_headers_preserves_existing_request_id() {
+        let url = "http://example.com/api/test";
+        let method = Method::GET;
+        let mut headers = HeaderMap::new();
+        headers.insert(REQUEST_ID_HEADER, HeaderValue::from_static("req-upstream-123"));
+
+        build_auth_headers(url, &method, &mut headers);
+
+        assert_eq!(headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-123"));
+    }
+
+    #[test]
+    fn test_build_auth_headers_may_set_request_id_from_trace_id() {
+        let url = "http://example.com/api/test";
+        let method = Method::GET;
+        let mut headers = HeaderMap::new();
+
+        let span = tracing::info_span!("rpc-test-span");
+        let _guard = span.enter();
+        build_auth_headers(url, &method, &mut headers);
+
+        if let Some(value) = headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()) {
+            assert!(!value.is_empty(), "request id should not be empty");
+        }
+    }
+
     #[test]
     fn test_verify_rpc_signature_success() {
         let url = "http://example.com/api/test";

crates/ecstore/src/rpc/mod.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod client;
+mod context_propagation;
 mod http_auth;
 mod peer_rest_client;
 mod peer_s3_client;

crates/ecstore/src/rpc/peer_s3_client.rs

@@ -18,13 +18,15 @@ use crate::disk::error::{Error, Result};
 use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
 use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration};
 use crate::global::GLOBAL_LOCAL_DISK_MAP;
-use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::rpc::client::{
+    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+};
 use crate::store::all_local_disk;
 use crate::store_utils::is_reserved_or_invalid_bucket;
 use crate::{
     disk::{
         self, VolumeInfo,
-        disk_store::{CHECK_EVERY, CHECK_TIMEOUT_DURATION, DiskHealthTracker},
+        disk_store::{DiskHealthTracker, get_drive_active_check_interval, get_drive_active_check_timeout},
     },
     endpoints::{EndpointServerPools, Node},
     store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
@@ -613,7 +615,7 @@ impl RemotePeerS3Client {
 
     /// Monitor remote peer health periodically
     async fn monitor_remote_peer_health(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());
 
         loop {
             tokio::select! {
@@ -682,7 +684,7 @@ impl RemotePeerS3Client {
         let port = url.port_or_known_default().unwrap_or(80);
 
         // Try to establish TCP connection
-        match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
+        match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
             Ok(Ok(_)) => Ok(()),
             _ => Err(Error::other(format!("Cannot connect to {host}:{port}"))),
         }
@@ -717,16 +719,39 @@ impl RemotePeerS3Client {
                     self.health.log_success();
                 }
                 self.health.decrement_waiting();
+                if let Err(err) = &operation_result
+                    && is_network_like_disk_error(err)
+                {
+                    self.mark_faulty_and_start_recovery("operation_network_error").await;
+                }
                 operation_result
             }
             Err(_) => {
                 // Timeout occurred, mark peer as potentially faulty
                 self.health.decrement_waiting();
+                self.mark_faulty_and_start_recovery("operation_timeout").await;
                 warn!("Remote peer operation timeout after {:?}", timeout_duration);
                 Err(Error::other(format!("Remote peer operation timeout after {timeout_duration:?}")))
             }
         }
     }
+
+    async fn mark_faulty_and_start_recovery(&self, reason: &'static str) {
+        if self.health.swap_ok_to_faulty() {
+            warn!(
+                addr = %self.addr,
+                reason,
+                "Remote peer marked faulty after network failure"
+            );
+
+            let health = Arc::clone(&self.health);
+            let cancel_token = self.cancel_token.clone();
+            let addr = self.addr.clone();
+            tokio::spawn(async move {
+                Self::monitor_remote_peer_recovery(addr, health, cancel_token).await;
+            });
+        }
+    }
 }
 
 #[async_trait]
@@ -1001,3 +1026,69 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResu
 async fn clone_drives() -> Vec<Option<DiskStore>> {
     GLOBAL_LOCAL_DISK_MAP.read().await.values().cloned().collect::<Vec<_>>()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn test_remote_peer(addr: &str) -> RemotePeerS3Client {
+        let node = Node {
+            url: url::Url::parse(addr).expect("test peer URL should parse"),
+            pools: vec![0],
+            is_local: false,
+            grid_host: addr.to_string(),
+        };
+
+        RemotePeerS3Client {
+            node: Some(node),
+            pools: Some(vec![0]),
+            addr: addr.to_string(),
+            health: Arc::new(DiskHealthTracker::new()),
+            cancel_token: CancellationToken::new(),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
+        let client = test_remote_peer("http://peer-network-error:9000");
+
+        let err = client
+            .execute_with_timeout(
+                || async {
+                    Err::<(), Error>(DiskError::Io(std::io::Error::new(
+                        std::io::ErrorKind::ConnectionRefused,
+                        "connection refused",
+                    )))
+                },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("network-like error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io network error, got {other:?}"),
+            },
+            std::io::ErrorKind::ConnectionRefused
+        );
+        assert!(client.health.is_faulty(), "network-like errors should mark remote peer faulty");
+
+        client.cancel_token.cancel();
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_keeps_remote_peer_online_for_business_error() {
+        let client = test_remote_peer("http://peer-business-error:9000");
+
+        let err = client
+            .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
+            .await
+            .expect_err("business error should fail");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(!client.health.is_faulty(), "business errors should not mark remote peer faulty");
+
+        client.cancel_token.cancel();
+    }
+}

crates/ecstore/src/rpc/remote_disk.rs

@@ -16,15 +16,17 @@ use crate::disk::{
     CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader,
     FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
     disk_store::{
-        CHECK_EVERY, CHECK_TIMEOUT_DURATION, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING,
-        SKIP_IF_SUCCESS_BEFORE, get_drive_disk_info_timeout, get_drive_list_dir_timeout, get_drive_metadata_timeout,
-        get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
+        DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
+        get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
+        get_drive_metadata_timeout, get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
     },
     endpoint::Endpoint,
     health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
-use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::rpc::client::{
+    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+};
 use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
 use crate::{
     disk::error::{Error, Result},
@@ -47,7 +49,7 @@ use rustfs_protos::proto_gen::node_service::{
 use rustfs_rio::{HttpReader, HttpWriter};
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
-    io::{Cursor, ErrorKind},
+    io::Cursor,
     path::PathBuf,
     sync::{
         Arc,
@@ -176,10 +178,10 @@ impl RemoteDisk {
         health: Arc<DiskHealthTracker>,
         cancel_token: CancellationToken,
     ) {
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());
 
         // Perform basic connectivity check
-        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
+        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
             warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
             // Start recovery monitoring
@@ -222,7 +224,7 @@ impl RemoteDisk {
                     }
 
                     // Perform basic connectivity check
-                    if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
+                    if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
                         warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
                         // Start recovery monitoring
@@ -263,7 +265,7 @@ impl RemoteDisk {
                             return;
                         }
                     } else {
-                        health.mark_failure(&endpoint, "connectivity_probe_failed");
+                        health.mark_offline(&endpoint, "connectivity_probe_failed");
                     }
                 }
             }
@@ -281,7 +283,7 @@ impl RemoteDisk {
         let port = url.port_or_known_default().unwrap_or(80);
 
         // Try to establish TCP connection
-        match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
+        match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
             Ok(Ok(stream)) => {
                 drop(stream);
                 Ok(())
@@ -334,10 +336,10 @@ impl RemoteDisk {
                 }
                 self.health.decrement_waiting();
                 if let Err(err) = &operation_result
-                    && Self::is_timeout_like_error(err)
+                    && is_network_like_disk_error(err)
                 {
                     counter!(
-                        "rustfs_drive_op_timeout_total",
+                        "rustfs_drive_op_network_error_total",
                         "endpoint" => self.endpoint.to_string(),
                         "op" => op.to_string()
                     )
@@ -347,9 +349,9 @@ impl RemoteDisk {
                         addr = %self.addr,
                         op,
                         timeout_ms = timeout_duration.as_millis(),
-                        "Remote disk operation returned a timeout-like error"
+                        "Remote disk operation returned a network-like error"
                     );
-                    self.mark_faulty_and_evict("operation_timeout_error").await;
+                    self.mark_faulty_and_evict("operation_network_error").await;
                 }
                 operation_result
             }
@@ -375,12 +377,8 @@ impl RemoteDisk {
         }
     }
 
-    fn is_timeout_like_error(err: &Error) -> bool {
-        matches!(err, DiskError::Timeout) || matches!(err, DiskError::Io(io_err) if io_err.kind() == ErrorKind::TimedOut)
-    }
-
     async fn mark_faulty_and_evict(&self, reason: &'static str) {
-        if self.health.mark_failure(&self.endpoint, reason) {
+        if self.health.mark_offline(&self.endpoint, reason) {
             self.spawn_recovery_monitor_if_needed();
             counter!(
                 "rustfs_drive_faulty_mark_total",
@@ -886,7 +884,7 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout_for_op(
             "write_metadata",
-            || async {
+            move || async move {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -896,8 +894,8 @@ impl DiskAPI for RemoteDisk {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
-                    file_info: file_info.clone(),
-                    file_info_bin: file_info_bin.clone(),
+                    file_info,
+                    file_info_bin: file_info_bin.into(),
                 });
 
                 let response = client.write_metadata(request).await?.into_inner();
@@ -951,7 +949,7 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout_for_op(
             "update_metadata",
-            || async {
+            move || async move {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -961,10 +959,10 @@ impl DiskAPI for RemoteDisk {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
-                    file_info: file_info.clone(),
-                    opts: opts_str.clone(),
-                    file_info_bin: file_info_bin.clone(),
-                    opts_bin: opts_bin.clone(),
+                    file_info,
+                    opts: opts_str,
+                    file_info_bin: file_info_bin.into(),
+                    opts_bin: opts_bin.into(),
                 });
 
                 let response = client.update_metadata(request).await?.into_inner();
@@ -994,7 +992,7 @@ impl DiskAPI for RemoteDisk {
         let opts_bin = encode_msgpack(opts)?;
 
         self.execute_with_timeout(
-            || async {
+            move || async {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -1005,8 +1003,8 @@ impl DiskAPI for RemoteDisk {
                     volume: volume.to_string(),
                     path: path.to_string(),
                     version_id: version_id.to_string(),
-                    opts: opts_str.clone(),
-                    opts_bin: opts_bin.clone(),
+                    opts: opts_str,
+                    opts_bin: opts_bin.into(),
                 });
 
                 let response = client.read_version(request).await?.into_inner();
@@ -1480,7 +1478,7 @@ impl DiskAPI for RemoteDisk {
                 let request = Request::new(ReadMultipleRequest {
                     disk,
                     read_multiple_req,
-                    read_multiple_req_bin,
+                    read_multiple_req_bin: read_multiple_req_bin.into(),
                 });
 
                 let response = client.read_multiple(request).await?.into_inner();
@@ -2073,6 +2071,96 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_faulty_on_network_like_error() {
+        let addr = "http://127.0.0.1:59993".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(
+                || async {
+                    Err::<(), Error>(DiskError::Io(std::io::Error::new(
+                        std::io::ErrorKind::ConnectionRefused,
+                        "connection refused",
+                    )))
+                },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("network-like operation error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io network error, got {other:?}"),
+            },
+            std::io::ErrorKind::ConnectionRefused
+        );
+        assert!(!remote_disk.is_online().await, "network-like errors should mark remote disk faulty");
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "network-like errors should evict cached connection"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_keeps_remote_disk_online_for_business_error() {
+        let addr = "http://127.0.0.1:59994".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
+            .await
+            .expect_err("business error should still fail the operation");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(remote_disk.is_online().await, "business errors should not mark remote disk faulty");
+        assert!(
+            GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "business errors should not evict cached connection"
+        );
+    }
+
     #[test]
     fn test_remote_disk_sync_properties() {
         let url = url::Url::parse("https://secure-remote:9000/data").unwrap();

crates/ecstore/src/rpc/remote_locker.rs

@@ -190,11 +190,6 @@ impl LockClient for RemoteClient {
             Err(err) => return Ok(Self::rpc_failure_response(request, &err)),
         };
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -204,7 +199,8 @@ impl LockClient for RemoteClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/ecstore/src/set_disk.rs

@@ -18,6 +18,7 @@
 use crate::batch_processor::{AsyncBatchProcessor, get_global_processors};
 use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
+use crate::bucket::object_lock::objectlock_sys::check_retention_for_modification;
 use crate::bucket::replication::check_replicate_delete;
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
@@ -1343,6 +1344,22 @@ impl BucketOperations for SetDisks {
     }
 }
 
+fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> {
+    if let Some(retention) = &opts.object_lock_retention
+        && check_retention_for_modification(
+            &obj_info.user_defined,
+            retention.mode.as_deref(),
+            retention.retain_until,
+            retention.bypass_governance,
+        )
+        .is_some()
+    {
+        return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
+    }
+
+    Ok(())
+}
+
 #[async_trait::async_trait]
 impl ObjectOperations for SetDisks {
     #[tracing::instrument(skip(self))]
@@ -1989,6 +2006,8 @@ impl ObjectOperations for SetDisks {
 
         let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
 
+        check_object_lock_retention_update(bucket, object, &obj_info, opts)?;
+
         for (k, v) in obj_info.user_defined {
             fi.metadata.insert(k, v);
         }
@@ -4129,6 +4148,7 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
                         total_space: res.total,
                         used_space: res.used,
                         available_space: res.free,
+                        physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
                         utilization: {
                             if res.total > 0 {
                                 res.used as f64 / res.total as f64 * 100_f64
@@ -5483,6 +5503,74 @@ mod tests {
         assert!(rendered.contains("object"), "{rendered}");
     }
 
+    #[test]
+    fn test_check_object_lock_retention_update_blocks_compliance_shorten() {
+        let now = OffsetDateTime::now_utc();
+        let existing_until = now + Duration::from_secs(60 * 60 * 24 * 60);
+        let requested_until = now + Duration::from_secs(60 * 60 * 24);
+
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
+        );
+
+        let obj_info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions {
+                mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()),
+                retain_until: Some(requested_until),
+                bypass_governance: true,
+            }),
+            ..Default::default()
+        };
+
+        let err = check_object_lock_retention_update("bucket", "object", &obj_info, &opts)
+            .expect_err("COMPLIANCE shortening must be blocked");
+
+        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
+    }
+
+    #[test]
+    fn test_check_object_lock_retention_update_allows_governance_shorten_with_bypass() {
+        let now = OffsetDateTime::now_utc();
+        let existing_until = now + Duration::from_secs(60 * 60 * 24 * 60);
+        let requested_until = now + Duration::from_secs(60 * 60 * 24);
+
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
+        );
+
+        let obj_info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions {
+                mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()),
+                retain_until: Some(requested_until),
+                bypass_governance: true,
+            }),
+            ..Default::default()
+        };
+
+        check_object_lock_retention_update("bucket", "object", &obj_info, &opts)
+            .expect("GOVERNANCE shortening with bypass should remain allowed");
+    }
+
     #[test]
     fn test_should_prevent_write() {
         let oi = ObjectInfo {
@@ -5576,6 +5664,100 @@ mod tests {
         assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string())));
     }
 
+    #[tokio::test]
+    async fn range_reads_use_shard_span_length_for_non_zero_offsets() {
+        use tokio::io::AsyncReadExt;
+        use uuid::Uuid;
+
+        let tempdir = tempfile::tempdir().expect("tempdir should be created");
+        let endpoint =
+            Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("disk should be created");
+
+        let bucket = "bucket";
+        let object = "object";
+        let payload = vec![b'x'; 3 * 1024 * 1024 + 1234];
+        let range_offset = 2 * 1024 * 1024 + 17;
+        let range_length = 512 * 1024;
+
+        disk.make_volume(bucket).await.expect("bucket should be created");
+
+        let mut fi = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
+        let data_dir = Uuid::new_v4();
+        fi.data_dir = Some(data_dir);
+        fi.size = payload.len() as i64;
+        fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None);
+
+        let erasure = erasure_coding::Erasure::new_with_options(
+            fi.erasure.data_blocks,
+            fi.erasure.parity_blocks,
+            fi.erasure.block_size,
+            fi.uses_legacy_checksum,
+        );
+        let shard_path = format!("{object}/{data_dir}/part.1");
+        let checksum_info = fi.erasure.get_checksum_info(1);
+
+        let mut bitrot_writer = create_bitrot_writer(
+            true,
+            None,
+            bucket,
+            &shard_path,
+            payload.len() as i64,
+            erasure.shard_size(),
+            checksum_info.algorithm.clone(),
+        )
+        .await
+        .expect("bitrot writer should be created");
+
+        for chunk in payload.chunks(erasure.shard_size()) {
+            bitrot_writer.write(chunk).await.expect("payload chunk should be written");
+        }
+
+        let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
+        disk.write_all(bucket, &shard_path, Bytes::from(encoded))
+            .await
+            .expect("encoded shard should be stored");
+
+        let files = vec![fi.clone()];
+        let disks = vec![Some(disk.clone())];
+        let (mut reader, mut writer) = tokio::io::duplex(range_length * 2);
+
+        let read_task = tokio::spawn(async move {
+            SetDisks::get_object_with_fileinfo(
+                bucket,
+                object,
+                range_offset,
+                range_length as i64,
+                &mut writer,
+                fi,
+                files,
+                &disks,
+                0,
+                0,
+                true,
+            )
+            .await
+        });
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.expect("range bytes should be readable");
+
+        read_task
+            .await
+            .expect("read task should complete")
+            .expect("range read should succeed");
+
+        assert_eq!(out, payload[range_offset..range_offset + range_length]);
+    }
+
     #[test]
     fn parts_after_marker_uses_marker_position() {
         let part_numbers = (1..=1002).collect::<Vec<_>>();

crates/ecstore/src/set_disk/multipart.rs

@@ -13,6 +13,87 @@
 // limitations under the License.
 
 use super::*;
+use std::future::Future;
+use std::time::Duration;
+use tokio::task::JoinSet;
+
+async fn collect_list_parts_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> disk::error::Result<(Vec<Option<DiskError>>, Vec<Vec<String>>)>
+where
+    F: Future<Output = disk::error::Result<Vec<String>>> + Send + 'static,
+{
+    let mut errs = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut object_parts = vec![Vec::new(); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(parts))) => {
+                errs[index] = None;
+                object_parts[index] = parts;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errs[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(DiskError::ErasureReadQuorum);
+        }
+    }
+
+    Ok((errs, object_parts))
+}
+
+fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize) -> Vec<usize> {
+    let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
+
+    for drive_parts in object_parts {
+        let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
+
+        // part files can be either part.N or part.N.meta
+        for part_path in drive_parts {
+            if let Some(num_str) = part_path.strip_prefix("part.") {
+                if let Some(meta_idx) = num_str.find(".meta") {
+                    if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
+                        *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
+                    }
+                } else if let Ok(part_num) = num_str.parse::<usize>() {
+                    *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
+                }
+            }
+        }
+
+        // Include only part.N.meta files with corresponding part.N
+        for (&part_num, &cnt) in &parts_with_meta_count {
+            if cnt >= 2 {
+                *part_quorum_map.entry(part_num).or_insert(0) += 1;
+            }
+        }
+    }
+
+    let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
+    for (part_num, count) in part_quorum_map {
+        if count >= read_quorum {
+            part_numbers.push(part_num);
+        }
+    }
+
+    part_numbers.sort();
+    part_numbers
+}
 
 impl SetDisks {
     pub(super) async fn list_parts(
@@ -21,10 +102,13 @@ impl SetDisks {
         read_quorum: usize,
     ) -> disk::error::Result<Vec<usize>> {
         let mut futures = Vec::with_capacity(disks.len());
-        for (i, disk) in disks.iter().enumerate() {
+        let part_path = part_path.to_string();
+        for disk in disks.iter() {
+            let disk = disk.clone();
+            let part_path = part_path.clone();
             futures.push(async move {
                 if let Some(disk) = disk {
-                    disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
+                    disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
                         .await
                 } else {
                     Err(DiskError::DiskNotFound)
@@ -35,60 +119,15 @@ impl SetDisks {
         let mut errs = Vec::with_capacity(disks.len());
         let mut object_parts = Vec::with_capacity(disks.len());
 
-        let results = join_all(futures).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    errs.push(None);
-                    object_parts.push(res);
-                }
-                Err(e) => {
-                    errs.push(Some(e));
-                    object_parts.push(vec![]);
-                }
-            }
-        }
+        let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?;
+        errs.extend(collected_errs);
+        object_parts.extend(collected_parts);
 
         if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
             return Err(err);
         }
 
-        let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
-
-        for drive_parts in object_parts {
-            let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
-
-            // part files can be either part.N or part.N.meta
-            for part_path in drive_parts {
-                if let Some(num_str) = part_path.strip_prefix("part.") {
-                    if let Some(meta_idx) = num_str.find(".meta") {
-                        if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
-                            *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
-                        }
-                    } else if let Ok(part_num) = num_str.parse::<usize>() {
-                        *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
-                    }
-                }
-            }
-
-            // Include only part.N.meta files with corresponding part.N
-            for (&part_num, &cnt) in &parts_with_meta_count {
-                if cnt >= 2 {
-                    *part_quorum_map.entry(part_num).or_insert(0) += 1;
-                }
-            }
-        }
-
-        let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
-        for (part_num, count) in part_quorum_map {
-            if count >= read_quorum {
-                part_numbers.push(part_num);
-            }
-        }
-
-        part_numbers.sort();
-
-        Ok(part_numbers)
+        Ok(reduce_quorum_part_numbers(object_parts, read_quorum))
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
@@ -145,3 +184,78 @@ impl SetDisks {
         Ok((fi, parts_metadata))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn collect_list_parts_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let err = collect_list_parts_results(tasks, 2)
+            .await
+            .expect_err("quorum should become impossible before slow tail completes");
+
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+        assert!(started.elapsed() < Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_list_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| async move {
+                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                if should_panic {
+                    panic!("simulated task panic");
+                }
+                Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])
+            })
+            .collect();
+
+        let (errs, object_parts) = collect_list_parts_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(errs.iter().filter(|err| err.is_none()).count(), 2);
+        assert_eq!(object_parts.iter().filter(|parts| !parts.is_empty()).count(), 2);
+    }
+
+    #[test]
+    fn reduce_quorum_part_numbers_only_keeps_parts_present_on_quorum_of_drives() {
+        let object_parts = vec![
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.2".to_string(),
+                "part.2.meta".to_string(),
+            ],
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.3".to_string(),
+                "part.3.meta".to_string(),
+            ],
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.2".to_string(),
+                "part.2.meta".to_string(),
+            ],
+        ];
+
+        let parts = reduce_quorum_part_numbers(object_parts, 2);
+        assert_eq!(parts, vec![1, 2]);
+    }
+}

crates/ecstore/src/set_disk/read.rs

@@ -14,6 +14,88 @@
 
 use super::*;
 use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
+use std::future::Future;
+use tokio::task::JoinSet;
+
+async fn collect_read_multiple_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> std::result::Result<(Vec<Option<Vec<ReadMultipleResp>>>, Vec<Option<DiskError>>), ()>
+where
+    F: Future<Output = disk::error::Result<Vec<ReadMultipleResp>>> + Send + 'static,
+{
+    let mut responses = vec![None; tasks.len()];
+    let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(resp))) => {
+                responses[index] = Some(resp);
+                errors[index] = None;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errors[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(());
+        }
+    }
+
+    Ok((responses, errors))
+}
+
+async fn collect_read_parts_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> std::result::Result<(Vec<Option<Vec<ObjectPartInfo>>>, Vec<Option<DiskError>>), ()>
+where
+    F: Future<Output = disk::error::Result<Vec<ObjectPartInfo>>> + Send + 'static,
+{
+    let mut responses = vec![None; tasks.len()];
+    let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(resp))) => {
+                responses[index] = Some(resp);
+                errors[index] = None;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errors[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(());
+        }
+    }
+
+    Ok((responses, errors))
+}
 
 impl SetDisks {
     pub(super) async fn read_parts(
@@ -25,9 +107,6 @@ impl SetDisks {
     ) -> disk::error::Result<Vec<ObjectPartInfo>> {
         let mut errs = Vec::with_capacity(disks.len());
         let mut object_parts = Vec::with_capacity(disks.len());
-
-        // Use batch processor for better performance
-        let processor = get_global_processors().read_processor();
         let bucket = bucket.to_string();
         let part_meta_paths = part_meta_paths.to_vec();
 
@@ -48,19 +127,13 @@ impl SetDisks {
             })
             .collect();
 
-        let results = processor.execute_batch(tasks).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    errs.push(None);
-                    object_parts.push(res);
-                }
-                Err(e) => {
-                    errs.push(Some(e));
-                    object_parts.push(vec![]);
-                }
-            }
-        }
+        let (responses, collected_errors) = match collect_read_parts_results(tasks, read_quorum).await {
+            Ok(collected) => collected,
+            Err(()) => return Err(DiskError::ErasureReadQuorum),
+        };
+
+        errs.extend(collected_errors);
+        object_parts.extend(responses.into_iter().map(|resp| resp.unwrap_or_default()));
 
         if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
             return Err(err);
@@ -384,10 +457,23 @@ impl SetDisks {
         read_quorum: usize,
     ) -> Vec<ReadMultipleResp> {
         let mut futures = Vec::with_capacity(disks.len());
-        let mut ress = Vec::with_capacity(disks.len());
-        let mut errors = Vec::with_capacity(disks.len());
+        let empty_quorum_result = || {
+            req.files
+                .iter()
+                .map(|want| ReadMultipleResp {
+                    bucket: req.bucket.clone(),
+                    prefix: req.prefix.clone(),
+                    file: want.clone(),
+                    exists: false,
+                    error: Error::ErasureReadQuorum.to_string(),
+                    data: Vec::new(),
+                    mod_time: None,
+                })
+                .collect::<Vec<_>>()
+        };
 
         for disk in disks.iter() {
+            let disk = disk.clone();
             let req = req.clone();
             futures.push(async move {
                 if let Some(disk) = disk {
@@ -398,19 +484,10 @@ impl SetDisks {
             });
         }
 
-        let results = join_all(futures).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    ress.push(Some(res));
-                    errors.push(None);
-                }
-                Err(e) => {
-                    ress.push(None);
-                    errors.push(Some(e));
-                }
-            }
-        }
+        let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
+            Ok(collected) => collected,
+            Err(()) => return empty_quorum_result(),
+        };
 
         // debug!("ReadMultipleResp ress {:?}", ress);
         // debug!("ReadMultipleResp errors {:?}", errors);
@@ -839,3 +916,181 @@ impl SetDisks {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let result = collect_read_multiple_results(tasks, 2).await;
+        assert!(result.is_err(), "quorum should become impossible before slow tail completes");
+        assert!(started.elapsed() < std::time::Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_returns_collected_responses_on_quorum() {
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1, 2, 3],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
+            (15, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
+            (250, Err(DiskError::DiskNotFound)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let (responses, errors) = collect_read_multiple_results(tasks, 2).await.expect("quorum should succeed");
+
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1, 2, 3],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| {
+                let resp = resp.clone();
+                async move {
+                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+                    if should_panic {
+                        panic!("simulated task panic");
+                    }
+                    Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])
+                }
+            })
+            .collect();
+
+        let (responses, errors) = collect_read_multiple_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let result = collect_read_parts_results(tasks, 2).await;
+        assert!(result.is_err(), "quorum should become impossible before slow tail completes");
+        assert!(started.elapsed() < std::time::Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_returns_collected_responses_on_quorum() {
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
+            (15, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
+            (250, Err(DiskError::DiskNotFound)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let (responses, errors) = collect_read_parts_results(tasks, 2).await.expect("quorum should succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| {
+                let part = part.clone();
+                async move {
+                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+                    if should_panic {
+                        panic!("simulated task panic");
+                    }
+                    Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])
+                }
+            })
+            .collect();
+
+        let (responses, errors) = collect_read_parts_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+}

crates/ecstore/src/store_api/types.rs

@@ -43,6 +43,13 @@ impl HTTPPreconditions {
     }
 }
 
+#[derive(Debug, Default, Clone)]
+pub struct ObjectLockRetentionOptions {
+    pub mode: Option<String>,
+    pub retain_until: Option<OffsetDateTime>,
+    pub bypass_governance: bool,
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -79,6 +86,7 @@ pub struct ObjectOptions {
     pub lifecycle_audit_event: LcAuditEvent,
 
     pub eval_metadata: Option<HashMap<String, String>>,
+    pub object_lock_retention: Option<ObjectLockRetentionOptions>,
 
     pub want_checksum: Option<Checksum>,
     pub skip_verify_bitrot: bool,

crates/ecstore/src/store_list_objects.rs

@@ -829,6 +829,7 @@ impl ECStore {
         let (merge_tx, mut merge_rx) = mpsc::channel::<MetaCacheEntry>(100);
 
         let bucket = bucket.to_owned();
+        let bucket_clone = bucket.clone();
 
         let vcf = match get_versioning_config(&bucket).await {
             Ok((res, _)) => Some(res),
@@ -839,7 +840,7 @@ impl ECStore {
             let mut sent_err = false;
             while let Some(entry) = merge_rx.recv().await {
                 if opts.latest_only {
-                    let fi = match entry.to_fileinfo(&bucket) {
+                    let fi = match entry.to_fileinfo(&bucket_clone) {
                         Ok(res) => res,
                         Err(err) => {
                             if !sent_err {
@@ -864,7 +865,7 @@ impl ECStore {
                     if let Some(filter) = opts.filter {
                         if filter(&fi) {
                             let item = ObjectInfoOrErr {
-                                item: Some(ObjectInfo::from_file_info(&fi, &bucket, &fi.name, {
+                                item: Some(ObjectInfo::from_file_info(&fi, &bucket_clone, &fi.name, {
                                     if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                                 })),
                                 err: None,
@@ -876,7 +877,7 @@ impl ECStore {
                         }
                     } else {
                         let item = ObjectInfoOrErr {
-                            item: Some(ObjectInfo::from_file_info(&fi, &bucket, &fi.name, {
+                            item: Some(ObjectInfo::from_file_info(&fi, &bucket_clone, &fi.name, {
                                 if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                             })),
                             err: None,
@@ -889,7 +890,7 @@ impl ECStore {
                     continue;
                 }
 
-                let fvs = match entry.file_info_versions(&bucket) {
+                let fvs = match entry.file_info_versions(&bucket_clone) {
                     Ok(res) => res,
                     Err(err) => {
                         let item = ObjectInfoOrErr {
@@ -912,7 +913,7 @@ impl ECStore {
                     if let Some(filter) = opts.filter {
                         if filter(fi) {
                             let item = ObjectInfoOrErr {
-                                item: Some(ObjectInfo::from_file_info(fi, &bucket, &fi.name, {
+                                item: Some(ObjectInfo::from_file_info(fi, &bucket_clone, &fi.name, {
                                     if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                                 })),
                                 err: None,
@@ -924,7 +925,7 @@ impl ECStore {
                         }
                     } else {
                         let item = ObjectInfoOrErr {
-                            item: Some(ObjectInfo::from_file_info(fi, &bucket, &fi.name, {
+                            item: Some(ObjectInfo::from_file_info(fi, &bucket_clone, &fi.name, {
                                 if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                             })),
                             err: None,
@@ -940,7 +941,18 @@ impl ECStore {
 
         tokio::spawn(async move { merge_entry_channels(rx, inputs, merge_tx, 1).await });
 
-        join_all(futures).await;
+        let walk_results = join_all(futures).await;
+        for (idx, walk_result) in walk_results.into_iter().enumerate() {
+            if let Err(err) = walk_result {
+                error!(
+                    bucket = %bucket,
+                    prefix = %prefix,
+                    set_task_index = idx,
+                    error = ?err,
+                    "walk_internal list_path_raw task failed"
+                );
+            }
+        }
 
         Ok(())
     }

crates/ecstore/tests/protobuf_bytes_regression_test.rs

@@ -0,0 +1,36 @@
+// Copyright (c) RustFS contributors
+// SPDX-License-Identifier: Apache-2.0
+
+use bytes::Bytes;
+use rustfs_protos::proto_gen::node_service::{
+    ReadMultipleRequest, ReadMultipleResponse, ReadVersionResponse, ReadXlResponse, UpdateMetadataRequest, WriteMetadataRequest,
+};
+
+fn expect_bytes(_: &Bytes) {}
+
+#[test]
+fn protobuf_bytes_fields_use_bytes_consistently() {
+    let update = UpdateMetadataRequest::default();
+    expect_bytes(&update.file_info_bin);
+    expect_bytes(&update.opts_bin);
+
+    let write = WriteMetadataRequest::default();
+    expect_bytes(&write.file_info_bin);
+
+    let version = ReadVersionResponse::default();
+    expect_bytes(&version.file_info_bin);
+
+    let read_xl = ReadXlResponse::default();
+    expect_bytes(&read_xl.raw_file_info_bin);
+
+    let read_multiple = ReadMultipleRequest::default();
+    expect_bytes(&read_multiple.read_multiple_req_bin);
+
+    let read_multiple_response = ReadMultipleResponse::default();
+    let first = read_multiple_response
+        .read_multiple_resps_bin
+        .first()
+        .cloned()
+        .unwrap_or_default();
+    expect_bytes(&first);
+}

crates/heal/src/heal/manager.rs

@@ -237,6 +237,14 @@ impl PriorityHealQueue {
     }
 }
 
+fn publish_active_heal_count(active_heals: &HashMap<String, Arc<HealTask>>) {
+    crate::set_heal_active_tasks(active_heals.len());
+}
+
+fn publish_heal_queue_length(queue: &PriorityHealQueue) {
+    crate::set_heal_queue_length(queue.len());
+}
+
 /// Heal config
 #[derive(Debug, Clone)]
 pub struct HealConfig {
@@ -420,6 +428,8 @@ impl HealManager {
             }
         }
         active_heals.clear();
+        publish_active_heal_count(&active_heals);
+        crate::set_heal_queue_length(0);
 
         // update state
         let mut state = self.state.write().await;
@@ -435,6 +445,7 @@ impl HealManager {
         let mut queue = self.heal_queue.lock().await;
 
         let queue_len = queue.len();
+        publish_heal_queue_length(&queue);
         let queue_capacity = config.queue_size;
 
         if queue.contains_key(&request) {
@@ -505,6 +516,7 @@ impl HealManager {
 
         let push_outcome = queue.push(request);
         debug_assert_eq!(push_outcome, QueuePushOutcome::Accepted);
+        publish_heal_queue_length(&queue);
 
         // Log queue statistics periodically (when adding high/urgent priority items)
         if matches!(priority, HealPriority::High | HealPriority::Urgent) {
@@ -544,7 +556,9 @@ impl HealManager {
 
     /// Get task progress
     pub async fn get_active_tasks_count(&self) -> usize {
-        self.active_heals.lock().await.len()
+        let active_heals = self.active_heals.lock().await;
+        publish_active_heal_count(&active_heals);
+        active_heals.len()
     }
 
     pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
@@ -564,6 +578,7 @@ impl HealManager {
         if let Some(task) = active_heals.get(task_id) {
             task.cancel().await?;
             active_heals.remove(task_id);
+            publish_active_heal_count(&active_heals);
             info!("Cancelled heal task: {}", task_id);
             Ok(())
         } else {
@@ -581,12 +596,14 @@ impl HealManager {
     /// Get active task count
     pub async fn get_active_task_count(&self) -> usize {
         let active_heals = self.active_heals.lock().await;
+        publish_active_heal_count(&active_heals);
         active_heals.len()
     }
 
     /// Get queue length
     pub async fn get_queue_length(&self) -> usize {
         let queue = self.heal_queue.lock().await;
+        publish_heal_queue_length(&queue);
         queue.len()
     }
 
@@ -731,6 +748,7 @@ impl HealManager {
                             );
                             let mut queue = heal_queue.lock().await;
                             if matches!(queue.push(req), QueuePushOutcome::Accepted) {
+                                publish_heal_queue_length(&queue);
                                 let config = config.read().await;
                                 if config.event_driven_scheduler_enable {
                                     notify.notify_one();
@@ -757,6 +775,7 @@ impl HealManager {
     ) {
         let config = config.read().await;
         let mut active_heals_guard = active_heals.lock().await;
+        publish_active_heal_count(&active_heals_guard);
 
         // Check if new heal tasks can be started
         let active_count = active_heals_guard.len();
@@ -769,6 +788,7 @@ impl HealManager {
 
         let mut queue = heal_queue.lock().await;
         let queue_len = queue.len();
+        publish_heal_queue_length(&queue);
 
         if queue_len == 0 {
             return;
@@ -804,6 +824,7 @@ impl HealManager {
                 let task = Arc::new(HealTask::from_request(request, storage.clone()));
                 let task_id = task.id.clone();
                 active_heals_guard.insert(task_id.clone(), task.clone());
+                publish_active_heal_count(&active_heals_guard);
                 update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
                 let active_heals_clone = active_heals.clone();
                 let statistics_clone = statistics.clone();
@@ -828,6 +849,7 @@ impl HealManager {
                     }
                     let mut active_heals_guard = active_heals_clone.lock().await;
                     if let Some(completed_task) = active_heals_guard.remove(&task_id) {
+                        publish_active_heal_count(&active_heals_guard);
                         update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
                         // update statistics
                         let mut stats = statistics_clone.write().await;
@@ -853,6 +875,8 @@ impl HealManager {
         let mut stats = statistics.write().await;
         stats.total_tasks += tasks_started as u64;
         stats.update_running_tasks(active_heals_guard.len() as u64);
+        publish_active_heal_count(&active_heals_guard);
+        publish_heal_queue_length(&queue);
 
         // Log queue status if items remain
         if !queue.is_empty() {

crates/heal/src/lib.rs

@@ -17,6 +17,7 @@ pub mod heal;
 
 pub use error::{Error, Result};
 pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor};
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, OnceLock};
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info};
@@ -55,6 +56,8 @@ static GLOBAL_HEAL_MANAGER: OnceLock<Arc<HealManager>> = OnceLock::new();
 
 /// Global heal channel processor instance
 static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock<Arc<tokio::sync::Mutex<HealChannelProcessor>>> = OnceLock::new();
+static GLOBAL_HEAL_ACTIVE_TASKS: AtomicU64 = AtomicU64::new(0);
+static GLOBAL_HEAL_QUEUE_LENGTH: AtomicU64 = AtomicU64::new(0);
 
 /// Initialize and start heal manager with channel processor
 pub async fn init_heal_manager(
@@ -107,3 +110,19 @@ pub fn get_heal_manager() -> Option<&'static Arc<HealManager>> {
 pub fn get_heal_channel_processor() -> Option<&'static Arc<tokio::sync::Mutex<HealChannelProcessor>>> {
     GLOBAL_HEAL_CHANNEL_PROCESSOR.get()
 }
+
+pub fn current_heal_active_tasks() -> u64 {
+    GLOBAL_HEAL_ACTIVE_TASKS.load(Ordering::Relaxed)
+}
+
+pub fn current_heal_queue_length() -> u64 {
+    GLOBAL_HEAL_QUEUE_LENGTH.load(Ordering::Relaxed)
+}
+
+pub(crate) fn set_heal_active_tasks(count: usize) {
+    GLOBAL_HEAL_ACTIVE_TASKS.store(count as u64, Ordering::Relaxed);
+}
+
+pub(crate) fn set_heal_queue_length(count: usize) {
+    GLOBAL_HEAL_QUEUE_LENGTH.store(count as u64, Ordering::Relaxed);
+}

crates/iam/src/manager.rs

@@ -36,7 +36,7 @@ use rustfs_policy::{
 use rustfs_utils::{get_env_opt_str, path::path_join_buf};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use std::sync::atomic::AtomicU8;
+use std::sync::atomic::{AtomicU8, AtomicU64};
 use std::{
     collections::{HashMap, HashSet},
     sync::{
@@ -93,6 +93,17 @@ pub struct IamCache<T> {
     pub roles: HashMap<ARN, Vec<String>>,
     pub send_chan: Sender<i64>,
     pub last_timestamp: AtomicI64,
+    pub sync_failures: AtomicU64,
+    pub sync_successes: AtomicU64,
+    pub last_sync_duration_millis: AtomicU64,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct IamSyncMetricsSnapshot {
+    pub last_sync_duration_millis: u64,
+    pub since_last_sync_millis: u64,
+    pub sync_failures: u64,
+    pub sync_successes: u64,
 }
 
 impl<T> IamCache<T>
@@ -116,6 +127,9 @@ where
             send_chan: sender,
             roles: HashMap::new(),
             last_timestamp: AtomicI64::new(0),
+            sync_failures: AtomicU64::new(0),
+            sync_successes: AtomicU64::new(0),
+            last_sync_duration_millis: AtomicU64::new(0),
         });
 
         sys.clone().init(receiver).await.unwrap();
@@ -200,11 +214,38 @@ where
     }
 
     async fn load(self: Arc<Self>) -> Result<()> {
-        // debug!("load iam to cache");
-        self.api.load_all(&self.cache).await?;
-        self.last_timestamp
-            .store(OffsetDateTime::now_utc().unix_timestamp(), Ordering::Relaxed);
-        Ok(())
+        let started_at = std::time::Instant::now();
+        match self.api.load_all(&self.cache).await {
+            Ok(()) => {
+                self.last_timestamp
+                    .store(OffsetDateTime::now_utc().unix_timestamp(), Ordering::Relaxed);
+                self.sync_successes.fetch_add(1, Ordering::Relaxed);
+                self.last_sync_duration_millis
+                    .store(started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, Ordering::Relaxed);
+                Ok(())
+            }
+            Err(err) => {
+                self.sync_failures.fetch_add(1, Ordering::Relaxed);
+                Err(err)
+            }
+        }
+    }
+
+    pub fn sync_metrics_snapshot(&self) -> IamSyncMetricsSnapshot {
+        let now_secs = OffsetDateTime::now_utc().unix_timestamp();
+        let last_sync_secs = self.last_timestamp.load(Ordering::Relaxed);
+        let since_last_sync_millis = if last_sync_secs > 0 && now_secs >= last_sync_secs {
+            ((now_secs - last_sync_secs) as u64).saturating_mul(1000)
+        } else {
+            0
+        };
+
+        IamSyncMetricsSnapshot {
+            last_sync_duration_millis: self.last_sync_duration_millis.load(Ordering::Relaxed),
+            since_last_sync_millis,
+            sync_failures: self.sync_failures.load(Ordering::Relaxed),
+            sync_successes: self.sync_successes.load(Ordering::Relaxed),
+        }
     }
 
     pub async fn load_user(&self, access_key: &str) -> Result<()> {
@@ -214,18 +255,30 @@ where
         let mut sts_policy_map = HashMap::new();
         let mut policy_docs_map = HashMap::new();
 
-        let _ = self.api.load_user(access_key, UserType::Svc, &mut users_map).await;
+        match self.api.load_user(access_key, UserType::Svc, &mut users_map).await {
+            Ok(()) => {}
+            Err(err) if is_err_no_such_user(&err) => {}
+            Err(err) => return Err(err),
+        }
 
         let parent_user = users_map.get(access_key).map(|svc| svc.credentials.parent_user.clone());
 
         if let Some(parent_user) = parent_user {
-            let _ = self.api.load_user(&parent_user, UserType::Reg, &mut users_map).await;
+            match self.api.load_user(&parent_user, UserType::Reg, &mut users_map).await {
+                Ok(()) => {}
+                Err(err) if is_err_no_such_user(&err) => {}
+                Err(err) => return Err(err),
+            }
             let _ = self
                 .api
                 .load_mapped_policy(&parent_user, UserType::Reg, false, &mut user_policy_map)
                 .await;
         } else {
-            let _ = self.api.load_user(access_key, UserType::Reg, &mut users_map).await;
+            match self.api.load_user(access_key, UserType::Reg, &mut users_map).await {
+                Ok(()) => {}
+                Err(err) if is_err_no_such_user(&err) => {}
+                Err(err) => return Err(err),
+            }
             if users_map.contains_key(access_key) {
                 let _ = self
                     .api
@@ -233,7 +286,11 @@ where
                     .await;
             }
 
-            let _ = self.api.load_user(access_key, UserType::Sts, &mut sts_users_map).await;
+            match self.api.load_user(access_key, UserType::Sts, &mut sts_users_map).await {
+                Ok(()) => {}
+                Err(err) if is_err_no_such_user(&err) => {}
+                Err(err) => return Err(err),
+            }
 
             let has_sts_user = sts_users_map.get(access_key);
 
@@ -588,6 +645,20 @@ where
             }
         }
 
+        let sts_accounts = self.cache.sts_accounts.load();
+        for (_, v) in sts_accounts.iter() {
+            if v.credentials.parent_user == access_key {
+                user_exists = true;
+                if v.credentials.is_temp() && !v.credentials.is_service_account() {
+                    let mut u = v.clone();
+                    u.credentials.secret_key = String::new();
+                    u.credentials.session_token = String::new();
+
+                    ret.push(u);
+                }
+            }
+        }
+
         if !user_exists {
             return Err(Error::NoSuchUser(access_key.to_string()));
         }
@@ -596,8 +667,8 @@ where
     }
 
     pub async fn list_sts_accounts(&self, access_key: &str) -> Result<Vec<Credentials>> {
-        let users = self.cache.users.load();
-        Ok(users
+        let sts_accounts = self.cache.sts_accounts.load();
+        Ok(sts_accounts
             .values()
             .filter_map(|x| {
                 if !access_key.is_empty()
@@ -1364,14 +1435,13 @@ where
     }
 
     pub async fn is_temp_user(&self, access_key: &str) -> Result<(bool, String)> {
-        let users = self.cache.users.load();
-        let u = match users.get(access_key) {
+        let u = match self.get_user(access_key).await {
             Some(u) => u,
             None => return Err(Error::NoSuchUser(access_key.to_string())),
         };
 
         if u.credentials.is_temp() {
-            Ok((true, u.credentials.parent_user.clone()))
+            Ok((true, u.credentials.parent_user))
         } else {
             Ok((false, String::new()))
         }

crates/iam/src/oidc.rs

@@ -18,11 +18,11 @@
 //! `openidconnect` crate for standards-compliant discovery, token exchange,
 //! and ID token verification.
 
-use crate::oidc_state::{OidcAuthSession, OidcStateStore};
-use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreProviderMetadata};
+use crate::oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore};
+use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken};
 use openidconnect::{
-    AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce,
-    PkceCodeChallenge, PkceCodeVerifier, RedirectUrl, Scope,
+    AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, LogoutRequest, Nonce,
+    PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, ProviderMetadataWithLogout, RedirectUrl, Scope,
 };
 use reqwest::Client;
 use rustfs_config::oidc::*;
@@ -31,11 +31,11 @@ use rustfs_ecstore::config::{Config as ServerConfig, KVS, get_global_server_conf
 use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::future::Future;
 use std::net::IpAddr;
 use std::pin::Pin;
-use std::sync::RwLock;
+use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
 use std::time::{Duration as StdDuration, Instant};
 use tokio::time::sleep;
 use tracing::{error, info, warn};
@@ -44,6 +44,127 @@ use url::Url;
 const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
 const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
 const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
+const OIDC_PLUGIN_AUTHN_WINDOW: StdDuration = StdDuration::from_secs(60);
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct OidcPluginAuthnMetricsSnapshot {
+    pub failed_requests_minute: u64,
+    pub last_fail_seconds: u64,
+    pub last_succ_seconds: u64,
+    pub succ_avg_rtt_ms_minute: u64,
+    pub succ_max_rtt_ms_minute: u64,
+    pub total_requests_minute: u64,
+}
+
+#[derive(Debug, Clone)]
+struct OidcPluginAuthnSample {
+    observed_at: Instant,
+    succeeded: bool,
+    rtt_ms: u64,
+}
+
+#[derive(Debug, Default)]
+struct OidcPluginAuthnMetrics {
+    samples: Mutex<VecDeque<OidcPluginAuthnSample>>,
+    last_fail_at: Mutex<Option<Instant>>,
+    last_succ_at: Mutex<Option<Instant>>,
+}
+
+fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex<T>, metric: &'static str) -> MutexGuard<'a, T> {
+    match mutex.lock() {
+        Ok(guard) => guard,
+        Err(err) => {
+            warn!("recovering poisoned OIDC plugin authn metrics lock: {}", metric);
+            err.into_inner()
+        }
+    }
+}
+
+fn seconds_since(now: Instant, observed_at: Option<Instant>) -> u64 {
+    observed_at
+        .map(|instant| now.duration_since(instant).as_secs())
+        .unwrap_or_default()
+}
+
+impl OidcPluginAuthnMetrics {
+    fn record(&self, rtt_ms: u64, succeeded: bool) {
+        let now = Instant::now();
+        let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples");
+        samples.push_back(OidcPluginAuthnSample {
+            observed_at: now,
+            succeeded,
+            rtt_ms,
+        });
+        while samples
+            .front()
+            .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW)
+        {
+            samples.pop_front();
+        }
+        drop(samples);
+
+        if succeeded {
+            *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at") = Some(now);
+        } else {
+            *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at") = Some(now);
+        }
+    }
+
+    fn snapshot(&self) -> OidcPluginAuthnMetricsSnapshot {
+        let now = Instant::now();
+        let (total_requests_minute, failed_requests_minute, succ_avg_rtt_ms_minute, succ_max_rtt_ms_minute) = {
+            let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples");
+            while samples
+                .front()
+                .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW)
+            {
+                samples.pop_front();
+            }
+
+            let mut failed_requests_minute = 0u64;
+            let mut successful_requests = 0u64;
+            let mut successful_rtt_sum = 0u64;
+            let mut succ_max_rtt_ms_minute = 0u64;
+
+            for sample in samples.iter() {
+                if sample.succeeded {
+                    successful_requests += 1;
+                    successful_rtt_sum += sample.rtt_ms;
+                    succ_max_rtt_ms_minute = succ_max_rtt_ms_minute.max(sample.rtt_ms);
+                } else {
+                    failed_requests_minute += 1;
+                }
+            }
+
+            let succ_avg_rtt_ms_minute = successful_rtt_sum.checked_div(successful_requests).unwrap_or_default();
+
+            (
+                samples.len() as u64,
+                failed_requests_minute,
+                succ_avg_rtt_ms_minute,
+                succ_max_rtt_ms_minute,
+            )
+        };
+
+        let last_fail_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at"));
+        let last_succ_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at"));
+
+        OidcPluginAuthnMetricsSnapshot {
+            failed_requests_minute,
+            last_fail_seconds,
+            last_succ_seconds,
+            succ_avg_rtt_ms_minute,
+            succ_max_rtt_ms_minute,
+            total_requests_minute,
+        }
+    }
+}
+
+static OIDC_PLUGIN_AUTHN_METRICS: LazyLock<OidcPluginAuthnMetrics> = LazyLock::new(OidcPluginAuthnMetrics::default);
+
+pub fn oidc_plugin_authn_metrics_snapshot() -> OidcPluginAuthnMetricsSnapshot {
+    OIDC_PLUGIN_AUTHN_METRICS.snapshot()
+}
 
 // ---- HTTP Client Adapter ----
 
@@ -120,6 +241,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
 
     fn call(&'c self, request: http::Request<Vec<u8>>) -> Self::Future {
         Box::pin(async move {
+            let started_at = Instant::now();
             let (parts, body) = request.into_parts();
             let uri = parts.uri.to_string();
             let client = self.client_for_uri(&uri);
@@ -128,8 +250,13 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
                 .headers(parts.headers)
                 .body(body)
                 .send()
-                .await
-                .map_err(OidcHttpError::Reqwest)?;
+                .await;
+
+            let elapsed_ms = started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
+            let succeeded = response.as_ref().is_ok_and(|resp| resp.status().is_success());
+            OIDC_PLUGIN_AUTHN_METRICS.record(elapsed_ms, succeeded);
+
+            let response = response.map_err(OidcHttpError::Reqwest)?;
 
             let status = response.status();
             let headers = response.headers().clone();
@@ -216,7 +343,7 @@ pub struct OidcClaims {
 /// on-the-fly from metadata when needed.
 #[derive(Clone)]
 struct ProviderState {
-    metadata: CoreProviderMetadata,
+    metadata: ProviderMetadataWithLogout,
     discovered_at: Instant,
 }
 
@@ -364,7 +491,7 @@ impl OidcSys {
         state: &str,
         code: &str,
         redirect_uri: &str,
-    ) -> Result<(OidcClaims, String, OidcAuthSession), String> {
+    ) -> Result<(OidcClaims, String, OidcAuthSession, String), String> {
         // Retrieve and consume the state (single-use)
         let session = self
             .state_store
@@ -449,7 +576,63 @@ impl OidcSys {
             raw,
         };
 
-        Ok((claims, session.provider_id.clone(), session))
+        Ok((claims, session.provider_id.clone(), session, raw_jwt))
+    }
+
+    /// Store a one-time logout session keyed by an opaque token so the console can
+    /// trigger browser logout without persisting the raw ID token.
+    pub async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result<String, String> {
+        if !self.configs.contains_key(provider_id) {
+            return Err(format!("unknown OIDC provider: {provider_id}"));
+        }
+
+        let token = CsrfToken::new_random().secret().clone();
+        self.state_store
+            .insert_logout(
+                token.clone(),
+                OidcLogoutSession {
+                    provider_id: provider_id.to_string(),
+                    id_token: id_token.to_string(),
+                },
+            )
+            .await;
+
+        Ok(token)
+    }
+
+    /// Build the RP-initiated logout URL for a previously issued logout token.
+    /// Returns `Ok(None)` when the provider does not advertise an end-session endpoint.
+    pub async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result<Option<String>, String> {
+        let session = self
+            .state_store
+            .take_logout(logout_token)
+            .await
+            .ok_or_else(|| "invalid or expired OIDC logout token".to_string())?;
+
+        let config = self
+            .configs
+            .get(&session.provider_id)
+            .ok_or_else(|| format!("unknown OIDC provider: {}", session.provider_id))?;
+        let state = self.ensure_provider_state(&session.provider_id, config).await?;
+        let Some(end_session_endpoint) = state.metadata.additional_metadata().end_session_endpoint.clone() else {
+            return Ok(None);
+        };
+
+        let id_token: CoreIdToken = session
+            .id_token
+            .parse()
+            .map_err(|e: serde_json::Error| format!("failed to parse ID token for logout: {e}"))?;
+        let post_logout_redirect_uri = PostLogoutRedirectUrl::new(post_logout_redirect_uri.to_string())
+            .map_err(|e| format!("invalid post logout redirect URI: {e}"))?;
+
+        let logout_url = LogoutRequest::from(end_session_endpoint)
+            .set_id_token_hint(&id_token)
+            .set_client_id(ClientId::new(config.client_id.clone()))
+            .set_post_logout_redirect_uri(post_logout_redirect_uri)
+            .http_get_url()
+            .to_string();
+
+        Ok(Some(logout_url))
     }
 
     /// Map OIDC claims to rustfs policy names.
@@ -930,7 +1113,7 @@ impl OidcSys {
             let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?;
 
             for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES {
-                match CoreProviderMetadata::discover_async(issuer_url.clone(), http_client)
+                match ProviderMetadataWithLogout::discover_async(issuer_url.clone(), http_client)
                     .await
                     .map_err(|e| format!("discovery failed: {e}"))
                 {
@@ -1416,7 +1599,9 @@ mod tests {
         use std::time::{Duration, Instant};
 
         // After the last completed response, exit if no new connection arrives within this window.
-        const IDLE_SHUTDOWN: Duration = Duration::from_millis(500);
+        // Keep the mock server alive long enough for slower CI/macOS test environments to finish
+        // discovery + JWKS requests without racing the shutdown timer.
+        const IDLE_SHUTDOWN: Duration = Duration::from_secs(1);
         const ABSOLUTE_CAP: Duration = Duration::from_secs(5);
 
         let listener = TcpListener::bind("127.0.0.1:0").unwrap();
@@ -1518,6 +1703,17 @@ mod tests {
         err.contains(base) && err.contains(&format!("{base}/")) && err.contains("discovery failed for all issuer variants")
     }
 
+    async fn validate_mocked_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
+        let http_client = ReqwestHttpClient::new()?;
+        let state = OidcSys::discover_provider(config, &http_client).await?;
+
+        Ok(OidcProviderValidationResult {
+            issuer: state.metadata.issuer().to_string(),
+            authorization_endpoint: state.metadata.authorization_endpoint().to_string(),
+            token_endpoint: state.metadata.token_endpoint().map(ToString::to_string),
+        })
+    }
+
     #[tokio::test]
     async fn test_validate_oidc_provider_config_retries_with_issuer_candidates() {
         // Discovery document must advertise the canonical issuer path. The first candidate has no
@@ -1526,7 +1722,7 @@ mod tests {
         let config_url = format!("{base}/application/o/rustfs");
         let config = build_mocked_oidc_provider_config("default", &config_url);
 
-        let result = validate_oidc_provider_config(&config).await;
+        let result = validate_mocked_oidc_provider_config(&config).await;
 
         let validation_result = result.expect("OIDC provider validation should succeed");
         assert_eq!(validation_result.issuer, format!("{base}/application/o/rustfs/"));
@@ -1539,7 +1735,7 @@ mod tests {
         let config_url = format!("{base}/application/o/rustfs");
         let config = build_mocked_oidc_provider_config("default", &config_url);
 
-        let err = validate_oidc_provider_config(&config)
+        let err = validate_mocked_oidc_provider_config(&config)
             .await
             .expect_err("OIDC provider validation should fail");
         assert!(discovery_error_contains_all_variants(&err, &base));

crates/iam/src/oidc_state.rs

@@ -32,11 +32,20 @@ pub struct OidcAuthSession {
     pub redirect_after: Option<String>,
 }
 
+/// Stores an ID token behind a one-time opaque handle so the console can trigger
+/// RP-initiated logout without persisting the raw token in browser storage.
+#[derive(Debug, Clone)]
+pub struct OidcLogoutSession {
+    pub provider_id: String,
+    pub id_token: String,
+}
+
 /// TTL cache for OIDC auth state (PKCE verifiers + nonces) during the authorization flow.
 /// Entries expire after 5 minutes and are single-use (removed on retrieval).
 #[derive(Clone)]
 pub struct OidcStateStore {
     cache: Cache<String, OidcAuthSession>,
+    logout_cache: Cache<String, OidcLogoutSession>,
     last_capacity_log_at: Arc<AtomicU64>,
 }
 
@@ -46,8 +55,13 @@ impl OidcStateStore {
             .max_capacity(OIDC_STATE_CAPACITY)
             .time_to_live(Duration::from_secs(300)) // 5 minute TTL
             .build();
+        let logout_cache = Cache::builder()
+            .max_capacity(OIDC_STATE_CAPACITY)
+            .time_to_live(Duration::from_secs(3600)) // 1 hour TTL to match console OIDC sessions
+            .build();
         Self {
             cache,
+            logout_cache,
             last_capacity_log_at: Arc::new(AtomicU64::new(0)),
         }
     }
@@ -98,6 +112,21 @@ impl OidcStateStore {
     pub async fn contains(&self, state: &str) -> bool {
         self.cache.get(state).await.is_some()
     }
+
+    /// Store a new one-time logout session keyed by an opaque logout token.
+    pub async fn insert_logout(&self, token: String, session: OidcLogoutSession) {
+        self.logout_cache.insert(token, session).await;
+    }
+
+    /// Retrieve and remove a logout session (single-use). Returns None if expired or not found.
+    pub async fn take_logout(&self, token: &str) -> Option<OidcLogoutSession> {
+        self.logout_cache.remove(token).await
+    }
+
+    /// Check if a logout token exists (without consuming it).
+    pub async fn contains_logout(&self, token: &str) -> bool {
+        self.logout_cache.get(token).await.is_some()
+    }
 }
 
 impl Default for OidcStateStore {
@@ -167,4 +196,24 @@ mod tests {
             assert!(store.take(&format!("state_{i}")).await.is_none());
         }
     }
+
+    #[tokio::test]
+    async fn test_logout_state_store_insert_and_take() {
+        let store = OidcStateStore::new();
+        let session = OidcLogoutSession {
+            provider_id: "okta".to_string(),
+            id_token: "jwt-token".to_string(),
+        };
+
+        store.insert_logout("logout_abc".to_string(), session.clone()).await;
+        assert!(store.contains_logout("logout_abc").await);
+
+        let retrieved = store.take_logout("logout_abc").await;
+        assert!(retrieved.is_some());
+        let retrieved = retrieved.unwrap();
+        assert_eq!(retrieved.provider_id, "okta");
+        assert_eq!(retrieved.id_token, "jwt-token");
+
+        assert!(store.take_logout("logout_abc").await.is_none());
+    }
 }

crates/iam/src/store/object.rs

@@ -227,20 +227,13 @@ impl ObjectStore {
         Ok(encrypted)
     }
 
-    fn encrypt_data(data: &[u8]) -> Result<Vec<u8>> {
+    fn prepare_data_for_storage(data: &[u8]) -> Result<Vec<u8>> {
         if keyring::encrypt_key().is_some() {
             let encrypted = Self::encrypt_data_with_master_key(data)?;
             return Ok(encrypted);
         }
 
-        let cred = get_global_action_cred().unwrap_or_default();
-        let password = if !cred.access_key.is_empty() && !cred.secret_key.is_empty() {
-            format!("{}:{}", cred.access_key, cred.secret_key).into_bytes()
-        } else {
-            cred.secret_key.into_bytes()
-        };
-        let en = rustfs_crypto::encrypt_stream_io(&password, data)?;
-        Ok(en)
+        Ok(data.to_vec())
     }
 
     fn should_lazy_rewrite(source: DecryptSource) -> bool {
@@ -343,8 +336,8 @@ impl ObjectStore {
     }
 
     #[cfg(test)]
-    fn encrypt_data_for_test(data: &[u8]) -> Result<Vec<u8>> {
-        Self::encrypt_data(data)
+    fn prepare_data_for_storage_for_test(data: &[u8]) -> Result<Vec<u8>> {
+        Self::prepare_data_for_storage(data)
     }
 
     async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef<str> + Send) -> Result<(Vec<u8>, ObjectInfo)> {
@@ -369,9 +362,17 @@ impl ObjectStore {
 
         let path = prefix.to_owned();
         tokio::spawn(async move {
-            store
+            if let Err(err) = store
                 .walk(ctx.clone(), Self::BUCKET_NAME, &path, tx, WalkOptions::default())
                 .await
+            {
+                error!(
+                    bucket = Self::BUCKET_NAME,
+                    prefix = %path,
+                    error = ?err,
+                    "list_iam_config_items walk task failed"
+                );
+            }
         });
 
         let prefix = prefix.to_owned();
@@ -648,7 +649,7 @@ impl Store for ObjectStore {
     #[tracing::instrument(skip(self, item, path))]
     async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()> {
         let mut data = serde_json::to_vec(&item)?;
-        data = Self::encrypt_data(&data)?;
+        data = Self::prepare_data_for_storage(&data)?;
 
         let mut attempts = 0;
         let max_attempts = 5;
@@ -1450,28 +1451,29 @@ mod tests {
     }
 
     #[test]
-    fn test_encrypt_data_produces_stream_io_format() {
-        let _ = test_cred();
+    #[serial]
+    fn test_prepare_data_defaults_to_plaintext_without_iam_master_key() {
         let plain = br#"{"Version":1,"policy":"readonly"}"#;
-        let encrypted = ObjectStore::encrypt_data_for_test(plain).expect("encrypt should succeed");
-        // stream_io header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes
-        const STREAM_IO_HEADER_LEN: usize = 41;
-        assert!(
-            encrypted.len() >= STREAM_IO_HEADER_LEN,
-            "encrypted should have at least 41-byte stream_io header"
+
+        with_vars(
+            [
+                (keyring::ENV_IAM_MASTER_KEY, None::<&str>),
+                (keyring::ENV_IAM_MASTER_KEY_OLD_KEYS, None::<&str>),
+            ],
+            || {
+                let stored = ObjectStore::prepare_data_for_storage_for_test(plain).expect("store bytes should build");
+                assert_eq!(stored, plain);
+
+                let decrypted = ObjectStore::decrypt_data_with_source(&stored).expect("plaintext should load");
+                assert_eq!(plain, decrypted.plain.as_slice());
+                assert_eq!(decrypted.source, DecryptSource::Plaintext);
+            },
         );
-        assert!(
-            encrypted[32] == 0x00 || encrypted[32] == 0x01 || encrypted[32] == 0x02,
-            "alg_id should be 0x00, 0x01, or 0x02"
-        );
-        // Round-trip: encrypt then decrypt
-        let decrypted = ObjectStore::decrypt_data_with_source(&encrypted).expect("decrypt should succeed");
-        assert_eq!(plain, decrypted.plain.as_slice());
     }
 
     #[test]
     #[serial]
-    fn test_encrypt_data_prefers_iam_master_key_roundtrip() {
+    fn test_prepare_data_uses_iam_master_key_roundtrip() {
         let _ = test_cred();
         let plain = br#"{"Version":1,"policy":"master-key"}"#;
         let master_key = "iam-master-key-roundtrip";
@@ -1482,7 +1484,7 @@ mod tests {
                 (keyring::ENV_IAM_MASTER_KEY_OLD_KEYS, None),
             ],
             || {
-                let encrypted = ObjectStore::encrypt_data_for_test(plain).expect("encrypt with iam master key");
+                let encrypted = ObjectStore::prepare_data_for_storage_for_test(plain).expect("encrypt with iam master key");
 
                 let by_master =
                     rustfs_crypto::decrypt_stream_io(master_key.as_bytes(), &encrypted).expect("decrypt via master key");

crates/iam/src/sys.rs

@@ -16,9 +16,9 @@ use crate::error::Error as IamError;
 use crate::error::is_err_no_such_account;
 use crate::error::is_err_no_such_temp_account;
 use crate::error::{Error, Result};
-use crate::manager::IamCache;
 use crate::manager::extract_jwt_claims;
 use crate::manager::get_default_policyes;
+use crate::manager::{IamCache, IamSyncMetricsSnapshot};
 use crate::store::GroupInfo;
 use crate::store::MappedPolicy;
 use crate::store::Store;
@@ -186,6 +186,10 @@ impl<T: Store> IamSys<T> {
         self.store.api.has_watcher()
     }
 
+    pub fn sync_metrics_snapshot(&self) -> IamSyncMetricsSnapshot {
+        self.store.sync_metrics_snapshot()
+    }
+
     pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
         let policy_plugin_client = get_policy_plugin_client();
         let mut guard = policy_plugin_client.write().await;
@@ -751,7 +755,7 @@ impl<T: Store> IamSys<T> {
                 Ok((Some(res), ok))
             }
             None => {
-                let _ = self.store.load_user(access_key).await;
+                self.store.load_user(access_key).await?;
 
                 if let Some(res) = self.store.get_user(access_key).await {
                     let ok = res.credentials.is_valid();
@@ -863,16 +867,6 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        let Ok((is_temp, parent_user)) = self.is_temp_user(args.account).await else {
-            return PreparedIamAuth {
-                needs_existing_object_tag: false,
-                mode: PreparedIamMode::Deny,
-            };
-        };
-        if is_temp {
-            return self.prepare_sts_auth(args, &parent_user).await;
-        }
-
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
             return PreparedIamAuth {
                 needs_existing_object_tag: false,
@@ -883,6 +877,16 @@ impl<T: Store> IamSys<T> {
             return self.prepare_service_account_auth(args, &parent_user).await;
         }
 
+        let Ok((is_temp, parent_user)) = self.is_temp_user(args.account).await else {
+            return PreparedIamAuth {
+                needs_existing_object_tag: false,
+                mode: PreparedIamMode::Deny,
+            };
+        };
+        if is_temp {
+            return self.prepare_sts_auth(args, &parent_user).await;
+        }
+
         self.prepare_regular_auth(args).await
     }
 
@@ -1291,7 +1295,7 @@ mod tests {
     use crate::manager::get_default_policyes;
     use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
     use rustfs_credentials::{Credentials, get_global_action_cred, init_global_action_credentials};
-    use rustfs_policy::auth::UserIdentity;
+    use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
     use rustfs_policy::policy::Args;
     use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
     use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
@@ -1368,6 +1372,10 @@ mod tests {
         }
 
         async fn load_user(&self, name: &str, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()> {
+            if user_type == UserType::Reg && name == "load-failure-user" {
+                return Err(Error::Io(std::io::Error::other("load user failed")));
+            }
+
             if user_type == UserType::Reg && name == "notify-user" {
                 let user = UserIdentity::from(Credentials {
                     access_key: name.to_string(),
@@ -1669,6 +1677,189 @@ mod tests {
         assert_eq!(claims.get("exp").and_then(|v| v.as_i64()), Some(updated_expiration.unix_timestamp()));
     }
 
+    #[tokio::test]
+    async fn test_created_access_token_authorizes_with_parent_policy() {
+        ensure_test_global_credentials();
+
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-fallback-test-parent";
+        let groups = Some(vec!["testgroup".to_string()]);
+        let (cred, _) = iam_sys
+            .new_service_account(
+                parent_user,
+                groups.clone(),
+                NewServiceAccountOpts {
+                    access_key: "ACCESSTOKENTESTUSER".to_string(),
+                    secret_key: "accessTokenTestSecret".to_string(),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("access token should be created");
+
+        let stored = iam_sys
+            .get_user(&cred.access_key)
+            .await
+            .expect("created access token should be cached");
+        assert!(stored.credentials.is_service_account());
+        assert_eq!(stored.credentials.parent_user, parent_user);
+
+        let claims = stored
+            .credentials
+            .claims
+            .as_ref()
+            .expect("created access token should have decoded JWT claims");
+        assert_eq!(claims.get("parent").and_then(Value::as_str), Some(parent_user));
+        assert_eq!(
+            claims.get(&iam_policy_claim_name_sa()).and_then(Value::as_str),
+            Some(INHERITED_POLICY_TYPE)
+        );
+
+        let (is_service_account, resolved_parent) = iam_sys
+            .is_service_account(&cred.access_key)
+            .await
+            .expect("created access token should be recognized as a service account");
+        assert!(is_service_account);
+        assert_eq!(resolved_parent, parent_user);
+
+        let (redacted, policy) = iam_sys
+            .get_service_account(&cred.access_key)
+            .await
+            .expect("created access token should be readable");
+        assert_eq!(redacted.access_key, cred.access_key);
+        assert_eq!(redacted.parent_user, parent_user);
+        assert!(redacted.secret_key.is_empty());
+        assert!(redacted.session_token.is_empty());
+        assert!(policy.is_none());
+
+        let args = Args {
+            account: &cred.access_key,
+            groups: &groups,
+            action: Action::S3Action(S3Action::ListBucketAction),
+            bucket: "mybucket",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims,
+            deny_only: false,
+        };
+
+        let prepared = iam_sys.prepare_auth(&args).await;
+        assert!(
+            matches!(prepared.mode, PreparedIamMode::ServiceAccount { .. }),
+            "created access token must use service-account authorization path"
+        );
+        assert!(
+            iam_sys.eval_prepared(&prepared, &args).await,
+            "created access token should be allowed through the parent's group policy"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_created_sts_credentials_authorize_with_session_token_claims() {
+        ensure_test_global_credentials();
+
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-fallback-test-parent";
+        let token_secret = get_global_action_cred()
+            .expect("global action credentials should be initialized")
+            .secret_key;
+        let mut claims = HashMap::new();
+        claims.insert("parent".to_string(), Value::String(parent_user.to_string()));
+        claims.insert(
+            "exp".to_string(),
+            Value::Number(serde_json::Number::from(
+                (OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp(),
+            )),
+        );
+
+        let mut cred = get_new_credentials_with_metadata(&claims, &token_secret).expect("STS credentials should be created");
+        cred.parent_user = parent_user.to_string();
+
+        iam_sys
+            .set_temp_user(&cred.access_key, &cred, None)
+            .await
+            .expect("STS credentials should be persisted in the temp-user cache");
+
+        let stored = iam_sys
+            .get_user(&cred.access_key)
+            .await
+            .expect("created STS credentials should be cached");
+        assert!(stored.credentials.is_temp());
+        assert!(!stored.credentials.is_service_account());
+        assert_eq!(stored.credentials.parent_user, parent_user);
+
+        let (is_temp, resolved_parent) = iam_sys
+            .is_temp_user(&cred.access_key)
+            .await
+            .expect("created STS credentials should be recognized as temp");
+        assert!(is_temp);
+        assert_eq!(resolved_parent, parent_user);
+
+        let listed = iam_sys
+            .list_sts_accounts(parent_user)
+            .await
+            .expect("created STS credentials should be listable by parent");
+        assert_eq!(listed.len(), 1);
+        assert_eq!(listed[0].access_key, cred.access_key);
+        assert_eq!(listed[0].parent_user, parent_user);
+        assert!(listed[0].secret_key.is_empty());
+        assert!(listed[0].session_token.is_empty());
+
+        let temp_accounts = iam_sys
+            .list_temp_accounts(parent_user)
+            .await
+            .expect("created STS credentials should be listable as temp accounts");
+        assert_eq!(temp_accounts.len(), 1);
+        assert_eq!(temp_accounts[0].credentials.access_key, cred.access_key);
+        assert_eq!(temp_accounts[0].credentials.parent_user, parent_user);
+        assert!(temp_accounts[0].credentials.secret_key.is_empty());
+        assert!(temp_accounts[0].credentials.session_token.is_empty());
+
+        let (redacted, policy) = iam_sys
+            .get_temporary_account(&cred.access_key)
+            .await
+            .expect("created STS credentials should be readable");
+        assert_eq!(redacted.access_key, cred.access_key);
+        assert_eq!(redacted.parent_user, parent_user);
+        assert!(redacted.secret_key.is_empty());
+        assert!(redacted.session_token.is_empty());
+        assert!(policy.is_none());
+
+        let decoded_claims = get_claims_from_token_with_secret(&cred.session_token, &token_secret)
+            .expect("created STS session token should decode with the active signing key");
+        assert_eq!(decoded_claims.get("parent").and_then(Value::as_str), Some(parent_user));
+
+        let groups: Option<Vec<String>> = None;
+        let args = Args {
+            account: &cred.access_key,
+            groups: &groups,
+            action: Action::S3Action(S3Action::ListBucketAction),
+            bucket: "mybucket",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims: &decoded_claims,
+            deny_only: false,
+        };
+
+        let prepared = iam_sys.prepare_auth(&args).await;
+        assert!(
+            matches!(prepared.mode, PreparedIamMode::Sts { .. }),
+            "created STS credentials must use STS authorization path"
+        );
+        assert!(
+            iam_sys.eval_prepared(&prepared, &args).await,
+            "created STS credentials should inherit the parent user's group policy"
+        );
+    }
+
     /// Regression test: temp credentials without groups in args still receive group-attached
     /// policies via the parent user (groups fallback). Without the fallback, policy_db_get
     /// would get None for groups and the user would have no group policies, so the action
@@ -1809,6 +2000,17 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_check_key_propagates_cache_miss_load_failure() {
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let result = iam_sys.check_key("load-failure-user").await;
+
+        assert!(matches!(result, Err(Error::Io(_))));
+    }
+
     #[tokio::test]
     async fn test_prepare_auth_eval_matches_prepare_sts_auth_for_parent_policy_fallback() {
         let store = StsTestMockStore { empty_policies: false };

crates/io-core/src/pool.rs

@@ -84,6 +84,12 @@ struct PoolTier {
     available_buffers: Mutex<Vec<BytesMut>>,
     /// Metrics for tracking this tier
     metrics: Mutex<Option<Arc<BytesPoolMetrics>>>,
+    /// Total acquisitions for this tier
+    tier_total_acquires: AtomicU64,
+    /// Total hits for this tier
+    tier_pool_hits: AtomicU64,
+    /// Current allocated bytes for this tier
+    tier_current_allocated_bytes: AtomicU64,
 }
 
 /// Pool metrics for monitoring and optimization.
@@ -291,6 +297,9 @@ impl PoolTier {
             name,
             available_buffers: Mutex::new(Vec::new()),
             metrics: Mutex::new(None),
+            tier_total_acquires: AtomicU64::new(0),
+            tier_pool_hits: AtomicU64::new(0),
+            tier_current_allocated_bytes: AtomicU64::new(0),
         }
     }
 
@@ -298,6 +307,65 @@ impl PoolTier {
         *self.metrics.lock().unwrap() = Some(metrics);
     }
 
+    fn take_or_allocate_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> (BytesMut, bool) {
+        let buffer_opt = {
+            let mut available = self.available_buffers.lock().unwrap();
+            available.pop()
+        };
+        let was_reused = buffer_opt.is_some();
+
+        let buffer = if let Some(mut buf) = buffer_opt {
+            let previous_capacity = buf.capacity();
+            buf.clear();
+            if previous_capacity < size {
+                buf.reserve(size - previous_capacity);
+            }
+            let current_capacity = buf.capacity();
+            if current_capacity > previous_capacity {
+                let delta = (current_capacity - previous_capacity) as u64;
+                pool_metrics.total_bytes_allocated.fetch_add(delta, Ordering::Relaxed);
+                pool_metrics.current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+                self.tier_current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+            }
+            buf
+        } else {
+            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
+            let allocated_bytes = buf.capacity() as u64;
+            pool_metrics
+                .total_bytes_allocated
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            pool_metrics
+                .current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            self.tier_current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            buf
+        };
+
+        (buffer, was_reused)
+    }
+
+    fn record_acquire_metrics(&self, pool_metrics: &BytesPoolMetrics, buffer_capacity: usize, was_reused: bool) {
+        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
+
+        if was_reused {
+            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
+            self.tier_pool_hits.fetch_add(1, Ordering::Relaxed);
+        } else {
+            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
+        }
+
+        let tier_total_acquires = self.tier_total_acquires.load(Ordering::Relaxed);
+        let tier_pool_hits = self.tier_pool_hits.load(Ordering::Relaxed);
+        let tier_hit_rate = if tier_total_acquires == 0 {
+            0.0
+        } else {
+            tier_pool_hits as f64 / tier_total_acquires as f64
+        };
+        rustfs_io_metrics::record_bytes_pool_hit_rate(self.name, tier_hit_rate);
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
+    }
+
     async fn acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> PooledBuffer {
         // Acquire semaphore permit (owned for storage in PooledBuffer)
         let permit = Arc::clone(&self.semaphore).acquire_owned().await.unwrap();
@@ -308,45 +376,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer - clear and ensure capacity
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -365,45 +399,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         Some(PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -421,8 +421,24 @@ impl PoolTier {
             if let Some(ref metrics) = *self.metrics.lock().unwrap() {
                 metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
             }
+        } else {
+            let released_bytes = buffer.capacity() as u64;
+            self.tier_current_allocated_bytes
+                .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                    Some(current.saturating_sub(released_bytes))
+                })
+                .ok();
+            if let Some(ref metrics) = *self.metrics.lock().unwrap() {
+                metrics
+                    .current_allocated_bytes
+                    .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                        Some(current.saturating_sub(released_bytes))
+                    })
+                    .ok();
+            }
         }
         // If pool is full, buffer is dropped and memory is freed
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 }
 
@@ -617,4 +633,47 @@ mod tests {
         let delta_hits = pool.metrics().pool_hits.load(Ordering::Relaxed) - initial_hits;
         assert_eq!(delta_hits, 1);
     }
+
+    #[tokio::test]
+    async fn test_tier_allocated_bytes_tracks_real_allocations() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire allocates one small-tier buffer.
+        let buf1 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // Return and reuse should not increase allocated bytes.
+        drop(buf1);
+        let buf2 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // A second in-flight buffer forces one more allocation.
+        let _buf3 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 2048);
+
+        drop(buf2);
+    }
+
+    #[tokio::test]
+    async fn test_tier_hit_rate_counters_track_reuse() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire is miss.
+        let buf1 = pool.acquire_buffer(512).await;
+        drop(buf1);
+
+        // Second acquire reuses previous buffer and counts as hit.
+        let _buf2 = pool.acquire_buffer(512).await;
+
+        assert_eq!(pool.small_pool.tier_total_acquires.load(Ordering::Relaxed), 2);
+        assert_eq!(pool.small_pool.tier_pool_hits.load(Ordering::Relaxed), 1);
+    }
 }

crates/io-metrics/src/adaptive_ttl.rs

@@ -31,14 +31,14 @@ use std::time::{Duration, Instant};
 pub fn record_ttl_adjustment(_key: &str, base_ttl: u64, adjusted_ttl: u64) {
     use metrics::{counter, gauge};
 
-    counter!("rustfs.cache.ttl.adjustments").increment(1);
-    gauge!("rustfs.cache.ttl.base").set(base_ttl as f64);
-    gauge!("rustfs.cache.ttl.adjusted").set(adjusted_ttl as f64);
+    counter!("rustfs_cache_ttl_adjustments").increment(1);
+    gauge!("rustfs_cache_ttl_base").set(base_ttl as f64);
+    gauge!("rustfs_cache_ttl_adjusted").set(adjusted_ttl as f64);
 
     if adjusted_ttl > base_ttl {
-        counter!("rustfs.cache.ttl.extensions").increment(1);
+        counter!("rustfs_cache_ttl_extensions").increment(1);
     } else if adjusted_ttl < base_ttl {
-        counter!("rustfs.cache.ttl.reductions").increment(1);
+        counter!("rustfs_cache_ttl_reductions").increment(1);
     }
 }
 
@@ -46,7 +46,7 @@ pub fn record_ttl_adjustment(_key: &str, base_ttl: u64, adjusted_ttl: u64) {
 #[inline(always)]
 pub fn record_ttl_expiration() {
     use metrics::counter;
-    counter!("rustfs.cache.ttl.expirations").increment(1);
+    counter!("rustfs_cache_ttl_expirations").increment(1);
 }
 
 /// Record early eviction.
@@ -57,7 +57,7 @@ pub fn record_ttl_expiration() {
 #[inline(always)]
 pub fn record_early_eviction(reason: &str) {
     use metrics::counter;
-    counter!("rustfs.cache.evictions.early", "reason" => reason.to_string()).increment(1);
+    counter!("rustfs_cache_evictions_early", "reason" => reason.to_string()).increment(1);
 }
 
 /// Record access pattern change.
@@ -69,7 +69,7 @@ pub fn record_early_eviction(reason: &str) {
 #[inline(always)]
 pub fn record_access_pattern_change(from: &str, to: &str) {
     use metrics::counter;
-    counter!("rustfs.cache.access_pattern.changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
+    counter!("rustfs_cache_access_pattern_changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
 }
 
 /// Adaptive TTL statistics.

crates/io-metrics/src/backpressure_metrics.rs

@@ -18,35 +18,35 @@
 #[inline(always)]
 pub fn record_backpressure_state_change(from: &str, to: &str) {
     use metrics::counter;
-    counter!("rustfs.backpressure.state.changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
+    counter!("rustfs_backpressure_state_changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
 }
 
 /// Record backpressure rejection.
 #[inline(always)]
 pub fn record_backpressure_rejection() {
     use metrics::counter;
-    counter!("rustfs.backpressure.rejections").increment(1);
+    counter!("rustfs_backpressure_rejections").increment(1);
 }
 
 /// Record concurrent operations count.
 #[inline(always)]
 pub fn record_concurrent_operations(count: usize) {
     use metrics::gauge;
-    gauge!("rustfs.backpressure.concurrent").set(count as f64);
+    gauge!("rustfs_backpressure_concurrent").set(count as f64);
 }
 
 /// Record backpressure activation.
 #[inline(always)]
 pub fn record_backpressure_activation() {
     use metrics::counter;
-    counter!("rustfs.backpressure.activations").increment(1);
+    counter!("rustfs_backpressure_activations").increment(1);
 }
 
 /// Record backpressure deactivation.
 #[inline(always)]
 pub fn record_backpressure_deactivation() {
     use metrics::counter;
-    counter!("rustfs.backpressure.deactivations").increment(1);
+    counter!("rustfs_backpressure_deactivations").increment(1);
 }
 
 #[cfg(test)]

crates/io-metrics/src/capacity_metrics.rs

@@ -20,35 +20,35 @@ use std::time::Duration;
 /// Record capacity cache hit.
 #[inline(always)]
 pub fn record_capacity_cache_hit() {
-    counter!("rustfs.capacity.cache.hits").increment(1);
+    counter!("rustfs_capacity_cache_hits").increment(1);
 }
 
 /// Record capacity cache miss.
 #[inline(always)]
 pub fn record_capacity_cache_miss() {
-    counter!("rustfs.capacity.cache.misses").increment(1);
+    counter!("rustfs_capacity_cache_misses").increment(1);
 }
 
 /// Record how capacity cache was served to the caller.
 #[inline(always)]
 pub fn record_capacity_cache_served(state: &'static str) {
-    counter!("rustfs.capacity.cache.served.total", "state" => state).increment(1);
+    counter!("rustfs_capacity_cache_served_total", "state" => state).increment(1);
 }
 
 /// Record current capacity gauge.
 #[inline(always)]
 pub fn record_capacity_current_bytes(used_bytes: u64) {
-    gauge!("rustfs.capacity.current").set(used_bytes as f64);
+    gauge!("rustfs_capacity_current_bytes").set(used_bytes as f64);
 }
 
 /// Record capacity update completion.
 #[inline(always)]
 pub fn record_capacity_update_completed(source: &'static str, duration: Duration, used_bytes: u64, is_estimated: bool) {
-    counter!("rustfs.capacity.update.total", "source" => source).increment(1);
-    histogram!("rustfs.capacity.update.duration.seconds", "source" => source).record(duration.as_secs_f64());
-    histogram!("rustfs.capacity.update.bytes", "source" => source).record(used_bytes as f64);
+    counter!("rustfs_capacity_update_total", "source" => source).increment(1);
+    histogram!("rustfs_capacity_update_duration_seconds", "source" => source).record(duration.as_secs_f64());
+    histogram!("rustfs_capacity_update_bytes", "source" => source).record(used_bytes as f64);
     counter!(
-        "rustfs.capacity.update.estimated.total",
+        "rustfs_capacity_update_estimated_total",
         "source" => source,
         "estimated" => if is_estimated { "true" } else { "false" }
     )
@@ -58,86 +58,86 @@ pub fn record_capacity_update_completed(source: &'static str, duration: Duration
 /// Record failed capacity update.
 #[inline(always)]
 pub fn record_capacity_update_failed(source: &'static str) {
-    counter!("rustfs.capacity.update.failures", "source" => source).increment(1);
+    counter!("rustfs_capacity_update_failures", "source" => source).increment(1);
 }
 
 /// Record a capacity refresh request.
 #[inline(always)]
 pub fn record_capacity_refresh_request(mode: &'static str, source: &'static str) {
-    counter!("rustfs.capacity.refresh.requests.total", "mode" => mode, "source" => source).increment(1);
+    counter!("rustfs_capacity_refresh_requests_total", "mode" => mode, "source" => source).increment(1);
 }
 
 /// Record a refresh joiner waiting for an inflight refresh.
 #[inline(always)]
 pub fn record_capacity_refresh_joiner(source: &'static str) {
-    counter!("rustfs.capacity.refresh.joiners.total", "source" => source).increment(1);
+    counter!("rustfs_capacity_refresh_joiners_total", "source" => source).increment(1);
 }
 
 /// Record the number of inflight capacity refreshes.
 #[inline(always)]
 pub fn record_capacity_refresh_inflight(count: usize) {
-    gauge!("rustfs.capacity.refresh.inflight").set(count as f64);
+    gauge!("rustfs_capacity_refresh_inflight").set(count as f64);
 }
 
 /// Record the final result of a capacity refresh.
 #[inline(always)]
 pub fn record_capacity_refresh_result(source: &'static str, result: &'static str, duration: Duration) {
-    counter!("rustfs.capacity.refresh.result.total", "source" => source, "result" => result).increment(1);
-    histogram!("rustfs.capacity.refresh.duration.seconds", "source" => source, "result" => result).record(duration.as_secs_f64());
+    counter!("rustfs_capacity_refresh_result_total", "source" => source, "result" => result).increment(1);
+    histogram!("rustfs_capacity_refresh_duration_seconds", "source" => source, "result" => result).record(duration.as_secs_f64());
 }
 
 /// Record the refresh scope selected for a capacity refresh.
 #[inline(always)]
 pub fn record_capacity_refresh_scope(scope: &'static str, disk_count: usize) {
-    counter!("rustfs.capacity.refresh.scope.total", "scope" => scope).increment(1);
-    histogram!("rustfs.capacity.refresh.scope.disks", "scope" => scope).record(disk_count as f64);
+    counter!("rustfs_capacity_refresh_scope_total", "scope" => scope).increment(1);
+    histogram!("rustfs_capacity_refresh_scope_disks", "scope" => scope).record(disk_count as f64);
 }
 
 /// Record the current number of dirty disks tracked by capacity management.
 #[inline(always)]
 pub fn record_capacity_dirty_disk_count(count: usize) {
-    gauge!("rustfs.capacity.dirty.disks").set(count as f64);
+    gauge!("rustfs_capacity_dirty_disks").set(count as f64);
 }
 
 /// Record capacity write activity.
 #[inline(always)]
 pub fn record_capacity_write_operation(write_frequency: usize) {
-    counter!("rustfs.capacity.write.operations").increment(1);
-    gauge!("rustfs.capacity.write.frequency").set(write_frequency as f64);
+    counter!("rustfs_capacity_write_operations").increment(1);
+    gauge!("rustfs_capacity_write_frequency").set(write_frequency as f64);
 }
 
 /// Record symlink accounting.
 #[inline(always)]
 pub fn record_capacity_symlink(size_bytes: u64) {
-    counter!("rustfs.capacity.symlinks.encountered").increment(1);
-    histogram!("rustfs.capacity.symlinks.size.bytes").record(size_bytes as f64);
+    counter!("rustfs_capacity_symlinks_encountered").increment(1);
+    histogram!("rustfs_capacity_symlinks_size_bytes").record(size_bytes as f64);
 }
 
 /// Record timeout fallback event.
 #[inline(always)]
 pub fn record_capacity_timeout_fallback() {
-    counter!("rustfs.capacity.timeout.fallback").increment(1);
+    counter!("rustfs_capacity_timeout_fallback").increment(1);
 }
 
 /// Record stall detection event.
 #[inline(always)]
 pub fn record_capacity_stall_detected() {
-    counter!("rustfs.capacity.timeout.stall").increment(1);
+    counter!("rustfs_capacity_timeout_stall").increment(1);
 }
 
 /// Record dynamic timeout usage.
 #[inline(always)]
 pub fn record_capacity_dynamic_timeout(timeout: Duration) {
-    counter!("rustfs.capacity.timeout.dynamic").increment(1);
-    histogram!("rustfs.capacity.timeout.dynamic.seconds").record(timeout.as_secs_f64());
+    counter!("rustfs_capacity_timeout_dynamic").increment(1);
+    histogram!("rustfs_capacity_timeout_dynamic_seconds").record(timeout.as_secs_f64());
 }
 
 /// Record scan sampling outcome.
 #[inline(always)]
 pub fn record_capacity_scan_sampling(sampled_count: usize, estimated: bool) {
-    histogram!("rustfs.capacity.scan.sampled.count").record(sampled_count as f64);
+    histogram!("rustfs_capacity_scan_sampled_count").record(sampled_count as f64);
     counter!(
-        "rustfs.capacity.scan.estimated.total",
+        "rustfs_capacity_scan_estimated_total",
         "estimated" => if estimated { "true" } else { "false" }
     )
     .increment(1);
@@ -146,7 +146,7 @@ pub fn record_capacity_scan_sampling(sampled_count: usize, estimated: bool) {
 /// Record the scan mode used for a capacity result.
 #[inline(always)]
 pub fn record_capacity_scan_mode(mode: &'static str) {
-    counter!("rustfs.capacity.scan.mode.total", "mode" => mode).increment(1);
+    counter!("rustfs_capacity_scan_mode_total", "mode" => mode).increment(1);
 }
 
 /// Record per-disk capacity scan statistics.
@@ -159,16 +159,16 @@ pub fn record_capacity_scan_disk(
     estimated: bool,
     partial_errors: bool,
 ) {
-    histogram!("rustfs.capacity.scan.disk.duration.seconds", "disk" => disk.to_owned()).record(duration.as_secs_f64());
-    histogram!("rustfs.capacity.scan.disk.files", "disk" => disk.to_owned()).record(file_count as f64);
-    histogram!("rustfs.capacity.scan.disk.sampled", "disk" => disk.to_owned()).record(sampled_count as f64);
+    histogram!("rustfs_capacity_scan_disk_duration_seconds", "disk" => disk.to_owned()).record(duration.as_secs_f64());
+    histogram!("rustfs_capacity_scan_disk_files", "disk" => disk.to_owned()).record(file_count as f64);
+    histogram!("rustfs_capacity_scan_disk_sampled", "disk" => disk.to_owned()).record(sampled_count as f64);
     counter!(
-        "rustfs.capacity.scan.disk.estimated.total",
+        "rustfs_capacity_scan_disk_estimated_total",
         "disk" => disk.to_owned(),
         "estimated" => if estimated { "true" } else { "false" }
     )
     .increment(1);
     if partial_errors {
-        counter!("rustfs.capacity.scan.disk.partial_errors.total", "disk" => disk.to_owned()).increment(1);
+        counter!("rustfs_capacity_scan_disk_partial_errors_total", "disk" => disk.to_owned()).increment(1);
     }
 }

crates/io-metrics/src/deadlock_metrics.rs

@@ -20,52 +20,52 @@ use std::time::Duration;
 #[inline(always)]
 pub fn record_deadlock_detected(cycle_length: usize) {
     use metrics::{counter, histogram};
-    counter!("rustfs.deadlock.detected").increment(1);
-    histogram!("rustfs.deadlock.cycle_length").record(cycle_length as f64);
+    counter!("rustfs_deadlock_detected_total").increment(1);
+    histogram!("rustfs_deadlock_cycle_length").record(cycle_length as f64);
 }
 
 /// Record long-held lock.
 #[inline(always)]
 pub fn record_long_held_lock(_lock_id: u64, hold_time: Duration) {
     use metrics::{counter, histogram};
-    counter!("rustfs.deadlock.long_held").increment(1);
-    histogram!("rustfs.deadlock.hold_time.secs").record(hold_time.as_secs_f64());
+    counter!("rustfs_deadlock_long_held").increment(1);
+    histogram!("rustfs_deadlock_hold_time_secs").record(hold_time.as_secs_f64());
 }
 
 /// Record lock acquisition.
 #[inline(always)]
 pub fn record_lock_acquisition(lock_type: &str) {
     use metrics::counter;
-    counter!("rustfs.lock.acquisitions", "type" => lock_type.to_string()).increment(1);
+    counter!("rustfs_lock_acquisitions", "type" => lock_type.to_string()).increment(1);
 }
 
 /// Record lock release.
 #[inline(always)]
 pub fn record_lock_release(lock_type: &str, hold_time: Duration) {
     use metrics::{counter, histogram};
-    counter!("rustfs.lock.releases", "type" => lock_type.to_string()).increment(1);
-    histogram!("rustfs.lock.hold_time.secs", "type" => lock_type.to_string()).record(hold_time.as_secs_f64());
+    counter!("rustfs_lock_releases", "type" => lock_type.to_string()).increment(1);
+    histogram!("rustfs_lock_hold_time_secs", "type" => lock_type.to_string()).record(hold_time.as_secs_f64());
 }
 
 /// Record lock contention.
 #[inline(always)]
 pub fn record_lock_contention(lock_type: &str) {
     use metrics::counter;
-    counter!("rustfs.lock.contentions", "type" => lock_type.to_string()).increment(1);
+    counter!("rustfs_lock_contentions", "type" => lock_type.to_string()).increment(1);
 }
 
 /// Record wait graph edge added.
 #[inline(always)]
 pub fn record_wait_edge_added() {
     use metrics::counter;
-    counter!("rustfs.deadlock.wait_edges.added").increment(1);
+    counter!("rustfs_deadlock_wait_edges_added").increment(1);
 }
 
 /// Record wait graph edge removed.
 #[inline(always)]
 pub fn record_wait_edge_removed() {
     use metrics::counter;
-    counter!("rustfs.deadlock.wait_edges.removed").increment(1);
+    counter!("rustfs_deadlock_wait_edges_removed").increment(1);
 }
 
 #[cfg(test)]

crates/io-metrics/src/io_metrics.rs

@@ -27,11 +27,11 @@
 pub fn record_io_scheduler_decision(buffer_size: usize, load_level: &str, strategy: &str) {
     use metrics::{counter, gauge, histogram};
 
-    counter!("rustfs.io.scheduler.decisions").increment(1);
-    gauge!("rustfs.io.scheduler.buffer_size").set(buffer_size as f64);
-    counter!("rustfs.io.scheduler.load", "level" => load_level.to_string()).increment(1);
-    counter!("rustfs.io.scheduler.strategy", "type" => strategy.to_string()).increment(1);
-    histogram!("rustfs.io.scheduler.buffer_size.histogram").record(buffer_size as f64);
+    counter!("rustfs_io_scheduler_decisions").increment(1);
+    gauge!("rustfs_io_scheduler_buffer_size").set(buffer_size as f64);
+    counter!("rustfs_io_scheduler_load", "level" => load_level.to_string()).increment(1);
+    counter!("rustfs_io_scheduler_strategy", "type" => strategy.to_string()).increment(1);
+    histogram!("rustfs_io_scheduler_buffer_size_histogram").record(buffer_size as f64);
 }
 
 /// Record I/O priority decision.
@@ -44,9 +44,9 @@ pub fn record_io_scheduler_decision(buffer_size: usize, load_level: &str, strate
 pub fn record_io_priority_decision(priority: &str, size: usize) {
     use metrics::{counter, histogram};
 
-    counter!("rustfs.io.priority.decisions").increment(1);
-    counter!("rustfs.io.priority.by_level", "priority" => priority.to_string()).increment(1);
-    histogram!("rustfs.io.priority.request_size").record(size as f64);
+    counter!("rustfs_io_priority_decisions").increment(1);
+    counter!("rustfs_io_priority_by_level", "priority" => priority.to_string()).increment(1);
+    histogram!("rustfs_io_priority_request_size").record(size as f64);
 }
 
 /// Record load level change.
@@ -58,7 +58,7 @@ pub fn record_io_priority_decision(priority: &str, size: usize) {
 #[inline(always)]
 pub fn record_load_level_change(from: &str, to: &str) {
     use metrics::counter;
-    counter!("rustfs.io.load.changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
+    counter!("rustfs_io_load_changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
 }
 
 /// Record bandwidth observation.
@@ -69,8 +69,8 @@ pub fn record_load_level_change(from: &str, to: &str) {
 #[inline(always)]
 pub fn record_bandwidth_observation(bps: u64) {
     use metrics::{gauge, histogram};
-    gauge!("rustfs.io.bandwidth.bps").set(bps as f64);
-    histogram!("rustfs.io.bandwidth.histogram").record(bps as f64);
+    gauge!("rustfs_io_bandwidth_bps").set(bps as f64);
+    histogram!("rustfs_io_bandwidth_histogram").record(bps as f64);
 }
 
 /// Record buffer size adjustment.
@@ -83,9 +83,9 @@ pub fn record_bandwidth_observation(bps: u64) {
 #[inline(always)]
 pub fn record_buffer_size_adjustment(original: usize, adjusted: usize, reason: &str) {
     use metrics::{counter, gauge};
-    counter!("rustfs.io.buffer.adjustments", "reason" => reason.to_string()).increment(1);
-    gauge!("rustfs.io.buffer.original").set(original as f64);
-    gauge!("rustfs.io.buffer.adjusted").set(adjusted as f64);
+    counter!("rustfs_io_buffer_adjustments", "reason" => reason.to_string()).increment(1);
+    gauge!("rustfs_io_buffer_original").set(original as f64);
+    gauge!("rustfs_io_buffer_adjusted").set(adjusted as f64);
 }
 
 /// Record queue operation.
@@ -98,8 +98,8 @@ pub fn record_buffer_size_adjustment(original: usize, adjusted: usize, reason: &
 #[inline(always)]
 pub fn record_queue_operation(operation: &str, priority: &str, queue_size: usize) {
     use metrics::{counter, gauge};
-    counter!("rustfs.io.queue.operations", "operation" => operation.to_string(), "priority" => priority.to_string()).increment(1);
-    gauge!("rustfs.io.queue.size", "priority" => priority.to_string()).set(queue_size as f64);
+    counter!("rustfs_io_queue_operations", "operation" => operation.to_string(), "priority" => priority.to_string()).increment(1);
+    gauge!("rustfs_io_queue_size", "priority" => priority.to_string()).set(queue_size as f64);
 }
 
 /// Record starvation event.
@@ -110,7 +110,7 @@ pub fn record_queue_operation(operation: &str, priority: &str, queue_size: usize
 #[inline(always)]
 pub fn record_starvation_event(priority: &str) {
     use metrics::counter;
-    counter!("rustfs.io.starvation.events", "priority" => priority.to_string()).increment(1);
+    counter!("rustfs_io_starvation_events", "priority" => priority.to_string()).increment(1);
 }
 
 /// I/O scheduler statistics.

crates/io-metrics/src/lib.rs

@@ -49,6 +49,8 @@
 #[macro_use]
 extern crate metrics;
 
+use std::sync::atomic::{AtomicU64, Ordering};
+
 // Public modules
 pub mod adaptive_ttl;
 pub mod autotuner;
@@ -137,6 +139,43 @@ pub use config::{
 pub use collector::MetricsCollector;
 pub use performance::PerformanceMetrics;
 
+static EC_ENCODE_INFLIGHT_BYTES: AtomicU64 = AtomicU64::new(0);
+static GET_OBJECT_BUFFERED_BYTES: AtomicU64 = AtomicU64::new(0);
+
+fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 {
+    let mut current = counter.load(Ordering::Relaxed);
+    loop {
+        let next = current.saturating_sub(bytes);
+        match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => return next,
+            Err(actual) => current = actual,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum TrackedMemoryGauge {
+    GetObjectBufferedBytes,
+}
+
+/// Drop-based guard for tracked in-memory payloads.
+#[derive(Debug)]
+pub struct MemoryGaugeGuard {
+    gauge: TrackedMemoryGauge,
+    bytes: u64,
+}
+
+impl Drop for MemoryGaugeGuard {
+    fn drop(&mut self) {
+        match self.gauge {
+            TrackedMemoryGauge::GetObjectBufferedBytes => {
+                let next = saturating_sub_atomic(&GET_OBJECT_BUFFERED_BYTES, self.bytes);
+                gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64);
+            }
+        }
+    }
+}
+
 /// Record GetObject request start.
 #[inline(always)]
 pub fn record_get_object_request_start(concurrent_requests: usize) {
@@ -217,9 +256,9 @@ pub fn record_get_object_io_state(
 /// * `duration_ms` - Time taken for the read operation in milliseconds
 #[inline(always)]
 pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) {
-    counter!("rustfs.zero_copy.reads.total").increment(1);
-    histogram!("rustfs.zero_copy.read.size.bytes").record(size_bytes as f64);
-    histogram!("rustfs.zero_copy.read.duration.ms").record(duration_ms);
+    counter!("rustfs_zero_copy_reads_total").increment(1);
+    histogram!("rustfs_zero_copy_read_size_bytes").record(size_bytes as f64);
+    histogram!("rustfs_zero_copy_read_duration_ms").record(duration_ms);
 }
 
 /// Record memory copies avoided by using zero-copy.
@@ -229,7 +268,7 @@ pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) {
 /// * `bytes_saved` - Number of bytes that would have been copied without zero-copy
 #[inline(always)]
 pub fn record_memory_copy_saved(bytes_saved: usize) {
-    counter!("rustfs.zero_copy.memory.saved.bytes").increment(bytes_saved as u64);
+    counter!("rustfs_zero_copy_memory_saved_bytes_total").increment(bytes_saved as u64);
 }
 
 /// Record a fallback from zero-copy to regular read.
@@ -242,7 +281,7 @@ pub fn record_memory_copy_saved(bytes_saved: usize) {
 /// * `reason` - Reason for the fallback (e.g., "mmap_unavailable", "file_too_large")
 #[inline(always)]
 pub fn record_zero_copy_fallback(reason: &str) {
-    counter!("rustfs.zero_copy.fallback.total", "reason" => reason.to_string()).increment(1);
+    counter!("rustfs_zero_copy_fallback_total", "reason" => reason.to_string()).increment(1);
 }
 
 // ============================================================================
@@ -258,13 +297,13 @@ pub fn record_zero_copy_fallback(reason: &str) {
 /// * `from_pool` - Whether buffer was reused from pool
 #[inline(always)]
 pub fn record_bytes_pool_acquire(tier: &str, size: usize, from_pool: bool) {
-    counter!("rustfs.bytes.pool.acquisitions.total", "tier" => tier.to_string()).increment(1);
-    gauge!("rustfs.bytes.pool.size.bytes", "tier" => tier.to_string()).set(size as f64);
+    counter!("rustfs_bytes_pool_acquisitions_total", "tier" => tier.to_string()).increment(1);
+    gauge!("rustfs_bytes_pool_size_bytes", "tier" => tier.to_string()).set(size as f64);
 
     if from_pool {
-        counter!("rustfs.bytes.pool.hits.total", "tier" => tier.to_string()).increment(1);
+        counter!("rustfs_bytes_pool_hits_total", "tier" => tier.to_string()).increment(1);
     } else {
-        counter!("rustfs.bytes.pool.misses.total", "tier" => tier.to_string()).increment(1);
+        counter!("rustfs_bytes_pool_misses_total", "tier" => tier.to_string()).increment(1);
     }
 }
 
@@ -275,7 +314,7 @@ pub fn record_bytes_pool_acquire(tier: &str, size: usize, from_pool: bool) {
 /// * `tier` - Pool tier ("small", "medium", "large", "xlarge")
 #[inline(always)]
 pub fn record_bytes_pool_return(tier: &str) {
-    counter!("rustfs.bytes.pool.returns.total", "tier" => tier.to_string()).increment(1);
+    counter!("rustfs_bytes_pool_returns_total", "tier" => tier.to_string()).increment(1);
 }
 
 /// Record current BytesPool allocated bytes.
@@ -286,7 +325,7 @@ pub fn record_bytes_pool_return(tier: &str) {
 /// * `bytes` - Currently allocated bytes
 #[inline(always)]
 pub fn record_bytes_pool_allocated(tier: &str, bytes: u64) {
-    gauge!("rustfs.bytes.pool.allocated.bytes", "tier" => tier.to_string()).set(bytes as f64);
+    gauge!("rustfs_bytes_pool_allocated_bytes", "tier" => tier.to_string()).set(bytes as f64);
 }
 
 /// Get BytesPool hit rate as a gauge metric.
@@ -297,7 +336,7 @@ pub fn record_bytes_pool_allocated(tier: &str, bytes: u64) {
 /// * `hit_rate` - Hit rate (0.0 - 1.0)
 #[inline(always)]
 pub fn record_bytes_pool_hit_rate(tier: &str, hit_rate: f64) {
-    gauge!("rustfs.bytes.pool.hit.rate", "tier" => tier.to_string()).set(hit_rate * 100.0);
+    gauge!("rustfs_bytes_pool_hit_rate", "tier" => tier.to_string()).set(hit_rate * 100.0);
 }
 
 /// Record zero-copy write operation.
@@ -308,9 +347,9 @@ pub fn record_bytes_pool_hit_rate(tier: &str, hit_rate: f64) {
 /// * `duration_ms` - Time taken for the write operation in milliseconds
 #[inline(always)]
 pub fn record_zero_copy_write(size_bytes: usize, duration_ms: f64) {
-    counter!("rustfs.zero_copy.write.total").increment(1);
-    histogram!("rustfs.zero_copy.write.size.bytes").record(size_bytes as f64);
-    histogram!("rustfs.zero_copy.write.duration.ms").record(duration_ms);
+    counter!("rustfs_zero_copy_write_total").increment(1);
+    histogram!("rustfs_zero_copy_write_size_bytes").record(size_bytes as f64);
+    histogram!("rustfs_zero_copy_write_duration_ms").record(duration_ms);
 }
 
 /// Record zero-copy write fallback.
@@ -322,7 +361,7 @@ pub fn record_zero_copy_write(size_bytes: usize, duration_ms: f64) {
 /// * `reason` - Reason for the fallback
 #[inline(always)]
 pub fn record_zero_copy_write_fallback(reason: &str) {
-    counter!("rustfs.zero_copy.write.fallback.total", "reason" => reason.to_string()).increment(1);
+    counter!("rustfs_zero_copy_write_fallback_total", "reason" => reason.to_string()).increment(1);
 }
 
 /// Record bytes saved from zero-copy.
@@ -332,7 +371,7 @@ pub fn record_zero_copy_write_fallback(reason: &str) {
 /// * `size_bytes` - Number of bytes saved from zero-copy
 #[inline(always)]
 pub fn record_bytes_saved(size_bytes: usize) {
-    counter!("rustfs.zero_copy.bytes.saved.total").increment(size_bytes as u64);
+    counter!("rustfs_zero_copy_bytes_saved_total").increment(size_bytes as u64);
 }
 
 // ============================================================================
@@ -350,11 +389,11 @@ pub fn record_bytes_saved(size_bytes: usize) {
 /// interpreted as the definitive source of truth for data-plane copy mode.
 #[inline(always)]
 pub fn record_get_object(duration_ms: f64, size_bytes: i64) {
-    counter!("rustfs.s3.get_object.total").increment(1);
-    histogram!("rustfs.s3.get_object.duration.ms").record(duration_ms);
+    counter!("rustfs_s3_get_object_total").increment(1);
+    histogram!("rustfs_s3_get_object_duration_ms").record(duration_ms);
 
     if size_bytes > 0 {
-        histogram!("rustfs.s3.get_object.size.bytes").record(size_bytes as f64);
+        histogram!("rustfs_s3_get_object_size_bytes").record(size_bytes as f64);
     }
 }
 
@@ -367,15 +406,15 @@ pub fn record_get_object(duration_ms: f64, size_bytes: i64) {
 /// * `zero_copy_enabled` - Whether zero-copy was enabled for this operation
 #[inline(always)]
 pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_enabled: bool) {
-    counter!("rustfs.s3.put_object.total").increment(1);
-    histogram!("rustfs.s3.put_object.duration.ms").record(duration_ms);
+    counter!("rustfs_s3_put_object_total").increment(1);
+    histogram!("rustfs_s3_put_object_duration_ms").record(duration_ms);
 
     if size_bytes > 0 {
-        histogram!("rustfs.s3.put_object.size.bytes").record(size_bytes as f64);
+        histogram!("rustfs_s3_put_object_size_bytes").record(size_bytes as f64);
     }
 
     if zero_copy_enabled {
-        counter!("rustfs.s3.put_object.zero_copy.enabled.total").increment(1);
+        counter!("rustfs_s3_put_object_zero_copy_enabled_total").increment(1);
     }
 }
 
@@ -388,12 +427,12 @@ pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_enabled: b
 /// * `is_truncated` - Whether the response was truncated
 #[inline(always)]
 pub fn record_list_objects(duration_ms: f64, objects_count: u64, is_truncated: bool) {
-    counter!("rustfs.s3.list_objects.total").increment(1);
-    histogram!("rustfs.s3.list_objects.duration.ms").record(duration_ms);
-    histogram!("rustfs.s3.list_objects.count").record(objects_count as f64);
+    counter!("rustfs_s3_list_objects_total").increment(1);
+    histogram!("rustfs_s3_list_objects_duration_ms").record(duration_ms);
+    histogram!("rustfs_s3_list_objects_count").record(objects_count as f64);
 
     if is_truncated {
-        counter!("rustfs.s3.list_objects.truncated.total").increment(1);
+        counter!("rustfs_s3_list_objects_truncated_total").increment(1);
     }
 }
 
@@ -405,11 +444,11 @@ pub fn record_list_objects(duration_ms: f64, objects_count: u64, is_truncated: b
 /// * `version_deleted` - Whether a specific version was deleted
 #[inline(always)]
 pub fn record_delete_object(duration_ms: f64, version_deleted: bool) {
-    counter!("rustfs.s3.delete_object.total").increment(1);
-    histogram!("rustfs.s3.delete_object.duration.ms").record(duration_ms);
+    counter!("rustfs_s3_delete_object_total").increment(1);
+    histogram!("rustfs_s3_delete_object_duration_ms").record(duration_ms);
 
     if version_deleted {
-        counter!("rustfs.s3.delete_object.version.total").increment(1);
+        counter!("rustfs_s3_delete_object_version_total").increment(1);
     }
 }
 
@@ -427,18 +466,18 @@ pub fn record_delete_object(duration_ms: f64, version_deleted: bool) {
 /// * `concurrent_requests` - Number of concurrent requests
 #[inline(always)]
 pub fn record_io_strategy(storage_media: &str, access_pattern: &str, buffer_size: usize, concurrent_requests: u64) {
-    counter!("rustfs.io.strategy.total",
+    counter!("rustfs_io_strategy_total",
         "storage_media" => storage_media.to_string(),
         "access_pattern" => access_pattern.to_string(),
     )
     .increment(1);
 
-    gauge!("rustfs.io.buffer.size.bytes",
+    gauge!("rustfs_io_buffer_size_bytes",
         "storage_media" => storage_media.to_string(),
     )
     .set(buffer_size as f64);
 
-    gauge!("rustfs.io.concurrent.requests").set(concurrent_requests as f64);
+    gauge!("rustfs_io_concurrent_requests").set(concurrent_requests as f64);
 }
 
 /// Record disk permit wait time (load tracking).
@@ -448,7 +487,7 @@ pub fn record_io_strategy(storage_media: &str, access_pattern: &str, buffer_size
 /// * `duration_ms` - Time spent waiting for disk permit
 #[inline(always)]
 pub fn record_permit_wait(duration_ms: f64) {
-    histogram!("rustfs.io.permit.wait.duration.ms").record(duration_ms);
+    histogram!("rustfs_io_permit_wait_duration_ms").record(duration_ms);
 }
 
 /// Record I/O load level.
@@ -459,12 +498,12 @@ pub fn record_permit_wait(duration_ms: f64) {
 /// * `concurrent_requests` - Number of concurrent requests
 #[inline(always)]
 pub fn record_io_load_level(load_level: &str, concurrent_requests: u64) {
-    counter!("rustfs.io.load.level",
+    counter!("rustfs_io_load_level",
         "level" => load_level.to_string(),
     )
     .increment(1);
 
-    gauge!("rustfs.io.concurrent.requests").set(concurrent_requests as f64);
+    gauge!("rustfs_io_concurrent_requests").set(concurrent_requests as f64);
 }
 
 /// Record cache size and entry count.
@@ -476,12 +515,12 @@ pub fn record_io_load_level(load_level: &str, concurrent_requests: u64) {
 /// * `entries` - Number of entries in the cache
 #[inline(always)]
 pub fn record_cache_size(tier: &str, size_bytes: usize, entries: u64) {
-    gauge!("rustfs.cache.size.bytes",
+    gauge!("rustfs_cache_size_bytes",
         "tier" => tier.to_string(),
     )
     .set(size_bytes as f64);
 
-    gauge!("rustfs.cache.entries",
+    gauge!("rustfs_cache_entries",
         "tier" => tier.to_string(),
     )
     .set(entries as f64);
@@ -499,13 +538,11 @@ pub fn record_cache_size(tier: &str, size_bytes: usize, entries: u64) {
 /// * `tier` - Bandwidth tier ("low", "medium", "high", "unknown")
 #[inline(always)]
 pub fn record_bandwidth(bytes_per_second: u64, tier: &str) {
-    gauge!("rustfs.bandwidth.current.bps").set(bytes_per_second as f64);
-    gauge!("rustfs.bandwidth.current.bps",
-        "tier" => tier.to_string(),
-    )
-    .set(bytes_per_second as f64);
+    let tier_label = if tier.is_empty() { "unknown" } else { tier };
+    gauge!("rustfs_bandwidth_current_bps", "tier" => "all").set(bytes_per_second as f64);
+    gauge!("rustfs_bandwidth_current_bps", "tier" => tier_label.to_string()).set(bytes_per_second as f64);
 
-    histogram!("rustfs.bandwidth.observed.bps").record(bytes_per_second as f64);
+    histogram!("rustfs_bandwidth_observed_bps").record(bytes_per_second as f64);
 }
 
 /// Record data transfer for bandwidth calculation.
@@ -516,12 +553,12 @@ pub fn record_bandwidth(bytes_per_second: u64, tier: &str) {
 /// * `duration_ms` - Duration of the transfer in milliseconds
 #[inline(always)]
 pub fn record_data_transfer(bytes: u64, duration_ms: f64) {
-    counter!("rustfs.io.transfer.bytes").increment(bytes);
-    histogram!("rustfs.io.transfer.duration.ms").record(duration_ms);
+    counter!("rustfs_io_transfer_bytes_total").increment(bytes);
+    histogram!("rustfs_io_transfer_duration_ms").record(duration_ms);
 
     if duration_ms > 0.0 {
         let bps = (bytes as f64 * 1000.0) / duration_ms;
-        histogram!("rustfs.io.transfer.bandwidth.bps").record(bps);
+        histogram!("rustfs_io_transfer_bandwidth_bps").record(bps);
     }
 }
 
@@ -537,15 +574,94 @@ pub fn record_data_transfer(bytes: u64, duration_ms: f64) {
 /// * `total_bytes` - Total memory in bytes
 #[inline(always)]
 pub fn record_memory_usage(used_bytes: u64, total_bytes: u64) {
-    gauge!("rustfs.memory.used.bytes").set(used_bytes as f64);
-    gauge!("rustfs.memory.total.bytes").set(total_bytes as f64);
+    gauge!("rustfs_memory_used_bytes").set(used_bytes as f64);
+    gauge!("rustfs_memory_total_bytes").set(total_bytes as f64);
 
     if total_bytes > 0 {
         let usage_percent = (used_bytes as f64 / total_bytes as f64) * 100.0;
-        gauge!("rustfs.memory.usage.percent").set(usage_percent);
+        gauge!("rustfs_memory_usage_percent").set(usage_percent);
     }
 }
 
+/// Record process-level memory split metrics.
+#[inline(always)]
+pub fn record_process_memory_split(resident_bytes: u64, virtual_bytes: u64) {
+    gauge!("rustfs_memory_process_resident_bytes").set(resident_bytes as f64);
+    gauge!("rustfs_memory_process_virtual_bytes").set(virtual_bytes as f64);
+}
+
+/// Record cgroup memory split metrics when available.
+#[inline(always)]
+pub fn record_cgroup_memory_split(
+    current_bytes: Option<u64>,
+    limit_bytes: Option<u64>,
+    anon_bytes: Option<u64>,
+    file_bytes: Option<u64>,
+    active_file_bytes: Option<u64>,
+    inactive_file_bytes: Option<u64>,
+) {
+    if let Some(current_bytes) = current_bytes {
+        gauge!("rustfs_memory_cgroup_current_bytes").set(current_bytes as f64);
+    }
+    if let Some(limit_bytes) = limit_bytes {
+        gauge!("rustfs_memory_cgroup_limit_bytes").set(limit_bytes as f64);
+    }
+    if let Some(anon_bytes) = anon_bytes {
+        gauge!("rustfs_memory_cgroup_anon_bytes").set(anon_bytes as f64);
+    }
+    if let Some(file_bytes) = file_bytes {
+        gauge!("rustfs_memory_cgroup_file_bytes").set(file_bytes as f64);
+    }
+    if let Some(active_file_bytes) = active_file_bytes {
+        gauge!("rustfs_memory_cgroup_active_file_bytes").set(active_file_bytes as f64);
+    }
+    if let Some(inactive_file_bytes) = inactive_file_bytes {
+        gauge!("rustfs_memory_cgroup_inactive_file_bytes").set(inactive_file_bytes as f64);
+    }
+}
+
+/// Track encoded bytes currently queued between erasure encode and disk writers.
+#[inline(always)]
+pub fn add_ec_encode_inflight_bytes(bytes: usize) {
+    let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
+    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
+}
+
+/// Remove encoded bytes from the tracked erasure encode in-flight gauge.
+#[inline(always)]
+pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
+    let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
+    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
+}
+
+/// Return the current tracked EC encode in-flight bytes.
+#[inline(always)]
+pub fn current_ec_encode_inflight_bytes() -> u64 {
+    EC_ENCODE_INFLIGHT_BYTES.load(Ordering::Relaxed)
+}
+
+/// Track whole-object buffering on the GET path.
+#[inline(always)]
+pub fn track_get_object_buffered_bytes(bytes: usize) -> Option<MemoryGaugeGuard> {
+    if bytes == 0 {
+        return None;
+    }
+
+    let next = GET_OBJECT_BUFFERED_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
+    gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64);
+
+    Some(MemoryGaugeGuard {
+        gauge: TrackedMemoryGauge::GetObjectBufferedBytes,
+        bytes: bytes as u64,
+    })
+}
+
+/// Return the current tracked GET whole-buffered bytes.
+#[inline(always)]
+pub fn current_get_object_buffered_bytes() -> u64 {
+    GET_OBJECT_BUFFERED_BYTES.load(Ordering::Relaxed)
+}
+
 /// Record CPU usage.
 ///
 /// # Arguments
@@ -553,7 +669,7 @@ pub fn record_memory_usage(used_bytes: u64, total_bytes: u64) {
 /// * `percent` - CPU usage percentage (0.0 - 100.0)
 #[inline(always)]
 pub fn record_cpu_usage(percent: f64) {
-    gauge!("rustfs.cpu.usage.percent").set(percent);
+    gauge!("rustfs_cpu_usage_percent").set(percent);
 }
 
 /// Record disk I/O statistics.
@@ -566,13 +682,10 @@ pub fn record_cpu_usage(percent: f64) {
 /// * `write_ops` - Number of write operations
 #[inline(always)]
 pub fn record_disk_io(read_bytes: u64, write_bytes: u64, read_ops: u64, write_ops: u64) {
-    counter!("rustfs.disk.read.bytes").increment(read_bytes);
-    counter!("rustfs.disk.write.bytes").increment(write_bytes);
-    counter!("rustfs.disk.read.ops").increment(read_ops);
-    counter!("rustfs.disk.write.ops").increment(write_ops);
-
-    gauge!("rustfs.disk.read.bytes_total").set(read_bytes as f64);
-    gauge!("rustfs.disk.write.bytes_total").set(write_bytes as f64);
+    counter!("rustfs_disk_read_bytes_total").increment(read_bytes);
+    counter!("rustfs_disk_write_bytes_total").increment(write_bytes);
+    counter!("rustfs_disk_read_ops_total").increment(read_ops);
+    counter!("rustfs_disk_write_ops_total").increment(write_ops);
 }
 
 // ============================================================================
@@ -587,7 +700,7 @@ pub fn record_disk_io(read_bytes: u64, write_bytes: u64, read_ops: u64, write_op
 /// * `error_type` - Error type (e.g., "timeout", "disk_error", "network")
 #[inline(always)]
 pub fn record_error(operation: &str, error_type: &str) {
-    counter!("rustfs.errors.total",
+    counter!("rustfs_errors_total",
         "operation" => operation.to_string(),
         "type" => error_type.to_string(),
     )
@@ -602,12 +715,12 @@ pub fn record_error(operation: &str, error_type: &str) {
 /// * `duration_ms` - Duration before timeout
 #[inline(always)]
 pub fn record_timeout(operation: &str, duration_ms: f64) {
-    counter!("rustfs.timeouts.total",
+    counter!("rustfs_timeouts_total",
         "operation" => operation.to_string(),
     )
     .increment(1);
 
-    histogram!("rustfs.timeouts.duration.ms",
+    histogram!("rustfs_timeouts_duration_ms",
         "operation" => operation.to_string(),
     )
     .record(duration_ms);
@@ -621,12 +734,12 @@ pub fn record_timeout(operation: &str, duration_ms: f64) {
 /// * `attempt_number` - Attempt number (1-based)
 #[inline(always)]
 pub fn record_retry(operation: &str, attempt_number: u32) {
-    counter!("rustfs.retries.total",
+    counter!("rustfs_retries_total",
         "operation" => operation.to_string(),
     )
     .increment(1);
 
-    histogram!("rustfs.retries.attempt",
+    histogram!("rustfs_retries_attempt",
         "operation" => operation.to_string(),
     )
     .record(attempt_number as f64);
@@ -643,7 +756,7 @@ pub fn record_retry(operation: &str, attempt_number: u32) {
 /// * `latency_ms` - I/O latency in milliseconds
 #[inline(always)]
 pub fn record_io_latency(latency_ms: f64) {
-    histogram!("rustfs.io.latency.ms").record(latency_ms);
+    histogram!("rustfs_io_latency_ms").record(latency_ms);
 }
 
 /// Record I/O latency P95 in milliseconds.
@@ -653,7 +766,7 @@ pub fn record_io_latency(latency_ms: f64) {
 /// * `latency_ms` - P95 I/O latency in milliseconds
 #[inline(always)]
 pub fn record_io_latency_p95(latency_ms: f64) {
-    gauge!("rustfs.io.latency.p95.ms").set(latency_ms);
+    gauge!("rustfs_io_latency_p95_ms").set(latency_ms);
 }
 
 /// Record I/O latency P99 in milliseconds.
@@ -663,7 +776,7 @@ pub fn record_io_latency_p95(latency_ms: f64) {
 /// * `latency_ms` - P99 I/O latency in milliseconds
 #[inline(always)]
 pub fn record_io_latency_p99(latency_ms: f64) {
-    gauge!("rustfs.io.latency.p99.ms").set(latency_ms);
+    gauge!("rustfs_io_latency_p99_ms").set(latency_ms);
 }
 
 #[cfg(test)]
@@ -763,6 +876,48 @@ mod tests {
         record_memory_usage(2 * 1024 * 1024 * 1024, 8 * 1024 * 1024 * 1024);
     }
 
+    #[test]
+    fn test_record_process_memory_split() {
+        record_process_memory_split(1024, 2048);
+        record_process_memory_split(4096, 8192);
+    }
+
+    #[test]
+    fn test_record_cgroup_memory_split() {
+        record_cgroup_memory_split(Some(1), Some(2), Some(3), Some(4), Some(5), Some(6));
+        record_cgroup_memory_split(None, None, None, None, None, None);
+    }
+
+    #[test]
+    fn test_ec_encode_inflight_bytes_tracking() {
+        EC_ENCODE_INFLIGHT_BYTES.store(0, Ordering::Relaxed);
+        add_ec_encode_inflight_bytes(1024);
+        add_ec_encode_inflight_bytes(2048);
+        remove_ec_encode_inflight_bytes(1024);
+        remove_ec_encode_inflight_bytes(2048);
+        remove_ec_encode_inflight_bytes(4096);
+        assert_eq!(current_ec_encode_inflight_bytes(), 0);
+    }
+
+    #[test]
+    fn test_get_object_buffered_bytes_guard() {
+        GET_OBJECT_BUFFERED_BYTES.store(0, Ordering::Relaxed);
+        drop(track_get_object_buffered_bytes(1024));
+        let guard = track_get_object_buffered_bytes(2048);
+        drop(guard);
+        assert_eq!(current_get_object_buffered_bytes(), 0);
+    }
+
+    #[test]
+    fn test_get_object_buffered_bytes_guard_saturates_on_underflow() {
+        GET_OBJECT_BUFFERED_BYTES.store(1024, Ordering::Relaxed);
+        drop(MemoryGaugeGuard {
+            gauge: TrackedMemoryGauge::GetObjectBufferedBytes,
+            bytes: 2048,
+        });
+        assert_eq!(current_get_object_buffered_bytes(), 0);
+    }
+
     #[test]
     fn test_record_cpu_usage() {
         record_cpu_usage(25.5);