test(get): reject range with part number (#2725)

Co-authored-by: houseme <housemecn@gmail.com>

.docker/compose/README.md

@@ -36,6 +36,24 @@ To start a 4-node cluster for distributed testing:
 docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```
 
+### Script-Based 4-Node Validation (Recommended)
+
+Use the local validation script when you need local-source image build, failover checks,
+and benchmark workflow in one command:
+
+```bash
+# Default mode: WAIT_PROBE_MODE=service
+# This avoids false negatives where /health/ready remains 503 locally
+# while the service path is already available.
+./scripts/run_four_node_cluster_failover_bench.sh
+```
+
+Strict mode is available when you explicitly want `/health/ready == 200` as the gate:
+
+```bash
+WAIT_PROBE_MODE=ready ./scripts/run_four_node_cluster_failover_bench.sh
+```
+
 ### (Deprecated) Minimal Observability
 
 ```bash

.docker/compose/docker-compose.cluster.local-build.yml

@@ -0,0 +1,144 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  node1:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node1
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node1_data_0:/data/rustfs0
+      - node1_data_1:/data/rustfs1
+      - node1_data_2:/data/rustfs2
+      - node1_data_3:/data/rustfs3
+    ports:
+      - "9000:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node2:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node2
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node2_data_0:/data/rustfs0
+      - node2_data_1:/data/rustfs1
+      - node2_data_2:/data/rustfs2
+      - node2_data_3:/data/rustfs3
+    ports:
+      - "9001:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node3:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node3
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node3_data_0:/data/rustfs0
+      - node3_data_1:/data/rustfs1
+      - node3_data_2:/data/rustfs2
+      - node3_data_3:/data/rustfs3
+    ports:
+      - "9002:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node4:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node4
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node4_data_0:/data/rustfs0
+      - node4_data_1:/data/rustfs1
+      - node4_data_2:/data/rustfs2
+      - node4_data_3:/data/rustfs3
+    ports:
+      - "9003:9000"
+    networks:
+      - rustfs-cluster-net
+
+volumes:
+  node1_data_0:
+  node1_data_1:
+  node1_data_2:
+  node1_data_3:
+  node2_data_0:
+  node2_data_1:
+  node2_data_2:
+  node2_data_3:
+  node3_data_0:
+  node3_data_1:
+  node3_data_2:
+  node3_data_3:
+  node4_data_0:
+  node4_data_1:
+  node4_data_2:
+  node4_data_3:
+
+networks:
+  rustfs-cluster-net:
+    driver: bridge

.docker/compose/docker-compose.observability.yaml

@@ -26,7 +26,7 @@ services:
     restart: "no"
 
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.10.5
     user: "10001"
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
@@ -36,9 +36,19 @@ services:
       - "3200:3200" # tempo
       - "4317"      # otlp grpc
       - "4318"      # otlp http
+      - "7946"      # memberlist
     restart: unless-stopped
     networks:
       - rustfs-network
+    depends_on:
+      tempo-init:
+        condition: service_completed_successfully
+    healthcheck:
+      test: [ "CMD", "/tempo", "-version" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
 
   otel-collector:
     image: otel/opentelemetry-collector-contrib:latest
@@ -61,6 +71,12 @@ services:
       - jaeger
       - prometheus
       - loki
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   jaeger:
     image: jaegertracing/jaeger:latest
@@ -72,12 +88,23 @@ services:
       - BADGER_DIRECTORY_KEY=/badger/key
       - COLLECTOR_OTLP_ENABLED=true
     volumes:
+      - ../../.docker/observability/jaeger.yaml:/etc/jaeger/config.yml:ro
       - jaeger-data:/badger
     ports:
       - "16686:16686" # Web UI
       - "14269:14269" # Admin/Metrics
+      - "4317"        # otlp grpc
+      - "4318"        # otlp http
+    command: [ "--config", "/etc/jaeger/config.yml" ]
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
 
   prometheus:
     image: prom/prometheus:latest
@@ -85,6 +112,7 @@ services:
       - TZ=Asia/Shanghai
     volumes:
       - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ../../.docker/observability/prometheus-rules:/etc/prometheus/rules:ro
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"
@@ -94,23 +122,45 @@ services:
       - '--web.enable-remote-write-receiver'
       - '--enable-feature=promql-experimental-functions'
       - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - '--storage.tsdb.retention.time=30d'
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   loki:
     image: grafana/loki:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/loki.yaml:/etc/loki/local-config.yaml:ro
+      - ../../.docker/observability/loki.yaml:/etc/loki/loki.yaml:ro
       - loki-data:/loki
     ports:
       - "3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/loki.yaml
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  pyroscope:
+    image: grafana/pyroscope:latest
+    ports:
+      - "4040:4040"
+    command:
+      - -self-profiling.disable-push=true
+    networks:
+      - rustfs-network
+    restart: unless-stopped
 
   grafana:
     image: grafana/grafana:latest
@@ -127,10 +177,17 @@ services:
     volumes:
       - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
       - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
     depends_on:
       - prometheus
       - tempo
       - loki
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   # --- RustFS Cluster ---
 
@@ -215,6 +272,7 @@ volumes:
   tempo-data:
   loki-data:
   jaeger-data:
+  grafana-data:
 
 networks:
   rustfs-network:

.docker/observability/README.md

@@ -13,6 +13,9 @@ The stack is composed of the following best-in-class open-source components:
 - **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
 - **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
 
+By default, this stack uses Tempo in single-binary mode and does not require Kafka/Redpanda.
+If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rustfs.yml` together with `docker-compose-tempo-ha-override.yml`.
+
 ## Architecture
 
 1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
@@ -46,6 +49,15 @@ Run the following command to start the entire stack:
 docker compose up -d
 ```
 
+### High Availability Tempo
+
+The default `docker-compose.yml` is the single-node stack.
+If you need the Kafka-backed HA Tempo configuration, start it with:
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### Access Dashboards
 
 | Service        | URL                                              | Credentials       | Description                    |

.docker/observability/README_ZH.md

@@ -13,6 +13,9 @@
 - **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
 - **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
 
+默认情况下，这套技术栈使用 Tempo 单二进制模式，不依赖 Kafka/Redpanda。
+如果需要基于 Kafka 的 HA Tempo 路径，请使用 `docker-compose-example-for-rustfs.yml` 配合 `docker-compose-tempo-ha-override.yml`。
+
 ## 架构
 
 1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
@@ -46,6 +49,15 @@
 docker compose up -d
 ```
 
+### Tempo 高可用模式
+
+默认的 `docker-compose.yml` 对应单机栈。
+如果需要基于 Kafka 的 HA Tempo 配置，请使用：
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### 访问仪表盘
 
 | 服务 | URL | 凭据 | 描述 |

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -85,10 +85,8 @@ services:
     networks:
       - otel-network
     restart: unless-stopped
-    depends_on:
-      - redpanda
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
@@ -205,7 +203,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/docker-compose.yml

@@ -16,10 +16,8 @@ services:
 
   # --- Tracing ---
   tempo:
-    image: grafana/tempo:2.10.3
+    image: grafana/tempo:2.10.5
     container_name: tempo
-    depends_on:
-      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -33,31 +31,11 @@ services:
       - otel-network
     restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
-  redpanda:
-    image: redpandadata/redpanda:latest
-    ports:
-      - "9092:9092" # Kafka API for clients
-    command: >
-      redpanda start --overprovisioned
-      --mode=dev-container
-      --kafka-addr=PLAINTEXT://0.0.0.0:9092
-      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
-
-  redpanda-console:
-    image: docker.redpanda.com/redpandadata/console:latest
-    environment:
-      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
-    volumes:
-      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
-    ports:
-      - "8080:8080"
-    depends_on:
-      - redpanda
 
   vulture:
     image: grafana/tempo-vulture:latest
@@ -106,6 +84,7 @@ services:
     container_name: prometheus
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus-rules:/etc/prometheus/rules:ro
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"
@@ -168,7 +147,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/grafana/provisioning/datasources.yaml

@@ -1,97 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    uid: prometheus
-    access: proxy
-    orgId: 1
-    url: http://prometheus:9090
-    isDefault: true
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      exemplarTraceIdDestinations:
-        - name: trace_id
-          datasourceUid: tempo
-
-  - name: Tempo
-    type: tempo
-    uid: tempo
-    access: proxy
-    orgId: 1
-    url: http://tempo:3200
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      serviceMap:
-        datasourceUid: prometheus
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false
-      tracesToMetrics:
-        datasourceUid: prometheus
-        tags: [ { key: 'service.name' }, { key: 'job' } ]
-        queries:
-          - name: 'Service-Level Latency'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
-          - name: 'Service-Level Calls'
-            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
-          - name: 'Service-Level Errors'
-            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
-      nodeGraph:
-        enabled: true
-
-  - name: Loki
-    type: loki
-    uid: loki
-    orgId: 1
-    url: http://loki:3100
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      derivedFields:
-        - datasourceUid: tempo
-          matcherRegex: 'trace_id=(\w+)'
-          name: 'TraceID'
-          url: '$${__value.raw}'
-
-  - name: Jaeger
-    type: jaeger
-    uid: jaeger
-    url: http://jaeger:16686
-    access: proxy
-    isDefault: false
-    editable: false
-    jsonData:
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false

.docker/observability/prometheus-rules/rustfs-dashboard.yml

@@ -0,0 +1,40 @@
+groups:
+  - name: rustfs-dashboard
+    interval: 30s
+    rules:
+      - record: rustfs:http_server_requests:rate5m
+        expr: sum by (job) (rate(rustfs_http_server_requests_total[5m]))
+
+      - record: rustfs:http_server_request_duration_seconds:p50_5m
+        expr: histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+      - record: rustfs:http_server_request_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+      - record: rustfs:http_server_request_duration_seconds:p99_5m
+        expr: histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+
+      - record: rustfs:http_server_response_body_size_bytes:p50_5m
+        expr: histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+      - record: rustfs:http_server_response_body_size_bytes:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+      - record: rustfs:http_server_response_body_size_bytes:p99_5m
+        expr: histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+
+      - record: rustfs:log_cleaner_runs:rate15m
+        expr: sum by (job) (rate(rustfs_log_cleaner_runs_total[15m]))
+      - record: rustfs:log_cleaner_failure_ratio:rate5m
+        expr: sum by (job) (rate(rustfs_log_cleaner_run_failures_total[5m])) / clamp_min(sum by (job) (rate(rustfs_log_cleaner_runs_total[5m])), 1e-9)
+      - record: rustfs:log_cleaner_rotation_failure_ratio:rate5m
+        expr: sum by (job) (rate(rustfs_log_cleaner_rotation_failures_total[5m])) / clamp_min(sum by (job) (rate(rustfs_log_cleaner_rotation_total[5m])), 1e-9)
+      - record: rustfs:log_cleaner_rotation_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_log_cleaner_rotation_duration_seconds_bucket[5m])))
+      - record: rustfs:log_cleaner_compress_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_log_cleaner_compress_duration_seconds_bucket[5m])))
+
+      - record: rustfs:scanner_objects_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_objects_scanned_total[5m]))
+      - record: rustfs:scanner_directories_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_directories_scanned_total[5m]))
+      - record: rustfs:scanner_buckets_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_buckets_scanned_total[5m]))
+      - record: rustfs:scanner_cycles_success:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_cycles_total{result="success"}[5m]))

.docker/observability/prometheus.yml

@@ -19,6 +19,9 @@ global:
     cluster: 'rustfs-dev'  # Label to identify the cluster
     replica: '1'  # Replica identifier
 
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
 scrape_configs:
   - job_name: 'otel-collector'
     static_configs:

.docker/observability/tempo.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-partition_ring_live_store: true
 stream_over_http_enabled: true
 
 server:
@@ -33,33 +32,17 @@ distributor:
           endpoint: "0.0.0.0:4317"
         http:
           endpoint: "0.0.0.0:4318"
-  #log_received_spans:
-  #    enabled: true
-  #  log_discarded_spans:
-  #    enabled: true
 
-backend_scheduler:
-  provider:
-    compaction:
-      compaction:
-        block_retention: 1h
-
-backend_worker:
-  backend_scheduler_addr: localhost:3200
-  compaction:
-    block_retention: 1h
-  ring:
-    kvstore:
-      store: memberlist
-
-querier:
-  query_live_store: true
+ingester:
+  max_block_duration: 5m
 
 metrics_generator:
   registry:
     external_labels:
       source: tempo
       cluster: docker-compose
+  traces_storage:
+    path: /var/tempo/generator/traces
   storage:
     path: /var/tempo/generator/wal
     remote_write:
@@ -85,15 +68,5 @@ overrides:
       processors: [ "span-metrics", "service-graphs", "local-blocks" ]
       generate_native_histograms: both
 
-ingest:
-  enabled: true
-  kafka:
-    address: redpanda:9092
-    topic: tempo-ingest
-
-block_builder:
-  consume_cycle_duration: 30s
-
 usage_report:
   reporting_enabled: false
-

.github/AGENTS.md

@@ -25,6 +25,12 @@ Current `test-and-lint` gate includes:
 
 - `cargo nextest run --all --exclude e2e_test`
 - `cargo test --all --doc`
+- `cargo test -p rustfs get_object_chunk_fast_path`
+- `cargo test -p rustfs materialize_chunk_stream_before_commit`
+- `touch rustfs/build.rs`
+- `cargo build -p rustfs --bins --jobs 2`
+- `cargo test -p e2e_test archive_multipart_roundtrip_preserves_bytes`
+- `cargo test -p e2e_test presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path`
 - `cargo fmt --all --check`
 - `cargo clippy --all-targets --all-features -- -D warnings`
 - `./scripts/check_layer_dependencies.sh`

.github/dependabot.yml

@@ -27,9 +27,8 @@ updates:
       timezone: "Asia/Shanghai"
       time: "08:00"
     assignees:
-      - "heihutu"
-    reviewers:
       - "houseme"
+    reviewers:
       - "overtrue"
       - "majinghe"
     ignore:
@@ -39,6 +38,8 @@ updates:
         versions: [ "0.23.x" ]
       - dependency-name: "ratelimit"
         versions: [ "1.x" ]
+      - dependency-name: "ratelimit"
+        versions: [ "2.x" ]
     groups:
       s3s:
         update-types:

.github/workflows/ci.yml

@@ -53,6 +53,8 @@ on:
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
       - ".github/workflows/performance.yml"
+  merge_group:
+    types: [checks_requested]
   schedule:
     - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
   workflow_dispatch:

.github/workflows/cla.yml

@@ -17,6 +17,8 @@ name: CLA Check
 on:
   pull_request_target:
     types: [opened, synchronize, reopened]
+  merge_group:
+    types: [checks_requested]
   issue_comment:
     types: [created, edited]
 
@@ -31,7 +33,26 @@ jobs:
     if: ${{ github.event_name != 'issue_comment' || github.event.issue.pull_request }}
     runs-on: ubuntu-latest
     steps:
+      - name: Report CLA result for merge queue
+        if: github.event_name == 'merge_group'
+        uses: actions/github-script@v8
+        with:
+          script: |
+            await github.rest.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'CLA Check',
+              head_sha: context.sha,
+              status: 'completed',
+              conclusion: 'success',
+              output: {
+                title: 'CLA requirements satisfied for merge queue',
+                summary: 'Queued pull requests must satisfy the required CLA check before they enter the merge queue. This reports the existing CLA result on the merge-group SHA.'
+              }
+            });
+
       - name: Create token for rustfs/cla
+        if: github.event_name != 'merge_group'
         id: registry-token
         uses: actions/create-github-app-token@v3
         with:
@@ -42,6 +63,7 @@ jobs:
           permission-contents: write
 
       - name: Run CLA Bot
+        if: github.event_name != 'merge_group'
         uses: overtrue/cla-bot@v0.0.9
         with:
           github-token: ${{ github.token }}

AGENTS.md

@@ -6,24 +6,39 @@ Use the nearest subdirectory `AGENTS.md` for path-specific guidance.
 ## Rule Precedence
 
 1. System/developer instructions.
-2. This file (global defaults).
-3. The nearest `AGENTS.md` in the current path (more specific scope wins).
+2. Current user/task instructions.
+3. The nearest `AGENTS.md` in the current path.
+4. This file (global defaults).
 
 If repo-level instructions conflict, follow the nearest file and keep behavior aligned with CI.
 
+## Execution Discipline
+
+- Read the relevant existing code, tests, and local guidance before changing behavior.
+- State assumptions when they affect the implementation or verification path.
+- If a task has multiple plausible interpretations, list the options briefly and choose the narrowest reasonable path; ask when the ambiguity would make the change risky.
+- For multi-step work, keep the plan minimal and tied to verifiable outcomes.
+- Avoid redundant file reads, repeated commands, and unnecessary exploratory work once enough context is available.
+- A good result is a minimal diff with clear assumptions, no over-engineering, and independent verification.
+
 ## Communication and Language
 
 - Respond in the same language used by the requester.
 - Keep source code, comments, commit messages, and PR title/body in English.
+- Be concise. Avoid sycophantic openers, closing fluff, and verbose status reporting.
 
 ## Change Style for Existing Logic
 
 - Prefer direct, local code over extracting one-off helpers.
 - Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow.
+- Solve only the requested problem; do not add speculative features, configurability, or adjacent improvements.
+- Prefer editing existing code over rewriting files or reshaping unrelated logic.
+- Modify only what is required and remove only artifacts introduced by your own changes.
 - Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths.
 - Do not refactor existing code only to make it easier to unit test.
 - Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths.
 - Keep code elegant, concise, and direct. Prefer minimal, readable implementations over over-engineering and excessive abstraction. Use comments to clarify non-obvious intent and invariants, not to compensate for unclear code.
+- Mention unrelated issues when useful, but do not fix them as part of a narrow task.
 
 ## Constant and String Usage
 
@@ -44,6 +59,8 @@ Reference the source files above instead.
 
 ## Verification Before PR
 
+Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion.
+
 For code changes, run and pass the following before opening a PR:
 
 ```bash
@@ -69,6 +86,7 @@ Do not open a PR with code changes when the required checks fail.
 - Use `N/A` for non-applicable template sections.
 - Include verification commands in the PR description.
 - When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
+- Do not include the literal sequence `\n` in any GitHub issue, pull request, or discussion comment.
 - After fixing code review comments or CI findings, always mark corresponding review
   comments/threads as resolved before returning to the user.
 - In handling review comments, confirm the underlying issue before changing code.

ARCHITECTURE.md

@@ -0,0 +1,376 @@
+# ARCHITECTURE.md
+
+> Last updated: 2026-04-13 · Revision: 1 (draft)
+>
+> This document describes the high-level architecture of RustFS.
+> If you want to familiarize yourself with the code base, you are in the right place!
+>
+> See also [CONTRIBUTING.md](CONTRIBUTING.md) for development workflow.
+
+## Bird's Eye View
+
+RustFS is a high-performance, S3-compatible distributed object storage system written
+in Rust. It uses erasure coding for data durability, supports multi-tenancy through
+IAM/STS, and provides a web-based admin console.
+
+A running RustFS node exposes:
+
+- **S3 API** (port 9000) — the primary data path for object CRUD
+- **Admin API** (port 9000, `/minio/` prefix) — cluster management, IAM, metrics
+- **Console** (port 9001) — web UI backed by the Admin API
+- **Inter-node RPC** (gRPC/tonic) — cluster communication for distributed mode
+
+The core data flow for a PUT request looks like:
+
+```
+HTTP request
+  → server (TLS, auth, routing, compression)
+    → app/object_usecase (validation, policy, lifecycle)
+      → storage/ecfs (erasure coding, encryption, checksums)
+        → ecstore (disk pool selection, data distribution)
+          → rio (reader pipeline: encrypt → compress → hash → write)
+            → io-core (zero-copy I/O, buffer pool, direct I/O)
+              → local disk / remote disk via RPC
+```
+
+## Code Map
+
+The repository is a Cargo workspace with a flat `crates/` layout:
+
+```
+rustfs/                      # Workspace root (virtual manifest)
+├── rustfs/                  # Main binary + library crate (75K lines)
+│   └── src/
+│       ├── main.rs          # Entry point, startup sequence
+│       ├── lib.rs           # Module tree root
+│       ├── server/          # HTTP server, TLS, routing, middleware
+│       ├── admin/           # Admin API handlers and console
+│       ├── app/             # Use-case layer (object, bucket, multipart)
+│       ├── storage/         # Storage engine interface and implementation
+│       ├── auth.rs          # S3 request authentication
+│       ├── config/          # CLI args, config parsing, workload profiles
+│       └── ...
+├── crates/                  # 39 library crates
+│   ├── ecstore/             # Erasure-coded storage engine (⚠️ 87K lines)
+│   ├── rio/                 # Reader I/O pipeline (encrypt, compress, hash)
+│   ├── io-core/             # Zero-copy I/O, scheduling, buffer pool
+│   ├── io-metrics/          # I/O metrics collection
+│   ├── common/              # Shared runtime state, globals, data usage types
+│   ├── config/              # Configuration types and parsing
+│   ├── utils/               # Pure utility functions
+│   ├── ...                  # (see "Crate Reference" below)
+│   └── e2e_test/            # End-to-end integration tests
+└── docs/                    # Design documents and analysis
+```
+
+### Main Crate Layers (`rustfs/src/`)
+
+The main crate is organized in layers, top to bottom:
+
+| Layer | Directory | Responsibility |
+|-------|-----------|----------------|
+| **Server** | `server/` | HTTP listener, TLS, CORS, compression, middleware, graceful shutdown |
+| **Admin** | `admin/` | Admin API routing, 30+ handler modules, web console |
+| **App** | `app/` | Use-case orchestration: object_usecase, bucket_usecase, multipart_usecase |
+| **Storage** | `storage/` | S3 API translation, erasure-coded FS, SSE encryption, RPC, concurrency |
+| **Auth** | `auth.rs` | S3 signature verification, credential validation |
+| **Config** | `config/` | CLI parsing, config struct, workload profiles |
+
+A request flows **downward** through the layers. No layer should reach upward
+(e.g., storage must not import from admin).
+
+### Crate Reference
+
+Crates are organized in a dependency DAG with 9 depth levels (0 = leaf, 8 = top):
+
+```
+Depth 0 — LEAF (no internal deps):
+  appauth, checksums, config, credentials, crypto, io-metrics,
+  madmin, s3-common, workers, zip
+
+Depth 1:
+  io-core (→ io-metrics)
+  policy (→ config, credentials, crypto)
+  utils (→ config)                        ⚠️ inverted: utils should be leaf
+
+Depth 2:
+  concurrency, filemeta, keystone, kms, lock, obs,
+  signer, targets, trusted-proxies
+
+Depth 3:
+  common (→ filemeta, madmin)             ⚠️ inverted: common should be leaf
+
+Depth 4:
+  object-capacity, protos, rio
+
+Depth 5 — CORE:
+  ecstore (16 internal deps, 11 dependents — the architectural heart)
+
+Depth 6:
+  audit, heal, iam, metrics, notify, s3select-api, scanner
+
+Depth 7:
+  object-io, protocols, s3select-query
+
+Depth 8 — TOP:
+  rustfs (35 internal deps — the binary, depends on almost everything)
+```
+
+#### By Domain
+
+**Core Infrastructure:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `config` | 3.3K | Configuration types and environment parsing |
+| `utils` | 8.7K | Pure utilities (paths, compression, network, retry) |
+| `common` | 4.4K | Shared runtime state, globals, data usage types, metrics |
+| `madmin` | 5.5K | Admin API request/response types |
+
+**I/O Pipeline:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `io-core` | 6.5K | Zero-copy I/O, buffer pool, direct I/O, scheduling, backpressure |
+| `io-metrics` | 4.5K | I/O operation metrics and counters |
+| `rio` | 6.9K | Composable reader chain (encrypt → compress → hash → limit) |
+| `object-io` | 2.4K | High-level object read/write using rio + ecstore |
+| `concurrency` | 1.8K | Concurrency control wrappers over io-core |
+
+**Storage Engine:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `ecstore` | 87K | ⚠️ Erasure-coded storage: disks, pools, buckets, replication, lifecycle |
+| `filemeta` | 10K | File/object metadata types and versioning |
+| `checksums` | 732 | Checksum computation |
+| `lock` | 7.1K | Distributed lock manager |
+| `heal` | 5.9K | Data healing / bitrot repair |
+| `scanner` | 5.4K | Background data usage scanner |
+| `object-capacity` | 2.5K | Capacity tracking and management |
+
+**Security & Auth:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `crypto` | 1.6K | Encryption primitives |
+| `credentials` | 713 | Credential types (access key / secret key) |
+| `signer` | 1.4K | S3 v4 request signing |
+| `iam` | 9.0K | Identity and access management |
+| `policy` | 8.8K | Policy engine (S3 bucket/IAM policies) |
+| `kms` | 8.1K | Key management service integration |
+| `keystone` | 1.9K | OpenStack Keystone auth |
+| `appauth` | 143 | Application-level auth tokens |
+
+**Protocol & API:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `protos` | 5.7K | Protobuf/gRPC definitions for inter-node RPC |
+| `protocols` | 18K | FTP/FTPS, WebDAV, Swift API support |
+| `s3-common` | 738 | Shared S3 types |
+| `s3select-api` | 1.9K | S3 Select interface |
+| `s3select-query` | 3.6K | S3 Select query engine |
+
+**Observability:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `metrics` | 8.4K | Prometheus metric collectors |
+| `io-metrics` | 4.5K | I/O-specific metrics |
+| `obs` | 5.6K | OpenTelemetry tracing and telemetry |
+| `audit` | 2.4K | Audit logging |
+
+**Events:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `notify` | 5.5K | Event notification system |
+| `targets` | 3.2K | Notification targets (Kafka, AMQP, webhook, etc.) |
+
+**Other:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `trusted-proxies` | 4.0K | Trusted proxy / IP forwarding |
+| `zip` | 986 | ZIP archive support for bulk downloads |
+| `workers` | 136 | Simple worker abstraction |
+
+## Architecture Invariants
+
+> These are rules that the codebase should follow. Some are currently violated
+> (marked with ⚠️). Documenting them here makes the violations explicit and
+> trackable.
+
+1. **Layers flow downward.** Server → Admin/App → Storage → ecstore → rio/io-core.
+   No upward imports.
+
+2. **Leaf crates have zero internal dependencies.** `config`, `credentials`, `crypto`,
+   `io-metrics`, `madmin`, `s3-common` should depend only on external crates.
+   - ⚠️ VIOLATED: `utils` depends on `config`, `common` depends on `filemeta` and `madmin`.
+
+3. **Each type has exactly one definition.** Types shared across crates must be defined
+   in one crate and re-exported or imported by others.
+   - ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
+     `BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
+
+4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
+   storage-level abstractions (objects, buckets, disks, pools).
+
+5. **The `rustfs` binary crate is the only place that wires everything together.**
+   Individual crates should be testable in isolation.
+
+6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
+   not bare `Error`).
+   - ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
+      `heal` use `anyhow` in library code.
+
+## Known Structural Issues
+
+> This section documents known problems in the current architecture.
+> It exists so the team can track and address them deliberately.
+
+### Critical
+
+- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
+  but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
+  types instead of importing them.
+
+- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
+  bucket management, erasure coding, replication, lifecycle, RPC, and configuration
+  — all in one crate. It should be decomposed along its existing subdirectories.
+
+### High
+
+- **Dependency inversions.** `utils → config` and `common → filemeta/madmin` break
+  the layering model. These need to be untangled.
+
+- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
+  concurrency, and rustfs/storage. Should be defined once with builder/composition.
+
+### Medium
+
+- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
+  mixed naming (bare `Error` vs descriptive names).
+
+- **Ambiguous common vs utils boundary.** Both described as "utilities and data
+  structures." Need clear ownership rules.
+
+## Cross-Cutting Concerns
+
+### Error Handling
+
+The project convention is `thiserror` for typed errors with descriptive names.
+See `AGENTS.md`: "Prefer thiserror for library-facing error types."
+
+```rust
+// GOOD
+#[derive(Debug, thiserror::Error)]
+pub enum StorageError {
+    #[error("disk not found: {0}")]
+    DiskNotFound(String),
+}
+
+// AVOID
+pub enum Error { ... }        // too generic
+anyhow::Result<T>             // in library code (OK in tests/CLI)
+```
+
+### Logging & Tracing
+
+- Use `tracing` crate (`info!`, `warn!`, `error!`, `debug!`, `trace!`)
+- Structured fields: `tracing::info!(bucket = %name, "created bucket")`
+- Spans for request-scoped context
+
+### Metrics
+
+- Prometheus-style metrics via `rustfs-obs` runtime and schema
+- I/O-specific counters via `rustfs-io-metrics`
+- Registration happens at crate level, collection/reporting in `rustfs-obs`
+
+### Testing
+
+- Unit tests: `#[cfg(test)] mod tests` in the same file
+- Integration tests: inside respective crates (not top-level `tests/`)
+- E2E tests: `crates/e2e_test/` — tests against a running server
+- Run all: `make test` or `cargo nextest run`
+
+## Startup Sequence
+
+The binary (`main.rs`) boots in this order:
+
+1. Environment variable compatibility (`MINIO_*` → `RUSTFS_*`)
+2. Tokio runtime construction
+3. CLI argument parsing
+4. License, observability, TLS, trusted proxies initialization
+5. Config parsing, server address resolution
+6. Credentials, endpoints, local disks, lock client initialization
+7. Capacity management initialization
+8. HTTP server start (S3 API + optional console)
+9. ECStore initialization (erasure coding storage engine)
+10. Global config, background replication, KMS
+11. Optional: FTP/FTPS/WebDAV servers
+12. Event notifier, audit system, deadlock detector
+13. Bucket metadata, IAM, Keystone, OIDC
+14. Scanner and heal manager
+15. Metrics system, mark `FullReady`
+16. Wait for shutdown signal → graceful shutdown
+
+## Dependency Diagram (Simplified)
+
+```
+                            ┌─────────┐
+                            │  rustfs │  (binary + lib, 75K lines)
+                            │  main   │
+                            └────┬────┘
+                                 │
+                 ┌───────────────┼───────────────┐
+                 │               │               │
+            ┌────▼────┐    ┌────▼────┐    ┌─────▼─────┐
+            │ server  │    │  admin  │    │    app     │
+            │ (HTTP)  │    │(console)│    │(use-cases) │
+            └────┬────┘    └────┬────┘    └─────┬─────┘
+                 │               │               │
+                 └───────────────┼───────────────┘
+                                 │
+                          ┌──────▼──────┐
+                          │   storage   │
+                          │ (ecfs, SSE, │
+                          │  RPC, ACL)  │
+                          └──────┬──────┘
+                                 │
+              ┌──────────────────┼──────────────────┐
+              │                  │                  │
+        ┌─────▼─────┐    ┌──────▼──────┐    ┌──────▼──────┐
+        │  ecstore   │    │     rio     │    │   io-core   │
+        │ (87K,core) │    │  (readers)  │    │ (zero-copy) │
+        └─────┬──────┘    └─────────────┘    └─────────────┘
+              │
+    ┌─────┬──┼──┬─────┬──────┐
+    │     │  │  │     │      │
+ common utils config policy filemeta ...
+```
+
+## How to Navigate
+
+- **"Where does S3 PutObject go?"**
+  `server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
+  `ecstore` distributes → `rio` encrypts/compresses → `io-core` writes
+
+- **"Where are bucket policies enforced?"**
+  `app/bucket_usecase` calls into `crates/policy/`
+
+- **"Where is replication configured?"**
+  `admin/handlers/replication.rs` and `admin/handlers/site_replication.rs` for API,
+  `ecstore/src/bucket/replication/` for engine
+
+- **"Where do I add a new admin endpoint?"**
+  Add handler in `admin/handlers/`, register in `admin/router.rs`
+
+- **"Where do I add a new metric?"**
+  Define descriptor/collector in `crates/obs/src/metrics/`, expose via `/minio/v2/metrics`
+
+---
+
+*Inspired by [matklad's ARCHITECTURE.md](https://matklad.github.io/2021/02/06/ARCHITECTURE.md.html)
+and [rust-analyzer's architecture.md](https://github.com/rust-analyzer/rust-analyzer/blob/master/docs/book/src/contributing/architecture.md).*

Cargo.toml

@@ -31,8 +31,6 @@ members = [
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
     "crates/madmin", # Management dashboard and admin API interface
-    "crates/mcp", # MCP server for S3 operations
-    "crates/metrics", # Metrics collection and reporting
     "crates/notify", # Notification system for events
     "crates/obs", # Observability utilities
     "crates/object-capacity", # Capacity scan and refresh core
@@ -40,7 +38,7 @@ members = [
     "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
     "crates/protos", # Protocol buffer definitions
     "crates/rio", # Rust I/O utilities and abstractions
-    "crates/concurrency", # Rust I/O utilities and abstractions
+    "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
     "crates/s3-common", # Common utilities and data structures for S3 compatibility
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -52,7 +50,6 @@ members = [
     "crates/workers", # Worker thread pools and task scheduling
     "crates/io-metrics", # Zero-copy metrics collection for performance analysis
     "crates/io-core", # Zero-copy core reader and writer implementations
-    "crates/object-io", # Object I/O policy and zero-copy helper primitives
     "crates/zip", # ZIP file handling and compression
 ]
 resolver = "3"
@@ -62,7 +59,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.93.0"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -74,68 +71,70 @@ unsafe_code = "deny"
 
 [workspace.lints.clippy]
 all = "warn"
+needless_collect = "warn"
+redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "0.0.5" }
-rustfs-heal = { path = "crates/heal", version = "0.0.5" }
-rustfs-appauth = { path = "crates/appauth", version = "0.0.5" }
-rustfs-audit = { path = "crates/audit", version = "0.0.5" }
-rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
-rustfs-common = { path = "crates/common", version = "0.0.5" }
-rustfs-config = { path = "./crates/config", version = "0.0.5" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "0.0.5" }
-rustfs-credentials = { path = "crates/credentials", version = "0.0.5" }
-rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
-rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
-rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
-rustfs-iam = { path = "crates/iam", version = "0.0.5" }
-rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
-rustfs-kms = { path = "crates/kms", version = "0.0.5" }
-rustfs-lock = { path = "crates/lock", version = "0.0.5" }
-rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
-rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
-rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
-rustfs-notify = { path = "crates/notify", version = "0.0.5" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
-rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
-rustfs-object-io = { path = "crates/object-io", version = "0.0.5" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
-rustfs-obs = { path = "crates/obs", version = "0.0.5" }
-rustfs-policy = { path = "crates/policy", version = "0.0.5" }
-rustfs-protos = { path = "crates/protos", version = "0.0.5" }
-rustfs-rio = { path = "crates/rio", version = "0.0.5" }
-rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
-rustfs-scanner = { path = "crates/scanner", version = "0.0.5" }
-rustfs-signer = { path = "crates/signer", version = "0.0.5" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "0.0.5" }
-rustfs-targets = { path = "crates/targets", version = "0.0.5" }
-rustfs-utils = { path = "crates/utils", version = "0.0.5" }
-rustfs-workers = { path = "crates/workers", version = "0.0.5" }
-rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
-rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
+rustfs = { path = "./rustfs", version = "1.0.0-beta.1" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.1" }
+rustfs-appauth = { path = "crates/appauth", version = "1.0.0-beta.1" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.1" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.1" }
+rustfs-common = { path = "crates/common", version = "1.0.0-beta.1" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-beta.1" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.1" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.1" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.1" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.1" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.1" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.1" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.1" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.1" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.1" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.1" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.1" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.1" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.1" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.1" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.1" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.1" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.1" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.1" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.1" }
+rustfs-s3-common = { path = "crates/s3-common", version = "1.0.0-beta.1" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.1" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.1" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.1" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.1" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.1" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.1" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.1" }
+rustfs-workers = { path = "crates/workers", version = "1.0.0-beta.1" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.1" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.41" }
+async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
-axum = "0.8.8"
+async-nats = "0.47.0"
+axum = "0.8.9"
 futures = "0.3.32"
 futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
+pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
 hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
-hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
+hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
-reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+rustfs-kafka-async = { version = "1.2.0" }
 socket2 = { version = "0.6.3", features = ["all"] }
-tokio = { version = "1.51.1", features = ["fs", "rt-multi-thread"] }
+tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
@@ -154,27 +153,25 @@ flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
 quick-xml = "0.39.2"
-rmcp = { version = "1.3.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
-schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
 argon2 = { version = "0.6.0-rc.8" }
-blake2 = "0.11.0-rc.5"
+blake2 = "0.11.0-rc.6"
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
-pbkdf2 = "0.13.0-rc.10"
-rsa = { version = "0.10.0-rc.17" }
-rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pki-types = "1.14.0"
+pbkdf2 = "0.13.0"
+rsa = { version = "0.10.0-rc.18" }
+rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls-pki-types = "1.14.1"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"
@@ -183,18 +180,18 @@ zeroize = { version = "1.8.2", features = ["derive"] }
 # Time and Date
 chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.23", features = ["serde"] }
+jiff = { version = "0.2.24", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
 anyhow = "1.0.102"
 arc-swap = "1.9.1"
-astral-tokio-tar = "0.6.0"
+astral-tokio-tar = "0.6.1"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.15" }
+aws-config = { version = "1.8.16" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.131.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
@@ -202,7 +199,7 @@ base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.6.0", features = ["derive", "env"] }
+clap = { version = "4.6.1", features = ["derive", "env"] }
 const-str = { version = "1.1.0", features = ["std", "proc"] }
 convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
@@ -210,23 +207,23 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "53.0.0"
+datafusion = "53.1.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
 glob = "0.3.3"
-google-cloud-storage = "1.10.0"
-google-cloud-auth = "1.8.0"
-hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
+google-cloud-storage = "1.11.0"
+google-cloud-auth = "1.9.0"
+hashbrown = { version = "0.17.0", features = ["serde", "rayon"] }
 hex = "0.4.3"
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.184"
+libc = "0.2.186"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.11"
+local-ip-address = "0.6.12"
 memmap2 = "0.9.10"
 lz4 = "1.28.1"
 matchit = "0.9.2"
@@ -244,19 +241,19 @@ path-clean = "1.0.1"
 percent-encoding = "2.3.2"
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
-rand = { version = "0.10.0", features = ["serde"] }
+rand = { version = "0.10.1", features = ["serde"] }
 ratelimit = "0.10.1"
-rayon = "1.11.0"
+rayon = "1.12.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
-rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] }
+rumqttc = { package = "rumqttc-next", version = "0.30.0", features = ["websocket"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
-s3s = { git = "https://github.com/rustfs/s3s", rev = "010ae03bdd8c93a9c046108190987bb0e6cb0692", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
 serial_test = "3.4.0"
-shadow-rs = { version = "1.7.1", default-features = false }
+shadow-rs = { version = "2.0.0", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
@@ -270,37 +267,37 @@ tempfile = "3.27.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
-tracing-appender = "0.2.4"
+tracing-appender = "0.2.5"
 tracing-error = "0.2.1"
 tracing-opentelemetry = "0.32.1"
 tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.23.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "8.5.1"
+zip = "8.6.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
-dial9-tokio-telemetry = { version = "0.2", git = "https://github.com/dial9-rs/dial9-tokio-telemetry.git", rev = "60502082601b647c4a51962595721f631b7bbce1" }
+dial9-tokio-telemetry = "0.3"
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
 opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }
-pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.0.3", features = ["backend-pprof-rs"] }
 
 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }
 unftp-core = "0.1.0"
-suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"
 
 # WebDAV
@@ -320,7 +317,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
 pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] }
 
 [workspace.metadata.cargo-shear]
-ignored = ["rustfs", "rustfs-mcp"]
+ignored = ["rustfs"]
 
 [profile.release]
 opt-level = 3

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM alpine:3.23 AS build
+FROM alpine:3.23.4 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
@@ -54,7 +54,7 @@ RUN set -eux; \
     rm -rf rustfs.zip /build/.tmp || true
 
 
-FROM alpine:3.23
+FROM alpine:3.23.4
 
 ARG RELEASE=latest
 ARG BUILD_DATE
@@ -73,7 +73,7 @@ LABEL name="RustFS" \
       license="Apache-2.0"
 
 RUN apk update && \
-    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
+    apk add --no-cache ca-certificates coreutils curl
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

Dockerfile.decommission-local

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.91-trixie
+FROM rust:1.93-trixie
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \

Dockerfile.source

@@ -30,7 +30,7 @@ ARG BUILDPLATFORM
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.91-trixie AS builder
+FROM rust:1.93-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
@@ -60,7 +60,15 @@ RUN set -eux; \
 
 # Optional: cross toolchain for aarch64 (only when targeting linux/arm64)
 RUN set -eux; \
-    if [ "${TARGETPLATFORM:-linux/amd64}" = "linux/arm64" ]; then \
+    target_platform="${TARGETPLATFORM:-}"; \
+    if [ -z "${target_platform}" ]; then \
+    case "$(uname -m)" in \
+    x86_64) target_platform="linux/amd64" ;; \
+    aarch64|arm64) target_platform="linux/arm64" ;; \
+    *) target_platform="linux/amd64" ;; \
+    esac; \
+    fi; \
+    if [ "${target_platform}" = "linux/arm64" ]; then \
     export DEBIAN_FRONTEND=noninteractive; \
     apt-get update; \
     apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu; \
@@ -120,7 +128,16 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
     --mount=type=cache,target=/usr/src/rustfs/target \
     set -eux; \
-    case "${TARGETPLATFORM:-linux/amd64}" in \
+    rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; \
+    target_platform="${TARGETPLATFORM:-}"; \
+    if [ -z "${target_platform}" ]; then \
+    case "$(uname -m)" in \
+    x86_64) target_platform="linux/amd64" ;; \
+    aarch64|arm64) target_platform="linux/arm64" ;; \
+    *) target_platform="linux/amd64" ;; \
+    esac; \
+    fi; \
+    case "${target_platform}" in \
     linux/amd64) \
     echo "Building for x86_64-unknown-linux-gnu"; \
     cargo build --release --locked --target x86_64-unknown-linux-gnu --bin rustfs -j "$(nproc)"; \
@@ -132,7 +149,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     install -m 0755 target/aarch64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \
     ;; \
     *) \
-    echo "Unsupported TARGETPLATFORM=${TARGETPLATFORM}" >&2; exit 1 \
+    echo "Unsupported target platform=${target_platform}" >&2; exit 1 \
     ;; \
     esac
 
@@ -178,7 +195,7 @@ CMD ["cargo", "run", "--bin", "rustfs", "--"]
 # -----------------------------
 # Runtime stage (Ubuntu minimal)
 # -----------------------------
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ARG BUILD_DATE
 ARG VCS_REF

README.md

@@ -100,28 +100,28 @@ To get started with RustFS, follow these steps:
 curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
 ```
 
-### 2\. Docker Quick Start (Option 2)
+### 2. Docker Quick Start (Option 2)
 
 The RustFS container runs as a non-root user `rustfs` (UID `10001`). If you run Docker with `-v` to mount a host directory, please ensure the host directory owner is set to `10001`, otherwise you will encounter permission denied errors.
 
 ```bash
- # Create data and logs directories
- mkdir -p data logs
+# Create data and logs directories
+mkdir -p data logs
 
- # Change the owner of these directories
- chown -R 10001:10001 data logs
+# Change the owner of these directories
+chown -R 10001:10001 data logs
 
- # Using latest version
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
+# Using latest version
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
- # Using specific version
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
+# Using specific version
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:v1.0.0-beta.1
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
 
 ```bash
- podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
+podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 ```
 
 You can also use Docker Compose. Using the `docker-compose.yml` file in the root directory:
@@ -219,6 +219,48 @@ rustfs --help
 
 **NOTE**: To access the RustFS instance via `https`, please refer to the [TLS Configuration Docs](https://docs.rustfs.com/integration/tls-configured.html).
 
+### OIDC Roles Claim (Microsoft Entra ID)
+
+RustFS supports mapping an OIDC claim containing role values into the existing
+authorization pipeline. The `roles_claim` setting is **optional**: when unset or
+empty, only the `groups` claim contributes to authorization (same as older
+RustFS releases). For Microsoft Entra ID app roles, set `roles_claim=roles` so
+both console admin checks and bucket IAM policies can evaluate those roles.
+
+Example environment configuration (opt-in roles claim):
+
+```bash
+RUSTFS_IDENTITY_OPENID_ENABLE=on
+RUSTFS_IDENTITY_OPENID_CONFIG_URL="https://login.microsoftonline.com/<tenant-id>/v2.0/.well-known/openid-configuration"
+RUSTFS_IDENTITY_OPENID_CLIENT_ID="<client-id>"
+RUSTFS_IDENTITY_OPENID_CLIENT_SECRET="<client-secret>"
+RUSTFS_IDENTITY_OPENID_SCOPES="openid,profile,email"
+RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM="groups"
+RUSTFS_IDENTITY_OPENID_ROLES_CLAIM="roles"
+```
+
+Policy condition example (evaluate app roles directly with `jwt:roles`; when
+`roles_claim` is configured, RustFS also merges those values into `jwt:groups`
+for backward compatibility with older policies):
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["admin:*"],
+      "Resource": ["arn:aws:s3:::*"],
+      "Condition": {
+        "ForAnyValue:StringEquals": {
+          "jwt:roles": ["RustFS.ConsoleAdmin"]
+        }
+      }
+    }
+  ]
+}
+```
+
 ## Documentation
 
 For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).

README_ZH.md

@@ -113,7 +113,7 @@ RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用
  docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
  # 使用指定版本运行
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.68
+ docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:v1.0.0-beta.1
 ```
 
 您也可以使用 Docker Compose。使用根目录下的 `docker-compose.yml` 文件：

_typos.toml

@@ -39,7 +39,7 @@ abd = "abd"
 mak = "mak"
 gae = "gae"
 GAE = "GAE"
-writeable = "writeable"
+thr = "thr"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
 consts = "consts"

crates/audit/Cargo.toml

@@ -41,11 +41,10 @@ serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
-url = { workspace = true }
-rumqttc = { workspace = true }
 
 [dev-dependencies]
 temp-env = { workspace = true }
+url = { workspace = true }
 
 [lints]
 workspace = true

crates/audit/src/factory.rs

@@ -14,28 +14,19 @@
 
 use crate::AuditEntry;
 use async_trait::async_trait;
-use hashbrown::HashSet;
-use rumqttc::QoS;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
-use rustfs_config::{
-    AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
-    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
-    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
-};
+use rustfs_config::AUDIT_DEFAULT_DIR;
+use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
-    error::TargetError,
-    target::{
-        mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
-        webhook::WebhookArgs,
+    config::{
+        build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+        validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
     },
+    error::TargetError,
+    target::TargetType,
 };
-use std::time::Duration;
-use tracing::{debug, warn};
-use url::Url;
+use std::collections::HashSet;
 
 /// Trait for creating targets from configuration
 #[async_trait]
@@ -49,10 +40,6 @@ pub trait TargetFactory: Send + Sync {
     /// Returns a set of valid configuration field names for this target type.
     /// This is used to filter environment variables.
     fn get_valid_fields(&self) -> HashSet<String>;
-
-    /// Returns a set of valid configuration env field names for this target type.
-    /// This is used to filter environment variables.
-    fn get_valid_env_fields(&self) -> HashSet<String>;
 }
 
 /// Factory for creating Webhook targets
@@ -61,71 +48,18 @@ pub struct WebhookTargetFactory;
 #[async_trait]
 impl TargetFactory for WebhookTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        // All config values are now read directly from the merged `config` KVS.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        let parsed_endpoint = endpoint.trim();
-        let endpoint_url = Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let args = WebhookArgs {
-            enable: true, // If we are here, it's already enabled.
-            endpoint: endpoint_url,
-            auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
-            queue_dir: config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(WEBHOOK_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
-            client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
-            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
-            skip_tls_verify: config
-                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
-                .and_then(|v| v.parse::<bool>().ok())
-                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        // Validation also uses the merged `config` KVS directly.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        debug!("endpoint: {}", endpoint);
-        let parsed_endpoint = endpoint.trim();
-        Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let client_cert = config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default();
-        let client_key = config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default();
-
-        if client_cert.is_empty() != client_key.is_empty() {
-            return Err(TargetError::Configuration(
-                "Both client_cert and client_key must be specified together".to_string(),
-            ));
-        }
-
-        let queue_dir = config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string());
-        if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
-            return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
-        }
-
-        Ok(())
+        validate_webhook_config(config, AUDIT_DEFAULT_DIR)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
     }
-
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
-    }
 }
 
 /// Factory for creating MQTT targets
@@ -134,112 +68,73 @@ pub struct MQTTTargetFactory;
 #[async_trait]
 impl TargetFactory for MQTTTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let broker_url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let topic = config
-            .lookup(MQTT_TOPIC)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
-
-        let args = MQTTArgs {
-            enable: true, // Assumed enabled.
-            broker: broker_url,
-            topic,
-            qos: config
-                .lookup(MQTT_QOS)
-                .and_then(|v| v.parse::<u8>().ok())
-                .map(|q| match q {
-                    0 => QoS::AtMostOnce,
-                    1 => QoS::AtLeastOnce,
-                    2 => QoS::ExactlyOnce,
-                    _ => QoS::AtLeastOnce,
-                })
-                .unwrap_or(QoS::AtLeastOnce),
-            username: config.lookup(MQTT_USERNAME).unwrap_or_default(),
-            password: config.lookup(MQTT_PASSWORD).unwrap_or_default(),
-            max_reconnect_interval: config
-                .lookup(MQTT_RECONNECT_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(5)),
-            keep_alive: config
-                .lookup(MQTT_KEEP_ALIVE_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(30)),
-            tls: MQTTTlsConfig::from_values(
-                config.lookup(MQTT_TLS_POLICY).as_deref(),
-                config.lookup(MQTT_TLS_CA).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-                config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-                config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-            )?,
-            queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(MQTT_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let tls = MQTTTlsConfig::from_values(
-            config.lookup(MQTT_TLS_POLICY).as_deref(),
-            config.lookup(MQTT_TLS_CA).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-            config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-            config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-        )?;
-        validate_mqtt_broker_url(&url, &tls)?;
-
-        if config.lookup(MQTT_TOPIC).is_none() {
-            return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
-        }
-
-        if let Some(qos_str) = config.lookup(MQTT_QOS) {
-            let qos = qos_str
-                .parse::<u8>()
-                .map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
-            if qos > 2 {
-                return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
-            }
-        }
-
-        let queue_dir = config.lookup(MQTT_QUEUE_DIR).unwrap_or_default();
-        if !queue_dir.is_empty() {
-            if !std::path::Path::new(&queue_dir).is_absolute() {
-                return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
-            }
-            if let Some(qos_str) = config.lookup(MQTT_QOS)
-                && qos_str == "0"
-            {
-                warn!("Using queue_dir with QoS 0 may result in event loss");
-            }
-        }
-
-        Ok(())
+        validate_mqtt_config(config)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
     }
+}
 
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
+pub struct NATSTargetFactory;
+
+#[async_trait]
+impl TargetFactory for NATSTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_nats_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_NATS_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct PulsarTargetFactory;
+
+#[async_trait]
+impl TargetFactory for PulsarTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_pulsar_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct KafkaTargetFactory;
+
+#[async_trait]
+impl TargetFactory for KafkaTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_kafka_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }

crates/audit/src/global.rs

@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{AuditEntry, AuditResult, AuditSystem};
+use crate::{AuditEntry, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
 use rustfs_ecstore::config::Config;
 use std::sync::{Arc, OnceLock};
 use tracing::{debug, error, trace, warn};
@@ -89,6 +89,15 @@ pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
     with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
 }
 
+/// Returns per-target audit delivery metrics for Prometheus collection.
+pub async fn audit_target_metrics() -> Vec<AuditTargetMetricSnapshot> {
+    if let Some(system) = audit_system() {
+        system.snapshot_target_metrics().await
+    } else {
+        Vec::new()
+    }
+}
+
 /// Check if the global audit system is running
 pub async fn is_audit_system_running() -> bool {
     if let Some(system) = audit_system() {

crates/audit/src/lib.rs

@@ -31,4 +31,4 @@ pub use error::{AuditError, AuditResult};
 pub use global::*;
 pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation};
 pub use registry::AuditRegistry;
-pub use system::AuditSystem;
+pub use system::{AuditSystem, AuditTargetMetricSnapshot};

crates/audit/src/registry.rs

@@ -14,18 +14,19 @@
 
 use crate::{
     AuditEntry, AuditError, AuditResult,
-    factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory},
+    factory::{
+        KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
+    },
 };
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
-use hashbrown::{HashMap, HashSet};
-use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, audit::AUDIT_ROUTE_PREFIX};
+use hashbrown::HashMap;
+use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_targets::arn::TargetID;
-use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
-use std::str::FromStr;
+use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
 use std::sync::Arc;
-use tracing::{debug, error, info, warn};
+use tracing::{error, info};
 
 /// Registry for managing audit targets
 pub struct AuditRegistry {
@@ -52,6 +53,9 @@ impl AuditRegistry {
         // Register built-in factories
         registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
+        registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
+        registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
+        registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
 
         registry
     }
@@ -105,143 +109,22 @@ impl AuditRegistry {
         &self,
         config: &Config,
     ) -> AuditResult<Vec<Box<dyn Target<AuditEntry> + Send + Sync>>> {
-        // Collect only environment variables with the relevant prefix to reduce memory usage
-        let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
-        // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
             info!("Start working on target types...");
-
-            // 2. Prepare the configuration source
-            // 2.1. Get the configuration segment in the file, e.g. 'audit_webhook'
-            let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
-            let file_configs = config.0.get(&section_name).cloned().unwrap_or_default();
-            // 2.2. Get the default configuration for that type
-            let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
-            debug!(?default_cfg, "Get the default configuration");
-
-            // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
-            debug!(?valid_fields, "Get the legitimate configuration fields");
-
-            // 3. Resolve instance IDs and configuration overrides from environment variables
-            let mut instance_ids_from_env = HashSet::new();
-            // 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
-            let enable_prefix =
-                format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
-                    .to_uppercase();
-            for (key, value) in &all_env {
-                if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
-                    && let Some(id) = key.strip_prefix(&enable_prefix)
-                    && !id.is_empty()
-                {
-                    instance_ids_from_env.insert(id.to_lowercase());
-                }
-            }
-
-            // 3.2. Parse all relevant environment variable configurations
-            // 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
-            let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
-            // 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
-            let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
-            for (key, value) in &all_env {
-                if let Some(rest) = key.strip_prefix(&env_prefix) {
-                    // Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
-                    // Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
-                    let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
-
-                    // The first part from the right is INSTANCE_ID
-                    let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
-                    // The remaining part is FIELD_NAME
-                    let field_name_part = parts.next();
-
-                    let (field_name, instance_id) = match field_name_part {
-                        // Case 1: The format is <FIELD_NAME>_<INSTANCE_ID>
-                        // e.g., rest = "ENDPOINT_PRIMARY" -> field_name="ENDPOINT", instance_id="PRIMARY"
-                        Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
-                        // Case 2: The format is <FIELD_NAME> (without INSTANCE_ID)
-                        // e.g., rest = "ENABLE" -> field_name="ENABLE", instance_id="" (Universal configuration `_ DEFAULT_DELIMITER`)
-                        None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
-                    };
-
-                    // *** Optimization point 2: Verify whether the parsed field_name is legal ***
-                    if !field_name.is_empty() && valid_fields.contains(&field_name) {
-                        debug!(
-                            instance_id = %if instance_id.is_empty() { DEFAULT_DELIMITER } else { &instance_id },
-                            %field_name,
-                            %value,
-                            "Parsing to environment variables"
-                        );
-                        env_overrides
-                            .entry(instance_id)
-                            .or_default()
-                            .insert(field_name, value.clone());
-                    } else {
-                        // Ignore illegal field names
-                        warn!(
-                            field_name = %field_name,
-                            "Ignore environment variable fields, not found in the list of valid fields for target type {}",
-                            target_type
-                        );
-                    }
-                }
-            }
-            debug!(?env_overrides, "Complete the environment variable analysis");
-
-            // 4. Determine all instance IDs that need to be processed
-            let mut all_instance_ids: HashSet<String> =
-                file_configs.keys().filter(|k| *k != DEFAULT_DELIMITER).cloned().collect();
-            all_instance_ids.extend(instance_ids_from_env);
-            debug!(?all_instance_ids, "Determine all instance IDs");
-
-            // 5. Merge configurations and create tasks for each instance
-            for id in all_instance_ids {
-                // 5.1. Merge configuration, priority: Environment variables > File instance configuration > File default configuration
-                let mut merged_config = default_cfg.clone();
-                // Instance-specific configuration in application files
-                if let Some(file_instance_cfg) = file_configs.get(&id) {
-                    merged_config.extend(file_instance_cfg.clone());
-                }
-                // Application instance-specific environment variable configuration
-                if let Some(env_instance_cfg) = env_overrides.get(&id) {
-                    // Convert HashMap<String, String> to KVS
-                    let mut kvs_from_env = KVS::new();
-                    for (k, v) in env_instance_cfg {
-                        kvs_from_env.insert(k.clone(), v.clone());
-                    }
-                    merged_config.extend(kvs_from_env);
-                }
-                debug!(instance_id = %id, ?merged_config, "Complete configuration merge");
-
-                // 5.2. Check if the instance is enabled
-                let enabled = merged_config
-                    .lookup(ENABLE_KEY)
-                    .map(|v| {
-                        EnableState::from_str(v.as_str())
-                            .ok()
-                            .map(|s| s.is_enabled())
-                            .unwrap_or(false)
-                    })
-                    .unwrap_or(false);
-
-                if enabled {
-                    info!(instance_id = %id, "Target is enabled, ready to create a task");
-                    // 5.3. Create asynchronous tasks for enabled instances
-                    let tid = id.clone();
-                    let merged_config_arc = Arc::new(merged_config);
-                    tasks.push(async move {
-                        let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (tid, result)
-                    });
-                } else {
-                    info!(instance_id = %id, "Skip disabled target");
-                }
+            for (id, merged_config) in collect_target_configs(config, AUDIT_ROUTE_PREFIX, target_type, &valid_fields) {
+                info!(instance_id = %id, "Target is enabled, ready to create a task");
+                let tid = id.clone();
+                let merged_config_arc = Arc::new(merged_config);
+                tasks.push(async move {
+                    let result = factory.create_target(tid.clone(), &merged_config_arc).await;
+                    (tid, result)
+                });
             }
         }
 
-        // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
         while let Some((id, result)) = tasks.next().await {
             match result {
@@ -317,8 +200,8 @@ impl AuditRegistry {
             }
         }
 
-        if !errors.is_empty() {
-            return Err(AuditError::Target(errors.into_iter().next().unwrap()));
+        if let Some(error) = errors.into_iter().next() {
+            return Err(AuditError::Target(error));
         }
 
         Ok(())

crates/audit/src/system.rs

@@ -24,6 +24,14 @@ use std::sync::Arc;
 use tokio::sync::{Mutex, RwLock, mpsc};
 use tracing::{error, info, warn};
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct AuditTargetMetricSnapshot {
+    pub failed_messages: u64,
+    pub queue_length: u64,
+    pub target_id: String,
+    pub total_messages: u64,
+}
+
 /// State of the audit system
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum AuditSystemState {
@@ -537,8 +545,14 @@ impl AuditSystem {
                                     TargetError::Timeout(_) => {
                                         warn!("Timeout sending to target {}, retrying...", target.id());
                                     }
+                                    TargetError::Dropped(reason) => {
+                                        warn!("Dropped queued payload for target {}: {}", target.id(), reason);
+                                        observability::record_target_failure();
+                                        break;
+                                    }
                                     _ => {
                                         error!("Permanent error for target {}: {}", target.id(), e);
+                                        target.record_final_failure();
                                         observability::record_target_failure();
                                         break;
                                     }
@@ -552,6 +566,7 @@ impl AuditSystem {
 
                     if retries >= MAX_RETRIES && !success {
                         warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id());
+                        target.record_final_failure();
                         observability::record_target_failure();
                     }
                 }
@@ -664,6 +679,25 @@ impl AuditSystem {
         registry.list_target_values()
     }
 
+    /// Returns per-target delivery metrics for Prometheus collection.
+    pub async fn snapshot_target_metrics(&self) -> Vec<AuditTargetMetricSnapshot> {
+        let targets = self.get_target_values().await;
+        let mut snapshots = Vec::with_capacity(targets.len());
+
+        for target in targets {
+            let delivery = target.delivery_snapshot();
+            snapshots.push(AuditTargetMetricSnapshot {
+                failed_messages: delivery.failed_messages,
+                queue_length: delivery.queue_length,
+                target_id: target.id().to_string(),
+                total_messages: delivery.total_messages,
+            });
+        }
+
+        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
+        snapshots
+    }
+
     /// Gets information about a specific target
     ///
     /// # Arguments

crates/common/src/data_usage.rs

@@ -51,14 +51,14 @@ impl AllTierStats {
     pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
         for (tier, st) in tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
     pub fn merge(&mut self, other: AllTierStats) {
         for (tier, st) in other.tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
@@ -673,7 +673,7 @@ impl DataUsageCache {
         let mut leaves = Vec::new();
         let mut remove = total - limit;
         add(self, path, &mut leaves);
-        leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
+        leaves.sort_by_key(|a| a.objects);
 
         while remove > 0 && !leaves.is_empty() {
             let Some(e) = leaves.first() else {
@@ -1097,7 +1097,7 @@ impl DataUsageInfo {
 
     /// Add bucket usage info
     pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) {
-        self.buckets_usage.insert(bucket.clone(), usage);
+        self.buckets_usage.insert(bucket, usage);
         self.buckets_count = self.buckets_usage.len() as u64;
         self.last_update = Some(SystemTime::now());
     }

crates/common/src/heal_channel.rs

@@ -18,7 +18,7 @@ use std::{
     fmt::{self, Display},
     sync::OnceLock,
 };
-use tokio::sync::{broadcast, mpsc};
+use tokio::sync::{broadcast, mpsc, oneshot};
 use uuid::Uuid;
 
 pub const HEAL_DELETE_DANGLING: bool = true;
@@ -206,11 +206,59 @@ pub struct HealOpts {
     pub set: Option<usize>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionDropReason {
+    QueueFull,
+    PolicyDropped,
+}
+
+impl HealAdmissionDropReason {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::QueueFull => "queue_full",
+            Self::PolicyDropped => "policy_dropped",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionResult {
+    Accepted,
+    Merged,
+    Full,
+    Dropped(HealAdmissionDropReason),
+}
+
+impl HealAdmissionResult {
+    pub fn result_label(self) -> &'static str {
+        match self {
+            Self::Accepted => "accepted",
+            Self::Merged => "merged",
+            Self::Full => "full",
+            Self::Dropped(_) => "dropped",
+        }
+    }
+
+    pub fn reason_label(self) -> &'static str {
+        match self {
+            Self::Dropped(reason) => reason.as_str(),
+            _ => "none",
+        }
+    }
+
+    pub fn is_admitted(self) -> bool {
+        matches!(self, Self::Accepted | Self::Merged)
+    }
+}
+
 /// Heal channel command type
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub enum HealChannelCommand {
     /// Start a new heal task
-    Start(HealChannelRequest),
+    Start {
+        request: HealChannelRequest,
+        response_tx: oneshot::Sender<Result<HealAdmissionResult, String>>,
+    },
     /// Query heal task status
     Query { heal_path: String, client_token: String },
     /// Cancel heal task
@@ -339,9 +387,22 @@ pub fn subscribe_heal_responses() -> broadcast::Receiver<HealChannelResponse> {
     heal_response_sender().subscribe()
 }
 
+/// Send heal start request and wait for structured admission feedback.
+pub async fn send_heal_request_with_admission(request: HealChannelRequest) -> Result<HealAdmissionResult, String> {
+    let (response_tx, response_rx) = oneshot::channel();
+    send_heal_command(HealChannelCommand::Start { request, response_tx }).await?;
+    response_rx
+        .await
+        .map_err(|e| format!("Failed to receive heal admission response: {e}"))?
+}
+
 /// Send heal start request
 pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> {
-    send_heal_command(HealChannelCommand::Start(request)).await
+    match send_heal_request_with_admission(request).await? {
+        HealAdmissionResult::Accepted | HealAdmissionResult::Merged => Ok(()),
+        HealAdmissionResult::Full => Err("Heal request queue is full".to_string()),
+        HealAdmissionResult::Dropped(reason) => Err(format!("Heal request dropped: {}", reason.as_str())),
+    }
 }
 
 /// Send heal query request
@@ -450,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str)
         }
 
         if let Some(e) = &rule.noncurrent_version_expiration {
-            if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
+            if e.noncurrent_days.is_some() {
                 return true;
             }
             if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
@@ -542,6 +603,19 @@ pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPri
 mod tests {
     use super::*;
 
+    #[test]
+    fn heal_admission_result_labels_are_stable() {
+        assert_eq!(HealAdmissionResult::Accepted.result_label(), "accepted");
+        assert_eq!(HealAdmissionResult::Merged.result_label(), "merged");
+        assert_eq!(HealAdmissionResult::Full.result_label(), "full");
+        assert_eq!(
+            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull).reason_label(),
+            "queue_full"
+        );
+        assert!(HealAdmissionResult::Merged.is_admitted());
+        assert!(!HealAdmissionResult::Full.is_admitted());
+    }
+
     #[tokio::test]
     async fn heal_response_broadcast_reaches_subscriber() {
         let mut receiver = subscribe_heal_responses();

crates/common/src/internode_metrics.rs

@@ -51,7 +51,7 @@ impl InternodeMetrics {
             return;
         }
         self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-        counter!("rustfs.internode.sent.bytes.total").increment(bytes);
+        counter!("rustfs_system_network_internode_sent_bytes_total").increment(bytes);
     }
 
     pub fn record_recv_bytes(&self, bytes: usize) {
@@ -60,22 +60,22 @@ impl InternodeMetrics {
             return;
         }
         self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-        counter!("rustfs.internode.recv.bytes.total").increment(bytes);
+        counter!("rustfs_system_network_internode_recv_bytes_total").increment(bytes);
     }
 
     pub fn record_outgoing_request(&self) {
         self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.requests.outgoing.total").increment(1);
+        counter!("rustfs_system_network_internode_requests_outgoing_total").increment(1);
     }
 
     pub fn record_incoming_request(&self) {
         self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.requests.incoming.total").increment(1);
+        counter!("rustfs_system_network_internode_requests_incoming_total").increment(1);
     }
 
     pub fn record_error(&self) {
         self.errors_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.errors.total").increment(1);
+        counter!("rustfs_system_network_internode_errors_total").increment(1);
     }
 
     pub fn record_dial_result(&self, duration: Duration, success: bool) {
@@ -83,11 +83,11 @@ impl InternodeMetrics {
         self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
         let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
         let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
-        gauge!("rustfs.internode.dial.avg_time.nanos").set(total as f64 / samples as f64);
+        gauge!("rustfs_system_network_internode_dial_avg_time_nanos").set(total as f64 / samples as f64);
 
         if !success {
             self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
-            counter!("rustfs.internode.dial.errors.total").increment(1);
+            counter!("rustfs_system_network_internode_dial_errors_total").increment(1);
         }
 
         let now_ms = SystemTime::now()
@@ -101,11 +101,7 @@ impl InternodeMetrics {
     pub fn snapshot(&self) -> InternodeMetricsSnapshot {
         let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
         let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
-        let dial_avg_time_nanos = if dial_samples_total == 0 {
-            0
-        } else {
-            dial_total_time_nanos / dial_samples_total
-        };
+        let dial_avg_time_nanos = dial_total_time_nanos.checked_div(dial_samples_total).unwrap_or(0);
 
         InternodeMetricsSnapshot {
             sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),

crates/common/src/metrics.rs

@@ -685,7 +685,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
     };
 
     let done_fn: CloseDiskFn = {
-        let disk = disk_name.clone();
+        let disk = disk_name;
         Arc::new(move || {
             let disk = disk.clone();
             Box::pin(async move {

crates/concurrency/Cargo.toml

@@ -26,7 +26,7 @@ thiserror = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]
-tokio = { workspace = true, features = ["full"] }
+tokio = { workspace = true, features = ["test-util","macros","rt-multi-thread"] }
 
 [features]
 default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"]

crates/config/README.md

@@ -53,8 +53,24 @@ Current guidance:
 
 ## Scanner environment aliases
 
+- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
+- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
 - `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
 - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
+- `RUSTFS_SCANNER_IDLE_MODE` (canonical)
+- `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
+
+## Drive timeout environment variables
+
+- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS`
+
+Legacy compatibility fallback:
+- `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
+  This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
 
 ## 📄 License
 

crates/config/src/audit/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Kafka Environment Variables
+pub const ENV_AUDIT_KAFKA_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_ENABLE";
+pub const ENV_AUDIT_KAFKA_BROKERS: &str = "RUSTFS_AUDIT_KAFKA_BROKERS";
+pub const ENV_AUDIT_KAFKA_TOPIC: &str = "RUSTFS_AUDIT_KAFKA_TOPIC";
+pub const ENV_AUDIT_KAFKA_ACKS: &str = "RUSTFS_AUDIT_KAFKA_ACKS";
+pub const ENV_AUDIT_KAFKA_TLS_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_TLS_ENABLE";
+pub const ENV_AUDIT_KAFKA_TLS_CA: &str = "RUSTFS_AUDIT_KAFKA_TLS_CA";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_KAFKA_QUEUE_DIR: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_DIR";
+pub const ENV_AUDIT_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_AUDIT_KAFKA_ENABLE,
+    ENV_AUDIT_KAFKA_BROKERS,
+    ENV_AUDIT_KAFKA_TOPIC,
+    ENV_AUDIT_KAFKA_ACKS,
+    ENV_AUDIT_KAFKA_TLS_ENABLE,
+    ENV_AUDIT_KAFKA_TLS_CA,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_CERT,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_KEY,
+    ENV_AUDIT_KAFKA_QUEUE_DIR,
+    ENV_AUDIT_KAFKA_QUEUE_LIMIT,
+];
+
+/// A list of all valid configuration keys for a Kafka audit target.
+pub const AUDIT_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/mod.rs

@@ -16,10 +16,16 @@
 //! This module defines the configuration for audit systems, including
 //! webhook and MQTT audit-related settings.
 
+mod kafka;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod webhook;
 
+pub use kafka::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use webhook::*;
 
 use crate::DEFAULT_DELIMITER;
@@ -29,8 +35,17 @@ pub const AUDIT_PREFIX: &str = "audit";
 pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
 
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
+pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
 pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
+pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
+pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
 
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
 #[allow(dead_code)]
-pub const AUDIT_SUB_SYSTEMS: &[&str] = &[AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS];
+pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
+    AUDIT_KAFKA_SUB_SYS,
+    AUDIT_MQTT_SUB_SYS,
+    AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_SUB_SYS,
+    AUDIT_WEBHOOK_SUB_SYS,
+];

crates/config/src/audit/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_NATS_ENABLE: &str = "RUSTFS_AUDIT_NATS_ENABLE";
+pub const ENV_AUDIT_NATS_ADDRESS: &str = "RUSTFS_AUDIT_NATS_ADDRESS";
+pub const ENV_AUDIT_NATS_SUBJECT: &str = "RUSTFS_AUDIT_NATS_SUBJECT";
+pub const ENV_AUDIT_NATS_USERNAME: &str = "RUSTFS_AUDIT_NATS_USERNAME";
+pub const ENV_AUDIT_NATS_PASSWORD: &str = "RUSTFS_AUDIT_NATS_PASSWORD";
+pub const ENV_AUDIT_NATS_TOKEN: &str = "RUSTFS_AUDIT_NATS_TOKEN";
+pub const ENV_AUDIT_NATS_CREDENTIALS_FILE: &str = "RUSTFS_AUDIT_NATS_CREDENTIALS_FILE";
+pub const ENV_AUDIT_NATS_TLS_CA: &str = "RUSTFS_AUDIT_NATS_TLS_CA";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
+pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
+pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
+    ENV_AUDIT_NATS_ENABLE,
+    ENV_AUDIT_NATS_ADDRESS,
+    ENV_AUDIT_NATS_SUBJECT,
+    ENV_AUDIT_NATS_USERNAME,
+    ENV_AUDIT_NATS_PASSWORD,
+    ENV_AUDIT_NATS_TOKEN,
+    ENV_AUDIT_NATS_CREDENTIALS_FILE,
+    ENV_AUDIT_NATS_TLS_CA,
+    ENV_AUDIT_NATS_TLS_CLIENT_CERT,
+    ENV_AUDIT_NATS_TLS_CLIENT_KEY,
+    ENV_AUDIT_NATS_TLS_REQUIRED,
+    ENV_AUDIT_NATS_QUEUE_DIR,
+    ENV_AUDIT_NATS_QUEUE_LIMIT,
+];
+
+pub const AUDIT_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_PULSAR_ENABLE: &str = "RUSTFS_AUDIT_PULSAR_ENABLE";
+pub const ENV_AUDIT_PULSAR_BROKER: &str = "RUSTFS_AUDIT_PULSAR_BROKER";
+pub const ENV_AUDIT_PULSAR_TOPIC: &str = "RUSTFS_AUDIT_PULSAR_TOPIC";
+pub const ENV_AUDIT_PULSAR_AUTH_TOKEN: &str = "RUSTFS_AUDIT_PULSAR_AUTH_TOKEN";
+pub const ENV_AUDIT_PULSAR_USERNAME: &str = "RUSTFS_AUDIT_PULSAR_USERNAME";
+pub const ENV_AUDIT_PULSAR_PASSWORD: &str = "RUSTFS_AUDIT_PULSAR_PASSWORD";
+pub const ENV_AUDIT_PULSAR_TLS_CA: &str = "RUSTFS_AUDIT_PULSAR_TLS_CA";
+pub const ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_AUDIT_PULSAR_QUEUE_DIR: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_DIR";
+pub const ENV_AUDIT_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_AUDIT_PULSAR_ENABLE,
+    ENV_AUDIT_PULSAR_BROKER,
+    ENV_AUDIT_PULSAR_TOPIC,
+    ENV_AUDIT_PULSAR_AUTH_TOKEN,
+    ENV_AUDIT_PULSAR_USERNAME,
+    ENV_AUDIT_PULSAR_PASSWORD,
+    ENV_AUDIT_PULSAR_TLS_CA,
+    ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_AUDIT_PULSAR_QUEUE_DIR,
+    ENV_AUDIT_PULSAR_QUEUE_LIMIT,
+];
+
+pub const AUDIT_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/constants/app.rs

@@ -131,6 +131,18 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable to explicitly bypass local physical disk independence checks.
+pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
+
+/// Compatibility alias used by legacy MinIO CI pipelines.
+///
+/// RustFS keeps this alias for backward compatibility only. Prefer
+/// `ENV_UNSAFE_BYPASS_DISK_CHECK` for explicit bypass control.
+pub const ENV_MINIO_CI: &str = "MINIO_CI";
+
+/// Default flag value for bypassing local physical disk independence checks.
+pub const DEFAULT_UNSAFE_BYPASS_DISK_CHECK: bool = false;
+
 /// Environment variable for server access key.
 pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
 
@@ -284,9 +296,9 @@ pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true;
 
 /// Default profiling export enabled
 /// It is used to enable or disable exporting profiles
-/// Default value: true
+/// Default value: false
 /// Environment variable: RUSTFS_OBS_PROFILING_EXPORT_ENABLED
-pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = true;
+pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = false;
 
 /// Default log local logging enabled for rustfs
 /// This is the default log local logging enabled for rustfs.

crates/config/src/constants/capacity.rs

@@ -39,6 +39,9 @@ pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
 /// Environment variable for sample rate
 pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
 
+/// Environment variable for metrics logging interval
+pub const ENV_CAPACITY_METRICS_INTERVAL: &str = "RUSTFS_CAPACITY_METRICS_INTERVAL";
+
 /// Environment variable for following symbolic links during capacity calculation
 pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
 
@@ -89,6 +92,10 @@ pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
 /// Default: 200
 pub const DEFAULT_SAMPLE_RATE: usize = 200;
 
+/// Capacity metrics logging interval in seconds
+/// Default: 600 seconds (10 minutes)
+pub const DEFAULT_CAPACITY_METRICS_INTERVAL_SECS: u64 = 600;
+
 /// Follow symbolic links during capacity calculation
 /// Default: false (disabled for safety)
 pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
@@ -130,6 +137,7 @@ mod tests {
         assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
         assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
         assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
+        assert_eq!(ENV_CAPACITY_METRICS_INTERVAL, "RUSTFS_CAPACITY_METRICS_INTERVAL");
         assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
         assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
         assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
@@ -147,6 +155,7 @@ mod tests {
         assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
         assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
         assert_eq!(DEFAULT_SAMPLE_RATE, 200);
+        assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
         assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
         assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2);
         assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15);

crates/config/src/constants/drive.rs

@@ -0,0 +1,68 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Legacy global drive timeout fallback.
+/// Deprecated in favor of per-operation drive timeout knobs.
+pub const ENV_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
+
+/// Default timeout in seconds for the legacy global drive timeout fallback.
+pub const DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS: u64 = 30;
+
+/// Timeout for metadata-oriented drive operations such as `read_metadata`.
+pub const ENV_DRIVE_METADATA_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_METADATA_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_METADATA_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `disk_info()` calls on local and remote drives.
+pub const ENV_DRIVE_DISK_INFO_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `list_dir()` style metadata listing operations.
+pub const ENV_DRIVE_LIST_DIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS: u64 = 5;
+
+/// Total timeout for `walk_dir()` operations.
+pub const ENV_DRIVE_WALKDIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
+
+/// Maximum time without forward progress while consuming a `walk_dir()` stream.
+pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
+
+/// Interval in seconds between active health probes for local and remote drives.
+pub const ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_INTERVAL_SECS";
+pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
+
+/// Timeout in seconds for a single active health probe.
+pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5;
+
+/// Number of consecutive failures before a suspect drive is classified as offline.
+pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
+pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
+
+/// Number of consecutive successful recovery probes before a returning drive is considered online again.
+pub const ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD: &str = "RUSTFS_DRIVE_RETURNING_SUCCESS_THRESHOLD";
+pub const DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD: u64 = 3;
+
+/// Probe interval in seconds while a drive is in the recovery path.
+pub const ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS: &str = "RUSTFS_DRIVE_RETURNING_PROBE_INTERVAL_SECS";
+pub const DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS: u64 = 2;
+
+/// Duration in seconds for classifying a recovered drive as a short offline event.
+pub const ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS: &str = "RUSTFS_DRIVE_OFFLINE_GRACE_PERIOD_SECS";
+pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
+
+/// Duration in seconds after which a recovered drive is classified as long offline.
+pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
+pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;

crates/config/src/constants/env.rs

@@ -26,6 +26,15 @@ pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";
 
+/// Global switch for enabling the audit module.
+pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
+/// Global switch for enabling the notify module.
+pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
+/// Default global audit switch (disabled by default).
+pub const DEFAULT_AUDIT_ENABLE: bool = false;
+/// Default global notify switch (disabled by default).
+pub const DEFAULT_NOTIFY_ENABLE: bool = false;
+
 /// Medium-drawn lines separator
 /// This is used to separate words in environment variable names.
 pub const ENV_WORD_DELIMITER_DASH: &str = "-";
@@ -290,4 +299,10 @@ mod tests {
             assert!(state.is_disabled());
         }
     }
+
+    #[test]
+    fn test_global_audit_notify_switch_constants() {
+        assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
+        assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
+    }
 }

crates/config/src/constants/heal.rs

@@ -56,6 +56,15 @@ pub const ENV_HEAL_TASK_TIMEOUT_SECS: &str = "RUSTFS_HEAL_TASK_TIMEOUT_SECS";
 /// - Note: A higher concurrency limit can speed up healing but may lead to resource contention.
 pub const ENV_HEAL_MAX_CONCURRENT_HEALS: &str = "RUSTFS_HEAL_MAX_CONCURRENT_HEALS";
 
+/// Environment variable name that specifies the maximum number of concurrent heal operations
+/// allowed for a single erasure set.
+///
+/// - Purpose: Prevent one degraded set from consuming all global heal slots.
+/// - Unit: number of operations (usize).
+/// - Valid values: any positive integer.
+/// - Example: `export RUSTFS_HEAL_MAX_CONCURRENT_PER_SET=1`
+pub const ENV_HEAL_MAX_CONCURRENT_PER_SET: &str = "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET";
+
 /// Default value for enabling authentication for heal operations if not specified in the environment variable.
 /// - Value: true (authentication enabled).
 /// - Rationale: Enabling authentication by default enhances security for heal operations.
@@ -86,3 +95,46 @@ pub const DEFAULT_HEAL_TASK_TIMEOUT_SECS: u64 = 300; // 5 minutes
 /// - Rationale: This default concurrency limit helps balance healing speed with resource usage, preventing system overload.
 /// - Adjustments: Users may modify this value via the `RUSTFS_HEAL_MAX_CONCURRENT_HEALS` environment variable based on their system capacity and expected heal workload.
 pub const DEFAULT_HEAL_MAX_CONCURRENT_HEALS: usize = 4;
+
+/// Default maximum number of concurrent heal operations per erasure set.
+///
+/// - Value: 1 concurrent heal operation per set.
+/// - Rationale: Keeps a degraded set from monopolizing the global heal scheduler.
+pub const DEFAULT_HEAL_MAX_CONCURRENT_PER_SET: usize = 1;
+
+/// Environment variable that controls whether low-priority heal requests should merge into
+/// an existing queued request with the same deduplication key.
+pub const ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE: &str = "RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE";
+
+/// Environment variable that allows low-priority heal requests to be dropped when the queue is full.
+pub const ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: &str = "RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL";
+
+/// Environment variable that controls concurrent object heals within a single erasure-set page.
+pub const ENV_HEAL_PAGE_OBJECT_CONCURRENCY: &str = "RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY";
+
+/// Environment variable that toggles notify-driven scheduler wakeups.
+pub const ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: &str = "RUSTFS_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE";
+
+/// Environment variable that toggles per-set bulkhead scheduling.
+pub const ENV_HEAL_SET_BULKHEAD_ENABLE: &str = "RUSTFS_HEAL_SET_BULKHEAD_ENABLE";
+
+/// Environment variable that toggles page-level parallel object healing for erasure-set repair.
+pub const ENV_HEAL_PAGE_PARALLEL_ENABLE: &str = "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE";
+
+/// Default behavior is to merge duplicate low-priority requests.
+pub const DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE: bool = true;
+
+/// Default behavior is to drop low-priority requests instead of blocking when the queue is full.
+pub const DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: bool = true;
+
+/// Default per-page object heal concurrency for erasure-set healing.
+pub const DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY: usize = 8;
+
+/// Default behavior is to keep notify-driven scheduler wakeups enabled.
+pub const DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: bool = true;
+
+/// Default behavior is to keep per-set bulkhead scheduling enabled.
+pub const DEFAULT_HEAL_SET_BULKHEAD_ENABLE: bool = true;
+
+/// Default behavior is to keep erasure-set page parallelism enabled.
+pub const DEFAULT_HEAL_PAGE_PARALLEL_ENABLE: bool = true;

crates/config/src/constants/health.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Enable or disable public `/health` and `/health/ready` endpoints.
+/// When disabled, the routes are not registered and return 404.
+pub const ENV_HEALTH_ENDPOINT_ENABLE: &str = "RUSTFS_HEALTH_ENDPOINT_ENABLE";
+pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
+
+/// Cache TTL for storage readiness runtime-state evaluation (milliseconds).
+/// This reduces storage-layer pressure when probes are called at high frequency.
+pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
+pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
+
+/// Enable minimal health payload mode for GET `/health*` responses.
+/// When enabled, only `status` and `ready` fields are returned.
+pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
+pub const DEFAULT_HEALTH_MINIMAL_RESPONSE_ENABLE: bool = false;

crates/config/src/constants/internode.rs

@@ -0,0 +1,62 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Timeout for establishing a new internode gRPC connection.
+pub const ENV_INTERNODE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS: u64 = 3;
+
+/// TCP keepalive interval for internode gRPC channels.
+pub const ENV_INTERNODE_TCP_KEEPALIVE_SECS: &str = "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS";
+pub const DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS: u64 = 10;
+
+/// HTTP/2 keepalive interval for internode gRPC channels.
+pub const ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS";
+pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5;
+
+/// HTTP/2 keepalive timeout for internode gRPC channels.
+pub const ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
+
+/// Overall timeout for a single internode gRPC request.
+pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn internode_timeout_defaults_stay_in_expected_bounds() {
+        assert_eq!(DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, 10);
+        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
+        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
+    }
+
+    #[test]
+    fn internode_timeout_env_names_are_stable() {
+        assert_eq!(ENV_INTERNODE_CONNECT_TIMEOUT_SECS, "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS");
+        assert_eq!(ENV_INTERNODE_TCP_KEEPALIVE_SECS, "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS");
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
+            "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS"
+        );
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
+            "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
+        );
+        assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
+    }
+}

crates/config/src/constants/mod.rs

@@ -17,8 +17,11 @@ pub(crate) mod body_limits;
 pub(crate) mod capacity;
 pub(crate) mod compress;
 pub(crate) mod console;
+pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
+pub(crate) mod health;
+pub(crate) mod internode;
 pub(crate) mod object;
 pub(crate) mod oidc;
 pub(crate) mod profiler;

crates/config/src/constants/oidc.rs

@@ -17,6 +17,7 @@ pub const OIDC_CONFIG_URL: &str = "config_url";
 pub const OIDC_CLIENT_ID: &str = "client_id";
 pub const OIDC_CLIENT_SECRET: &str = "client_secret";
 pub const OIDC_SCOPES: &str = "scopes";
+pub const OIDC_OTHER_AUDIENCES: &str = "other_audiences";
 pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
 pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
 pub const OIDC_CLAIM_NAME: &str = "claim_name";
@@ -24,6 +25,7 @@ pub const OIDC_CLAIM_PREFIX: &str = "claim_prefix";
 pub const OIDC_ROLE_POLICY: &str = "role_policy";
 pub const OIDC_DISPLAY_NAME: &str = "display_name";
 pub const OIDC_GROUPS_CLAIM: &str = "groups_claim";
+pub const OIDC_ROLES_CLAIM: &str = "roles_claim";
 pub const OIDC_EMAIL_CLAIM: &str = "email_claim";
 pub const OIDC_USERNAME_CLAIM: &str = "username_claim";
 
@@ -33,6 +35,7 @@ pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_
 pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
 pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
 pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
+pub const ENV_IDENTITY_OPENID_OTHER_AUDIENCES: &str = "RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
 pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
@@ -40,16 +43,18 @@ pub const ENV_IDENTITY_OPENID_CLAIM_PREFIX: &str = "RUSTFS_IDENTITY_OPENID_CLAIM
 pub const ENV_IDENTITY_OPENID_ROLE_POLICY: &str = "RUSTFS_IDENTITY_OPENID_ROLE_POLICY";
 pub const ENV_IDENTITY_OPENID_DISPLAY_NAME: &str = "RUSTFS_IDENTITY_OPENID_DISPLAY_NAME";
 pub const ENV_IDENTITY_OPENID_GROUPS_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM";
+pub const ENV_IDENTITY_OPENID_ROLES_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_ROLES_CLAIM";
 pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM";
 pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
 
 /// List of all environment variable keys for an OIDC provider.
-pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 16] = &[
     ENV_IDENTITY_OPENID_ENABLE,
     ENV_IDENTITY_OPENID_CONFIG_URL,
     ENV_IDENTITY_OPENID_CLIENT_ID,
     ENV_IDENTITY_OPENID_CLIENT_SECRET,
     ENV_IDENTITY_OPENID_SCOPES,
+    ENV_IDENTITY_OPENID_OTHER_AUDIENCES,
     ENV_IDENTITY_OPENID_REDIRECT_URI,
     ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
     ENV_IDENTITY_OPENID_CLAIM_NAME,
@@ -57,6 +62,7 @@ pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
     ENV_IDENTITY_OPENID_ROLE_POLICY,
     ENV_IDENTITY_OPENID_DISPLAY_NAME,
     ENV_IDENTITY_OPENID_GROUPS_CLAIM,
+    ENV_IDENTITY_OPENID_ROLES_CLAIM,
     ENV_IDENTITY_OPENID_EMAIL_CLAIM,
     ENV_IDENTITY_OPENID_USERNAME_CLAIM,
 ];
@@ -68,6 +74,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
     OIDC_CLIENT_ID,
     OIDC_CLIENT_SECRET,
     OIDC_SCOPES,
+    OIDC_OTHER_AUDIENCES,
     OIDC_REDIRECT_URI,
     OIDC_REDIRECT_URI_DYNAMIC,
     OIDC_CLAIM_NAME,
@@ -75,6 +82,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
     OIDC_ROLE_POLICY,
     OIDC_DISPLAY_NAME,
     OIDC_GROUPS_CLAIM,
+    OIDC_ROLES_CLAIM,
     OIDC_EMAIL_CLAIM,
     OIDC_USERNAME_CLAIM,
     crate::COMMENT_KEY,
@@ -84,6 +92,8 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
 pub const OIDC_DEFAULT_SCOPES: &str = "openid,profile,email";
 pub const OIDC_DEFAULT_CLAIM_NAME: &str = "groups";
 pub const OIDC_DEFAULT_GROUPS_CLAIM: &str = "groups";
+/// Empty means do not merge a secondary claim into groups (legacy behavior). Set to e.g. `roles` to opt in.
+pub const OIDC_DEFAULT_ROLES_CLAIM: &str = "";
 pub const OIDC_DEFAULT_EMAIL_CLAIM: &str = "email";
 pub const OIDC_DEFAULT_USERNAME_CLAIM: &str = "preferred_username";
 

crates/config/src/constants/proxy.rs

@@ -20,6 +20,11 @@ pub const ENV_TRUSTED_PROXY_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_ENABLED";
 /// Trusted proxy middleware is enabled by default.
 pub const DEFAULT_TRUSTED_PROXY_ENABLED: bool = true;
 
+/// Environment variable to select the trusted proxy implementation.
+pub const ENV_TRUSTED_PROXY_IMPLEMENTATION: &str = "RUSTFS_TRUSTED_PROXY_IMPLEMENTATION";
+/// The simplified implementation is used by default.
+pub const DEFAULT_TRUSTED_PROXY_IMPLEMENTATION: &str = "simple";
+
 /// Environment variable for the proxy validation mode.
 pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDATION_MODE";
 /// Default validation mode is "hop_by_hop".

crates/config/src/constants/runtime.rs

@@ -59,9 +59,28 @@ pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10;
 pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling
 // Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option<String>)
 
+// Allocator reclaim configuration
+pub const ENV_ALLOCATOR_RECLAIM_ENABLED: &str = "RUSTFS_ALLOCATOR_RECLAIM_ENABLED";
+pub const ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS: &str = "RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS";
+pub const ENV_ALLOCATOR_RECLAIM_FORCE: &str = "RUSTFS_ALLOCATOR_RECLAIM_FORCE";
+pub const ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS: &str = "RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS";
+pub const DEFAULT_ALLOCATOR_RECLAIM_ENABLED: bool = false;
+pub const DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS: u64 = 30;
+pub const DEFAULT_ALLOCATOR_RECLAIM_FORCE: bool = true;
+pub const DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS: u64 = 3;
+
+// File page-cache reclaim configuration
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE";
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE";
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD";
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false;
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = false;
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024;
+
 /// Threshold for small object seek support in megabytes.
 ///
 /// When an object is smaller than this size, rustfs will provide seek support.
 ///
 /// Default is set to 10MB.
+pub const ENV_OBJECT_SEEK_SUPPORT_THRESHOLD: &str = "RUSTFS_OBJECT_SEEK_SUPPORT_THRESHOLD";
 pub const DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD: usize = 10 * 1024 * 1024;

crates/config/src/constants/scanner.rs

@@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS"
 #[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
 pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
 
+/// Environment variable that specifies the scanner cycle interval in seconds.
+/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour)
+pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE";
+
 /// Environment variable that selects the scanner speed preset.
 /// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
 /// Controls the sleep factor, maximum sleep duration, and cycle interval.
@@ -39,9 +45,24 @@ pub const DEFAULT_SCANNER_SPEED: &str = "default";
 /// - Example: `export RUSTFS_SCANNER_IDLE_MODE=false`
 pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
 
+/// Environment variable that controls scanner cache save timeout in seconds.
+/// The scanner enforces a minimum value of `1`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
+pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
+
 /// Default scanner idle mode.
 pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 
+/// Compatibility flag kept for Patch 3 rollback windows.
+///
+/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
+/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
+pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
+
+/// Default inline scanner heal compatibility mode.
+pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
+
 /// Scanner speed preset controlling throttling behavior.
 ///
 /// Each preset defines three parameters:
@@ -56,6 +77,8 @@ pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 /// | `default` | 2x     | 1 second  | 1 minute       |
 /// | `slow`    | 10x    | 15 seconds| 1 minute       |
 /// | `slowest` | 100x   | 15 seconds| 30 minutes     |
+///
+/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum ScannerSpeed {
     Fastest,

crates/config/src/constants/targets.rs

@@ -40,6 +40,39 @@ pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
 pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
 pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
+pub const KAFKA_BROKERS: &str = "brokers";
+pub const KAFKA_TOPIC: &str = "topic";
+pub const KAFKA_ACKS: &str = "acks";
+pub const KAFKA_QUEUE_DIR: &str = "queue_dir";
+pub const KAFKA_QUEUE_LIMIT: &str = "queue_limit";
+pub const KAFKA_TLS_ENABLE: &str = "tls_enable";
+pub const KAFKA_TLS_CA: &str = "tls_ca";
+pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key";
+
+pub const NATS_ADDRESS: &str = "address";
+pub const NATS_SUBJECT: &str = "subject";
+pub const NATS_USERNAME: &str = "username";
+pub const NATS_PASSWORD: &str = "password";
+pub const NATS_TOKEN: &str = "token";
+pub const NATS_CREDENTIALS_FILE: &str = "credentials_file";
+pub const NATS_TLS_CA: &str = "tls_ca";
+pub const NATS_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
+pub const NATS_TLS_REQUIRED: &str = "tls_required";
+pub const NATS_QUEUE_DIR: &str = "queue_dir";
+pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
+
+pub const PULSAR_BROKER: &str = "broker";
+pub const PULSAR_TOPIC: &str = "topic";
+pub const PULSAR_AUTH_TOKEN: &str = "auth_token";
+pub const PULSAR_USERNAME: &str = "username";
+pub const PULSAR_PASSWORD: &str = "password";
+pub const PULSAR_TLS_CA: &str = "tls_ca";
+pub const PULSAR_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure";
+pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification";
+pub const PULSAR_QUEUE_DIR: &str = "queue_dir";
+pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit";
 
 /// Environment variable controlling whether target queue files are Snappy-compressed.
 /// Applies to both notify and audit target queue stores.

crates/config/src/constants/zero_copy.rs

@@ -17,24 +17,6 @@
 //! This module defines environment variables and default values for zero-copy
 //! read operations, which use memory mapping (mmap) to avoid data copying.
 
-// =============================================================================
-// GET Fast Path Configuration
-// =============================================================================
-
-/// Environment variable for the GetObject chunk fast path master switch.
-///
-/// When disabled, `GetObject` bypasses the chunk-streaming fast path entirely and
-/// always uses the legacy reader path. This provides an operational stopgap for
-/// regressions in the streaming data plane while keeping zero-copy internals
-/// configurable independently for future opt-in validation.
-pub const ENV_OBJECT_GET_CHUNK_FAST_PATH_ENABLE: &str = "RUSTFS_OBJECT_GET_CHUNK_FAST_PATH_ENABLE";
-
-/// Default: GetObject chunk fast path is disabled.
-///
-/// The legacy reader path remains the safe default until the chunk-streaming
-/// path has sufficient regression coverage for full-body delivery semantics.
-pub const DEFAULT_OBJECT_GET_CHUNK_FAST_PATH_ENABLE: bool = false;
-
 // =============================================================================
 // Zero-Copy Configuration
 // =============================================================================
@@ -67,34 +49,6 @@ pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
 /// to regular I/O without errors.
 pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true;
 
-/// Environment variable for zero-copy read operating mode.
-///
-/// Supported values:
-/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path
-/// - `conservative`: allow a single mmap window per request
-/// - `balanced`: allow multiple mmap windows with the default size guardrails
-/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff
-pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE";
-
-/// Default zero-copy read mode.
-pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced";
-
-/// Environment variable for the maximum mmap window size used by the chunk fast path.
-///
-/// This controls the visible bytes per mapped chunk before the implementation emits a new window.
-pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES";
-
-/// Default mmap window size for chunk fast path reads: 8 MiB.
-pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024;
-
-/// Environment variable for the maximum total active mmap bytes.
-///
-/// Requests that would exceed this active window budget fall back to the compatibility path.
-pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES";
-
-/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB.
-pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024;
-
 // =============================================================================
 // Direct I/O Configuration
 // =============================================================================

crates/config/src/lib.rs

@@ -25,10 +25,16 @@ pub use constants::compress::*;
 #[cfg(feature = "constants")]
 pub use constants::console::*;
 #[cfg(feature = "constants")]
+pub use constants::drive::*;
+#[cfg(feature = "constants")]
 pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::heal::*;
 #[cfg(feature = "constants")]
+pub use constants::health::*;
+#[cfg(feature = "constants")]
+pub use constants::internode::*;
+#[cfg(feature = "constants")]
 pub use constants::object::*;
 #[cfg(feature = "constants")]
 pub use constants::profiler::*;

crates/config/src/notify/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// A list of all valid configuration keys for a Kafka target.
+pub const NOTIFY_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+// Kafka Environment Variables
+pub const ENV_NOTIFY_KAFKA_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_ENABLE";
+pub const ENV_NOTIFY_KAFKA_BROKERS: &str = "RUSTFS_NOTIFY_KAFKA_BROKERS";
+pub const ENV_NOTIFY_KAFKA_TOPIC: &str = "RUSTFS_NOTIFY_KAFKA_TOPIC";
+pub const ENV_NOTIFY_KAFKA_ACKS: &str = "RUSTFS_NOTIFY_KAFKA_ACKS";
+pub const ENV_NOTIFY_KAFKA_TLS_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_TLS_ENABLE";
+pub const ENV_NOTIFY_KAFKA_TLS_CA: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CA";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_KAFKA_QUEUE_DIR: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_DIR";
+pub const ENV_NOTIFY_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_NOTIFY_KAFKA_ENABLE,
+    ENV_NOTIFY_KAFKA_BROKERS,
+    ENV_NOTIFY_KAFKA_TOPIC,
+    ENV_NOTIFY_KAFKA_ACKS,
+    ENV_NOTIFY_KAFKA_TLS_ENABLE,
+    ENV_NOTIFY_KAFKA_TLS_CA,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY,
+    ENV_NOTIFY_KAFKA_QUEUE_DIR,
+    ENV_NOTIFY_KAFKA_QUEUE_LIMIT,
+];

crates/config/src/notify/mod.rs

@@ -13,12 +13,18 @@
 // limitations under the License.
 
 mod arn;
+mod kafka;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod store;
 mod webhook;
 
 pub use arn::*;
+pub use kafka::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use store::*;
 pub use webhook::*;
 
@@ -64,9 +70,14 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
 pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
 
 #[allow(dead_code)]
-pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
+pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
+    NOTIFY_KAFKA_SUB_SYS,
+    NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_SUB_SYS,
+    NOTIFY_WEBHOOK_SUB_SYS,
+];
 
-#[allow(dead_code)]
 pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
 pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
 #[allow(dead_code)]
@@ -83,4 +94,5 @@ pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
 pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
 #[allow(dead_code)]
 pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
+pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
 pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";

crates/config/src/notify/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_NATS_ENABLE: &str = "RUSTFS_NOTIFY_NATS_ENABLE";
+pub const ENV_NOTIFY_NATS_ADDRESS: &str = "RUSTFS_NOTIFY_NATS_ADDRESS";
+pub const ENV_NOTIFY_NATS_SUBJECT: &str = "RUSTFS_NOTIFY_NATS_SUBJECT";
+pub const ENV_NOTIFY_NATS_USERNAME: &str = "RUSTFS_NOTIFY_NATS_USERNAME";
+pub const ENV_NOTIFY_NATS_PASSWORD: &str = "RUSTFS_NOTIFY_NATS_PASSWORD";
+pub const ENV_NOTIFY_NATS_TOKEN: &str = "RUSTFS_NOTIFY_NATS_TOKEN";
+pub const ENV_NOTIFY_NATS_CREDENTIALS_FILE: &str = "RUSTFS_NOTIFY_NATS_CREDENTIALS_FILE";
+pub const ENV_NOTIFY_NATS_TLS_CA: &str = "RUSTFS_NOTIFY_NATS_TLS_CA";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
+pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
+pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
+    ENV_NOTIFY_NATS_ENABLE,
+    ENV_NOTIFY_NATS_ADDRESS,
+    ENV_NOTIFY_NATS_SUBJECT,
+    ENV_NOTIFY_NATS_USERNAME,
+    ENV_NOTIFY_NATS_PASSWORD,
+    ENV_NOTIFY_NATS_TOKEN,
+    ENV_NOTIFY_NATS_CREDENTIALS_FILE,
+    ENV_NOTIFY_NATS_TLS_CA,
+    ENV_NOTIFY_NATS_TLS_CLIENT_CERT,
+    ENV_NOTIFY_NATS_TLS_CLIENT_KEY,
+    ENV_NOTIFY_NATS_TLS_REQUIRED,
+    ENV_NOTIFY_NATS_QUEUE_DIR,
+    ENV_NOTIFY_NATS_QUEUE_LIMIT,
+];

crates/config/src/notify/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_PULSAR_ENABLE: &str = "RUSTFS_NOTIFY_PULSAR_ENABLE";
+pub const ENV_NOTIFY_PULSAR_BROKER: &str = "RUSTFS_NOTIFY_PULSAR_BROKER";
+pub const ENV_NOTIFY_PULSAR_TOPIC: &str = "RUSTFS_NOTIFY_PULSAR_TOPIC";
+pub const ENV_NOTIFY_PULSAR_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_PULSAR_AUTH_TOKEN";
+pub const ENV_NOTIFY_PULSAR_USERNAME: &str = "RUSTFS_NOTIFY_PULSAR_USERNAME";
+pub const ENV_NOTIFY_PULSAR_PASSWORD: &str = "RUSTFS_NOTIFY_PULSAR_PASSWORD";
+pub const ENV_NOTIFY_PULSAR_TLS_CA: &str = "RUSTFS_NOTIFY_PULSAR_TLS_CA";
+pub const ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_NOTIFY_PULSAR_QUEUE_DIR: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_DIR";
+pub const ENV_NOTIFY_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_NOTIFY_PULSAR_ENABLE,
+    ENV_NOTIFY_PULSAR_BROKER,
+    ENV_NOTIFY_PULSAR_TOPIC,
+    ENV_NOTIFY_PULSAR_AUTH_TOKEN,
+    ENV_NOTIFY_PULSAR_USERNAME,
+    ENV_NOTIFY_PULSAR_PASSWORD,
+    ENV_NOTIFY_PULSAR_TLS_CA,
+    ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_NOTIFY_PULSAR_QUEUE_DIR,
+    ENV_NOTIFY_PULSAR_QUEUE_LIMIT,
+];

crates/config/src/observability/mod.rs

@@ -22,6 +22,14 @@ pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
 pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT";
 pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT";
 pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT";
+pub const ENV_OBS_ENDPOINT_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_HEADERS";
+pub const ENV_OBS_ENDPOINT_TRACES_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS";
+pub const ENV_OBS_ENDPOINT_METRICS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS";
+pub const ENV_OBS_ENDPOINT_LOGS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS";
+pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
 pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
@@ -108,6 +116,14 @@ mod tests {
         assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT");
         assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT");
         assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT");
+        assert_eq!(ENV_OBS_ENDPOINT_HEADERS, "RUSTFS_OBS_ENDPOINT_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_HEADERS, "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_HEADERS, "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_HEADERS, "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS");
         assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT");
         assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
         assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");

crates/credentials/src/credentials.rs

@@ -20,7 +20,7 @@ use std::collections::HashMap;
 use std::env;
 use std::fmt;
 use std::io::Error;
-use std::sync::OnceLock;
+use std::sync::{LazyLock, OnceLock};
 use time::OffsetDateTime;
 
 /// Global active credentials
@@ -204,10 +204,11 @@ pub fn gen_secret_key(length: usize) -> std::io::Result<String> {
     let mut key = vec![0u8; URL_SAFE_NO_PAD.estimated_decoded_length(length)];
     rng.fill_bytes(&mut key);
 
+    // URL_SAFE_NO_PAD uses "-" and "_" instead of "+" and "/", so "/" never
+    // appears in the output. The .replace("/", "+") was a dead no-op.
     let encoded = URL_SAFE_NO_PAD.encode_to_string(&key);
-    let key_str = encoded.replace("/", "+");
 
-    Ok(key_str)
+    Ok(encoded)
 }
 
 /// Get the RPC authentication token from environment variable
@@ -313,6 +314,17 @@ impl fmt::Debug for Credentials {
 }
 
 impl Credentials {
+    /// Returns a reference to this credential's claims, or a shared empty map
+    /// when the credential has no claims attached. Avoids per-call allocation
+    /// at call sites that need an `&HashMap<String, Value>`.
+    pub fn claims_or_empty(&self) -> &HashMap<String, Value> {
+        static EMPTY: LazyLock<HashMap<String, Value>> = LazyLock::new(HashMap::new);
+        match &self.claims {
+            Some(c) => c,
+            None => &EMPTY,
+        }
+    }
+
     pub fn is_expired(&self) -> bool {
         if self.expiration.is_none() {
             return false;
@@ -447,7 +459,7 @@ mod tests {
             // Initialize
             let test_ak = "test_access_key".to_string();
             let test_sk = "test_secret_key_123456".to_string();
-            init_global_action_credentials(Some(test_ak.clone()), Some(test_sk.clone())).ok();
+            init_global_action_credentials(Some(test_ak), Some(test_sk)).ok();
         }
 
         // Verify the state after initialization
@@ -473,6 +485,16 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_gen_secret_key_uses_url_safe_base64_without_padding() {
+        let key = gen_secret_key(32).expect("secret key should generate");
+
+        assert_eq!(key.len(), 32);
+        assert!(!key.contains('/'));
+        assert!(!key.contains('+'));
+        assert!(!key.contains('='));
+    }
+
     #[test]
     fn test_masked_debug() {
         // Test None

crates/e2e_test/Cargo.toml

@@ -20,11 +20,6 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 
-[[bin]]
-name = "small_put_bench"
-path = "src/bin/small_put_bench.rs"
-test = false
-
 [lints]
 workspace = true
 
@@ -46,7 +41,6 @@ serde_json.workspace = true
 tonic = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
-clap = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
 bytes.workspace = true
@@ -58,8 +52,6 @@ async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
 flate2.workspace = true
 http.workspace = true
-http-body.workspace = true
-http-body-util.workspace = true
 reqwest = { workspace = true }
 rustfs-signer.workspace = true
 tracing = { workspace = true }
@@ -81,3 +73,4 @@ rcgen.workspace = true
 anyhow.workspace = true
 rustls.workspace = true
 zip.workspace = true
+clap.workspace = true

crates/e2e_test/src/archive_download_integrity_test.rs

@@ -15,6 +15,8 @@
 #[cfg(test)]
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
+    use aws_sdk_s3::Client as S3Client;
+    use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
     use http::header::{CONTENT_TYPE, HOST};
@@ -216,12 +218,157 @@ mod tests {
         Ok(builder.send().await?)
     }
 
+    async fn signed_get_request_with_headers(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        extra_headers: &[(&str, &str)],
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let mut request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+        for (name, value) in extra_headers {
+            request = request.header(*name, *value);
+        }
+
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut builder = client.get(url);
+        for (name, value) in signed.headers() {
+            builder = builder.header(name, value);
+        }
+
+        Ok(builder.send().await?)
+    }
+
+    async fn assert_archive_object_content_encoding(
+        client: &S3Client,
+        bucket: &str,
+        key: &str,
+        expected_content_encoding: Option<&str>,
+        expected_body: &[u8],
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let head_resp = client.head_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(head_resp.content_encoding(), expected_content_encoding);
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(get_resp.content_encoding(), expected_content_encoding);
+        let body = get_resp.body.collect().await?.into_bytes();
+        assert_eq!(body.as_ref(), expected_body);
+
+        Ok(())
+    }
+
+    async fn complete_archive_multipart_upload_with_content_encoding(
+        client: &S3Client,
+        bucket: &str,
+        key: &str,
+        content_encoding: Option<&str>,
+    ) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let payload = random_bytes(MULTIPART_PART_SIZE + 256 * 1024);
+        let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
+        assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
+
+        let mut create_builder = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .content_type("application/zip");
+        if let Some(content_encoding) = content_encoding {
+            create_builder = create_builder.content_encoding(content_encoding);
+        }
+        let create_output = create_builder.send().await?;
+        let upload_id = create_output.upload_id().expect("multipart upload id");
+
+        let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
+        let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
+
+        let upload_part_1 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(first_part))
+            .send()
+            .await?;
+
+        let upload_part_2 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(2)
+            .body(ByteStream::from(second_part))
+            .send()
+            .await?;
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload_part_1.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload_part_2.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await?;
+
+        Ok(zip_bytes)
+    }
+
     #[tokio::test]
     #[serial]
-    async fn test_archive_put_rejects_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
+    async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
         let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server_without_cleanup(vec![]).await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
+        let response =
+            signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip")
+                .await?;
+
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        let head_resp = client
+            .head_object()
+            .bucket(ARCHIVE_TEST_BUCKET)
+            .key("bundle.zip")
+            .send()
+            .await?;
+        assert_eq!(head_resp.content_encoding(), Some("gzip"));
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
         env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
         let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
         let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
@@ -232,7 +379,145 @@ mod tests {
         assert_eq!(response.status(), StatusCode::BAD_REQUEST);
         let body = response.text().await?;
         assert!(
-            body.contains("InvalidArgument") || body.contains("Content-Encoding"),
+            body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
+            "unexpected error body: {body}"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked-gzip.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked,gzip",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-aws-chunked-gzip.zip",
+            Some("gzip"),
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-strict-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked-gzip.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes,
+            "application/zip",
+            "aws-chunked,gzip",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+        let body = response.text().await?;
+        assert!(
+            body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
             "unexpected error body: {body}"
         );
 
@@ -400,11 +685,139 @@ mod tests {
 
     #[tokio::test]
     #[serial]
-    async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path()
+    async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let key = "multipart-empty-conditional-headers.zip";
+        let zip_bytes =
+            complete_archive_multipart_upload_with_content_encoding(&client, MULTIPART_ARCHIVE_TEST_BUCKET, key, None).await?;
+        let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, key);
+
+        let response = signed_get_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            &[("if-match", ""), ("if-none-match", "")],
+        )
+        .await?;
+        let status = response.status();
+        let body = response.bytes().await?;
+
+        assert_eq!(
+            status,
+            StatusCode::OK,
+            "unexpected multipart GET status {status}, body: {}",
+            String::from_utf8_lossy(body.as_ref())
+        );
+        assert_eq!(body.as_ref(), zip_bytes.as_slice());
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
     -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
         let mut env = RustFSTestEnvironment::new().await?;
-        start_rustfs_server_with_env(&mut env, &[("RUSTFS_OBJECT_GET_CHUNK_FAST_PATH_ENABLE", "true")]).await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-aws-chunked-gzip.zip",
+            Some("aws-chunked,gzip"),
+        )
+        .await?;
+        assert_archive_object_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-aws-chunked-gzip.zip",
+            Some("gzip"),
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-strict-aws-chunked.zip",
+            Some("aws-chunked"),
+        )
+        .await?;
+        assert_archive_object_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-strict-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let create_result = client
+            .create_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-strict-aws-chunked-gzip.zip")
+            .content_type("application/zip")
+            .content_encoding("aws-chunked,gzip")
+            .send()
+            .await;
+        let err = create_result.expect_err("strict mode should reject effective archive content encoding");
+        assert_eq!(err.code(), Some("InvalidArgument"));
+        assert!(
+            err.message().is_some_and(|message| {
+                message.contains("Content-Encoding") && message.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING")
+            }),
+            "unexpected error metadata: code={:?}, message={:?}",
+            err.code(),
+            err.message()
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
         env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
 
         let client = env.create_s3_client();

crates/e2e_test/src/bin/small_put_bench.rs

@@ -1,441 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use anyhow::{Context, Result, anyhow, bail};
-use aws_sdk_s3::config::{Credentials, Region};
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{Delete, ObjectIdentifier};
-use aws_sdk_s3::{Client, Config};
-use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
-use bytes::Bytes;
-use clap::Parser;
-use serde::Serialize;
-use std::path::PathBuf;
-use std::time::{Duration, Instant};
-
-#[derive(Parser, Debug)]
-#[command(name = "small_put_bench")]
-#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")]
-struct Args {
-    #[arg(long, env = "RUSTFS_BENCH_ENDPOINT")]
-    endpoint: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")]
-    access_key: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")]
-    secret_key: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")]
-    region: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")]
-    bucket: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")]
-    sizes: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)]
-    concurrency: usize,
-
-    #[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)]
-    duration_secs: u64,
-
-    #[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)]
-    timeout_secs: u64,
-
-    #[arg(long, env = "RUSTFS_BENCH_PREFIX")]
-    prefix: Option<String>,
-
-    #[arg(long)]
-    output_json: Option<PathBuf>,
-
-    #[arg(long, default_value_t = false)]
-    cleanup: bool,
-}
-
-#[derive(Clone, Debug)]
-struct SizeSpec {
-    label: String,
-    slug: String,
-    bytes: usize,
-}
-
-#[derive(Debug)]
-struct Sample {
-    ok: bool,
-    duration_ms: f64,
-}
-
-#[derive(Debug, Serialize)]
-struct SizeSummary {
-    label: String,
-    bytes: usize,
-    total: usize,
-    succeeded: usize,
-    failed: usize,
-    wall_secs: f64,
-    object_rate: f64,
-    throughput_mib_per_sec: f64,
-    avg_ms: Option<f64>,
-    p50_ms: Option<f64>,
-    p90_ms: Option<f64>,
-    p99_ms: Option<f64>,
-}
-
-#[derive(Debug, Serialize)]
-struct RunSummary {
-    run_id: String,
-    endpoint: String,
-    bucket: String,
-    concurrency: usize,
-    duration_secs: u64,
-    timeout_secs: u64,
-    sizes: Vec<SizeSummary>,
-}
-
-fn main() -> Result<()> {
-    let runtime = tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()
-        .context("failed to build tokio runtime")?;
-    runtime.block_on(async_main())
-}
-
-async fn async_main() -> Result<()> {
-    let args = Args::parse();
-    validate_args(&args)?;
-
-    let sizes = parse_size_list(&args.sizes)?;
-    let run_id = args.prefix.clone().unwrap_or_else(default_run_id);
-    let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region);
-
-    ensure_bucket(&client, &args.bucket).await?;
-
-    let mut size_summaries = Vec::with_capacity(sizes.len());
-    for size in &sizes {
-        let summary = run_size_benchmark(
-            client.clone(),
-            args.bucket.clone(),
-            run_id.clone(),
-            size.clone(),
-            args.concurrency,
-            Duration::from_secs(args.duration_secs),
-            Duration::from_secs(args.timeout_secs),
-        )
-        .await?;
-        print_size_summary(&summary);
-        size_summaries.push(summary);
-    }
-
-    if args.cleanup {
-        cleanup_prefix(&client, &args.bucket, &run_id).await?;
-    }
-
-    let summary = RunSummary {
-        run_id,
-        endpoint: args.endpoint,
-        bucket: args.bucket,
-        concurrency: args.concurrency,
-        duration_secs: args.duration_secs,
-        timeout_secs: args.timeout_secs,
-        sizes: size_summaries,
-    };
-
-    if let Some(path) = args.output_json {
-        let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?;
-        std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?;
-        println!("Wrote summary to {}", path.display());
-    }
-
-    Ok(())
-}
-
-fn validate_args(args: &Args) -> Result<()> {
-    if args.concurrency == 0 {
-        bail!("--concurrency must be greater than zero");
-    }
-    if args.duration_secs == 0 {
-        bail!("--duration-secs must be greater than zero");
-    }
-    if args.timeout_secs == 0 {
-        bail!("--timeout-secs must be greater than zero");
-    }
-    Ok(())
-}
-
-fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client {
-    let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench");
-    let mut config = Config::builder()
-        .credentials_provider(credentials)
-        .region(Region::new(region.to_string()))
-        .endpoint_url(endpoint)
-        .force_path_style(true)
-        .behavior_version_latest();
-
-    if endpoint.starts_with("http://") {
-        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
-    }
-
-    Client::from_conf(config.build())
-}
-
-async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> {
-    if client.head_bucket().bucket(bucket).send().await.is_ok() {
-        return Ok(());
-    }
-
-    match client.create_bucket().bucket(bucket).send().await {
-        Ok(_) => Ok(()),
-        Err(err) => {
-            let rendered = err.to_string();
-            if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") {
-                Ok(())
-            } else {
-                Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}"))
-            }
-        }
-    }
-}
-
-async fn run_size_benchmark(
-    client: Client,
-    bucket: String,
-    run_id: String,
-    size: SizeSpec,
-    concurrency: usize,
-    duration: Duration,
-    timeout: Duration,
-) -> Result<SizeSummary> {
-    let payload = Bytes::from(vec![0_u8; size.bytes]);
-    let deadline = Instant::now() + duration;
-    let wall_start = Instant::now();
-
-    let mut handles = Vec::with_capacity(concurrency);
-    for worker in 0..concurrency {
-        let client = client.clone();
-        let bucket = bucket.clone();
-        let payload = payload.clone();
-        let prefix = format!("{run_id}/{}/worker-{worker}", size.slug);
-        handles.push(tokio::spawn(async move {
-            let mut samples = Vec::new();
-            let mut idx = 0usize;
-
-            while Instant::now() < deadline {
-                let key = format!("{prefix}/obj-{idx}.bin");
-                let started_at = Instant::now();
-                let request = client
-                    .put_object()
-                    .bucket(&bucket)
-                    .key(key)
-                    .body(ByteStream::from(payload.clone()))
-                    .content_type("application/octet-stream");
-
-                let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_)));
-                samples.push(Sample {
-                    ok,
-                    duration_ms: started_at.elapsed().as_secs_f64() * 1000.0,
-                });
-                idx += 1;
-            }
-
-            samples
-        }));
-    }
-
-    let mut samples = Vec::new();
-    for handle in handles {
-        samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?);
-    }
-
-    Ok(build_size_summary(&size, samples, wall_start.elapsed()))
-}
-
-fn build_size_summary(size: &SizeSpec, mut samples: Vec<Sample>, wall_elapsed: Duration) -> SizeSummary {
-    let total = samples.len();
-    let succeeded = samples.iter().filter(|sample| sample.ok).count();
-    let failed = total.saturating_sub(succeeded);
-    let wall_secs = wall_elapsed.as_secs_f64();
-    let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 };
-    let throughput_mib_per_sec = if wall_secs > 0.0 {
-        ((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs
-    } else {
-        0.0
-    };
-
-    let avg_ms = if total > 0 {
-        Some(samples.iter().map(|sample| sample.duration_ms).sum::<f64>() / total as f64)
-    } else {
-        None
-    };
-
-    samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms));
-    let durations: Vec<f64> = samples.into_iter().map(|sample| sample.duration_ms).collect();
-
-    SizeSummary {
-        label: size.label.clone(),
-        bytes: size.bytes,
-        total,
-        succeeded,
-        failed,
-        wall_secs,
-        object_rate,
-        throughput_mib_per_sec,
-        avg_ms,
-        p50_ms: percentile(&durations, 0.50),
-        p90_ms: percentile(&durations, 0.90),
-        p99_ms: percentile(&durations, 0.99),
-    }
-}
-
-async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
-    let mut continuation_token = None;
-    loop {
-        let response = client
-            .list_objects_v2()
-            .bucket(bucket)
-            .prefix(prefix)
-            .set_continuation_token(continuation_token.clone())
-            .send()
-            .await
-            .with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?;
-
-        let objects: Vec<ObjectIdentifier> = response
-            .contents
-            .unwrap_or_default()
-            .into_iter()
-            .filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok()))
-            .flatten()
-            .collect();
-
-        for chunk in objects.chunks(1_000) {
-            if chunk.is_empty() {
-                continue;
-            }
-
-            client
-                .delete_objects()
-                .bucket(bucket)
-                .delete(
-                    Delete::builder()
-                        .set_objects(Some(chunk.to_vec()))
-                        .quiet(true)
-                        .build()
-                        .context("failed to build delete request")?,
-                )
-                .send()
-                .await
-                .with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?;
-        }
-
-        if response.is_truncated.unwrap_or(false) {
-            continuation_token = response.next_continuation_token;
-        } else {
-            break;
-        }
-    }
-
-    Ok(())
-}
-
-fn parse_size_list(input: &str) -> Result<Vec<SizeSpec>> {
-    input
-        .split(',')
-        .map(str::trim)
-        .filter(|item| !item.is_empty())
-        .map(parse_size_spec)
-        .collect()
-}
-
-fn parse_size_spec(input: &str) -> Result<SizeSpec> {
-    let normalized = input.trim();
-    let lower = normalized.to_ascii_lowercase();
-
-    let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") {
-        (value, 1024usize)
-    } else if let Some(value) = lower.strip_suffix("mib") {
-        (value, 1024usize * 1024usize)
-    } else if let Some(value) = lower.strip_suffix('b') {
-        (value, 1usize)
-    } else {
-        (lower.as_str(), 1usize)
-    };
-
-    let value = number_part
-        .trim()
-        .parse::<usize>()
-        .with_context(|| format!("invalid size component: {input}"))?;
-    let bytes = value
-        .checked_mul(multiplier)
-        .ok_or_else(|| anyhow!("size overflow for {input}"))?;
-
-    Ok(SizeSpec {
-        label: normalized.to_string(),
-        slug: normalized
-            .chars()
-            .filter(|ch| ch.is_ascii_alphanumeric())
-            .collect::<String>()
-            .to_ascii_lowercase(),
-        bytes,
-    })
-}
-
-fn percentile(values: &[f64], percentile: f64) -> Option<f64> {
-    if values.is_empty() {
-        return None;
-    }
-
-    let index = ((values.len() - 1) as f64 * percentile).floor() as usize;
-    values.get(index).copied()
-}
-
-fn default_run_id() -> String {
-    format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))
-}
-
-fn print_size_summary(summary: &SizeSummary) {
-    println!(
-        "{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}",
-        summary.label,
-        summary.succeeded,
-        summary.failed,
-        summary.object_rate,
-        summary.throughput_mib_per_sec,
-        summary.avg_ms,
-        summary.p50_ms,
-        summary.p90_ms,
-        summary.p99_ms,
-    );
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn parse_size_spec_supports_binary_units() {
-        let four_kib = parse_size_spec("4KiB").expect("4KiB should parse");
-        assert_eq!(four_kib.bytes, 4 * 1024);
-
-        let one_mib = parse_size_spec("1MiB").expect("1MiB should parse");
-        assert_eq!(one_mib.bytes, 1024 * 1024);
-    }
-
-    #[test]
-    fn percentile_returns_expected_bucket() {
-        let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
-        assert_eq!(percentile(&values, 0.50), Some(30.0));
-        assert_eq!(percentile(&values, 0.90), Some(40.0));
-    }
-}

crates/e2e_test/src/bin/tls_gen.rs

@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod collectors;
-pub mod constants;
-pub mod format;
-mod global;
-mod metrics_type;
+use clap::Parser;
+use e2e_test::tls_gen::{Args, run};
 
-pub use format::report_metrics;
-pub use global::init_metrics_system;
-pub use metrics_type::*;
+fn main() -> anyhow::Result<()> {
+    let out_dir = run(Args::parse())?;
+    println!("Generated RustFS TLS bundle in {}", out_dir.display());
+    Ok(())
+}

crates/e2e_test/src/checksum_upload_test.rs

@@ -19,13 +19,9 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
-    use aws_sdk_s3::primitives::{ByteStream, SdkBody};
+    use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
     use base64::Engine;
-    use bytes::Bytes;
-    use futures::StreamExt;
-    use http_body::Frame;
-    use http_body_util::StreamBody;
     use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
@@ -68,53 +64,6 @@ mod tests {
             .encoded
     }
 
-    fn streamed_body_70kib_of_a() -> ByteStream {
-        let bytes = Bytes::from_static(&[b'a'; 1024]);
-        let stream = futures::stream::repeat_with(move || {
-            let frame = Frame::data(bytes.clone());
-            Ok::<_, std::io::Error>(frame)
-        });
-        let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
-        ByteStream::new(SdkBody::from_body_1_x(body))
-    }
-
-    struct WithSizeHint<T> {
-        inner: T,
-        size_hint: usize,
-    }
-
-    impl<T> WithSizeHint<T> {
-        fn new(inner: T, size_hint: usize) -> Self {
-            Self { inner, size_hint }
-        }
-    }
-
-    impl<T> http_body::Body for WithSizeHint<T>
-    where
-        T: http_body::Body + Unpin,
-    {
-        type Data = T::Data;
-        type Error = T::Error;
-
-        fn poll_frame(
-            self: std::pin::Pin<&mut Self>,
-            cx: &mut std::task::Context<'_>,
-        ) -> std::task::Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
-            let this = self.get_mut();
-            std::pin::Pin::new(&mut this.inner).poll_frame(cx)
-        }
-
-        fn is_end_stream(&self) -> bool {
-            self.inner.is_end_stream()
-        }
-
-        fn size_hint(&self) -> http_body::SizeHint {
-            let mut hint = self.inner.size_hint();
-            hint.set_exact(self.size_hint as u64);
-            hint
-        }
-    }
-
     /// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
     #[tokio::test]
     #[serial]
@@ -187,121 +136,6 @@ mod tests {
         info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
     }
 
-    /// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect
-    /// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum.
-    #[tokio::test]
-    #[serial]
-    async fn test_put_object_with_checksum_algorithm_only() {
-        init_logging();
-        info!("TEST: PutObject with checksum_algorithm only");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-checksum-algorithm-only";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let key = "obj-with-checksum-algorithm-only.txt";
-        let content = vec![b'a'; 70 * 1024];
-
-        let put_resp = client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .body(ByteStream::from(content.clone()))
-            .send()
-            .await
-            .expect("PutObject with checksum_algorithm should succeed");
-
-        let put_checksum = put_resp
-            .checksum_crc32()
-            .expect("PutObject should return checksum_crc32 when checksum_algorithm is used")
-            .to_string();
-
-        let mut get_resp = client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_mode(ChecksumMode::Enabled)
-            .send()
-            .await
-            .expect("GetObject should succeed");
-
-        let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty()))
-            .collect()
-            .await
-            .expect("collect body")
-            .into_bytes();
-
-        assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content");
-        assert_eq!(
-            get_resp.checksum_crc32().map(str::to_string),
-            Some(put_checksum),
-            "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum"
-        );
-    }
-
-    /// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec<u8>)`.
-    #[tokio::test]
-    #[serial]
-    async fn test_put_object_with_checksum_algorithm_only_streaming_body() {
-        init_logging();
-        info!("TEST: PutObject with checksum_algorithm only using streaming body");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-checksum-algorithm-streaming";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let key = "obj-with-checksum-algorithm-streaming.txt";
-        let expected_content = vec![b'a'; 70 * 1024];
-
-        let put_resp = client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .body(streamed_body_70kib_of_a())
-            .send()
-            .await
-            .expect("PutObject with streaming checksum_algorithm should succeed");
-
-        let put_checksum = put_resp
-            .checksum_crc32()
-            .expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads")
-            .to_string();
-
-        let mut get_resp = client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_mode(ChecksumMode::Enabled)
-            .send()
-            .await
-            .expect("GetObject should succeed");
-
-        let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty()))
-            .collect()
-            .await
-            .expect("collect body")
-            .into_bytes();
-
-        assert_eq!(
-            body_bytes.as_ref(),
-            expected_content.as_slice(),
-            "GetObject body must match uploaded content"
-        );
-        assert_eq!(
-            get_resp.checksum_crc32().map(str::to_string),
-            Some(put_checksum),
-            "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads"
-        );
-    }
-
     /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
     /// Uses part size >= 5MB (server minimum) for two parts.
     #[tokio::test]
@@ -400,114 +234,6 @@ mod tests {
         info!("PASSED: MultipartUpload with checksum and GetObject content match");
     }
 
-    /// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation,
-    /// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed.
-    #[tokio::test]
-    #[serial]
-    async fn test_multipart_upload_with_crc32_algorithm_only() {
-        init_logging();
-        info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-multipart-checksum-crc32-auto";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let key = "multipart-with-crc32-auto.bin";
-        let part1_content = "a".repeat(5 * 1024 * 1024 + 1);
-        let part2_content = "b".repeat(1024);
-
-        let create_resp = client
-            .create_multipart_upload()
-            .bucket(bucket)
-            .key(key)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .send()
-            .await
-            .expect("CreateMultipartUpload should succeed");
-
-        let upload_id = create_resp.upload_id().expect("upload_id should be present");
-
-        let part1_resp = client
-            .upload_part()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(upload_id)
-            .part_number(1)
-            .body(ByteStream::from(part1_content.clone().into_bytes()))
-            .send()
-            .await
-            .expect("UploadPart 1 should succeed");
-        let part1_checksum = part1_resp
-            .checksum_crc32()
-            .expect("UploadPart 1 should return checksum_crc32")
-            .to_string();
-
-        let part2_resp = client
-            .upload_part()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(upload_id)
-            .part_number(2)
-            .body(ByteStream::from(part2_content.clone().into_bytes()))
-            .send()
-            .await
-            .expect("UploadPart 2 should succeed");
-        let part2_checksum = part2_resp
-            .checksum_crc32()
-            .expect("UploadPart 2 should return checksum_crc32")
-            .to_string();
-
-        let completed_upload = CompletedMultipartUpload::builder()
-            .parts(
-                CompletedPart::builder()
-                    .part_number(1)
-                    .e_tag(part1_resp.e_tag().expect("etag part 1"))
-                    .checksum_crc32(part1_checksum)
-                    .build(),
-            )
-            .parts(
-                CompletedPart::builder()
-                    .part_number(2)
-                    .e_tag(part2_resp.e_tag().expect("etag part 2"))
-                    .checksum_crc32(part2_checksum)
-                    .build(),
-            )
-            .build();
-
-        client
-            .complete_multipart_upload()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(upload_id)
-            .multipart_upload(completed_upload)
-            .send()
-            .await
-            .expect("CompleteMultipartUpload should succeed");
-
-        let body_bytes = client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .send()
-            .await
-            .expect("GetObject should succeed")
-            .body
-            .collect()
-            .await
-            .expect("collect body")
-            .into_bytes();
-
-        let expected_content = format!("{part1_content}{part2_content}");
-        assert_eq!(
-            body_bytes.as_ref(),
-            expected_content.as_bytes(),
-            "completed multipart object must match concatenated parts"
-        );
-    }
-
     /// Regression test for issue #2282:
     /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
     #[tokio::test]

crates/e2e_test/src/content_encoding_test.rs

@@ -138,4 +138,63 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding,
+    /// only the effective encoding should roundtrip through GET/HEAD.
+    #[tokio::test]
+    #[serial]
+    async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() {
+        init_logging();
+        info!("aws-chunked,gzip should persist only gzip");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "content-encoding-aws-chunked-gzip-test";
+        let key = "streamed/object.txt";
+        let content = b"streaming upload body with effective gzip encoding";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .content_type("text/plain")
+            .content_encoding("aws-chunked,gzip")
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PUT failed");
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
+        assert_eq!(
+            get_resp.content_encoding(),
+            Some("gzip"),
+            "GET must return only the effective content encoding after aws-chunked is stripped"
+        );
+        let body = get_resp.body.collect().await.unwrap().into_bytes();
+        assert_eq!(body.as_ref(), content, "Body content mismatch");
+
+        let head_resp = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("HEAD failed");
+        assert_eq!(
+            head_resp.content_encoding(),
+            Some("gzip"),
+            "HEAD must return only the effective content encoding after aws-chunked is stripped"
+        );
+
+        env.stop_server();
+    }
 }

crates/e2e_test/src/kms/kms_local_test.rs

@@ -344,9 +344,6 @@ async fn test_local_kms_multipart_upload() {
     test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET)
         .await
         .expect("SSE-C multipart upload test failed");
-    test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET)
-        .await
-        .expect("SSE-C multipart wrong-key download test failed");
 
     // Test 4: Large multipart upload (test streaming encryption with multiple blocks)
     // TODO: Re-enable after fixing streaming encryption issues with large files
@@ -651,99 +648,6 @@ async fn test_multipart_upload_with_sse_c(
     Ok(())
 }
 
-async fn test_multipart_download_with_wrong_sse_c_key_fails(
-    s3_client: &aws_sdk_s3::Client,
-    bucket: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let object_key = "multipart-sse-c-bad-download-test";
-    let part_size = 5 * 1024 * 1024;
-    let total_parts = 2;
-    let total_size = part_size * total_parts;
-
-    let encryption_key = "01234567890123456789012345678901";
-    let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
-    let key_md5 = sse_customer_key_md5_base64(encryption_key);
-
-    let wrong_key = "abcdefghijklmnopqrstuvwxyz012345";
-    let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
-    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
-
-    let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect();
-
-    let create_multipart_output = s3_client
-        .create_multipart_upload()
-        .bucket(bucket)
-        .key(object_key)
-        .sse_customer_algorithm("AES256")
-        .sse_customer_key(&key_b64)
-        .sse_customer_key_md5(&key_md5)
-        .send()
-        .await?;
-
-    let upload_id = create_multipart_output.upload_id().unwrap();
-    let mut completed_parts = Vec::new();
-
-    for part_number in 1..=total_parts {
-        let start = (part_number - 1) * part_size;
-        let end = std::cmp::min(start + part_size, total_size);
-        let part_data = &test_data[start..end];
-
-        let upload_part_output = s3_client
-            .upload_part()
-            .bucket(bucket)
-            .key(object_key)
-            .upload_id(upload_id)
-            .part_number(part_number as i32)
-            .body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()))
-            .sse_customer_algorithm("AES256")
-            .sse_customer_key(&key_b64)
-            .sse_customer_key_md5(&key_md5)
-            .send()
-            .await?;
-
-        completed_parts.push(
-            aws_sdk_s3::types::CompletedPart::builder()
-                .part_number(part_number as i32)
-                .e_tag(upload_part_output.e_tag().unwrap())
-                .build(),
-        );
-    }
-
-    let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
-        .set_parts(Some(completed_parts))
-        .build();
-
-    s3_client
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(object_key)
-        .upload_id(upload_id)
-        .multipart_upload(completed_multipart_upload)
-        .send()
-        .await?;
-
-    let err = s3_client
-        .get_object()
-        .bucket(bucket)
-        .key(object_key)
-        .sse_customer_algorithm("AES256")
-        .sse_customer_key(&wrong_key_b64)
-        .sse_customer_key_md5(&wrong_key_md5)
-        .send()
-        .await
-        .expect_err("multipart SSE-C download with the wrong key should fail");
-
-    let service_err = err.into_service_error();
-    assert_eq!(
-        service_err.meta().code(),
-        Some("InvalidRequest"),
-        "wrong-key multipart SSE-C download should return InvalidRequest, got {:?}",
-        service_err.meta().code()
-    );
-
-    Ok(())
-}
-
 /// Test large multipart upload to verify streaming encryption works correctly
 #[allow(dead_code)]
 async fn test_large_multipart_upload(

crates/e2e_test/src/kms/test_runner.rs

@@ -450,11 +450,7 @@ impl KMSTestSuite {
         if failed > 0 {
             warn!("❌ Failing tests:");
             for result in results.iter().filter(|r| !r.success) {
-                warn!(
-                    "  - {}: {}",
-                    result.test_name,
-                    result.error_message.as_ref().unwrap_or(&"Unknown error".to_string())
-                );
+                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
             }
         }
     }

crates/e2e_test/src/lib.rs

@@ -100,10 +100,6 @@ mod cluster_concurrency_test;
 #[cfg(test)]
 mod checksum_upload_test;
 
-// Range request regression tests
-#[cfg(test)]
-mod range_request_test;
-
 // Group deletion tests
 #[cfg(test)]
 mod group_delete_test;
@@ -129,3 +125,5 @@ mod replication_extension_test;
 
 #[cfg(test)]
 mod snowball_auto_extract_test;
+
+pub mod tls_gen;

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -19,6 +19,7 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
     use serial_test::serial;
     use tracing::info;
@@ -179,4 +180,84 @@ mod tests {
             "Delete marker should no longer be latest after the second put"
         );
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_prefix_with_marker_object_returns_children() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns prefix children when a marker object also exists");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-versions-prefix-marker";
+        let marker_key = "data01";
+        let child_keys = [
+            "data01/meta/dump-2026-04-08-053205.json.gz",
+            "data01/meta/dump-2026-04-08-063209.json.gz",
+        ];
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to suspend versioning");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(marker_key)
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to put marker object");
+
+        for key in child_keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(ByteStream::from_static(b"payload"))
+                .send()
+                .await
+                .expect("Failed to put child object");
+        }
+
+        let listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix("data01/")
+            .send()
+            .await
+            .expect("Failed to list object versions by prefix");
+
+        let version_keys: Vec<_> = listing.versions().iter().filter_map(|version| version.key()).collect();
+
+        assert_eq!(
+            version_keys.len(),
+            child_keys.len(),
+            "ListObjectVersions with a trailing slash prefix should include child objects even when the marker object exists"
+        );
+
+        for key in child_keys {
+            assert!(
+                version_keys.contains(&key),
+                "ListObjectVersions(prefix=data01/) should include child object {key}"
+            );
+        }
+    }
 }

crates/e2e_test/src/range_request_test.rs

@@ -1,80 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! End-to-end regression test for invalid GET object ranges.
-
-#[cfg(test)]
-mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
-    use aws_sdk_s3::Client;
-    use aws_sdk_s3::error::SdkError;
-    use aws_sdk_s3::primitives::ByteStream;
-    use serial_test::serial;
-    use tracing::info;
-
-    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
-        env.create_s3_client()
-    }
-
-    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        match client.create_bucket().bucket(bucket).send().await {
-            Ok(_) => Ok(()),
-            Err(err) => {
-                if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") {
-                    Ok(())
-                } else {
-                    Err(Box::new(err))
-                }
-            }
-        }
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() {
-        init_logging();
-        info!("TEST: GetObject invalid range should return InvalidRange/416");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-invalid-range";
-        let key = "range.txt";
-        let content = b"testcontent";
-
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-        client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(ByteStream::from_static(content))
-            .send()
-            .await
-            .expect("PutObject should succeed");
-
-        let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await;
-
-        let err = result.expect_err("GetObject with an unsatisfiable range should fail");
-        match err {
-            SdkError::ServiceError(service_err) => {
-                assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416");
-
-                let s3_err = service_err.into_err();
-                assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange");
-            }
-            other_err => panic!("Expected S3 service error, got: {other_err:?}"),
-        }
-    }
-}

crates/e2e_test/src/reliant/conditional_writes.rs

@@ -259,7 +259,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
     let upload_id = initiate_response
         .upload_id()
-        .ok_or(std::io::Error::other("No upload ID returned"))?;
+        .ok_or_else(|| std::io::Error::other("No upload ID returned"))?;
 
     // Upload parts
     for part_number in 1..=num_parts {
@@ -277,7 +277,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
         let part_etag = upload_part_response
             .e_tag()
-            .ok_or(std::io::Error::other("Do not have etag"))?
+            .ok_or_else(|| std::io::Error::other("Do not have etag"))?
             .to_string();
 
         let completed_part = CompletedPart::builder().part_number(part_number).e_tag(part_etag).build();

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -120,11 +120,6 @@ impl LockClient for GrpcLockClient {
             .map_err(|e| LockError::internal(e.to_string()))?
             .into_inner();
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -134,7 +129,8 @@ impl LockClient for GrpcLockClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -50,6 +50,18 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
     }
 }
 
+fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> GenerallyLockResult {
+    if success {
+        GenerallyLockResult {
+            success: true,
+            error_info: None,
+            lock_info: None,
+        }
+    } else {
+        lock_result_from_error(format!("lock not found for release: {lock_id}"))
+    }
+}
+
 /// Minimal NodeService implementation that only supports Lock RPCs
 /// Used for testing distributed lock scenarios with real gRPC
 #[derive(Debug)]
@@ -102,7 +114,7 @@ impl NodeService for MinimalLockNodeService {
                 let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
                 Ok(Response::new(GenerallyLockResponse {
                     success: result.success,
-                    error_info: None,
+                    error_info: result.error,
                     lock_info: lock_info_json,
                 }))
             }
@@ -131,11 +143,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -161,11 +176,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.force_release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -271,10 +289,9 @@ impl NodeService for MinimalLockNodeService {
                 Ok(batch_results) => {
                     for (result_idx, success) in batch_results.into_iter().enumerate() {
                         if let Some(request_idx) = valid_indices.get(result_idx) {
-                            results[*request_idx] = GenerallyLockResult {
-                                success,
-                                error_info: None,
-                                lock_info: None,
+                            results[*request_idx] = match lock_ids.get(result_idx) {
+                                Some(lock_id) => lock_result_from_release(lock_id, success),
+                                None => lock_result_from_error(format!("unlock response index out of range: {result_idx}")),
                             };
                         }
                     }

crates/e2e_test/src/reliant/head_tls_bodyless_test.rs

@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for TLS/HTTP2 `HEAD` responses on missing objects.
+//!
+//! Before the fix, RustFS returned `404` for a missing object but still wrote
+//! the XML error payload on a `HEAD` request. Under HTTP/2 this emitted DATA
+//! frames after the response headers, which clients surfaced as a protocol
+//! error. This test keeps the request at the raw HTTPS layer so it can validate
+//! the final wire-facing behavior rather than SDK-level error mapping.
+
+#![cfg(test)]
+
+use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use http::Version;
+use http::header::HOST;
+use rcgen::generate_simple_self_signed;
+use reqwest::{Certificate, Client, Response, StatusCode};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+use std::path::Path;
+use std::process::Command;
+use tokio::fs;
+use tokio::time::{Duration, sleep};
+use tracing::info;
+
+const ACCESS_KEY: &str = "rustfsadmin";
+const SECRET_KEY: &str = "rustfsadmin";
+const BUCKET: &str = "test-head-tls-bodyless-bucket";
+
+async fn generate_tls_bundle(tls_dir: &Path) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    fs::create_dir_all(tls_dir).await?;
+    let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
+    let cert_pem = cert.cert.pem();
+    let key_pem = cert.signing_key.serialize_pem();
+
+    fs::write(tls_dir.join("rustfs_cert.pem"), cert_pem.as_bytes()).await?;
+    fs::write(tls_dir.join("rustfs_key.pem"), key_pem.as_bytes()).await?;
+
+    Ok(cert_pem.into_bytes())
+}
+
+fn local_https_h2_client(ca_pem: &[u8]) -> Result<Client, Box<dyn Error + Send + Sync>> {
+    let _ca_cert = Certificate::from_pem(ca_pem)?;
+    Ok(Client::builder()
+        .no_proxy()
+        .no_gzip()
+        .no_brotli()
+        .no_zstd()
+        .no_deflate()
+        .danger_accept_invalid_certs(true)
+        .build()?)
+}
+
+async fn signed_empty_request(
+    client: &Client,
+    method: http::Method,
+    url: &str,
+) -> Result<Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let request = http::Request::builder()
+        .method(method.as_str())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+
+    let signed = sign_v4(request, 0, ACCESS_KEY, SECRET_KEY, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        builder = builder.header(name, value);
+    }
+
+    Ok(builder.send().await?)
+}
+
+async fn ensure_bucket_exists(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let bucket_url = format!("{endpoint}/{BUCKET}/");
+    let response = signed_empty_request(client, http::Method::HEAD, &bucket_url).await?;
+
+    if response.status() == StatusCode::OK {
+        return Ok(());
+    }
+
+    let response = signed_empty_request(client, http::Method::PUT, &bucket_url).await?;
+    match response.status() {
+        StatusCode::OK => Ok(()),
+        StatusCode::CONFLICT => Ok(()),
+        status => Err(format!("unexpected bucket setup status: {status}").into()),
+    }
+}
+
+async fn wait_for_tls_server_ready(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let ready_url = format!("{endpoint}/");
+    for _attempt in 0..60 {
+        match signed_empty_request(client, http::Method::GET, &ready_url).await {
+            Ok(response) if response.status().is_success() => return Ok(()),
+            Ok(_) | Err(_) => sleep(Duration::from_millis(500)).await,
+        }
+    }
+
+    Err("RustFS TLS server failed to become ready within 30 seconds".into())
+}
+
+async fn start_tls_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let binary_path = rustfs_binary_path();
+    let mut command = Command::new(&binary_path);
+    command
+        .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .current_dir(&env.temp_dir);
+
+    for key in [
+        "RUSTFS_ADDRESS",
+        "RUSTFS_VOLUMES",
+        "RUSTFS_ACCESS_KEY",
+        "RUSTFS_SECRET_KEY",
+        "RUSTFS_TLS_PATH",
+        "RUSTFS_OBS_LOG_DIRECTORY",
+    ] {
+        command.env_remove(key);
+    }
+
+    let process = command
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .env("RUSTFS_CONSOLE_ENABLE", "false")
+        .args([
+            "--address",
+            &env.address,
+            "--access-key",
+            &env.access_key,
+            "--secret-key",
+            &env.secret_key,
+            &env.temp_dir,
+        ])
+        .spawn()?;
+
+    env.process = Some(process);
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_head_missing_object_over_tls_http2_is_bodyless() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let tls_dir = std::path::PathBuf::from(&env.temp_dir).join("tls");
+    let ca_pem = generate_tls_bundle(&tls_dir).await?;
+    start_tls_rustfs_server(&mut env, &tls_dir).await?;
+
+    let endpoint = format!("https://{}", env.address);
+    let client = local_https_h2_client(&ca_pem)?;
+    wait_for_tls_server_ready(&client, &endpoint).await?;
+    ensure_bucket_exists(&client, &endpoint).await?;
+
+    let missing_key = "head-does-not-exist.txt";
+    let object_url = format!("{endpoint}/{BUCKET}/{missing_key}");
+
+    let get_response = signed_empty_request(&client, http::Method::GET, &object_url).await?;
+    assert_eq!(get_response.status(), StatusCode::NOT_FOUND);
+    let get_version = get_response.version();
+    let get_body = get_response.bytes().await?;
+    let get_body_text = String::from_utf8_lossy(&get_body);
+    assert!(
+        get_body_text.contains("<Code>NoSuchKey</Code>") || get_body_text.contains("<Code>NoSuchObject</Code>"),
+        "GET missing-object error body should expose NoSuchKey/NoSuchObject, got: {}",
+        get_body_text
+    );
+    info!("GET missing object over TLS used {:?} and returned {} bytes", get_version, get_body.len());
+
+    let head_response = signed_empty_request(&client, http::Method::HEAD, &object_url).await?;
+    assert_eq!(head_response.status(), StatusCode::NOT_FOUND);
+    assert_eq!(head_response.version(), Version::HTTP_2, "HEAD regression test must exercise HTTP/2");
+    let head_body = head_response.bytes().await?;
+    assert!(
+        head_body.is_empty(),
+        "HEAD missing-object response must not send body bytes over TLS/HTTP2, got {} bytes: {:?}",
+        head_body.len(),
+        head_body
+    );
+
+    Ok(())
+}

crates/e2e_test/src/reliant/lifecycle.rs

@@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 #[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
+async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
     use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
 
     let client = create_aws_s3_client().await?;
@@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::er
         .build()?;
     let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
 
-    let err = client
+    client
         .put_bucket_lifecycle_configuration()
         .bucket(BUCKET)
         .lifecycle_configuration(lifecycle)
         .send()
-        .await
-        .expect_err("zero-day lifecycle expiration should be rejected");
-
-    let err_msg = format!("{err:?}");
-    assert!(
-        err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"),
-        "unexpected error: {err_msg}"
-    );
+        .await?;
 
     Ok(())
 }

crates/e2e_test/src/reliant/lock.rs

@@ -275,6 +275,41 @@ async fn test_grpc_lock_client_batch_acquire_and_release() {
     handle.abort();
 }
 
+#[tokio::test]
+async fn test_grpc_lock_client_uses_request_lock_id_and_reports_missing_unlock() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+    let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client = GrpcLockClient::new(addr);
+    let request = LockRequest::new(test_resource(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(2));
+
+    let response = grpc_client.acquire_lock(&request).await.expect("gRPC acquire should succeed");
+    let lock_info = response.lock_info.expect("gRPC acquire should include lock info");
+    assert_eq!(lock_info.id, request.lock_id);
+
+    assert!(
+        grpc_client
+            .release(&request.lock_id)
+            .await
+            .expect("gRPC release should succeed"),
+        "release should find the request lock id"
+    );
+
+    let missing_release = grpc_client
+        .release(&request.lock_id)
+        .await
+        .expect_err("second release should report missing lock");
+    assert!(
+        missing_release.to_string().contains("lock not found for release"),
+        "missing release should preserve server error, got: {missing_release}"
+    );
+
+    handle.abort();
+}
+
 #[tokio::test]
 async fn test_distributed_lock_4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes() {
     let manager1 = Arc::new(GlobalLockManager::new());

crates/e2e_test/src/reliant/mod.rs

@@ -17,6 +17,7 @@ mod get_deleted_object_test;
 mod grpc_lock_client;
 mod grpc_lock_server;
 mod head_deleted_object_versioning_test;
+mod head_tls_bodyless_test;
 mod lifecycle;
 mod lock;
 mod node_interact_test;

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -154,7 +154,7 @@ async fn walk_dir() -> Result<(), Box<dyn Error>> {
             match response.next().await {
                 Some(Ok(resp)) => {
                     if !resp.success {
-                        println!("{}", resp.error_info.unwrap_or("".to_string()));
+                        println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
                     }
                     let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
                         .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))

crates/e2e_test/src/reliant/sql.rs

@@ -281,8 +281,11 @@ async fn test_select_object_content_csv_limit() -> Result<(), Box<dyn Error>> {
     println!("CSV Limit result: {result_str}");
 
     // Verify only first 2 records are returned
-    let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
-    assert_eq!(lines.len(), 2, "Should return exactly 2 records");
+    assert_eq!(
+        result_str.lines().filter(|line| !line.trim().is_empty()).count(),
+        2,
+        "Should return exactly 2 records"
+    );
 
     Ok(())
 }
@@ -321,8 +324,10 @@ async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>>
     println!("CSV Order By result: {result_str}");
 
     // Verify ordered by age descending
-    let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
-    assert!(lines.len() >= 2, "Should return at least 2 records");
+    assert!(
+        result_str.lines().filter(|line| !line.trim().is_empty()).count() >= 2,
+        "Should return at least 2 records"
+    );
 
     // Check if contains highest age records
     assert!(result_str.contains("Charlie,35"));

crates/e2e_test/src/special_chars_test.rs

@@ -26,10 +26,16 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
     use aws_sdk_s3::Client;
     use aws_sdk_s3::primitives::ByteStream;
+    use http::StatusCode;
+    use http::header::HOST;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
     use serial_test::serial;
+    use std::error::Error;
     use tracing::{debug, info};
 
     /// Helper function to create an S3 client for testing
@@ -56,6 +62,30 @@ mod tests {
         }
     }
 
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
     /// Test PUT and GET with space character in path
     ///
     /// This reproduces Part A of the issue:
@@ -274,6 +304,73 @@ mod tests {
         info!("Test completed successfully");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_missing_object_with_trailing_equals_returns_no_such_key() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-missing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let url = format!("{}/{}/path/sitemap.xmlage=", env.url, bucket);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::NOT_FOUND,
+            "missing object key ending with '=' should pass signature validation before object lookup"
+        );
+
+        let body = response.text().await?;
+        assert!(body.contains("<Code>NoSuchKey</Code>"), "expected NoSuchKey XML response, got: {body}");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_existing_object_with_trailing_equals_returns_content() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-existing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let key = "path/sitemap.xmlage=";
+        let content = b"object content for raw signed URL with trailing equals";
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}/{}", env.url, bucket, key);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::OK,
+            "existing object key ending with '=' should pass signature validation and return content"
+        );
+
+        let body = response.bytes().await?;
+        assert_eq!(body.as_ref(), content);
+
+        env.stop_server();
+        Ok(())
+    }
+
     /// Test DELETE operation with special characters
     #[tokio::test]
     #[serial]

crates/e2e_test/src/tls_gen.rs

@@ -0,0 +1,259 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::{Context, Result, bail};
+use clap::Parser;
+use rcgen::{
+    BasicConstraints, CertificateParams, CertifiedIssuer, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose,
+    SanType,
+};
+use std::fs;
+use std::net::IpAddr;
+use std::path::{Path, PathBuf};
+use time::{Duration, OffsetDateTime};
+
+pub const DEFAULT_OUT_DIR: &str = "target/tls";
+pub const OUTPUT_FILES: [&str; 7] = [
+    "rustfs_cert.pem",
+    "rustfs_key.pem",
+    "ca.crt",
+    "public.crt",
+    "client_ca.crt",
+    "client_cert.pem",
+    "client_key.pem",
+];
+
+#[derive(Debug, Parser)]
+#[command(name = "tls_gen", about = "Generate a full RustFS TLS bundle for local TLS and mTLS tests.")]
+pub struct Args {
+    #[arg(long, default_value = DEFAULT_OUT_DIR)]
+    pub out_dir: PathBuf,
+    #[arg(long, default_value_t = 365)]
+    pub days: i64,
+    #[arg(long)]
+    pub force: bool,
+}
+
+pub fn run(args: Args) -> Result<PathBuf> {
+    if args.days <= 0 {
+        bail!("--days must be a positive integer");
+    }
+
+    write_bundle(&args.out_dir, args.force, args.days)?;
+    Ok(args.out_dir)
+}
+
+pub fn ensure_writable(out_dir: &Path, force: bool) -> Result<()> {
+    if force {
+        return Ok(());
+    }
+
+    let existing: Vec<_> = OUTPUT_FILES
+        .iter()
+        .map(|name| out_dir.join(name))
+        .filter(|path| path.exists())
+        .collect();
+
+    if existing.is_empty() {
+        return Ok(());
+    }
+
+    let existing_list = existing
+        .iter()
+        .map(|path| path.file_name().and_then(|name| name.to_str()).unwrap_or("<unknown>"))
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    bail!(
+        "Refusing to overwrite existing files in {}: {}. Re-run with --force to replace them.",
+        out_dir.display(),
+        existing_list
+    )
+}
+
+fn write_bundle(out_dir: &Path, force: bool, days: i64) -> Result<()> {
+    fs::create_dir_all(out_dir).with_context(|| format!("failed to create output directory {}", out_dir.display()))?;
+    ensure_writable(out_dir, force)?;
+
+    let ca_key = generate_private_key()?;
+    let ca = build_ca_certificate(ca_key, days)?;
+
+    let server_key = generate_private_key()?;
+    let server_cert = build_leaf_certificate(
+        &server_key,
+        "localhost",
+        &[SanType::DnsName("localhost".try_into()?)],
+        &[
+            SanType::IpAddress(IpAddr::V4("127.0.0.1".parse()?)),
+            SanType::IpAddress(IpAddr::V6("::1".parse()?)),
+        ],
+        ExtendedKeyUsagePurpose::ServerAuth,
+        &ca,
+        days,
+    )?;
+
+    let client_key = generate_private_key()?;
+    let client_cert = build_leaf_certificate(
+        &client_key,
+        "rustfs-test-client",
+        &[SanType::DnsName("rustfs-test-client".try_into()?)],
+        &[],
+        ExtendedKeyUsagePurpose::ClientAuth,
+        &ca,
+        days,
+    )?;
+
+    let ca_pem = ca.pem();
+    let bundle = [
+        ("rustfs_cert.pem", server_cert.pem()),
+        ("rustfs_key.pem", server_key.serialize_pem()),
+        ("ca.crt", ca_pem.clone()),
+        ("public.crt", ca_pem.clone()),
+        ("client_ca.crt", ca_pem),
+        ("client_cert.pem", client_cert.pem()),
+        ("client_key.pem", client_key.serialize_pem()),
+    ];
+
+    for (name, content) in bundle {
+        fs::write(out_dir.join(name), content).with_context(|| format!("failed to write {}", out_dir.join(name).display()))?;
+    }
+
+    Ok(())
+}
+
+fn build_ca_certificate(signing_key: KeyPair, days: i64) -> Result<CertifiedIssuer<'static, KeyPair>> {
+    let mut params = base_params("RustFS Test CA", days)?;
+    params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+    params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
+
+    CertifiedIssuer::self_signed(params, signing_key).context("failed to create CA certificate")
+}
+
+fn build_leaf_certificate(
+    signing_key: &KeyPair,
+    common_name: &str,
+    dns_names: &[SanType],
+    ip_addresses: &[SanType],
+    usage: ExtendedKeyUsagePurpose,
+    issuer: &CertifiedIssuer<'_, KeyPair>,
+    days: i64,
+) -> Result<rcgen::Certificate> {
+    let mut params = base_params(common_name, days)?;
+    params.is_ca = IsCa::ExplicitNoCa;
+    params.key_usages = vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment];
+    params.extended_key_usages = vec![usage];
+    params.use_authority_key_identifier_extension = true;
+    params.subject_alt_names.extend_from_slice(dns_names);
+    params.subject_alt_names.extend_from_slice(ip_addresses);
+
+    params
+        .signed_by(signing_key, issuer)
+        .with_context(|| format!("failed to create leaf certificate for {common_name}"))
+}
+
+fn base_params(common_name: &str, days: i64) -> Result<CertificateParams> {
+    let mut params = CertificateParams::default();
+    let issued_at = OffsetDateTime::now_utc() - Duration::minutes(5);
+    params.not_before = issued_at;
+    params.not_after = issued_at + Duration::days(days);
+    params.distinguished_name.push(DnType::CountryName, "US");
+    params.distinguished_name.push(DnType::OrganizationName, "RustFS");
+    params.distinguished_name.push(DnType::CommonName, common_name);
+    Ok(params)
+}
+
+fn generate_private_key() -> Result<KeyPair> {
+    KeyPair::generate().context("failed to generate private key")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Args, OUTPUT_FILES, ensure_writable, run};
+    use std::fs;
+    use std::path::{Path, PathBuf};
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn unique_temp_dir() -> PathBuf {
+        let suffix = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("system time must be after unix epoch")
+            .as_nanos();
+        std::env::temp_dir().join(format!("rustfs-tls-gen-{suffix}"))
+    }
+
+    struct TempDir {
+        path: PathBuf,
+    }
+
+    impl TempDir {
+        fn new() -> Self {
+            let path = unique_temp_dir();
+            fs::create_dir_all(&path).expect("temporary directory should be created");
+            Self { path }
+        }
+
+        fn path(&self) -> &Path {
+            &self.path
+        }
+    }
+
+    impl Drop for TempDir {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.path);
+        }
+    }
+
+    #[test]
+    fn run_writes_full_bundle() {
+        let temp_dir = TempDir::new();
+
+        let out_dir = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 365,
+            force: false,
+        })
+        .expect("bundle generation should succeed");
+
+        for name in OUTPUT_FILES {
+            let content = fs::read(out_dir.join(name)).unwrap_or_else(|error| panic!("{name} should exist: {error}"));
+            assert!(!content.is_empty(), "{name} should not be empty");
+        }
+    }
+
+    #[test]
+    fn ensure_writable_rejects_existing_files_without_force() {
+        let temp_dir = TempDir::new();
+        let existing = temp_dir.path().join(OUTPUT_FILES[0]);
+        fs::write(&existing, "existing").expect("existing file should be created");
+
+        let error = ensure_writable(temp_dir.path(), false).expect_err("existing files must be rejected");
+        let message = format!("{error:#}");
+
+        assert!(message.contains("Refusing to overwrite existing files"));
+        assert!(message.contains(OUTPUT_FILES[0]));
+    }
+
+    #[test]
+    fn run_rejects_non_positive_days() {
+        let temp_dir = TempDir::new();
+        let error = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 0,
+            force: false,
+        })
+        .expect_err("non-positive days must fail");
+
+        assert_eq!(format!("{error:#}"), "--days must be a positive integer");
+    }
+}

crates/ecstore/Cargo.toml

@@ -55,6 +55,7 @@ flatbuffers.workspace = true
 futures.workspace = true
 futures-util.workspace = true
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
 serde.workspace = true
 time.workspace = true
 bytesize.workspace = true
@@ -62,6 +63,7 @@ serde_json.workspace = true
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
 s3s.workspace = true
 http.workspace = true
+opentelemetry.workspace = true
 http-body = { workspace = true }
 http-body-util.workspace = true
 url.workspace = true
@@ -70,7 +72,6 @@ reed-solomon-erasure = { workspace = true }
 reed-solomon-simd = { workspace = true }
 lazy_static.workspace = true
 rustfs-lock.workspace = true
-rustfs-io-core.workspace = true
 rustfs-io-metrics.workspace = true
 regex = { workspace = true }
 path-absolutize = { workspace = true }
@@ -87,7 +88,7 @@ hyper.workspace = true
 hyper-util.workspace = true
 hyper-rustls.workspace = true
 rustls.workspace = true
-tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
+tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower.workspace = true
@@ -99,6 +100,7 @@ pin-project-lite.workspace = true
 md-5.workspace = true
 memmap2 = { workspace = true }
 libc.workspace = true
+rustix = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-workers.workspace = true
 reqwest = { workspace = true }
@@ -125,9 +127,10 @@ metrics = { workspace = true }
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true }
 serial_test = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 
 [build-dependencies]
 shadow-rs = { workspace = true, features = ["build", "metadata"] }
@@ -140,17 +143,5 @@ harness = false
 name = "comparison_benchmark"
 harness = false
 
-[[bench]]
-name = "direct_chunk_benchmark"
-harness = false
-
-[[bench]]
-name = "reconstructed_chunk_benchmark"
-harness = false
-
-[[bench]]
-name = "bitrot_chunk_benchmark"
-harness = false
-
 [lib]
 doctest = false

crates/ecstore/README.md

@@ -32,43 +32,6 @@
 
 For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
-## 📈 Benchmarks
-
-ECStore ships several Criterion benchmarks under [`crates/ecstore/benches/`](./benches/).
-
-### Direct Chunk Path
-
-Use the direct chunk benchmark to compare the current slice-forwarding path against the previous assembled-copy path:
-
-```bash
-cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark
-```
-
-To run only the end-to-end ECStore range-read benchmark:
-
-```bash
-cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark ecstore_get_object_chunks
-```
-
-To run the reconstructed multi-disk range-read benchmark:
-
-```bash
-cargo bench -p rustfs-ecstore --bench reconstructed_chunk_benchmark
-```
-
-### Saved Comparison Points
-
-Latest local measurements on this branch:
-
-- `direct_chunk_path/slice_forwarding/single_block_aligned`: about `477 ns`
-- `direct_chunk_path/assembled_copy/single_block_aligned`: about `3.25 us`
-- `direct_chunk_path/slice_forwarding/multi_block_unaligned`: about `963 ns`
-- `direct_chunk_path/assembled_copy/multi_block_unaligned`: about `7.25 us`
-- `ecstore_get_object_chunks/drain/multi_disk_range`: about `644-654 us`, throughput about `2.86-2.90 GiB/s`
-- `reconstructed_chunk_path/drain/multi_disk_missing_shard`: about `1.292-1.304 ms`, throughput about `1.43-1.45 GiB/s`
-
-These numbers are intended as branch-local reference points. Re-run the benchmark on your target machine before treating them as a regression baseline.
-
 ## 📄 License
 
 This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/ecstore/benches/bitrot_chunk_benchmark.rs

@@ -1,106 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use bytes::Bytes;
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use rustfs_ecstore::bitrot::decode_bitrot_chunk_source_for_bench;
-use rustfs_io_core::IoChunk;
-use rustfs_utils::HashAlgorithm;
-use std::hint::black_box;
-
-struct BitrotChunkBenchCase {
-    name: &'static str,
-    source_chunks: Vec<IoChunk>,
-    shard_size: usize,
-    expected_decoded_len: usize,
-    expected_copied: bool,
-}
-
-fn encode_shard(checksum_algo: HashAlgorithm, shard: &[u8]) -> Vec<u8> {
-    let mut encoded = Vec::with_capacity(checksum_algo.size() + shard.len());
-    encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
-    encoded.extend_from_slice(shard);
-    encoded
-}
-
-fn bitrot_chunk_bench_cases() -> [BitrotChunkBenchCase; 2] {
-    let checksum_algo = HashAlgorithm::Md5;
-    let shard_one = b"abcd";
-    let shard_two = b"efgh";
-    let encoded_one = encode_shard(checksum_algo.clone(), shard_one);
-    let encoded_two = encode_shard(checksum_algo.clone(), shard_two);
-
-    let mut cross_chunk = Vec::with_capacity(encoded_one.len() + encoded_two.len());
-    cross_chunk.extend_from_slice(&encoded_one);
-    cross_chunk.extend_from_slice(&encoded_two);
-    let split = checksum_algo.size() + 2;
-
-    [
-        BitrotChunkBenchCase {
-            name: "aligned_multi_chunk_no_copy",
-            source_chunks: vec![
-                IoChunk::Shared(Bytes::from(encoded_one)),
-                IoChunk::Shared(Bytes::from(encoded_two)),
-            ],
-            shard_size: shard_one.len(),
-            expected_decoded_len: shard_one.len() + shard_two.len(),
-            expected_copied: false,
-        },
-        BitrotChunkBenchCase {
-            name: "cross_chunk_frame_copy",
-            source_chunks: vec![
-                IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[..split])),
-                IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[split..])),
-            ],
-            shard_size: shard_one.len(),
-            expected_decoded_len: shard_one.len() + shard_two.len(),
-            expected_copied: true,
-        },
-    ]
-}
-
-fn bench_bitrot_chunk_decode(c: &mut Criterion) {
-    let checksum_algo = HashAlgorithm::Md5;
-    let mut group = c.benchmark_group("bitrot_chunk_decode");
-    group.sample_size(20);
-
-    for case in bitrot_chunk_bench_cases() {
-        let (decoded, copied) =
-            decode_bitrot_chunk_source_for_bench(&case.source_chunks, case.shard_size, checksum_algo.clone(), false)
-                .expect("decode bitrot source");
-        let decoded_len: usize = decoded.iter().map(IoChunk::len).sum();
-
-        assert_eq!(decoded_len, case.expected_decoded_len);
-        assert_eq!(copied, case.expected_copied);
-
-        group.throughput(Throughput::Bytes(case.expected_decoded_len as u64));
-        group.bench_with_input(BenchmarkId::new("decode", case.name), &case, |b, case| {
-            b.iter(|| {
-                let result = decode_bitrot_chunk_source_for_bench(
-                    black_box(&case.source_chunks),
-                    black_box(case.shard_size),
-                    checksum_algo.clone(),
-                    false,
-                )
-                .expect("decode bitrot source");
-                black_box(result);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-criterion_group!(benches, bench_bitrot_chunk_decode);
-criterion_main!(benches);

crates/ecstore/benches/direct_chunk_benchmark.rs

@@ -1,390 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use bytes::{Bytes, BytesMut};
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use futures_util::StreamExt;
-use futures_util::stream;
-use http::HeaderMap;
-use rustfs_ecstore::bucket::metadata_sys;
-use rustfs_ecstore::disk::endpoint::Endpoint;
-use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
-use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
-use rustfs_ecstore::set_disk::collect_direct_data_shard_chunks_for_benchmark;
-use rustfs_ecstore::store::{ECStore, init_local_disks};
-use rustfs_ecstore::store_api::{
-    BucketOperations, BucketOptions, ChunkNativePutData, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions, ObjectIO,
-    ObjectOptions,
-};
-use rustfs_io_core::{BoxChunkStream, IoChunk, MappedChunk};
-use std::hint::black_box;
-use std::net::SocketAddr;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU16, Ordering};
-use tempfile::TempDir;
-use tokio::runtime::Runtime;
-use tokio_util::sync::CancellationToken;
-
-#[derive(Clone)]
-struct BenchCase {
-    name: &'static str,
-    data_shards: usize,
-    block_size: usize,
-    blocks: usize,
-    offset: usize,
-    length: usize,
-}
-
-fn bench_cases() -> [BenchCase; 2] {
-    [
-        BenchCase {
-            name: "single_block_aligned",
-            data_shards: 4,
-            block_size: 256 * 1024,
-            blocks: 1,
-            offset: 0,
-            length: 256 * 1024,
-        },
-        BenchCase {
-            name: "multi_block_unaligned",
-            data_shards: 4,
-            block_size: 256 * 1024,
-            blocks: 8,
-            offset: 123_457,
-            length: 2 * 256 * 1024 + 33_333,
-        },
-    ]
-}
-
-#[derive(Clone)]
-struct EcstoreBenchCase {
-    name: &'static str,
-    disk_count: usize,
-    payload_len: usize,
-    range: HTTPRangeSpec,
-    expected_copy_mode: GetObjectChunkCopyMode,
-}
-
-struct EcstoreBenchEnv {
-    _temp_dir: TempDir,
-    store: Arc<ECStore>,
-    bucket: String,
-    key: String,
-    range: HTTPRangeSpec,
-    opts: ObjectOptions,
-    expected_len: usize,
-    expected_copy_mode: GetObjectChunkCopyMode,
-}
-
-fn ecstore_bench_cases() -> [EcstoreBenchCase; 1] {
-    [EcstoreBenchCase {
-        name: "multi_disk_range",
-        disk_count: 4,
-        payload_len: 3 * 1024 * 1024 + 137,
-        range: HTTPRangeSpec {
-            is_suffix_length: false,
-            start: 123_457,
-            end: 2 * 1024 * 1024 + 33_333,
-        },
-        expected_copy_mode: expected_direct_copy_mode(),
-    }]
-}
-
-#[cfg(unix)]
-const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
-    GetObjectChunkCopyMode::TrueZeroCopy
-}
-
-#[cfg(not(unix))]
-const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
-    GetObjectChunkCopyMode::SharedBytes
-}
-
-fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
-    HTTPRangeSpec {
-        is_suffix_length: range.is_suffix_length,
-        start: range.start,
-        end: range.end,
-    }
-}
-
-fn next_loopback_addr() -> SocketAddr {
-    static NEXT_PORT: AtomicU16 = AtomicU16::new(39013);
-    let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
-    SocketAddr::from(([127, 0, 0, 1], port))
-}
-
-fn build_endpoint_pools(paths: &[std::path::PathBuf]) -> EndpointServerPools {
-    let mut endpoints = Vec::with_capacity(paths.len());
-    for (idx, disk_path) in paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(idx);
-        endpoints.push(endpoint);
-    }
-
-    EndpointServerPools(vec![PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: paths.len(),
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "bench".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    }])
-}
-
-async fn build_ecstore_bench_env(case: &EcstoreBenchCase) -> EcstoreBenchEnv {
-    let temp_dir = tempfile::tempdir().expect("tempdir");
-    let mut disk_paths = Vec::with_capacity(case.disk_count);
-    for idx in 0..case.disk_count {
-        let path = temp_dir.path().join(format!("disk{}", idx + 1));
-        tokio::fs::create_dir_all(&path).await.expect("create disk dir");
-        disk_paths.push(path);
-    }
-
-    let endpoint_pools = build_endpoint_pools(&disk_paths);
-    GLOBAL_LOCAL_DISK_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
-    init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
-
-    let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
-        .await
-        .expect("create ecstore");
-
-    let buckets = store
-        .list_bucket(&BucketOptions {
-            no_metadata: true,
-            ..Default::default()
-        })
-        .await
-        .expect("list buckets")
-        .into_iter()
-        .map(|bucket| bucket.name)
-        .collect();
-    metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
-
-    let object_id = case.name.replace('_', "-");
-    let bucket = format!("bench-direct-{object_id}");
-    let key = format!("objects/{object_id}.bin");
-    store
-        .make_bucket(
-            &bucket,
-            &MakeBucketOptions {
-                versioning_enabled: true,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("make bucket");
-
-    let payload: Vec<u8> = (0..case.payload_len).map(|idx| (idx % 251) as u8).collect();
-    let mut reader = ChunkNativePutData::from_vec(payload);
-    let put_info = store
-        .put_object(&bucket, &key, &mut reader, &ObjectOptions::default())
-        .await
-        .expect("put object");
-    if case.disk_count > 1 {
-        assert!(put_info.data_blocks > 1, "expected multi-data-shard object");
-    }
-
-    let (_, expected_len) = case.range.get_offset_length(case.payload_len as i64).expect("range length");
-
-    EcstoreBenchEnv {
-        _temp_dir: temp_dir,
-        store,
-        bucket,
-        key,
-        range: clone_range_spec(&case.range),
-        opts: ObjectOptions::default(),
-        expected_len: expected_len as usize,
-        expected_copy_mode: case.expected_copy_mode,
-    }
-}
-
-async fn run_ecstore_get_object_chunks_bench(env: &EcstoreBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
-    let mut result = env
-        .store
-        .get_object_chunks(&env.bucket, &env.key, Some(clone_range_spec(&env.range)), HeaderMap::new(), &env.opts)
-        .await
-        .expect("get object chunks");
-    let copy_mode = result.copy_mode;
-    let mut total_len = 0usize;
-    let mut chunk_count = 0usize;
-    while let Some(chunk) = result.stream.next().await {
-        let chunk = chunk.expect("chunk");
-        total_len += chunk.len();
-        chunk_count += 1;
-    }
-
-    (copy_mode, total_len, chunk_count)
-}
-
-fn build_shard_bytes(case: &BenchCase) -> Vec<Vec<Bytes>> {
-    let total_len = case.block_size * case.blocks;
-    let payload: Vec<u8> = (0..total_len).map(|idx| (idx % 251) as u8).collect();
-    let mut shards = vec![Vec::with_capacity(case.blocks); case.data_shards];
-
-    for block in 0..case.blocks {
-        let block_start = block * case.block_size;
-        let block_slice = &payload[block_start..block_start + case.block_size];
-        let shard_width = case.block_size / case.data_shards;
-        for (shard_idx, shard) in shards.iter_mut().enumerate().take(case.data_shards) {
-            let shard_start = shard_idx * shard_width;
-            let shard_end = shard_start + shard_width;
-            shard.push(Bytes::copy_from_slice(&block_slice[shard_start..shard_end]));
-        }
-    }
-
-    shards
-}
-
-fn build_mapped_streams(shards: &[Vec<Bytes>]) -> Vec<BoxChunkStream> {
-    shards
-        .iter()
-        .map(|shard_chunks| {
-            let chunks: Vec<_> = shard_chunks
-                .iter()
-                .map(|chunk| {
-                    let mapped = MappedChunk::new(chunk.clone(), 0, chunk.len()).expect("mapped chunk");
-                    Ok::<IoChunk, std::io::Error>(IoChunk::Mapped(mapped))
-                })
-                .collect();
-            Box::pin(stream::iter(chunks)) as BoxChunkStream
-        })
-        .collect()
-}
-
-fn collect_old_assembly(
-    shards: &[Vec<Bytes>],
-    data_shards: usize,
-    block_size: usize,
-    offset: usize,
-    length: usize,
-) -> Vec<IoChunk> {
-    if length == 0 {
-        return Vec::new();
-    }
-
-    let start_block = offset / block_size;
-    let end_block = offset.saturating_add(length - 1) / block_size;
-    let mut result = Vec::with_capacity(end_block - start_block + 1);
-
-    for block_index in start_block..=end_block {
-        let block_offset = if block_index == start_block { offset % block_size } else { 0 };
-        let block_length = if start_block == end_block {
-            length
-        } else if block_index == start_block {
-            block_size - (offset % block_size)
-        } else if block_index == end_block {
-            (offset + length) % block_size
-        } else {
-            block_size
-        };
-
-        if block_length == 0 {
-            break;
-        }
-
-        let mut block = BytesMut::with_capacity(block_length);
-        let mut write_left = block_length;
-        let mut skip = block_offset;
-
-        for shard in shards.iter().take(data_shards) {
-            let shard_chunk = &shard[block_index];
-            if skip >= shard_chunk.len() {
-                skip -= shard_chunk.len();
-                continue;
-            }
-
-            let available = &shard_chunk[skip..];
-            skip = 0;
-            let take = available.len().min(write_left);
-            block.extend_from_slice(&available[..take]);
-            write_left -= take;
-
-            if write_left == 0 {
-                break;
-            }
-        }
-
-        result.push(IoChunk::Shared(block.freeze()));
-    }
-
-    result
-}
-
-fn bench_direct_chunk_path(c: &mut Criterion) {
-    let runtime = Runtime::new().expect("tokio runtime");
-    let mut group = c.benchmark_group("direct_chunk_path");
-
-    for case in bench_cases() {
-        let shard_bytes = build_shard_bytes(&case);
-        group.throughput(Throughput::Bytes(case.length as u64));
-
-        group.bench_with_input(BenchmarkId::new("slice_forwarding", case.name), &case, |b, case| {
-            b.iter(|| {
-                let streams = build_mapped_streams(&shard_bytes);
-                let chunks = runtime
-                    .block_on(collect_direct_data_shard_chunks_for_benchmark(
-                        streams,
-                        case.data_shards,
-                        case.block_size,
-                        case.blocks * case.block_size,
-                        false,
-                        case.offset,
-                        case.length,
-                    ))
-                    .expect("collect direct chunks");
-                black_box(chunks);
-            });
-        });
-
-        group.bench_with_input(BenchmarkId::new("assembled_copy", case.name), &case, |b, case| {
-            b.iter(|| {
-                let chunks = collect_old_assembly(&shard_bytes, case.data_shards, case.block_size, case.offset, case.length);
-                black_box(chunks);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-fn bench_ecstore_get_object_chunks(c: &mut Criterion) {
-    let runtime = Runtime::new().expect("tokio runtime");
-    let mut group = c.benchmark_group("ecstore_get_object_chunks");
-    group.sample_size(10);
-
-    for case in ecstore_bench_cases() {
-        let env = runtime.block_on(build_ecstore_bench_env(&case));
-        let (copy_mode, total_len, _) = runtime.block_on(run_ecstore_get_object_chunks_bench(&env));
-        assert_eq!(copy_mode, env.expected_copy_mode);
-        assert_eq!(total_len, env.expected_len);
-
-        group.throughput(Throughput::Bytes(env.expected_len as u64));
-        group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
-            b.iter(|| {
-                let result = runtime.block_on(run_ecstore_get_object_chunks_bench(env));
-                black_box(result);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-criterion_group!(benches, bench_direct_chunk_path, bench_ecstore_get_object_chunks);
-criterion_main!(benches);

crates/ecstore/benches/reconstructed_chunk_benchmark.rs

@@ -1,393 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use futures_util::StreamExt;
-use http::HeaderMap;
-use rustfs_ecstore::bucket::metadata_sys;
-use rustfs_ecstore::disk::endpoint::Endpoint;
-use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
-use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
-use rustfs_ecstore::store::{ECStore, init_local_disks};
-use rustfs_ecstore::store_api::{
-    BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions,
-    MultipartOperations, ObjectOperations, ObjectOptions,
-};
-use std::hint::black_box;
-use std::net::SocketAddr;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU16, Ordering};
-use tempfile::TempDir;
-use tokio::runtime::Runtime;
-use tokio_util::sync::CancellationToken;
-
-#[derive(Clone)]
-enum ReconstructedReadSpec {
-    PartNumber {
-        part_number: usize,
-        missing_part_name: &'static str,
-    },
-    Range {
-        start: u64,
-        end: u64,
-        missing_part_name: &'static str,
-    },
-}
-
-#[derive(Clone)]
-struct ReconstructedBenchCase {
-    object_id: &'static str,
-    name: &'static str,
-    read_spec: ReconstructedReadSpec,
-}
-
-struct ReconstructedBenchEnv {
-    store: Arc<ECStore>,
-    bucket: String,
-    key: String,
-    range: HTTPRangeSpec,
-    opts: ObjectOptions,
-    expected_len: usize,
-}
-
-struct ReconstructedBenchSuite {
-    _temp_dir: TempDir,
-    store: Arc<ECStore>,
-    disk_paths: Vec<PathBuf>,
-}
-
-const MULTIPART_PART_ONE_LEN: usize = 5 * 1024 * 1024;
-const MULTIPART_PART_TWO_LEN: usize = 5 * 1024 * 1024 + 137;
-const MULTIPART_PART_THREE_LEN: usize = 1024 * 1024 + 77;
-
-fn reconstructed_bench_cases() -> [ReconstructedBenchCase; 4] {
-    [
-        ReconstructedBenchCase {
-            object_id: "mp-part2",
-            name: "multi_disk_missing_shard_multipart_part2",
-            read_spec: ReconstructedReadSpec::PartNumber {
-                part_number: 2,
-                missing_part_name: "part.2",
-            },
-        },
-        ReconstructedBenchCase {
-            object_id: "mp-cross-range",
-            name: "multi_disk_missing_shard_multipart_cross_part_range",
-            read_spec: ReconstructedReadSpec::Range {
-                start: (MULTIPART_PART_ONE_LEN - 32 * 1024) as u64,
-                end: (MULTIPART_PART_ONE_LEN + 96 * 1024) as u64,
-                missing_part_name: "part.2",
-            },
-        },
-        ReconstructedBenchCase {
-            object_id: "mp-part3",
-            name: "multi_disk_missing_shard_multipart_part3",
-            read_spec: ReconstructedReadSpec::PartNumber {
-                part_number: 3,
-                missing_part_name: "part.3",
-            },
-        },
-        ReconstructedBenchCase {
-            object_id: "mp-cross-final-range",
-            name: "multi_disk_missing_shard_multipart_cross_final_part_range",
-            read_spec: ReconstructedReadSpec::Range {
-                start: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN - 32 * 1024) as u64,
-                end: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN + 96 * 1024) as u64,
-                missing_part_name: "part.3",
-            },
-        },
-    ]
-}
-
-fn next_loopback_addr() -> SocketAddr {
-    static NEXT_PORT: AtomicU16 = AtomicU16::new(39113);
-    let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
-    SocketAddr::from(([127, 0, 0, 1], port))
-}
-
-fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
-    HTTPRangeSpec {
-        is_suffix_length: range.is_suffix_length,
-        start: range.start,
-        end: range.end,
-    }
-}
-
-fn build_endpoint_pools(paths: &[PathBuf]) -> EndpointServerPools {
-    let mut endpoints = Vec::with_capacity(paths.len());
-    for (idx, disk_path) in paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(idx);
-        endpoints.push(endpoint);
-    }
-
-    EndpointServerPools(vec![PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: paths.len(),
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "bench".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    }])
-}
-
-fn find_part_files(root: &Path, part_name: &str, out: &mut Vec<PathBuf>) {
-    let Ok(entries) = std::fs::read_dir(root) else {
-        return;
-    };
-    for entry in entries.flatten() {
-        let path = entry.path();
-        if path.is_dir() {
-            find_part_files(&path, part_name, out);
-            continue;
-        }
-
-        if path.file_name().and_then(|name| name.to_str()) == Some(part_name) {
-            out.push(path);
-        }
-    }
-}
-
-async fn remove_part_files(root: &Path, part_name: &str) -> Vec<(PathBuf, Vec<u8>)> {
-    let mut paths = Vec::new();
-    find_part_files(root, part_name, &mut paths);
-
-    let mut removed = Vec::with_capacity(paths.len());
-    for path in paths {
-        let content = tokio::fs::read(&path).await.expect("read part file before removal");
-        tokio::fs::remove_file(&path).await.expect("remove part file");
-        removed.push((path, content));
-    }
-
-    removed
-}
-
-async fn restore_part_files(files: Vec<(PathBuf, Vec<u8>)>) {
-    for (path, content) in files {
-        if let Some(parent) = path.parent() {
-            tokio::fs::create_dir_all(parent)
-                .await
-                .expect("ensure parent for part restore");
-        }
-        tokio::fs::write(&path, content).await.expect("restore part file");
-    }
-}
-
-async fn run_reconstructed_get_object_chunks(
-    store: &Arc<ECStore>,
-    bucket: &str,
-    key: &str,
-    range: &HTTPRangeSpec,
-    opts: &ObjectOptions,
-) -> (GetObjectChunkCopyMode, usize, usize) {
-    let mut result = store
-        .get_object_chunks(bucket, key, Some(clone_range_spec(range)), HeaderMap::new(), opts)
-        .await
-        .expect("get object chunks");
-
-    let copy_mode = result.copy_mode;
-    let mut total_len = 0usize;
-    let mut chunk_count = 0usize;
-    while let Some(chunk) = result.stream.next().await {
-        let chunk = chunk.expect("chunk");
-        total_len += chunk.len();
-        chunk_count += 1;
-    }
-
-    (copy_mode, total_len, chunk_count)
-}
-
-async fn create_multipart_object(store: &Arc<ECStore>, bucket: &str, key: &str, parts: &[Vec<u8>]) -> usize {
-    let upload = store
-        .new_multipart_upload(bucket, key, &ObjectOptions::default())
-        .await
-        .expect("new multipart upload");
-
-    let mut completed_parts = Vec::with_capacity(parts.len());
-    for (idx, part) in parts.iter().enumerate() {
-        let mut reader = ChunkNativePutData::from_vec(part.clone());
-        let part_info = store
-            .put_object_part(bucket, key, &upload.upload_id, idx + 1, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("put object part");
-        completed_parts.push(CompletePart {
-            part_num: idx + 1,
-            etag: part_info.etag,
-            ..Default::default()
-        });
-    }
-
-    store
-        .clone()
-        .complete_multipart_upload(bucket, key, &upload.upload_id, completed_parts, &ObjectOptions::default())
-        .await
-        .expect("complete multipart upload");
-
-    parts.iter().map(Vec::len).sum()
-}
-
-async fn build_reconstructed_bench_suite() -> ReconstructedBenchSuite {
-    let temp_dir = tempfile::tempdir().expect("tempdir");
-    let disk_paths: Vec<_> = (1..=4).map(|idx| temp_dir.path().join(format!("disk{idx}"))).collect();
-    for disk_path in &disk_paths {
-        tokio::fs::create_dir_all(disk_path).await.expect("create disk dir");
-    }
-
-    let endpoint_pools = build_endpoint_pools(&disk_paths);
-    GLOBAL_LOCAL_DISK_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
-    init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
-
-    let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
-        .await
-        .expect("create ecstore");
-
-    let buckets = store
-        .list_bucket(&BucketOptions {
-            no_metadata: true,
-            ..Default::default()
-        })
-        .await
-        .expect("list buckets")
-        .into_iter()
-        .map(|bucket| bucket.name)
-        .collect();
-    metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
-
-    ReconstructedBenchSuite {
-        _temp_dir: temp_dir,
-        store,
-        disk_paths,
-    }
-}
-
-async fn build_reconstructed_bench_env(suite: &ReconstructedBenchSuite, case: &ReconstructedBenchCase) -> ReconstructedBenchEnv {
-    let bucket = format!("bench-r-{}", case.object_id);
-    let key = format!("objects/{}.bin", case.object_id);
-    suite
-        .store
-        .make_bucket(
-            &bucket,
-            &MakeBucketOptions {
-                versioning_enabled: true,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("make bucket");
-
-    let part_one: Vec<u8> = (0..MULTIPART_PART_ONE_LEN).map(|idx| (idx % 251) as u8).collect();
-    let part_two: Vec<u8> = (0..MULTIPART_PART_TWO_LEN).map(|idx| ((idx + 11) % 251) as u8).collect();
-    let part_three: Vec<u8> = (0..MULTIPART_PART_THREE_LEN).map(|idx| ((idx + 29) % 251) as u8).collect();
-    let payload_len = create_multipart_object(&suite.store, &bucket, &key, &[part_one, part_two, part_three]).await;
-    let info = suite
-        .store
-        .get_object_info(&bucket, &key, &ObjectOptions::default())
-        .await
-        .expect("get object info");
-    let (range, opts, missing_part_name) = match case.read_spec.clone() {
-        ReconstructedReadSpec::PartNumber {
-            part_number,
-            missing_part_name,
-        } => {
-            let range = HTTPRangeSpec::from_object_info(&info, part_number).expect("part_number range");
-            let opts = ObjectOptions {
-                part_number: Some(part_number),
-                ..Default::default()
-            };
-            (range, opts, missing_part_name)
-        }
-        ReconstructedReadSpec::Range {
-            start,
-            end,
-            missing_part_name,
-        } => (
-            HTTPRangeSpec {
-                is_suffix_length: false,
-                start: start as i64,
-                end: end as i64,
-            },
-            ObjectOptions::default(),
-            missing_part_name,
-        ),
-    };
-
-    let (_, expected_len) = range.get_offset_length(payload_len as i64).expect("range length");
-    let mut selected_reconstructed = false;
-    for disk_path in &suite.disk_paths {
-        let object_root = disk_path
-            .join(&bucket)
-            .join("objects")
-            .join(format!("{}.bin", case.object_id));
-        let removed_parts = remove_part_files(&object_root, missing_part_name).await;
-        if removed_parts.is_empty() {
-            continue;
-        }
-
-        let (copy_mode, total_len, _) = run_reconstructed_get_object_chunks(&suite.store, &bucket, &key, &range, &opts).await;
-        if copy_mode == GetObjectChunkCopyMode::Reconstructed && total_len == expected_len as usize {
-            selected_reconstructed = true;
-            break;
-        }
-
-        restore_part_files(removed_parts).await;
-    }
-    assert!(
-        selected_reconstructed,
-        "failed to select a missing shard placement that triggers reconstructed copy mode for benchmark case {}",
-        case.name
-    );
-
-    ReconstructedBenchEnv {
-        store: suite.store.clone(),
-        bucket,
-        key,
-        range: clone_range_spec(&range),
-        opts,
-        expected_len: expected_len as usize,
-    }
-}
-
-async fn run_reconstructed_get_object_chunks_bench(env: &ReconstructedBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
-    run_reconstructed_get_object_chunks(&env.store, &env.bucket, &env.key, &env.range, &env.opts).await
-}
-
-fn bench_reconstructed_chunk_path(c: &mut Criterion) {
-    let runtime = Runtime::new().expect("tokio runtime");
-    let suite = runtime.block_on(build_reconstructed_bench_suite());
-    let mut group = c.benchmark_group("reconstructed_chunk_path");
-    group.sample_size(10);
-    for case in reconstructed_bench_cases() {
-        let env = runtime.block_on(build_reconstructed_bench_env(&suite, &case));
-        let (copy_mode, total_len, _) = runtime.block_on(run_reconstructed_get_object_chunks_bench(&env));
-        assert_eq!(copy_mode, GetObjectChunkCopyMode::Reconstructed);
-        assert_eq!(total_len, env.expected_len);
-
-        group.throughput(Throughput::Bytes(env.expected_len as u64));
-        group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
-            b.iter(|| {
-                let result = runtime.block_on(run_reconstructed_get_object_chunks_bench(env));
-                black_box(result);
-            });
-        });
-    }
-    group.finish();
-}
-
-criterion_group!(benches, bench_reconstructed_chunk_path);
-criterion_main!(benches);

crates/ecstore/run_benchmarks.sh

@@ -119,16 +119,6 @@ run_large_data_test() {
     print_success "Large-dataset tests completed"
 }
 
-# Run direct chunk path benchmarks
-run_direct_chunk_benchmark() {
-    print_info "📦 Starting direct chunk path benchmarks..."
-    echo "================================================"
-
-    cargo bench --bench direct_chunk_benchmark
-
-    print_success "Direct chunk path benchmarks completed"
-}
-
 # Generate comparison report
 generate_comparison_report() {
     print_info "📊 Generating performance report..."
@@ -178,7 +168,6 @@ show_help() {
     echo "  full         Run the full benchmark suite"
     echo "  performance  Run detailed performance tests"
     echo "  simd         Run the SIMD-only tests"
-    echo "  direct       Run the direct chunk path benchmarks"
     echo "  large        Run large-dataset tests"
     echo "  clean        Remove previous results"
     echo "  help         Show this help message"
@@ -188,7 +177,6 @@ show_help() {
     echo "  $0 performance        # Detailed performance test"
     echo "  $0 full              # Full benchmark suite"
     echo "  $0 simd              # SIMD-only benchmark"
-    echo "  $0 direct            # Direct chunk path benchmark"
     echo "  $0 large             # Large-dataset benchmark"
     echo ""
     echo "Features:"
@@ -252,11 +240,6 @@ main() {
             run_simd_benchmark
             generate_comparison_report
             ;;
-        "direct")
-            cleanup
-            run_direct_chunk_benchmark
-            generate_comparison_report
-            ;;
         "large")
             cleanup
             run_large_data_test

crates/ecstore/src/admin_server_info.rs

@@ -162,8 +162,9 @@ pub async fn get_local_server_property() -> ServerProperties {
 
     let mut props = ServerProperties {
         endpoint: addr,
-        uptime: SystemTime::now()
-            .duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
+        uptime: GLOBAL_BOOT_TIME
+            .get()
+            .and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok())
             .unwrap_or_default()
             .as_secs(),
         network,

crates/ecstore/src/batch_processor.rs

@@ -87,25 +87,70 @@ impl AsyncBatchProcessor {
         T: Send + 'static,
         F: Future<Output = Result<T>> + Send + 'static,
     {
-        let results = self.execute_batch(tasks).await;
-        let mut successes = Vec::new();
+        if required_successes == 0 {
+            return Ok(Vec::new());
+        }
 
-        for value in results.into_iter().flatten() {
-            successes.push(value);
-            if successes.len() >= required_successes {
-                return Ok(successes);
+        if tasks.is_empty() {
+            return Err(Error::other(format!(
+                "Insufficient successful results: got 0, needed {required_successes}"
+            )));
+        }
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
+        let mut join_set = JoinSet::new();
+        let mut successes = Vec::new();
+        let mut pending_tasks = tasks.len();
+        let mut first_error = None;
+
+        for task in tasks {
+            let sem = semaphore.clone();
+            join_set.spawn(async move {
+                let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
+                task.await
+            });
+        }
+
+        while let Some(join_result) = join_set.join_next().await {
+            pending_tasks = pending_tasks.saturating_sub(1);
+
+            match join_result {
+                Ok(Ok(value)) => {
+                    successes.push(value);
+                    if successes.len() >= required_successes {
+                        return Ok(successes);
+                    }
+                }
+                Ok(Err(err)) => {
+                    if first_error.is_none() {
+                        first_error = Some(err);
+                    }
+                }
+                Err(join_error) => {
+                    if first_error.is_none() {
+                        first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
+                    }
+                }
+            }
+
+            if successes.len() + pending_tasks < required_successes {
+                return Err(first_error.unwrap_or_else(|| {
+                    Error::other(format!(
+                        "Insufficient successful results: got {}, needed {}",
+                        successes.len(),
+                        required_successes
+                    ))
+                }));
             }
         }
 
-        if successes.len() >= required_successes {
-            Ok(successes)
-        } else {
-            Err(Error::other(format!(
+        Err(first_error.unwrap_or_else(|| {
+            Error::other(format!(
                 "Insufficient successful results: got {}, needed {}",
                 successes.len(),
                 required_successes
-            )))
-        }
+            ))
+        }))
     }
 }
 
@@ -228,4 +273,52 @@ mod tests {
         let successes = results.unwrap();
         assert!(successes.len() >= 2);
     }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_returns_before_slow_tail() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))]
+            .into_iter()
+            .map(|(delay_ms, outcome)| async move {
+                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                outcome
+            })
+            .collect();
+
+        let results = processor
+            .execute_batch_with_quorum(tasks, 2)
+            .await
+            .expect("quorum should succeed");
+        assert_eq!(results.len(), 2);
+        assert!(started.elapsed() < Duration::from_millis(100));
+    }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok(1_i32)),
+            (15, Err(Error::other("first failure"))),
+            (20, Err(Error::other("second failure"))),
+            (250, Ok(4)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let err = processor
+            .execute_batch_with_quorum(tasks, 3)
+            .await
+            .expect_err("quorum should fail once it becomes impossible");
+
+        assert!(err.to_string().contains("first failure"));
+        assert!(started.elapsed() < Duration::from_millis(120));
+    }
 }

crates/ecstore/src/bitrot.rs

@@ -14,328 +14,13 @@
 
 use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError};
 use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
-use crate::store_api::{GetObjectChunkCopyMode, GetObjectChunkPath, GetObjectChunkResult};
-use bytes::{Bytes, BytesMut};
-use futures_util::{StreamExt, stream};
-use rustfs_io_core::{BoxChunkStream, IoChunk};
+use bytes::Bytes;
 use rustfs_utils::HashAlgorithm;
-use std::collections::VecDeque;
 use std::io::Cursor;
 use std::time::Instant;
 use tokio::io::AsyncRead;
 use tracing::debug;
 
-const BITROT_READ_OPERATION: &str = "bitrot_read";
-
-fn classify_chunk_copy_mode(source_direct: bool, copied: bool) -> GetObjectChunkCopyMode {
-    if copied {
-        GetObjectChunkCopyMode::SingleCopy
-    } else if source_direct {
-        GetObjectChunkCopyMode::TrueZeroCopy
-    } else {
-        GetObjectChunkCopyMode::SharedBytes
-    }
-}
-
-struct ChunkSpan {
-    bytes: Bytes,
-    chunk: IoChunk,
-    copied: bool,
-}
-
-fn take_contiguous_chunk_span(chunk: &IoChunk, offset: usize, len: usize) -> std::io::Result<ChunkSpan> {
-    match chunk {
-        IoChunk::Shared(bytes) => {
-            let bytes = bytes.slice(offset..offset + len);
-            Ok(ChunkSpan {
-                bytes: bytes.clone(),
-                chunk: IoChunk::Shared(bytes),
-                copied: false,
-            })
-        }
-        IoChunk::Mapped(mapped) => {
-            let chunk = IoChunk::Mapped(mapped.slice(offset, len)?);
-            let bytes = chunk.as_bytes();
-            Ok(ChunkSpan {
-                bytes,
-                chunk,
-                copied: false,
-            })
-        }
-        IoChunk::Pooled(pooled) => {
-            let chunk = IoChunk::Pooled(pooled.slice(offset, len)?);
-            let bytes = chunk.as_bytes();
-            Ok(ChunkSpan {
-                bytes,
-                chunk,
-                copied: false,
-            })
-        }
-    }
-}
-
-struct BitrotChunkSource {
-    source_stream: BoxChunkStream,
-    source_chunks: VecDeque<IoChunk>,
-    source_chunk_offset: usize,
-    source_buffered_bytes: usize,
-    source_done: bool,
-}
-
-struct BitrotChunkStreamState {
-    source: BitrotChunkSource,
-    decoded_remaining: usize,
-    trim_prefix: usize,
-    output_remaining: usize,
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-}
-
-struct ChunkCursor<'a> {
-    chunks: &'a [IoChunk],
-    chunk_index: usize,
-    chunk_offset: usize,
-    consumed: usize,
-    total_len: usize,
-}
-
-impl<'a> ChunkCursor<'a> {
-    fn new(chunks: &'a [IoChunk]) -> Self {
-        Self {
-            chunks,
-            chunk_index: 0,
-            chunk_offset: 0,
-            consumed: 0,
-            total_len: chunks.iter().map(IoChunk::len).sum(),
-        }
-    }
-
-    fn remaining(&self) -> usize {
-        self.total_len.saturating_sub(self.consumed)
-    }
-
-    fn skip_empty_chunks(&mut self) {
-        while let Some(chunk) = self.chunks.get(self.chunk_index) {
-            if self.chunk_offset < chunk.len() {
-                break;
-            }
-            self.chunk_index += 1;
-            self.chunk_offset = 0;
-        }
-    }
-
-    fn advance(&mut self, len: usize) {
-        self.consumed += len;
-        self.chunk_offset += len;
-        self.skip_empty_chunks();
-    }
-
-    fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
-        self.skip_empty_chunks();
-        if self.remaining() < len {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-        }
-
-        let Some(chunk) = self.chunks.get(self.chunk_index) else {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
-        };
-        let available = chunk.len().saturating_sub(self.chunk_offset);
-
-        if len <= available {
-            let span = take_contiguous_chunk_span(chunk, self.chunk_offset, len)?;
-            self.advance(len);
-            return Ok(span);
-        }
-
-        let mut aggregate = BytesMut::with_capacity(len);
-        let mut remaining = len;
-        while remaining > 0 {
-            self.skip_empty_chunks();
-            let Some(chunk) = self.chunks.get(self.chunk_index) else {
-                return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-            };
-            let available = chunk.len().saturating_sub(self.chunk_offset);
-            let take = available.min(remaining);
-            aggregate.extend_from_slice(&chunk.as_bytes()[self.chunk_offset..self.chunk_offset + take]);
-            self.advance(take);
-            remaining -= take;
-        }
-
-        let bytes = aggregate.freeze();
-        Ok(ChunkSpan {
-            bytes: bytes.clone(),
-            chunk: IoChunk::Shared(bytes),
-            copied: true,
-        })
-    }
-}
-
-impl BitrotChunkSource {
-    fn new(source_stream: BoxChunkStream, source_chunks: VecDeque<IoChunk>, source_done: bool) -> Self {
-        let source_buffered_bytes = source_chunks.iter().map(IoChunk::len).sum();
-        Self {
-            source_stream,
-            source_chunks,
-            source_chunk_offset: 0,
-            source_buffered_bytes,
-            source_done,
-        }
-    }
-
-    fn skip_empty_chunks(&mut self) {
-        while let Some(chunk) = self.source_chunks.front() {
-            if self.source_chunk_offset < chunk.len() {
-                break;
-            }
-            self.source_chunks.pop_front();
-            self.source_chunk_offset = 0;
-        }
-    }
-
-    async fn fill(&mut self, min_bytes: usize) -> std::io::Result<()> {
-        while self.source_buffered_bytes < min_bytes && !self.source_done {
-            match self.source_stream.next().await {
-                Some(Ok(chunk)) => {
-                    self.source_buffered_bytes += chunk.len();
-                    self.source_chunks.push_back(chunk);
-                }
-                Some(Err(err)) => return Err(err),
-                None => self.source_done = true,
-            }
-        }
-
-        if self.source_buffered_bytes < min_bytes {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-        }
-
-        Ok(())
-    }
-
-    fn advance(&mut self, len: usize) {
-        self.source_buffered_bytes = self.source_buffered_bytes.saturating_sub(len);
-        self.source_chunk_offset += len;
-        self.skip_empty_chunks();
-    }
-
-    fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
-        self.skip_empty_chunks();
-        if self.source_buffered_bytes < len {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-        }
-
-        let Some(chunk) = self.source_chunks.front() else {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
-        };
-        let available = chunk.len().saturating_sub(self.source_chunk_offset);
-
-        if len <= available {
-            let span = take_contiguous_chunk_span(chunk, self.source_chunk_offset, len)?;
-            self.advance(len);
-            return Ok(span);
-        }
-
-        let mut aggregate = BytesMut::with_capacity(len);
-        let mut remaining = len;
-        while remaining > 0 {
-            self.skip_empty_chunks();
-            let Some(chunk) = self.source_chunks.front() else {
-                return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-            };
-            let available = chunk.len().saturating_sub(self.source_chunk_offset);
-            let take = available.min(remaining);
-            aggregate.extend_from_slice(&chunk.as_bytes()[self.source_chunk_offset..self.source_chunk_offset + take]);
-            self.advance(take);
-            remaining -= take;
-        }
-
-        let bytes = aggregate.freeze();
-        Ok(ChunkSpan {
-            bytes: bytes.clone(),
-            chunk: IoChunk::Shared(bytes),
-            copied: true,
-        })
-    }
-}
-
-impl BitrotChunkStreamState {
-    #[allow(clippy::too_many_arguments)]
-    fn new(
-        source_stream: BoxChunkStream,
-        source_chunks: VecDeque<IoChunk>,
-        source_done: bool,
-        decoded_remaining: usize,
-        trim_prefix: usize,
-        output_remaining: usize,
-        shard_size: usize,
-        checksum_algo: HashAlgorithm,
-        skip_verify: bool,
-    ) -> Self {
-        Self {
-            source: BitrotChunkSource::new(source_stream, source_chunks, source_done),
-            decoded_remaining,
-            trim_prefix,
-            output_remaining,
-            shard_size,
-            checksum_algo,
-            skip_verify,
-        }
-    }
-
-    fn hash_size(&self) -> usize {
-        self.checksum_algo.size()
-    }
-
-    async fn next_verified_chunk(&mut self) -> std::io::Result<Option<IoChunk>> {
-        let hash_size = self.hash_size();
-
-        while self.output_remaining > 0 && self.decoded_remaining > 0 {
-            let data_len = self.shard_size.min(self.decoded_remaining);
-
-            let expected_hash = if hash_size > 0 {
-                self.source.fill(hash_size).await?;
-                Some(self.source.take_span(hash_size)?)
-            } else {
-                None
-            };
-
-            self.source.fill(data_len).await?;
-            let data_span = self.source.take_span(data_len)?;
-
-            if let Some(expected_hash) = expected_hash
-                && !self.skip_verify
-                && self.checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref()
-            {
-                return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
-            }
-
-            self.decoded_remaining -= data_len;
-
-            if self.trim_prefix >= data_len {
-                self.trim_prefix -= data_len;
-                continue;
-            }
-
-            let start = self.trim_prefix;
-            self.trim_prefix = 0;
-            let take = (data_len - start).min(self.output_remaining);
-            self.output_remaining -= take;
-
-            let chunk = if start == 0 && take == data_len {
-                data_span.chunk
-            } else {
-                data_span.chunk.slice(start, take)?
-            };
-
-            if !chunk.is_empty() {
-                return Ok(Some(chunk));
-            }
-        }
-
-        Ok(None)
-    }
-}
-
 /// Create a BitrotReader from either inline data or disk file stream
 ///
 /// # Parameters
@@ -379,40 +64,21 @@ pub async fn create_bitrot_reader(
         Ok(Some(reader))
     } else if let Some(disk) = disk {
         // Read from disk
-        if use_zero_copy {
-            if !disk.is_local() {
-                rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
-                rustfs_io_metrics::record_io_fallback(
-                    rustfs_io_metrics::IoStage::ReadSetup,
-                    rustfs_io_metrics::FallbackReason::NonLocalBackend,
-                );
-
-                let rd = disk.read_file_stream(bucket, path, offset, length).await?;
-                let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
-                return Ok(Some(reader));
-            }
-
+        if use_zero_copy && disk.is_local() {
             // Try zero-copy read first (uses mmap on Unix)
             let start = Instant::now();
             match disk.read_file_zero_copy(bucket, path, offset, length).await {
                 Ok(bytes) => {
                     let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
 
-                    rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Fast);
-                    // `read_file_zero_copy()` returns a shared `Bytes` view, but it may still
-                    // internally aggregate multiple chunk windows. The exact chunk-native copy
-                    // mode is only preserved by `create_bitrot_chunk_stream()`.
-                    rustfs_io_metrics::record_io_copy_mode(
-                        BITROT_READ_OPERATION,
-                        rustfs_io_metrics::CopyMode::SharedBytes,
-                        bytes.len(),
-                    );
+                    // Record zero-copy metrics
+                    rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
 
+                    // Log successful zero-copy read
                     debug!(
                         size = bytes.len(),
-                        duration_ms,
                         path = %path,
-                        "bitrot_fast_read_success"
+                        "zero_copy_read_success"
                     );
 
                     // Wrap Bytes in Cursor for AsyncRead
@@ -427,16 +93,14 @@ pub async fn create_bitrot_reader(
                     Ok(Some(reader))
                 }
                 Err(e) => {
-                    rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
-                    rustfs_io_metrics::record_io_fallback(
-                        rustfs_io_metrics::IoStage::ReadSetup,
-                        rustfs_io_metrics::FallbackReason::Unknown,
-                    );
+                    // Record zero-copy fallback
+                    rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e));
 
+                    // Log zero-copy fallback
                     debug!(
-                        reason = %e,
+                        reason = %format!("{:?}", e),
                         path = %path,
-                        "bitrot_fast_read_fallback"
+                        "zero_copy_fallback"
                     );
 
                     // Fall back to regular stream read on error
@@ -453,7 +117,6 @@ pub async fn create_bitrot_reader(
                 }
             }
         } else {
-            rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
             // Use regular stream read
             match disk.read_file_stream(bucket, path, offset, length).await {
                 Ok(rd) => {
@@ -469,198 +132,6 @@ pub async fn create_bitrot_reader(
     }
 }
 
-/// Create a chunk stream from bitrot-encoded data, preserving source chunk provenance when possible.
-#[allow(clippy::too_many_arguments)]
-pub async fn create_bitrot_chunk_stream(
-    inline_data: Option<&[u8]>,
-    disk: Option<&DiskStore>,
-    bucket: &str,
-    path: &str,
-    offset: usize,
-    length: usize,
-    total_data_size: usize,
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-    use_zero_copy: bool,
-) -> disk::error::Result<Option<GetObjectChunkResult>> {
-    let fetch_start = (offset / shard_size) * shard_size;
-    let fetch_end = (offset + length).div_ceil(shard_size) * shard_size;
-    let fetch_end = fetch_end.min(total_data_size);
-    let fetch_length = fetch_end.saturating_sub(fetch_start);
-    let trim_prefix = offset.saturating_sub(fetch_start);
-    let hash_size = checksum_algo.size();
-    let encoded_length = fetch_length.div_ceil(shard_size) * hash_size + fetch_length;
-    let encoded_offset = fetch_start.div_ceil(shard_size) * hash_size + fetch_start;
-
-    let mut source_done = false;
-    let (source_stream, mut prefetched_chunks, source_direct) = if let Some(data) = inline_data {
-        source_done = true;
-        let mut chunks = VecDeque::new();
-        chunks.push_back(IoChunk::Shared(
-            Bytes::copy_from_slice(data).slice(encoded_offset..encoded_offset + encoded_length),
-        ));
-        let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
-        (source_stream, chunks, false)
-    } else if let Some(disk) = disk {
-        if use_zero_copy {
-            let mut source_stream = disk.read_file_chunks(bucket, path, encoded_offset, encoded_length).await?;
-            let mut prefetched_chunks = VecDeque::new();
-            let mut direct = true;
-            while prefetched_chunks.len() < 2 {
-                let Some(chunk) = source_stream.next().await else {
-                    source_done = true;
-                    break;
-                };
-                let chunk = chunk?;
-                direct &= matches!(chunk, IoChunk::Mapped(_));
-                prefetched_chunks.push_back(chunk);
-            }
-            (source_stream, prefetched_chunks, direct)
-        } else {
-            source_done = true;
-            let bytes = disk.read_file_zero_copy(bucket, path, encoded_offset, encoded_length).await?;
-            let mut chunks = VecDeque::new();
-            chunks.push_back(IoChunk::Shared(bytes));
-            let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
-            (source_stream, chunks, false)
-        }
-    } else {
-        return Ok(None);
-    };
-
-    let copied = predicted_stream_copy(encoded_length, shard_size, checksum_algo.size(), &prefetched_chunks, source_done);
-    let state = BitrotChunkStreamState::new(
-        source_stream,
-        std::mem::take(&mut prefetched_chunks),
-        source_done,
-        fetch_length,
-        trim_prefix,
-        length,
-        shard_size,
-        checksum_algo,
-        skip_verify,
-    );
-    let stream = stream::unfold(Some(state), |state| async move {
-        let mut state = match state {
-            Some(state) => state,
-            None => return None,
-        };
-
-        match state.next_verified_chunk().await {
-            Ok(Some(chunk)) => {
-                let next_state = if state.output_remaining == 0 { None } else { Some(state) };
-                Some((Ok::<IoChunk, std::io::Error>(chunk), next_state))
-            }
-            Ok(None) => None,
-            Err(err) => Some((Err(err), None)),
-        }
-    });
-    Ok(Some(GetObjectChunkResult {
-        stream: Box::pin(stream),
-        path: GetObjectChunkPath::Direct,
-        copy_mode: classify_chunk_copy_mode(source_direct, copied),
-    }))
-}
-
-fn predicted_stream_copy(
-    encoded_length: usize,
-    shard_size: usize,
-    hash_size: usize,
-    prefetched_chunks: &VecDeque<IoChunk>,
-    source_done: bool,
-) -> bool {
-    if prefetched_chunks.is_empty() {
-        return false;
-    }
-
-    if source_done && prefetched_chunks.len() == 1 {
-        return false;
-    }
-
-    let full_frame_len = hash_size + shard_size;
-    if full_frame_len == 0 {
-        return false;
-    }
-
-    let first_window_len = prefetched_chunks.front().map(IoChunk::len).unwrap_or(encoded_length);
-    encoded_length > first_window_len && !first_window_len.is_multiple_of(full_frame_len)
-}
-
-fn trim_chunk_vec(chunks: Vec<IoChunk>, offset: usize, length: usize) -> std::io::Result<Vec<IoChunk>> {
-    let mut skip = offset;
-    let mut remaining = length;
-    let mut result = Vec::new();
-
-    for chunk in chunks {
-        if remaining == 0 {
-            break;
-        }
-
-        let chunk_len = chunk.len();
-        if skip >= chunk_len {
-            skip -= chunk_len;
-            continue;
-        }
-
-        let start = skip;
-        let take = (chunk_len - start).min(remaining);
-        result.push(chunk.slice(start, take)?);
-        remaining -= take;
-        skip = 0;
-    }
-
-    Ok(result)
-}
-
-fn decode_bitrot_chunk_source(
-    source_chunks: &[IoChunk],
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-) -> std::io::Result<(Vec<IoChunk>, bool)> {
-    let hash_size = checksum_algo.size();
-    let mut cursor = ChunkCursor::new(source_chunks);
-    let mut result = Vec::new();
-    let mut copied = false;
-
-    while cursor.remaining() > 0 {
-        let expected_hash = if hash_size > 0 {
-            Some(cursor.take_span(hash_size)?)
-        } else {
-            None
-        };
-
-        let data_len = shard_size.min(cursor.remaining());
-        if data_len == 0 {
-            break;
-        }
-
-        let data_span = cursor.take_span(data_len)?;
-        copied |= data_span.copied;
-        if let Some(expected_hash) = expected_hash {
-            copied |= expected_hash.copied;
-            if !skip_verify && checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref() {
-                return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
-            }
-        }
-
-        result.push(data_span.chunk);
-    }
-
-    Ok((result, copied))
-}
-
-#[doc(hidden)]
-pub fn decode_bitrot_chunk_source_for_bench(
-    source_chunks: &[IoChunk],
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-) -> std::io::Result<(Vec<IoChunk>, bool)> {
-    decode_bitrot_chunk_source(source_chunks, shard_size, checksum_algo, skip_verify)
-}
-
 /// Create a new BitrotWriterWrapper based on the provided parameters
 ///
 /// # Parameters
@@ -705,7 +176,6 @@ pub async fn create_bitrot_writer(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use futures_util::StreamExt;
 
     #[tokio::test]
     async fn test_create_bitrot_reader_with_inline_data() {
@@ -756,246 +226,6 @@ mod tests {
         assert!(result.unwrap().is_some());
     }
 
-    #[tokio::test]
-    async fn test_create_bitrot_chunk_stream_with_inline_data() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::HighwayHash256S;
-        let shard1 = b"abcd";
-        let shard2 = b"ef";
-
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        encoded.extend_from_slice(shard1);
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        encoded.extend_from_slice(shard2);
-
-        let mut stream = create_bitrot_chunk_stream(
-            Some(&encoded),
-            None,
-            "test-bucket",
-            "test-path",
-            0,
-            shard1.len() + shard2.len(),
-            shard1.len() + shard2.len(),
-            shard_size,
-            checksum_algo,
-            false,
-            false,
-        )
-        .await
-        .unwrap()
-        .unwrap()
-        .stream;
-
-        let mut collected = Vec::new();
-        while let Some(chunk) = stream.next().await {
-            collected.extend_from_slice(&chunk.unwrap().as_bytes());
-        }
-
-        assert_eq!(collected, b"abcdef");
-    }
-
-    #[tokio::test]
-    async fn test_create_bitrot_chunk_stream_detects_hash_mismatch() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::HighwayHash256S;
-        let shard = b"abcd";
-
-        let mut encoded = Vec::new();
-        let mut bad_hash = checksum_algo.hash_encode(shard).as_ref().to_vec();
-        bad_hash[0] ^= 0xFF;
-        encoded.extend_from_slice(&bad_hash);
-        encoded.extend_from_slice(shard);
-
-        let result = create_bitrot_chunk_stream(
-            Some(&encoded),
-            None,
-            "test-bucket",
-            "test-path",
-            0,
-            shard.len(),
-            shard.len(),
-            shard_size,
-            checksum_algo,
-            false,
-            false,
-        )
-        .await;
-
-        let mut stream = result.unwrap().unwrap().stream;
-        let err = stream.next().await.unwrap().unwrap_err();
-        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
-        assert!(err.to_string().contains("bitrot hash mismatch"));
-    }
-
-    #[tokio::test]
-    async fn test_create_bitrot_chunk_stream_trims_range_after_decode() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::HighwayHash256S;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        encoded.extend_from_slice(shard1);
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        encoded.extend_from_slice(shard2);
-
-        let mut stream = create_bitrot_chunk_stream(
-            Some(&encoded),
-            None,
-            "test-bucket",
-            "test-path",
-            1,
-            5,
-            shard1.len() + shard2.len(),
-            shard_size,
-            checksum_algo,
-            false,
-            false,
-        )
-        .await
-        .unwrap()
-        .unwrap()
-        .stream;
-
-        let mut collected = Vec::new();
-        while let Some(chunk) = stream.next().await {
-            collected.extend_from_slice(&chunk.unwrap().as_bytes());
-        }
-
-        assert_eq!(collected, b"bcdef");
-    }
-
-    #[test]
-    fn test_decode_bitrot_chunk_source_preserves_aligned_multi_chunk_slices() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let mut encoded_chunk_one = Vec::new();
-        encoded_chunk_one.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        encoded_chunk_one.extend_from_slice(shard1);
-
-        let mut encoded_chunk_two = Vec::new();
-        encoded_chunk_two.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        encoded_chunk_two.extend_from_slice(shard2);
-
-        let source_chunks = vec![
-            IoChunk::Shared(Bytes::from(encoded_chunk_one)),
-            IoChunk::Shared(Bytes::from(encoded_chunk_two)),
-        ];
-        let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
-
-        assert!(!copied, "frame-aligned multi-chunk source should not require aggregate copies");
-        assert_eq!(decoded.len(), 2);
-        assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
-        assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
-    }
-
-    #[test]
-    fn test_decode_bitrot_chunk_source_marks_cross_chunk_frame_as_copied() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let hash1 = checksum_algo.hash_encode(shard1).as_ref().to_vec();
-        let hash2 = checksum_algo.hash_encode(shard2).as_ref().to_vec();
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(&hash1);
-        encoded.extend_from_slice(shard1);
-        encoded.extend_from_slice(&hash2);
-        encoded.extend_from_slice(shard2);
-
-        let split = hash1.len() + 2;
-        let source_chunks = vec![
-            IoChunk::Shared(Bytes::copy_from_slice(&encoded[..split])),
-            IoChunk::Shared(Bytes::copy_from_slice(&encoded[split..])),
-        ];
-        let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
-
-        assert!(copied, "cross-chunk frame should be classified as requiring a copy");
-        assert_eq!(decoded.len(), 2);
-        assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
-        assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
-    }
-
-    #[test]
-    fn test_decode_bitrot_chunk_source_preserves_pooled_single_chunk_slice() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard = b"abcd";
-
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
-        encoded.extend_from_slice(shard);
-
-        let source_chunks = vec![IoChunk::Pooled(rustfs_io_core::PooledChunk::from_vec(encoded))];
-        let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
-
-        assert!(!copied, "single pooled chunk slice should preserve provenance without copy");
-        assert_eq!(decoded.len(), 1);
-        assert!(matches!(&decoded[0], IoChunk::Pooled(_)));
-        assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
-    }
-
-    #[tokio::test]
-    async fn test_bitrot_chunk_source_marks_cross_chunk_take_as_copied() {
-        let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
-            Ok(IoChunk::Shared(Bytes::from_static(b"ab"))),
-            Ok(IoChunk::Shared(Bytes::from_static(b"cd"))),
-        ]));
-        let mut source = BitrotChunkSource::new(source_stream, VecDeque::new(), false);
-
-        source.fill(4).await.expect("source fill should succeed");
-        let span = source.take_span(4).expect("cross-chunk take should succeed");
-
-        assert!(span.copied, "cross-chunk take should be classified as copied");
-        assert_eq!(span.bytes, Bytes::from_static(b"abcd"));
-        assert_eq!(span.chunk.as_bytes(), Bytes::from_static(b"abcd"));
-    }
-
-    #[tokio::test]
-    async fn test_bitrot_chunk_stream_state_yields_verified_prefix_before_later_truncation() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let mut first_frame = Vec::new();
-        first_frame.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        first_frame.extend_from_slice(shard1);
-
-        let mut second_frame_prefix = Vec::new();
-        second_frame_prefix.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        second_frame_prefix.extend_from_slice(&shard2[..2]);
-
-        let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
-            Ok(IoChunk::Shared(Bytes::from(first_frame))),
-            Ok(IoChunk::Shared(Bytes::from(second_frame_prefix))),
-        ]));
-        let mut state = BitrotChunkStreamState::new(
-            source_stream,
-            VecDeque::new(),
-            false,
-            shard1.len() + shard2.len(),
-            0,
-            shard1.len() + shard2.len(),
-            shard_size,
-            checksum_algo,
-            false,
-        );
-
-        let first = state.next_verified_chunk().await.unwrap().unwrap();
-        assert_eq!(first.as_bytes(), Bytes::from_static(b"abcd"));
-
-        let err = state.next_verified_chunk().await.unwrap_err();
-        assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
-        assert!(err.to_string().contains("truncated bitrot chunk source"));
-    }
-
     #[tokio::test]
     async fn test_create_bitrot_reader_with_inline_offset_starts_at_requested_shard() {
         let shard_size = 4;

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -395,8 +395,27 @@ impl BucketTargetSys {
         }
     }
 
-    pub async fn set_target(&self, bucket: &str, target: &BucketTarget, update: bool) -> Result<(), BucketTargetError> {
-        if !target.target_type.is_valid() && !update {
+    pub async fn set_target(
+        &self,
+        bucket: &str,
+        target: &BucketTarget,
+        update: bool,
+    ) -> Result<BucketTargets, BucketTargetError> {
+        self.validate_target(bucket, target).await?;
+
+        let mut bucket_targets = match self.list_bucket_targets(bucket).await {
+            Ok(targets) => targets,
+            Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
+            Err(err) => return Err(err),
+        };
+
+        Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
+
+        Ok(bucket_targets)
+    }
+
+    pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
+        if !target.target_type.is_valid() {
             return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
                 bucket: bucket.to_string(),
             });
@@ -450,52 +469,44 @@ impl BucketTargetSys {
             }
         }
 
-        {
-            let mut targets_map = self.targets_map.write().await;
-            let bucket_targets = targets_map.entry(bucket.to_string()).or_insert_with(Vec::new);
-            let mut found = false;
+        Ok(())
+    }
 
-            for (idx, existing_target) in bucket_targets.iter().enumerate() {
-                if existing_target.target_type.to_string() == target.target_type.to_string() {
-                    if existing_target.arn == target.arn {
-                        if !update {
-                            return Err(BucketTargetError::BucketRemoteAlreadyExists {
-                                bucket: existing_target.target_bucket.clone(),
-                            });
-                        }
-                        bucket_targets[idx] = target.clone();
-                        found = true;
-                        break;
-                    }
-                    if existing_target.endpoint == target.endpoint {
+    fn upsert_target_entry(
+        bucket_targets: &mut Vec<BucketTarget>,
+        target: &BucketTarget,
+        update: bool,
+    ) -> Result<(), BucketTargetError> {
+        let mut found = false;
+
+        for (idx, existing_target) in bucket_targets.iter().enumerate() {
+            if existing_target.target_type.to_string() == target.target_type.to_string() {
+                if existing_target.arn == target.arn {
+                    if !update {
                         return Err(BucketTargetError::BucketRemoteAlreadyExists {
                             bucket: existing_target.target_bucket.clone(),
                         });
                     }
+                    bucket_targets[idx] = target.clone();
+                    found = true;
+                    break;
+                }
+                if existing_target.endpoint == target.endpoint {
+                    return Err(BucketTargetError::BucketRemoteAlreadyExists {
+                        bucket: existing_target.target_bucket.clone(),
+                    });
                 }
             }
-
-            if !found && !update {
-                bucket_targets.push(target.clone());
-            }
         }
 
-        {
-            let mut arn_remotes_map = self.arn_remotes_map.write().await;
-            arn_remotes_map.insert(
-                target.arn.clone(),
-                ArnTarget {
-                    client: Some(Arc::new(target_client)),
-                    last_refresh: OffsetDateTime::now_utc(),
-                },
-            );
+        if !found && !update {
+            bucket_targets.push(target.clone());
         }
 
-        self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
         Ok(())
     }
 
-    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<(), BucketTargetError> {
+    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<BucketTargets, BucketTargetError> {
         if arn_str.is_empty() {
             return Err(BucketTargetError::BucketRemoteArnInvalid {
                 bucket: bucket.to_string(),
@@ -524,33 +535,16 @@ impl BucketTargetSys {
             }
         }
 
-        {
-            let mut targets_map = self.targets_map.write().await;
+        let targets = self.list_bucket_targets(bucket).await?;
+        let new_targets: Vec<BucketTarget> = targets.targets.iter().filter(|t| t.arn != arn_str).cloned().collect();
 
-            let Some(targets) = targets_map.get(bucket) else {
-                return Err(BucketTargetError::BucketRemoteTargetNotFound {
-                    bucket: bucket.to_string(),
-                });
-            };
-
-            let new_targets: Vec<BucketTarget> = targets.iter().filter(|t| t.arn != arn_str).cloned().collect();
-
-            if new_targets.len() == targets.len() {
-                return Err(BucketTargetError::BucketRemoteTargetNotFound {
-                    bucket: bucket.to_string(),
-                });
-            }
-
-            targets_map.insert(bucket.to_string(), new_targets);
+        if new_targets.len() == targets.targets.len() {
+            return Err(BucketTargetError::BucketRemoteTargetNotFound {
+                bucket: bucket.to_string(),
+            });
         }
 
-        {
-            self.arn_remotes_map.write().await.remove(arn_str);
-        }
-
-        self.update_bandwidth_limit(bucket, arn_str, 0);
-
-        Ok(())
+        Ok(BucketTargets { targets: new_targets })
     }
 
     pub async fn mark_refresh_in_progress(&self, bucket: &str, arn: &str) {
@@ -593,7 +587,7 @@ impl BucketTargetSys {
         };
 
         if let Some(cli) = cli {
-            return Some(cli.clone());
+            return Some(cli);
         }
 
         // TODO: spawn a task to reload the target
@@ -603,7 +597,7 @@ impl BucketTargetSys {
 
         if let Some(last_refresh) = last_refresh {
             let now = OffsetDateTime::now_utc();
-            if now - last_refresh > Duration::from_secs(60 * 5) {
+            if now - last_refresh < Duration::from_secs(60 * 5) {
                 return None;
             }
         }
@@ -619,6 +613,16 @@ impl BucketTargetSys {
             }
         };
 
+        let cli = self
+            .arn_remotes_map
+            .read()
+            .await
+            .get(arn)
+            .and_then(|target| target.client.clone());
+        if cli.is_some() {
+            return cli;
+        }
+
         self.inc_arn_errs(bucket, arn).await;
         None
     }
@@ -786,7 +790,10 @@ impl BucketTargetSys {
                         && tgt
                             .credentials
                             .as_ref()
-                            .map(|c| c.access_key == target.credentials.as_ref().unwrap_or(&Credentials::default()).access_key)
+                            .map(|c| {
+                                let default_creds = Credentials::default();
+                                c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
+                            })
                             .unwrap_or(false)
                     {
                         return (tgt.arn.clone(), true);
@@ -894,6 +901,41 @@ pub struct RemoveObjectOptions {
     pub replication_validity_check: bool,
 }
 
+fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOptions) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    if opts.force_delete {
+        insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
+    }
+    if opts.governance_bypass {
+        headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
+    }
+
+    if opts.replication_delete_marker {
+        insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
+    }
+
+    if let Some(t) = opts.replication_mtime {
+        insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
+    }
+
+    if !opts.replication_status.is_empty() {
+        headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
+    }
+
+    if let Some(version_id) = version_id {
+        insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
+    }
+
+    if opts.replication_request {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
+    }
+    if opts.replication_validity_check {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
+    }
+
+    headers
+}
+
 #[derive(Debug, Clone)]
 pub struct AdvancedPutOptions {
     pub source_version_id: String,
@@ -1424,39 +1466,15 @@ impl TargetClient {
         version_id: Option<String>,
         opts: RemoveObjectOptions,
     ) -> Result<(), S3ClientError> {
-        let mut headers = HeaderMap::new();
-        if opts.force_delete {
-            insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
-        }
-        if opts.governance_bypass {
-            headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
-        }
-
-        if opts.replication_delete_marker {
-            insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
-        }
-
-        if let Some(t) = opts.replication_mtime {
-            insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
-        }
-
-        if !opts.replication_status.is_empty() {
-            headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
-        }
-
-        if opts.replication_request {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
-        }
-        if opts.replication_validity_check {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
-        }
+        let headers = build_remove_object_headers(version_id.as_deref(), &opts);
+        let api_version_id = if opts.replication_request { None } else { version_id };
 
         match self
             .client
             .delete_object()
             .bucket(bucket)
             .key(object)
-            .set_version_id(version_id)
+            .set_version_id(api_version_id)
             .customize()
             .map_request(move |mut req| {
                 for (k, v) in headers.clone().into_iter() {
@@ -1550,3 +1568,53 @@ impl From<std::io::Error> for BucketTargetError {
 }
 
 impl Error for BucketTargetError {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: true,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert_eq!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(),
+            Some(version_id.as_str()),
+            "replication delete requests must preserve the version id in internal headers"
+        );
+    }
+
+    #[test]
+    fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: false,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(),
+            "delete-marker version purges must not masquerade as delete-marker creations"
+        );
+    }
+}

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -14,8 +14,9 @@
 
 pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
+pub mod core;
 pub mod evaluator;
-pub mod lifecycle;
+pub use self::core as lifecycle;
 pub mod rule;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;

crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -83,7 +83,7 @@ impl LastDayTierStats {
     #[allow(dead_code)]
     fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
         let mut cl = self.clone();
-        let mut cm = m.clone();
+        let mut cm = m;
         let mut merged = LastDayTierStats::default();
 
         if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() {