test(get): reject range with part number (#2725)

Co-authored-by: houseme <housemecn@gmail.com>

.docker/compose/README.md

@@ -36,6 +36,24 @@ To start a 4-node cluster for distributed testing:
 docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
 ```
 
+### Script-Based 4-Node Validation (Recommended)
+
+Use the local validation script when you need local-source image build, failover checks,
+and benchmark workflow in one command:
+
+```bash
+# Default mode: WAIT_PROBE_MODE=service
+# This avoids false negatives where /health/ready remains 503 locally
+# while the service path is already available.
+./scripts/run_four_node_cluster_failover_bench.sh
+```
+
+Strict mode is available when you explicitly want `/health/ready == 200` as the gate:
+
+```bash
+WAIT_PROBE_MODE=ready ./scripts/run_four_node_cluster_failover_bench.sh
+```
+
 ### (Deprecated) Minimal Observability
 
 ```bash

.docker/compose/docker-compose.cluster.local-build.yml

@@ -0,0 +1,144 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  node1:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node1
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node1_data_0:/data/rustfs0
+      - node1_data_1:/data/rustfs1
+      - node1_data_2:/data/rustfs2
+      - node1_data_3:/data/rustfs3
+    ports:
+      - "9000:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node2:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node2
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node2_data_0:/data/rustfs0
+      - node2_data_1:/data/rustfs1
+      - node2_data_2:/data/rustfs2
+      - node2_data_3:/data/rustfs3
+    ports:
+      - "9001:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node3:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node3
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node3_data_0:/data/rustfs0
+      - node3_data_1:/data/rustfs1
+      - node3_data_2:/data/rustfs2
+      - node3_data_3:/data/rustfs3
+    ports:
+      - "9002:9000"
+    networks:
+      - rustfs-cluster-net
+
+  node4:
+    image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
+    build:
+      context: ../..
+      dockerfile: Dockerfile.source
+    hostname: node4
+    environment:
+      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_ADDRESS=:9000
+      - RUSTFS_CONSOLE_ENABLE=true
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
+      - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
+      - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - node4_data_0:/data/rustfs0
+      - node4_data_1:/data/rustfs1
+      - node4_data_2:/data/rustfs2
+      - node4_data_3:/data/rustfs3
+    ports:
+      - "9003:9000"
+    networks:
+      - rustfs-cluster-net
+
+volumes:
+  node1_data_0:
+  node1_data_1:
+  node1_data_2:
+  node1_data_3:
+  node2_data_0:
+  node2_data_1:
+  node2_data_2:
+  node2_data_3:
+  node3_data_0:
+  node3_data_1:
+  node3_data_2:
+  node3_data_3:
+  node4_data_0:
+  node4_data_1:
+  node4_data_2:
+  node4_data_3:
+
+networks:
+  rustfs-cluster-net:
+    driver: bridge

.docker/compose/docker-compose.observability.yaml

@@ -26,7 +26,7 @@ services:
     restart: "no"
 
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.10.5
     user: "10001"
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
@@ -36,9 +36,19 @@ services:
       - "3200:3200" # tempo
       - "4317"      # otlp grpc
       - "4318"      # otlp http
+      - "7946"      # memberlist
     restart: unless-stopped
     networks:
       - rustfs-network
+    depends_on:
+      tempo-init:
+        condition: service_completed_successfully
+    healthcheck:
+      test: [ "CMD", "/tempo", "-version" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
 
   otel-collector:
     image: otel/opentelemetry-collector-contrib:latest
@@ -61,6 +71,12 @@ services:
       - jaeger
       - prometheus
       - loki
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   jaeger:
     image: jaegertracing/jaeger:latest
@@ -72,12 +88,23 @@ services:
       - BADGER_DIRECTORY_KEY=/badger/key
       - COLLECTOR_OTLP_ENABLED=true
     volumes:
+      - ../../.docker/observability/jaeger.yaml:/etc/jaeger/config.yml:ro
       - jaeger-data:/badger
     ports:
       - "16686:16686" # Web UI
       - "14269:14269" # Admin/Metrics
+      - "4317"        # otlp grpc
+      - "4318"        # otlp http
+    command: [ "--config", "/etc/jaeger/config.yml" ]
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
 
   prometheus:
     image: prom/prometheus:latest
@@ -85,6 +112,7 @@ services:
       - TZ=Asia/Shanghai
     volumes:
       - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ../../.docker/observability/prometheus-rules:/etc/prometheus/rules:ro
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"
@@ -94,23 +122,45 @@ services:
       - '--web.enable-remote-write-receiver'
       - '--enable-feature=promql-experimental-functions'
       - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - '--storage.tsdb.retention.time=30d'
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   loki:
     image: grafana/loki:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
-      - ../../.docker/observability/loki.yaml:/etc/loki/local-config.yaml:ro
+      - ../../.docker/observability/loki.yaml:/etc/loki/loki.yaml:ro
       - loki-data:/loki
     ports:
       - "3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/loki.yaml
     networks:
       - rustfs-network
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+  pyroscope:
+    image: grafana/pyroscope:latest
+    ports:
+      - "4040:4040"
+    command:
+      - -self-profiling.disable-push=true
+    networks:
+      - rustfs-network
+    restart: unless-stopped
 
   grafana:
     image: grafana/grafana:latest
@@ -127,10 +177,17 @@ services:
     volumes:
       - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
       - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
     depends_on:
       - prometheus
       - tempo
       - loki
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
 
   # --- RustFS Cluster ---
 
@@ -215,6 +272,7 @@ volumes:
   tempo-data:
   loki-data:
   jaeger-data:
+  grafana-data:
 
 networks:
   rustfs-network:

.docker/observability/docker-compose.yml

@@ -84,6 +84,7 @@ services:
     container_name: prometheus
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus-rules:/etc/prometheus/rules:ro
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"

.docker/observability/grafana/dashboards/rustfs.json

@@ -157,7 +157,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(increase(rustfs_api_requests_total{job=~\"$job\"}[$__range]))",
+          "expr": "sum(increase(rustfs_http_server_requests_total{job=~\"$job\"}[$__range]))",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -518,7 +518,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rate(rustfs_api_requests_total{job=~\"$job\"}[5m]))",
+          "expr": "sum(rate(rustfs_http_server_requests_total{job=~\"$job\"}[5m]))",
           "hide": false,
           "instant": false,
           "legendFormat": "__auto",
@@ -605,7 +605,7 @@
       },
       "id": 11,
       "panels": [],
-      "title": "Requests",
+      "title": "API RED",
       "type": "row"
     },
     {
@@ -699,8 +699,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (request_method) (rate(rustfs_api_requests_total{job=~\"$job\", request_method=~\"$method\"}[$__rate_interval]))",
-          "legendFormat": "{{request_method}}",
+          "expr": "sum by (method) (rate(rustfs_http_server_requests_total{job=~\"$job\", method=~\"$method\"}[$__rate_interval]))",
+          "legendFormat": "{{method}}",
           "range": true,
           "refId": "A"
         }
@@ -851,7 +851,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.50, sum by (le, job) (rate(rustfs_request_latency_ms_bucket{job=~\"$job\"}[5m])))",
+          "expr": "1000 * histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket{job=~\"$job\"}[5m])))",
           "legendFormat": "P50",
           "range": true,
           "refId": "A"
@@ -862,7 +862,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.95, sum by (le, job) (rate(rustfs_request_latency_ms_bucket{job=~\"$job\"}[5m])))",
+          "expr": "1000 * histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket{job=~\"$job\"}[5m])))",
           "legendFormat": "P95",
           "range": true,
           "refId": "B"
@@ -873,7 +873,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum by (le, job) (rate(rustfs_request_latency_ms_bucket{job=~\"$job\"}[5m])))",
+          "expr": "1000 * histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket{job=~\"$job\"}[5m])))",
           "legendFormat": "P99",
           "range": true,
           "refId": "C"
@@ -1025,7 +1025,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.50, sum by (le, job) (rate(rustfs_request_body_len_bucket{job=~\"$job\"}[5m])))",
+          "expr": "histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket{job=~\"$job\"}[5m])))",
           "legendFormat": "P50",
           "range": true,
           "refId": "A"
@@ -1036,7 +1036,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.95, sum by (le, job) (rate(rustfs_request_body_len_bucket{job=~\"$job\"}[5m])))",
+          "expr": "histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket{job=~\"$job\"}[5m])))",
           "legendFormat": "P95",
           "range": true,
           "refId": "B"
@@ -1047,13 +1047,13 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum by (le, job) (rate(rustfs_request_body_len_bucket{job=~\"$job\"}[5m])))",
+          "expr": "histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket{job=~\"$job\"}[5m])))",
           "legendFormat": "P99",
           "range": true,
           "refId": "C"
         }
       ],
-      "title": "Request Body Percentiles",
+      "title": "Response Body Percentiles",
       "type": "timeseries"
     },
     {
@@ -1566,7 +1566,7 @@
       },
       "id": 34,
       "panels": [],
-      "title": "Buckets",
+      "title": "Storage and Capacity",
       "type": "row"
     },
     {
@@ -2314,7 +2314,7 @@
       },
       "id": 12,
       "panels": [],
-      "title": "Resource Usage",
+      "title": "Host and Process USE",
       "type": "row"
     },
     {
@@ -2534,7 +2534,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"})",
+          "expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"}) or sum by (job) (rustfs_memory_process_resident_bytes{job=~\"$job\"})",
           "legendFormat": "process rss - {{job}}",
           "range": true,
           "refId": "A"
@@ -2545,8 +2545,8 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_system_memory_used_bytes{job=~\"$job\"}",
-          "legendFormat": "system used - {{job}}",
+          "expr": "sum by (job) (rustfs_memory_process_virtual_bytes{job=~\"$job\"})",
+          "legendFormat": "process virtual - {{job}}",
           "range": true,
           "refId": "B"
         },
@@ -2556,13 +2556,755 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_system_memory_used_perc{job=~\"$job\"}",
-          "legendFormat": "system used percent - {{job}}",
+          "expr": "sum by (job) (rustfs_memory_cgroup_anon_bytes{job=~\"$job\"})",
+          "legendFormat": "anon split - {{job}}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_cgroup_file_bytes{job=~\"$job\"})",
+          "legendFormat": "file split - {{job}}",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Memory Split",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 80
+      },
+      "id": 501,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_delete_tail_activity_total_inflight_current{job=~\"$job\"})",
+          "legendFormat": "delete tail inflight - {{job}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_allocator_reclaim_scanner_activity_current{job=~\"$job\"})",
+          "legendFormat": "scanner activity - {{job}}",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_allocator_reclaim_heal_activity_current{job=~\"$job\"})",
+          "legendFormat": "heal activity - {{job}}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_allocator_reclaim_reclaimable_work_current{job=~\"$job\"})",
+          "legendFormat": "reclaimable work - {{job}}",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Tail / Reclaim Activity",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 80
+      },
+      "id": 502,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_allocator_reclaim_idle_streak{job=~\"$job\"})",
+          "legendFormat": "idle streak - {{job}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rate(rustfs_memory_allocator_reclaim_total{job=~\"$job\",result=\"ok\"}[5m]))",
+          "legendFormat": "reclaim ok rate - {{job}}",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job,reason) (rate(rustfs_memory_allocator_reclaim_skipped_total{job=~\"$job\"}[5m]))",
+          "legendFormat": "reclaim skipped {{reason}} - {{job}}",
           "range": true,
           "refId": "C"
         }
       ],
-      "title": "Memory Usage",
+      "title": "Allocator Reclaim Health",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 87
+      },
+      "id": 503,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_cgroup_current_bytes{job=~\"$job\"})",
+          "legendFormat": "cgroup current - {{job}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_cgroup_limit_bytes{job=~\"$job\"})",
+          "legendFormat": "cgroup limit - {{job}}",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_cgroup_active_file_bytes{job=~\"$job\"})",
+          "legendFormat": "active file - {{job}}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_cgroup_inactive_file_bytes{job=~\"$job\"})",
+          "legendFormat": "inactive file - {{job}}",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Memory Cgroup Detail",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 87
+      },
+      "id": 504,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_ec_encode_inflight_bytes_current{job=~\"$job\"})",
+          "legendFormat": "ec inflight bytes - {{job}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_get_object_buffered_bytes_current{job=~\"$job\"})",
+          "legendFormat": "get buffered bytes - {{job}}",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_allocator_reclaim_active_requests{job=~\"$job\"})",
+          "legendFormat": "allocator active requests - {{job}}",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job) (rustfs_memory_allocator_reclaim_delete_tail_activity_current{job=~\"$job\"})",
+          "legendFormat": "allocator tail activity - {{job}}",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Heap Amplification Signals",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 94
+      },
+      "id": 505,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job,stage) (rustfs_delete_tail_activity_inflight_current{job=~\"$job\"})",
+          "legendFormat": "{{stage}} inflight - {{job}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job,stage) (rate(rustfs_delete_tail_activity_started_total{job=~\"$job\"}[5m]))",
+          "legendFormat": "{{stage}} started rate - {{job}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Delete Tail by Stage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 94
+      },
+      "id": 506,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job,kind,result) (rate(rustfs_page_cache_reclaim_requests_total{job=~\"$job\"}[5m]))",
+          "legendFormat": "{{kind}} reclaim {{result}} rate - {{job}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job,kind) (rate(rustfs_page_cache_reclaim_bytes_total{job=~\"$job\"}[5m]))",
+          "legendFormat": "{{kind}} reclaim bytes/s - {{job}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Page Cache Reclaim",
       "type": "timeseries"
     },
     {
@@ -4964,7 +5706,7 @@
       },
       "id": 114,
       "panels": [],
-      "title": "Delivery Targets",
+      "title": "Security and Delivery",
       "type": "row"
     },
     {
@@ -5463,7 +6205,7 @@
       },
       "id": 121,
       "panels": [],
-      "title": "Scanner Activity",
+      "title": "Background Services",
       "type": "row"
     },
     {
@@ -6566,7 +7308,7 @@
           },
           "editorMode": "code",
           "expr": "rustfs_bucket_replication_proxied_put_tagging_requests_total{job=~\"$job\",bucket=~\"$bucket\"}",
-          "legendFormat": "put - {{bucket}}",
+          "legendFormat": "put via tagging - {{bucket}}",
           "range": true,
           "refId": "E"
         },
@@ -6577,12 +7319,12 @@
           },
           "editorMode": "code",
           "expr": "rustfs_bucket_replication_proxied_put_tagging_requests_failures_total{job=~\"$job\",bucket=~\"$bucket\"}",
-          "legendFormat": "put failures - {{bucket}}",
+          "legendFormat": "put via tagging failures - {{bucket}}",
           "range": true,
           "refId": "F"
         }
       ],
-      "title": "Bucket Replication Proxy Requests (Get/Head/Put)",
+      "title": "Bucket Replication Proxy Requests (Get/Head/Put+Tagging)",
       "type": "timeseries"
     },
     {
@@ -6954,7 +7696,7 @@
       },
       "id": 137,
       "panels": [],
-      "title": "System / Cluster Coverage Gap (By Subsystem)",
+      "title": "Debug / Raw Explorer",
       "type": "row"
     },
     {
@@ -8155,7 +8897,7 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(rustfs_api_requests_total, job)",
+        "definition": "label_values(rustfs_http_server_requests_total, job)",
         "includeAll": true,
         "label": "Job",
         "multi": true,
@@ -8163,7 +8905,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs_api_requests_total, job)",
+          "query": "label_values(rustfs_http_server_requests_total, job)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,
@@ -8181,7 +8923,7 @@
           "type": "prometheus",
           "uid": "${datasource}"
         },
-        "definition": "label_values(rustfs_api_requests_total,request_method)",
+        "definition": "label_values(rustfs_http_server_requests_total,method)",
         "includeAll": true,
         "label": "Method",
         "multi": true,
@@ -8189,7 +8931,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs_api_requests_total,request_method)",
+          "query": "label_values(rustfs_http_server_requests_total,method)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,

.docker/observability/prometheus-rules/rustfs-dashboard.yml

@@ -0,0 +1,40 @@
+groups:
+  - name: rustfs-dashboard
+    interval: 30s
+    rules:
+      - record: rustfs:http_server_requests:rate5m
+        expr: sum by (job) (rate(rustfs_http_server_requests_total[5m]))
+
+      - record: rustfs:http_server_request_duration_seconds:p50_5m
+        expr: histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+      - record: rustfs:http_server_request_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+      - record: rustfs:http_server_request_duration_seconds:p99_5m
+        expr: histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_request_duration_seconds_bucket[5m])))
+
+      - record: rustfs:http_server_response_body_size_bytes:p50_5m
+        expr: histogram_quantile(0.50, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+      - record: rustfs:http_server_response_body_size_bytes:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+      - record: rustfs:http_server_response_body_size_bytes:p99_5m
+        expr: histogram_quantile(0.99, sum by (le, job) (rate(rustfs_http_server_response_body_size_bytes_bucket[5m])))
+
+      - record: rustfs:log_cleaner_runs:rate15m
+        expr: sum by (job) (rate(rustfs_log_cleaner_runs_total[15m]))
+      - record: rustfs:log_cleaner_failure_ratio:rate5m
+        expr: sum by (job) (rate(rustfs_log_cleaner_run_failures_total[5m])) / clamp_min(sum by (job) (rate(rustfs_log_cleaner_runs_total[5m])), 1e-9)
+      - record: rustfs:log_cleaner_rotation_failure_ratio:rate5m
+        expr: sum by (job) (rate(rustfs_log_cleaner_rotation_failures_total[5m])) / clamp_min(sum by (job) (rate(rustfs_log_cleaner_rotation_total[5m])), 1e-9)
+      - record: rustfs:log_cleaner_rotation_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_log_cleaner_rotation_duration_seconds_bucket[5m])))
+      - record: rustfs:log_cleaner_compress_duration_seconds:p95_5m
+        expr: histogram_quantile(0.95, sum by (le, job) (rate(rustfs_log_cleaner_compress_duration_seconds_bucket[5m])))
+
+      - record: rustfs:scanner_objects_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_objects_scanned_total[5m]))
+      - record: rustfs:scanner_directories_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_directories_scanned_total[5m]))
+      - record: rustfs:scanner_buckets_scanned:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_buckets_scanned_total[5m]))
+      - record: rustfs:scanner_cycles_success:rate5m
+        expr: sum by (job) (rate(rustfs_scanner_cycles_total{result="success"}[5m]))

.docker/observability/prometheus.yml

@@ -19,6 +19,9 @@ global:
     cluster: 'rustfs-dev'  # Label to identify the cluster
     replica: '1'  # Replica identifier
 
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
 scrape_configs:
   - job_name: 'otel-collector'
     static_configs:

.github/dependabot.yml

@@ -27,9 +27,8 @@ updates:
       timezone: "Asia/Shanghai"
       time: "08:00"
     assignees:
-      - "heihutu"
-    reviewers:
       - "houseme"
+    reviewers:
       - "overtrue"
       - "majinghe"
     ignore:
@@ -39,6 +38,8 @@ updates:
         versions: [ "0.23.x" ]
       - dependency-name: "ratelimit"
         versions: [ "1.x" ]
+      - dependency-name: "ratelimit"
+        versions: [ "2.x" ]
     groups:
       s3s:
         update-types:

Cargo.lock

@@ -40,18 +40,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b657e772794c6b04730ea897b66a058ccd866c16d1967da05eeeecec39043fe"
 dependencies = [
  "crypto-common 0.2.1",
- "inout 0.2.2",
-]
-
-[[package]]
-name = "aes"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
-dependencies = [
- "cfg-if",
- "cipher 0.4.4",
- "cpufeatures 0.2.17",
+ "inout",
 ]
 
 [[package]]
@@ -60,7 +49,7 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "66bd29a732b644c0431c6140f370d097879203d79b80c94a6747ba0872adaef8"
 dependencies = [
- "cipher 0.5.1",
+ "cipher",
  "cpubits",
  "cpufeatures 0.3.0",
 ]
@@ -72,8 +61,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e22c0c90bbe8d4f77c3ca9ddabe41a1f8382d6fc1f7cea89459d0f320371f972"
 dependencies = [
  "aead",
- "aes 0.9.0",
- "cipher 0.5.1",
+ "aes",
+ "cipher",
  "ctr",
  "ghash",
  "subtle",
@@ -529,9 +518,9 @@ dependencies = [
 
 [[package]]
 name = "astral-tokio-tar"
-version = "0.6.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c23f3af104b40a3430ccb90ed5f7bd877a8dc5c26fc92fde51a22b40890dcf9"
+checksum = "4ce73b17c62717c4b6a9af10b43e87c578b0cac27e00666d48304d3b7d2c0693"
 dependencies = [
  "filetime",
  "futures-core",
@@ -1323,6 +1312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
 dependencies = [
  "hybrid-array",
+ "zeroize",
 ]
 
 [[package]]
@@ -1483,12 +1473,6 @@ dependencies = [
  "shlex",
 ]
 
-[[package]]
-name = "cesu8"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.4"
@@ -1508,7 +1492,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
 dependencies = [
  "cfg-if",
- "cipher 0.5.1",
+ "cipher",
  "cpufeatures 0.3.0",
  "rand_core 0.10.1",
 ]
@@ -1521,7 +1505,7 @@ checksum = "1c9ed179664f12fd6f155f6dd632edf5f3806d48c228c67ff78366f2a0eb6b5e"
 dependencies = [
  "aead",
  "chacha20",
- "cipher 0.5.1",
+ "cipher",
  "poly1305",
 ]
 
@@ -1576,16 +1560,6 @@ dependencies = [
  "half",
 ]
 
-[[package]]
-name = "cipher"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
-dependencies = [
- "crypto-common 0.1.6",
- "inout 0.1.4",
-]
-
 [[package]]
 name = "cipher"
 version = "0.5.1"
@@ -1594,7 +1568,7 @@ checksum = "e34d8227fe1ba289043aeb13792056ff80fd6de1a9f49137a5f499de8e8c78ea"
 dependencies = [
  "block-buffer 0.12.0",
  "crypto-common 0.2.1",
- "inout 0.2.2",
+ "inout",
 ]
 
 [[package]]
@@ -2102,7 +2076,7 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17469f8eb9bdbfad10f71f4cfddfd38b01143520c0e717d8796ccb4d44d44e42"
 dependencies = [
- "cipher 0.5.1",
+ "cipher",
 ]
 
 [[package]]
@@ -2114,6 +2088,12 @@ dependencies = [
  "cmov",
 ]
 
+[[package]]
+name = "cty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
+
 [[package]]
 name = "curve25519-dalek"
 version = "4.1.3"
@@ -3189,6 +3169,7 @@ dependencies = [
  "const-oid 0.10.2",
  "crypto-common 0.2.1",
  "ctutils",
+ "zeroize",
 ]
 
 [[package]]
@@ -3243,7 +3224,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
 
 [[package]]
 name = "e2e_test"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -3263,7 +3244,7 @@ dependencies = [
  "md5",
  "rand 0.10.1",
  "rcgen",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rmp-serde",
  "rustfs-common",
  "rustfs-ecstore",
@@ -3888,7 +3869,7 @@ dependencies = [
  "hmac 0.13.0",
  "http 1.4.0",
  "jsonwebtoken",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustc_version",
  "rustls",
  "rustls-pki-types",
@@ -3946,7 +3927,7 @@ dependencies = [
  "pin-project",
  "prost 0.14.3",
  "prost-types 0.14.3",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustc_version",
  "serde",
  "serde_json",
@@ -4620,9 +4601,9 @@ dependencies = [
 
 [[package]]
 name = "idna_adapter"
-version = "1.2.1"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
+checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
 dependencies = [
  "icu_normalizer",
  "icu_properties",
@@ -4691,15 +4672,6 @@ dependencies = [
  "str_stack",
 ]
 
-[[package]]
-name = "inout"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
-dependencies = [
- "generic-array",
-]
-
 [[package]]
 name = "inout"
 version = "0.2.2"
@@ -4876,27 +4848,32 @@ dependencies = [
 
 [[package]]
 name = "jni"
-version = "0.21.1"
+version = "0.22.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
 dependencies = [
- "cesu8",
  "cfg-if",
  "combine",
- "jni-sys 0.3.1",
+ "jni-macros",
+ "jni-sys",
  "log",
- "thiserror 1.0.69",
+ "simd_cesu8",
+ "thiserror 2.0.18",
  "walkdir",
- "windows-sys 0.45.0",
+ "windows-link",
 ]
 
 [[package]]
-name = "jni-sys"
-version = "0.3.1"
+name = "jni-macros"
+version = "0.22.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
 dependencies = [
- "jni-sys 0.4.1",
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "simd_cesu8",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -5161,6 +5138,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
 dependencies = [
  "cc",
+ "cty",
 ]
 
 [[package]]
@@ -6438,16 +6416,6 @@ dependencies = [
  "once_cell",
 ]
 
-[[package]]
-name = "pbkdf2"
-version = "0.12.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
-dependencies = [
- "digest 0.10.7",
- "hmac 0.12.1",
-]
-
 [[package]]
 name = "pbkdf2"
 version = "0.13.0"
@@ -6631,9 +6599,9 @@ dependencies = [
 
 [[package]]
 name = "pkcs8"
-version = "0.11.0-rc.11"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12922b6296c06eb741b02d7b5161e3aaa22864af38dfa025a1a3ba3f68c84577"
+checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
 dependencies = [
  "der 0.8.0",
  "spki 0.8.0",
@@ -7095,9 +7063,9 @@ dependencies = [
 
 [[package]]
 name = "pyroscope"
-version = "2.0.2"
+version = "2.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ef335beaacecb830dc21ab54c8b4a75677874bb78a375ad880715ce385ccb35"
+checksum = "fc050356140f6c47dcaab95b6554ce12d461e4902c0a5baf4e888fcc86dcfaf4"
 dependencies = [
  "lazy_static",
  "libc",
@@ -7106,7 +7074,7 @@ dependencies = [
  "names",
  "pprof-pyroscope-fork",
  "prost 0.14.3",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "serde_json",
  "thiserror 2.0.18",
  "url",
@@ -7586,9 +7554,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.13.2"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801"
+checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
 dependencies = [
  "base64 0.22.1",
  "bytes",
@@ -7721,16 +7689,16 @@ dependencies = [
 
 [[package]]
 name = "rsa"
-version = "0.10.0-rc.17"
+version = "0.10.0-rc.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ed3e93fc7e473e464b9726f4759659e72bc8665e4b8ea227547024f416d905"
+checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf"
 dependencies = [
  "const-oid 0.10.2",
  "crypto-bigint 0.7.3",
  "crypto-primes",
  "digest 0.11.2",
  "pkcs1 0.8.0-rc.4",
- "pkcs8 0.11.0-rc.11",
+ "pkcs8 0.11.0",
  "rand_core 0.10.1",
  "signature 3.0.0-rc.10",
  "spki 0.8.0",
@@ -7847,7 +7815,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -7878,6 +7846,7 @@ dependencies = [
  "jemalloc_pprof",
  "jiff",
  "libc",
+ "libmimalloc-sys",
  "libsystemd",
  "matchit 0.9.2",
  "md5",
@@ -7890,7 +7859,7 @@ dependencies = [
  "pin-project-lite",
  "pprof-pyroscope-fork",
  "rand 0.10.1",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rmp-serde",
  "rust-embed",
  "rustfs-appauth",
@@ -7962,18 +7931,18 @@ dependencies = [
 
 [[package]]
 name = "rustfs-appauth"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "base64-simd",
  "rand 0.10.1",
- "rsa 0.10.0-rc.17",
+ "rsa 0.10.0-rc.18",
  "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "rustfs-audit"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-trait",
  "chrono",
@@ -7996,7 +7965,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-checksums"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "base64-simd",
  "bytes",
@@ -8010,7 +7979,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-common"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-trait",
  "chrono",
@@ -8029,7 +7998,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-concurrency"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "rustfs-io-core",
  "rustfs-io-metrics",
@@ -8041,14 +8010,14 @@ dependencies = [
 
 [[package]]
 name = "rustfs-config"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "const-str",
 ]
 
 [[package]]
 name = "rustfs-credentials"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "base64-simd",
  "rand 0.10.1",
@@ -8059,14 +8028,14 @@ dependencies = [
 
 [[package]]
 name = "rustfs-crypto"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "aes-gcm",
  "argon2",
  "cfg-if",
  "chacha20poly1305",
  "jsonwebtoken",
- "pbkdf2 0.13.0",
+ "pbkdf2",
  "rand 0.10.1",
  "serde_json",
  "sha2 0.11.0",
@@ -8077,7 +8046,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-ecstore"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-channel",
  "async-recursion",
@@ -8127,7 +8096,7 @@ dependencies = [
  "reed-solomon-erasure",
  "reed-solomon-simd",
  "regex",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rmp",
  "rmp-serde",
  "rustfs-checksums",
@@ -8145,6 +8114,7 @@ dependencies = [
  "rustfs-signer",
  "rustfs-utils",
  "rustfs-workers",
+ "rustix 1.1.4",
  "rustls",
  "s3s",
  "serde",
@@ -8174,7 +8144,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-filemeta"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "byteorder",
  "bytes",
@@ -8197,7 +8167,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-heal"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -8225,7 +8195,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-iam"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -8237,7 +8207,7 @@ dependencies = [
  "openidconnect",
  "pollster",
  "rand 0.10.1",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustfs-config",
  "rustfs-credentials",
  "rustfs-crypto",
@@ -8259,7 +8229,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-io-core"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "bytes",
  "memmap2 0.9.10",
@@ -8270,7 +8240,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-io-metrics"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "criterion",
  "metrics",
@@ -8294,7 +8264,7 @@ dependencies = [
  "indexmap 2.14.0",
  "kafka-protocol",
  "metrics",
- "pbkdf2 0.13.0",
+ "pbkdf2",
  "rand 0.10.1",
  "rustls",
  "rustls-native-certs",
@@ -8317,7 +8287,7 @@ dependencies = [
  "hmac 0.13.0",
  "kafka-protocol",
  "metrics",
- "pbkdf2 0.13.0",
+ "pbkdf2",
  "rand 0.10.1",
  "rustfs-kafka",
  "rustls",
@@ -8331,7 +8301,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-keystone"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "bytes",
  "futures",
@@ -8340,7 +8310,7 @@ dependencies = [
  "http-body-util",
  "hyper",
  "moka",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustfs-credentials",
  "rustfs-policy",
  "rustfs-utils",
@@ -8356,7 +8326,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-kms"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "aes-gcm",
  "arc-swap",
@@ -8367,7 +8337,7 @@ dependencies = [
  "md5",
  "moka",
  "rand 0.10.1",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustfs-utils",
  "serde",
  "serde_json",
@@ -8385,7 +8355,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-lock"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-trait",
  "crossbeam-queue",
@@ -8406,7 +8376,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-madmin"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "chrono",
  "humantime",
@@ -8419,7 +8389,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-notify"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -8450,7 +8420,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-object-capacity"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "criterion",
  "futures",
@@ -8469,8 +8439,9 @@ dependencies = [
 
 [[package]]
 name = "rustfs-obs"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
+ "chrono",
  "crossbeam-channel",
  "crossbeam-deque",
  "crossbeam-utils",
@@ -8490,8 +8461,10 @@ dependencies = [
  "percent-encoding",
  "pyroscope",
  "rustfs-audit",
+ "rustfs-common",
  "rustfs-config",
  "rustfs-ecstore",
+ "rustfs-iam",
  "rustfs-io-metrics",
  "rustfs-notify",
  "rustfs-utils",
@@ -8512,7 +8485,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-policy"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-trait",
  "base64-simd",
@@ -8523,7 +8496,7 @@ dependencies = [
  "moka",
  "pollster",
  "regex",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustfs-config",
  "rustfs-credentials",
  "rustfs-crypto",
@@ -8540,7 +8513,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-protocols"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "astral-tokio-tar",
  "async-compression",
@@ -8590,11 +8563,13 @@ dependencies = [
 
 [[package]]
 name = "rustfs-protos"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "flatbuffers",
  "prost 0.14.3",
  "rustfs-common",
+ "rustfs-config",
+ "rustfs-utils",
  "tonic",
  "tonic-prost",
  "tonic-prost-build",
@@ -8603,7 +8578,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-rio"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "aes-gcm",
  "axum",
@@ -8618,7 +8593,7 @@ dependencies = [
  "md-5 0.11.0",
  "pin-project-lite",
  "rand 0.10.1",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustfs-common",
  "rustfs-config",
  "rustfs-utils",
@@ -8636,7 +8611,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3-common"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "metrics",
  "serde",
@@ -8645,7 +8620,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3select-api"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-trait",
  "bytes",
@@ -8672,7 +8647,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3select-query"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -8689,7 +8664,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-scanner"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-trait",
  "chrono",
@@ -8720,7 +8695,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-signer"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "base64-simd",
  "bytes",
@@ -8729,20 +8704,21 @@ dependencies = [
  "rustfs-utils",
  "s3s",
  "serde_urlencoded",
+ "thiserror 2.0.18",
  "time",
  "tracing",
 ]
 
 [[package]]
 name = "rustfs-targets"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-nats",
  "async-trait",
  "criterion",
  "hyper-rustls",
  "pulsar",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rumqttc-next",
  "rustfs-config",
  "rustfs-ecstore",
@@ -8764,7 +8740,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-trusted-proxies"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "async-trait",
  "axum",
@@ -8773,7 +8749,7 @@ dependencies = [
  "metrics",
  "moka",
  "regex",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustfs-config",
  "rustfs-utils",
  "serde",
@@ -8788,7 +8764,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-utils"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "base64-simd",
  "blake2 0.11.0-rc.6",
@@ -8832,7 +8808,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-workers"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "tokio",
  "tracing",
@@ -8840,7 +8816,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-zip"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 dependencies = [
  "astral-tokio-tar",
  "async-compression",
@@ -8867,7 +8843,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "http 1.4.0",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustify_derive",
  "serde",
  "serde_json",
@@ -8957,9 +8933,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-platform-verifier"
-version = "0.6.2"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
 dependencies = [
  "core-foundation 0.10.1",
  "core-foundation-sys",
@@ -9498,6 +9474,16 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
 
+[[package]]
+name = "simd_cesu8"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
+dependencies = [
+ "rustc_version",
+ "simdutf8",
+]
+
 [[package]]
 name = "simdutf8"
 version = "0.1.5"
@@ -10713,7 +10699,7 @@ dependencies = [
  "async-trait",
  "derive_builder",
  "http 1.4.0",
- "reqwest 0.13.2",
+ "reqwest 0.13.3",
  "rustify",
  "rustify_derive",
  "serde",
@@ -11091,15 +11077,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "windows-sys"
-version = "0.45.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
-dependencies = [
- "windows-targets 0.42.2",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.52.0"
@@ -11136,21 +11113,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "windows-targets"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
-dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
-]
-
 [[package]]
 name = "windows-targets"
 version = "0.52.6"
@@ -11193,12 +11155,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
-
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.52.6"
@@ -11211,12 +11167,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.52.6"
@@ -11229,12 +11179,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.52.6"
@@ -11259,12 +11203,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.52.6"
@@ -11277,12 +11215,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.52.6"
@@ -11295,12 +11227,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
 
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
-
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.52.6"
@@ -11313,12 +11239,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
 
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.52.6"
@@ -11641,24 +11561,24 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "8.5.1"
+version = "8.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcab981e19633ebcf0b001ddd37dd802996098bc1864f90b7c5d970ce76c1d59"
+checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
 dependencies = [
- "aes 0.8.4",
+ "aes",
  "bzip2",
  "constant_time_eq",
  "crc32fast",
  "deflate64",
  "flate2",
  "getrandom 0.4.2",
- "hmac 0.12.1",
+ "hmac 0.13.0",
  "indexmap 2.14.0",
  "lzma-rust2",
  "memchr",
- "pbkdf2 0.12.2",
+ "pbkdf2",
  "ppmd-rust",
- "sha1 0.10.6",
+ "sha1 0.11.0",
  "time",
  "typed-path",
  "zeroize",

Cargo.toml

@@ -59,7 +59,7 @@ edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
 rust-version = "1.93.0"
-version = "0.0.5"
+version = "1.0.0-beta.1"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -76,42 +76,42 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "0.0.5" }
-rustfs-heal = { path = "crates/heal", version = "0.0.5" }
-rustfs-appauth = { path = "crates/appauth", version = "0.0.5" }
-rustfs-audit = { path = "crates/audit", version = "0.0.5" }
-rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
-rustfs-common = { path = "crates/common", version = "0.0.5" }
-rustfs-config = { path = "./crates/config", version = "0.0.5" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "0.0.5" }
-rustfs-credentials = { path = "crates/credentials", version = "0.0.5" }
-rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
-rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
-rustfs-filemeta = { path = "crates/filemeta", version = "0.0.5" }
-rustfs-iam = { path = "crates/iam", version = "0.0.5" }
-rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
-rustfs-kms = { path = "crates/kms", version = "0.0.5" }
-rustfs-lock = { path = "crates/lock", version = "0.0.5" }
-rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
-rustfs-notify = { path = "crates/notify", version = "0.0.5" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
-rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
-rustfs-obs = { path = "crates/obs", version = "0.0.5" }
-rustfs-policy = { path = "crates/policy", version = "0.0.5" }
-rustfs-protos = { path = "crates/protos", version = "0.0.5" }
-rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
-rustfs-rio = { path = "crates/rio", version = "0.0.5" }
-rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
-rustfs-scanner = { path = "crates/scanner", version = "0.0.5" }
-rustfs-signer = { path = "crates/signer", version = "0.0.5" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "0.0.5" }
-rustfs-targets = { path = "crates/targets", version = "0.0.5" }
-rustfs-utils = { path = "crates/utils", version = "0.0.5" }
-rustfs-workers = { path = "crates/workers", version = "0.0.5" }
-rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
+rustfs = { path = "./rustfs", version = "1.0.0-beta.1" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.1" }
+rustfs-appauth = { path = "crates/appauth", version = "1.0.0-beta.1" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.1" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.1" }
+rustfs-common = { path = "crates/common", version = "1.0.0-beta.1" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-beta.1" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.1" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.1" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.1" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.1" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.1" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.1" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.1" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.1" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.1" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.1" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.1" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.1" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.1" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.1" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.1" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.1" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.1" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.1" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.1" }
+rustfs-s3-common = { path = "crates/s3-common", version = "1.0.0-beta.1" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.1" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.1" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.1" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.1" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.1" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.1" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.1" }
+rustfs-workers = { path = "crates/workers", version = "1.0.0-beta.1" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.1" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
@@ -131,7 +131,7 @@ hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-g
 http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
-reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 rustfs-kafka-async = { version = "1.2.0" }
 socket2 = { version = "0.6.3", features = ["all"] }
 tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
@@ -169,7 +169,7 @@ hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
 pbkdf2 = "0.13.0"
-rsa = { version = "0.10.0-rc.17" }
+rsa = { version = "0.10.0-rc.18" }
 rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = "1.14.1"
 sha1 = "0.11.0"
@@ -186,7 +186,7 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros
 # Utilities and Tools
 anyhow = "1.0.102"
 arc-swap = "1.9.1"
-astral-tokio-tar = "0.6.0"
+astral-tokio-tar = "0.6.1"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.8.16" }
@@ -280,7 +280,7 @@ walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "8.5.1"
+zip = "8.6.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
@@ -292,7 +292,7 @@ opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rus
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }
-pyroscope = { version = "2.0.2", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.0.3", features = ["backend-pprof-rs"] }
 
 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }

Dockerfile.decommission-local

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.91-trixie
+FROM rust:1.93-trixie
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \

Dockerfile.source

@@ -30,7 +30,7 @@ ARG BUILDPLATFORM
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.91-trixie AS builder
+FROM rust:1.93-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
@@ -60,7 +60,15 @@ RUN set -eux; \
 
 # Optional: cross toolchain for aarch64 (only when targeting linux/arm64)
 RUN set -eux; \
-    if [ "${TARGETPLATFORM:-linux/amd64}" = "linux/arm64" ]; then \
+    target_platform="${TARGETPLATFORM:-}"; \
+    if [ -z "${target_platform}" ]; then \
+    case "$(uname -m)" in \
+    x86_64) target_platform="linux/amd64" ;; \
+    aarch64|arm64) target_platform="linux/arm64" ;; \
+    *) target_platform="linux/amd64" ;; \
+    esac; \
+    fi; \
+    if [ "${target_platform}" = "linux/arm64" ]; then \
     export DEBIAN_FRONTEND=noninteractive; \
     apt-get update; \
     apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu; \
@@ -120,7 +128,16 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
     --mount=type=cache,target=/usr/src/rustfs/target \
     set -eux; \
-    case "${TARGETPLATFORM:-linux/amd64}" in \
+    rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; \
+    target_platform="${TARGETPLATFORM:-}"; \
+    if [ -z "${target_platform}" ]; then \
+    case "$(uname -m)" in \
+    x86_64) target_platform="linux/amd64" ;; \
+    aarch64|arm64) target_platform="linux/arm64" ;; \
+    *) target_platform="linux/amd64" ;; \
+    esac; \
+    fi; \
+    case "${target_platform}" in \
     linux/amd64) \
     echo "Building for x86_64-unknown-linux-gnu"; \
     cargo build --release --locked --target x86_64-unknown-linux-gnu --bin rustfs -j "$(nproc)"; \
@@ -132,7 +149,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     install -m 0755 target/aarch64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \
     ;; \
     *) \
-    echo "Unsupported TARGETPLATFORM=${TARGETPLATFORM}" >&2; exit 1 \
+    echo "Unsupported target platform=${target_platform}" >&2; exit 1 \
     ;; \
     esac
 
@@ -178,7 +195,7 @@ CMD ["cargo", "run", "--bin", "rustfs", "--"]
 # -----------------------------
 # Runtime stage (Ubuntu minimal)
 # -----------------------------
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ARG BUILD_DATE
 ARG VCS_REF

README.md

@@ -115,7 +115,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:v1.0.0-beta.1
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command

README_ZH.md

@@ -113,7 +113,7 @@ RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用
  docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
  # 使用指定版本运行
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.68
+ docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:v1.0.0-beta.1
 ```
 
 您也可以使用 Docker Compose。使用根目录下的 `docker-compose.yml` 文件：

crates/common/src/internode_metrics.rs

@@ -51,7 +51,7 @@ impl InternodeMetrics {
             return;
         }
         self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-        counter!("rustfs.internode.sent.bytes.total").increment(bytes);
+        counter!("rustfs_system_network_internode_sent_bytes_total").increment(bytes);
     }
 
     pub fn record_recv_bytes(&self, bytes: usize) {
@@ -60,22 +60,22 @@ impl InternodeMetrics {
             return;
         }
         self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-        counter!("rustfs.internode.recv.bytes.total").increment(bytes);
+        counter!("rustfs_system_network_internode_recv_bytes_total").increment(bytes);
     }
 
     pub fn record_outgoing_request(&self) {
         self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.requests.outgoing.total").increment(1);
+        counter!("rustfs_system_network_internode_requests_outgoing_total").increment(1);
     }
 
     pub fn record_incoming_request(&self) {
         self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.requests.incoming.total").increment(1);
+        counter!("rustfs_system_network_internode_requests_incoming_total").increment(1);
     }
 
     pub fn record_error(&self) {
         self.errors_total.fetch_add(1, Ordering::Relaxed);
-        counter!("rustfs.internode.errors.total").increment(1);
+        counter!("rustfs_system_network_internode_errors_total").increment(1);
     }
 
     pub fn record_dial_result(&self, duration: Duration, success: bool) {
@@ -83,11 +83,11 @@ impl InternodeMetrics {
         self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
         let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
         let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
-        gauge!("rustfs.internode.dial.avg_time.nanos").set(total as f64 / samples as f64);
+        gauge!("rustfs_system_network_internode_dial_avg_time_nanos").set(total as f64 / samples as f64);
 
         if !success {
             self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
-            counter!("rustfs.internode.dial.errors.total").increment(1);
+            counter!("rustfs_system_network_internode_dial_errors_total").increment(1);
         }
 
         let now_ms = SystemTime::now()

crates/config/src/constants/app.rs

@@ -296,9 +296,9 @@ pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true;
 
 /// Default profiling export enabled
 /// It is used to enable or disable exporting profiles
-/// Default value: true
+/// Default value: false
 /// Environment variable: RUSTFS_OBS_PROFILING_EXPORT_ENABLED
-pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = true;
+pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = false;
 
 /// Default log local logging enabled for rustfs
 /// This is the default log local logging enabled for rustfs.

crates/config/src/constants/drive.rs

@@ -39,6 +39,14 @@ pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
 pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
 pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
 
+/// Interval in seconds between active health probes for local and remote drives.
+pub const ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_INTERVAL_SECS";
+pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
+
+/// Timeout in seconds for a single active health probe.
+pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5;
+
 /// Number of consecutive failures before a suspect drive is classified as offline.
 pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
 pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;

crates/config/src/constants/internode.rs

@@ -0,0 +1,62 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Timeout for establishing a new internode gRPC connection.
+pub const ENV_INTERNODE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS: u64 = 3;
+
+/// TCP keepalive interval for internode gRPC channels.
+pub const ENV_INTERNODE_TCP_KEEPALIVE_SECS: &str = "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS";
+pub const DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS: u64 = 10;
+
+/// HTTP/2 keepalive interval for internode gRPC channels.
+pub const ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS";
+pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5;
+
+/// HTTP/2 keepalive timeout for internode gRPC channels.
+pub const ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
+
+/// Overall timeout for a single internode gRPC request.
+pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
+pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn internode_timeout_defaults_stay_in_expected_bounds() {
+        assert_eq!(DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, 10);
+        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
+        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
+    }
+
+    #[test]
+    fn internode_timeout_env_names_are_stable() {
+        assert_eq!(ENV_INTERNODE_CONNECT_TIMEOUT_SECS, "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS");
+        assert_eq!(ENV_INTERNODE_TCP_KEEPALIVE_SECS, "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS");
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
+            "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS"
+        );
+        assert_eq!(
+            ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
+            "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
+        );
+        assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
+    }
+}

crates/config/src/constants/mod.rs

@@ -21,6 +21,7 @@ pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
 pub(crate) mod health;
+pub(crate) mod internode;
 pub(crate) mod object;
 pub(crate) mod oidc;
 pub(crate) mod profiler;

crates/config/src/constants/runtime.rs

@@ -59,9 +59,28 @@ pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10;
 pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling
 // Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option<String>)
 
+// Allocator reclaim configuration
+pub const ENV_ALLOCATOR_RECLAIM_ENABLED: &str = "RUSTFS_ALLOCATOR_RECLAIM_ENABLED";
+pub const ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS: &str = "RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS";
+pub const ENV_ALLOCATOR_RECLAIM_FORCE: &str = "RUSTFS_ALLOCATOR_RECLAIM_FORCE";
+pub const ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS: &str = "RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS";
+pub const DEFAULT_ALLOCATOR_RECLAIM_ENABLED: bool = false;
+pub const DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS: u64 = 30;
+pub const DEFAULT_ALLOCATOR_RECLAIM_FORCE: bool = true;
+pub const DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS: u64 = 3;
+
+// File page-cache reclaim configuration
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE";
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE";
+pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD";
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false;
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = false;
+pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024;
+
 /// Threshold for small object seek support in megabytes.
 ///
 /// When an object is smaller than this size, rustfs will provide seek support.
 ///
 /// Default is set to 10MB.
+pub const ENV_OBJECT_SEEK_SUPPORT_THRESHOLD: &str = "RUSTFS_OBJECT_SEEK_SUPPORT_THRESHOLD";
 pub const DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD: usize = 10 * 1024 * 1024;

crates/config/src/lib.rs

@@ -33,6 +33,8 @@ pub use constants::heal::*;
 #[cfg(feature = "constants")]
 pub use constants::health::*;
 #[cfg(feature = "constants")]
+pub use constants::internode::*;
+#[cfg(feature = "constants")]
 pub use constants::object::*;
 #[cfg(feature = "constants")]
 pub use constants::profiler::*;

crates/e2e_test/src/replication_extension_test.rs

@@ -21,6 +21,7 @@ use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use aws_sdk_s3::{Client, Config};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
 use rustfs_madmin::{
     AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
     ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
@@ -255,6 +256,60 @@ async fn put_bucket_replication(
     Ok(())
 }
 
+async fn put_bucket_replication_rules(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    target_arns: &[&str],
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let mut rules = String::new();
+    for (idx, target_arn) in target_arns.iter().enumerate() {
+        rules.push_str(&format!(
+            r#"
+  <Rule>
+    <ID>rule-{}</ID>
+    <Priority>{}</Priority>
+    <Status>Enabled</Status>
+    <DeleteMarkerReplication>
+      <Status>Enabled</Status>
+    </DeleteMarkerReplication>
+    <ExistingObjectReplication>
+      <Status>Enabled</Status>
+    </ExistingObjectReplication>
+    <Destination>
+      <Bucket>{}</Bucket>
+    </Destination>
+  </Rule>"#,
+            idx + 1,
+            idx + 1,
+            target_arn
+        ));
+    }
+
+    let body = format!(
+        r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Role></Role>{rules}
+</ReplicationConfiguration>"#
+    );
+    let url = format!("{}/{bucket}?replication", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.into_bytes()),
+        Some("application/xml"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("put bucket replication with multiple rules failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
 async fn delete_bucket_replication(
     env: &RustFSTestEnvironment,
     bucket: &str,
@@ -415,6 +470,33 @@ async fn admin_attach_policy_to_group(
     Ok(())
 }
 
+async fn wait_for_replicated_object(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    expected_body: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
+
+    loop {
+        match client.get_object().bucket(bucket).key(key).send().await {
+            Ok(output) => {
+                let body = output.body.collect().await?.into_bytes();
+                let body = String::from_utf8(body.to_vec())?;
+                if body == expected_body {
+                    return Ok(());
+                }
+                return Err(format!("replicated object body mismatch: expected {expected_body}, got {body}").into());
+            }
+            Err(_err) if tokio::time::Instant::now() < deadline => {
+                sleep(Duration::from_secs(1)).await;
+                continue;
+            }
+            Err(err) => return Err(err.into()),
+        }
+    }
+}
+
 async fn run_replication_check(
     env: &RustFSTestEnvironment,
     bucket: &str,
@@ -1518,6 +1600,139 @@ async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Bo
     Ok(())
 }
 
+#[tokio::test]
+#[serial]
+async fn test_single_bucket_replication_fans_out_to_multiple_targets() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env_a = RustFSTestEnvironment::new().await?;
+    target_env_a.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let mut target_env_b = RustFSTestEnvironment::new().await?;
+    target_env_b.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-fanout-src";
+    let target_bucket_a = "replication-fanout-dst-a";
+    let target_bucket_b = "replication-fanout-dst-b";
+    let object_key = "fanout.txt";
+    let body = "payload-fanout";
+
+    let source_client = source_env.create_s3_client();
+    let target_client_a = target_env_a.create_s3_client();
+    let target_client_b = target_env_b.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client_a.create_bucket().bucket(target_bucket_a).send().await?;
+    target_client_b.create_bucket().bucket(target_bucket_b).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env_a, target_bucket_a).await?;
+    enable_bucket_versioning(&target_env_b, target_bucket_b).await?;
+
+    let target_arn_a = set_replication_target(&source_env, source_bucket, &target_env_a, target_bucket_a).await?;
+    let target_arn_b = set_replication_target(&source_env, source_bucket, &target_env_b, target_bucket_b).await?;
+    put_bucket_replication_rules(&source_env, source_bucket, &[target_arn_a.as_str(), target_arn_b.as_str()]).await?;
+
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(object_key)
+        .body(ByteStream::from(body.as_bytes().to_vec()))
+        .send()
+        .await?;
+
+    wait_for_replicated_object(&target_client_a, target_bucket_a, object_key, body).await?;
+    wait_for_replicated_object(&target_client_b, target_bucket_b, object_key, body).await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sequential_bucket_replication_succeeds_for_multiple_buckets() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    for idx in 1..=5 {
+        let source_bucket = format!("replication-multi-src-{idx}");
+        let target_bucket = format!("replication-multi-dst-{idx}");
+        let object_key = format!("probe-{idx}.txt");
+        let body = format!("payload-{idx}");
+
+        source_client.create_bucket().bucket(&source_bucket).send().await?;
+        target_client.create_bucket().bucket(&target_bucket).send().await?;
+        enable_bucket_versioning(&source_env, &source_bucket).await?;
+        enable_bucket_versioning(&target_env, &target_bucket).await?;
+
+        let target_arn = set_replication_target(&source_env, &source_bucket, &target_env, &target_bucket).await?;
+        put_bucket_replication(&source_env, &source_bucket, &target_arn).await?;
+
+        source_client
+            .put_object()
+            .bucket(&source_bucket)
+            .key(&object_key)
+            .body(ByteStream::from(body.clone().into_bytes()))
+            .send()
+            .await?;
+
+        wait_for_replicated_object(&target_client, &target_bucket, &object_key, &body).await?;
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_recovers_after_runtime_target_cache_is_cleared() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-refresh-src";
+    let target_bucket = "replication-refresh-dst";
+    let object_key = "probe-refresh.txt";
+    let body = "payload-refresh";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    BucketTargetSys::get().delete(source_bucket).await;
+
+    source_client
+        .put_object()
+        .bucket(source_bucket)
+        .key(object_key)
+        .body(ByteStream::from(body.as_bytes().to_vec()))
+        .send()
+        .await?;
+
+    wait_for_replicated_object(&target_client, target_bucket, object_key, body).await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {

crates/ecstore/Cargo.toml

@@ -100,6 +100,7 @@ pin-project-lite.workspace = true
 md-5.workspace = true
 memmap2 = { workspace = true }
 libc.workspace = true
+rustix = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-workers.workspace = true
 reqwest = { workspace = true }

crates/ecstore/src/batch_processor.rs

@@ -87,25 +87,70 @@ impl AsyncBatchProcessor {
         T: Send + 'static,
         F: Future<Output = Result<T>> + Send + 'static,
     {
-        let results = self.execute_batch(tasks).await;
-        let mut successes = Vec::new();
+        if required_successes == 0 {
+            return Ok(Vec::new());
+        }
 
-        for value in results.into_iter().flatten() {
-            successes.push(value);
-            if successes.len() >= required_successes {
-                return Ok(successes);
+        if tasks.is_empty() {
+            return Err(Error::other(format!(
+                "Insufficient successful results: got 0, needed {required_successes}"
+            )));
+        }
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
+        let mut join_set = JoinSet::new();
+        let mut successes = Vec::new();
+        let mut pending_tasks = tasks.len();
+        let mut first_error = None;
+
+        for task in tasks {
+            let sem = semaphore.clone();
+            join_set.spawn(async move {
+                let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
+                task.await
+            });
+        }
+
+        while let Some(join_result) = join_set.join_next().await {
+            pending_tasks = pending_tasks.saturating_sub(1);
+
+            match join_result {
+                Ok(Ok(value)) => {
+                    successes.push(value);
+                    if successes.len() >= required_successes {
+                        return Ok(successes);
+                    }
+                }
+                Ok(Err(err)) => {
+                    if first_error.is_none() {
+                        first_error = Some(err);
+                    }
+                }
+                Err(join_error) => {
+                    if first_error.is_none() {
+                        first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
+                    }
+                }
+            }
+
+            if successes.len() + pending_tasks < required_successes {
+                return Err(first_error.unwrap_or_else(|| {
+                    Error::other(format!(
+                        "Insufficient successful results: got {}, needed {}",
+                        successes.len(),
+                        required_successes
+                    ))
+                }));
             }
         }
 
-        if successes.len() >= required_successes {
-            Ok(successes)
-        } else {
-            Err(Error::other(format!(
+        Err(first_error.unwrap_or_else(|| {
+            Error::other(format!(
                 "Insufficient successful results: got {}, needed {}",
                 successes.len(),
                 required_successes
-            )))
-        }
+            ))
+        }))
     }
 }
 
@@ -228,4 +273,52 @@ mod tests {
         let successes = results.unwrap();
         assert!(successes.len() >= 2);
     }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_returns_before_slow_tail() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))]
+            .into_iter()
+            .map(|(delay_ms, outcome)| async move {
+                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                outcome
+            })
+            .collect();
+
+        let results = processor
+            .execute_batch_with_quorum(tasks, 2)
+            .await
+            .expect("quorum should succeed");
+        assert_eq!(results.len(), 2);
+        assert!(started.elapsed() < Duration::from_millis(100));
+    }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok(1_i32)),
+            (15, Err(Error::other("first failure"))),
+            (20, Err(Error::other("second failure"))),
+            (250, Ok(4)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let err = processor
+            .execute_batch_with_quorum(tasks, 3)
+            .await
+            .expect_err("quorum should fail once it becomes impossible");
+
+        assert!(err.to_string().contains("first failure"));
+        assert!(started.elapsed() < Duration::from_millis(120));
+    }
 }

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -395,8 +395,27 @@ impl BucketTargetSys {
         }
     }
 
-    pub async fn set_target(&self, bucket: &str, target: &BucketTarget, update: bool) -> Result<(), BucketTargetError> {
-        if !target.target_type.is_valid() && !update {
+    pub async fn set_target(
+        &self,
+        bucket: &str,
+        target: &BucketTarget,
+        update: bool,
+    ) -> Result<BucketTargets, BucketTargetError> {
+        self.validate_target(bucket, target).await?;
+
+        let mut bucket_targets = match self.list_bucket_targets(bucket).await {
+            Ok(targets) => targets,
+            Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
+            Err(err) => return Err(err),
+        };
+
+        Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
+
+        Ok(bucket_targets)
+    }
+
+    pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
+        if !target.target_type.is_valid() {
             return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
                 bucket: bucket.to_string(),
             });
@@ -450,52 +469,44 @@ impl BucketTargetSys {
             }
         }
 
-        {
-            let mut targets_map = self.targets_map.write().await;
-            let bucket_targets = targets_map.entry(bucket.to_string()).or_insert_with(Vec::new);
-            let mut found = false;
+        Ok(())
+    }
 
-            for (idx, existing_target) in bucket_targets.iter().enumerate() {
-                if existing_target.target_type.to_string() == target.target_type.to_string() {
-                    if existing_target.arn == target.arn {
-                        if !update {
-                            return Err(BucketTargetError::BucketRemoteAlreadyExists {
-                                bucket: existing_target.target_bucket.clone(),
-                            });
-                        }
-                        bucket_targets[idx] = target.clone();
-                        found = true;
-                        break;
-                    }
-                    if existing_target.endpoint == target.endpoint {
+    fn upsert_target_entry(
+        bucket_targets: &mut Vec<BucketTarget>,
+        target: &BucketTarget,
+        update: bool,
+    ) -> Result<(), BucketTargetError> {
+        let mut found = false;
+
+        for (idx, existing_target) in bucket_targets.iter().enumerate() {
+            if existing_target.target_type.to_string() == target.target_type.to_string() {
+                if existing_target.arn == target.arn {
+                    if !update {
                         return Err(BucketTargetError::BucketRemoteAlreadyExists {
                             bucket: existing_target.target_bucket.clone(),
                         });
                     }
+                    bucket_targets[idx] = target.clone();
+                    found = true;
+                    break;
+                }
+                if existing_target.endpoint == target.endpoint {
+                    return Err(BucketTargetError::BucketRemoteAlreadyExists {
+                        bucket: existing_target.target_bucket.clone(),
+                    });
                 }
             }
-
-            if !found && !update {
-                bucket_targets.push(target.clone());
-            }
         }
 
-        {
-            let mut arn_remotes_map = self.arn_remotes_map.write().await;
-            arn_remotes_map.insert(
-                target.arn.clone(),
-                ArnTarget {
-                    client: Some(Arc::new(target_client)),
-                    last_refresh: OffsetDateTime::now_utc(),
-                },
-            );
+        if !found && !update {
+            bucket_targets.push(target.clone());
         }
 
-        self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
         Ok(())
     }
 
-    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<(), BucketTargetError> {
+    pub async fn remove_target(&self, bucket: &str, arn_str: &str) -> Result<BucketTargets, BucketTargetError> {
         if arn_str.is_empty() {
             return Err(BucketTargetError::BucketRemoteArnInvalid {
                 bucket: bucket.to_string(),
@@ -524,33 +535,16 @@ impl BucketTargetSys {
             }
         }
 
-        {
-            let mut targets_map = self.targets_map.write().await;
+        let targets = self.list_bucket_targets(bucket).await?;
+        let new_targets: Vec<BucketTarget> = targets.targets.iter().filter(|t| t.arn != arn_str).cloned().collect();
 
-            let Some(targets) = targets_map.get(bucket) else {
-                return Err(BucketTargetError::BucketRemoteTargetNotFound {
-                    bucket: bucket.to_string(),
-                });
-            };
-
-            let new_targets: Vec<BucketTarget> = targets.iter().filter(|t| t.arn != arn_str).cloned().collect();
-
-            if new_targets.len() == targets.len() {
-                return Err(BucketTargetError::BucketRemoteTargetNotFound {
-                    bucket: bucket.to_string(),
-                });
-            }
-
-            targets_map.insert(bucket.to_string(), new_targets);
+        if new_targets.len() == targets.targets.len() {
+            return Err(BucketTargetError::BucketRemoteTargetNotFound {
+                bucket: bucket.to_string(),
+            });
         }
 
-        {
-            self.arn_remotes_map.write().await.remove(arn_str);
-        }
-
-        self.update_bandwidth_limit(bucket, arn_str, 0);
-
-        Ok(())
+        Ok(BucketTargets { targets: new_targets })
     }
 
     pub async fn mark_refresh_in_progress(&self, bucket: &str, arn: &str) {
@@ -603,7 +597,7 @@ impl BucketTargetSys {
 
         if let Some(last_refresh) = last_refresh {
             let now = OffsetDateTime::now_utc();
-            if now - last_refresh > Duration::from_secs(60 * 5) {
+            if now - last_refresh < Duration::from_secs(60 * 5) {
                 return None;
             }
         }
@@ -619,6 +613,16 @@ impl BucketTargetSys {
             }
         };
 
+        let cli = self
+            .arn_remotes_map
+            .read()
+            .await
+            .get(arn)
+            .and_then(|target| target.client.clone());
+        if cli.is_some() {
+            return cli;
+        }
+
         self.inc_arn_errs(bucket, arn).await;
         None
     }

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -54,8 +54,8 @@ use rustfs_filemeta::{
 use rustfs_s3_common::EventName;
 use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
 use s3s::dto::{
-    BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
-    Timestamp,
+    BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
+    RestoreRequestType, RestoreStatus, Timestamp,
 };
 use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
 use sha2::{Digest, Sha256};
@@ -92,6 +92,7 @@ const ENV_STALE_UPLOADS_EXPIRY: &str = "RUSTFS_API_STALE_UPLOADS_EXPIRY";
 const ENV_STALE_UPLOADS_CLEANUP_INTERVAL: &str = "RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL";
 const DEFAULT_STALE_UPLOADS_EXPIRY: StdDuration = StdDuration::from_secs(24 * 60 * 60);
 const DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL: StdDuration = StdDuration::from_secs(6 * 60 * 60);
+const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
 
 lazy_static! {
     pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
@@ -1241,6 +1242,29 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket:
     }
 }
 
+fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
+    lc.rules.iter().any(|rule| {
+        rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
+            && rule.id.as_deref() == Some(rule_id)
+            && rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
+    })
+}
+
+fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
+    lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
+        let updated_at = OffsetDateTime::from(updated_at.clone());
+        now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
+    })
+}
+
+async fn apply_existing_object_expiry(api: Arc<ECStore>, object: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
+    if object.is_remote() {
+        apply_expiry_on_transitioned_object(api, object, event, src).await;
+    } else {
+        apply_expiry_on_non_transitioned_objects(api, object, event, src).await;
+    }
+}
+
 pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
     let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
         return Ok(());
@@ -1253,6 +1277,8 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
     let mut marker = None;
     let mut version_marker = None;
     let src = LcEventSrc::Scanner;
+    let defer_date_expiry_once = should_defer_date_expiry_for_recent_config_update(&lc, OffsetDateTime::now_utc());
+    let mut date_expiry_deferred_once = false;
 
     loop {
         let page = api
@@ -1269,15 +1295,16 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
                 | IlmAction::DeleteRestoredVersionAction
                 | IlmAction::DeleteAllVersionsAction
                 | IlmAction::DelMarkerDeleteAllVersionsAction => {
-                    if event
-                        .due
-                        .is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp())
-                    {
-                        if object.is_remote() {
-                            apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await;
-                        } else {
-                            apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await;
+                    let now = OffsetDateTime::now_utc();
+                    if event.due.is_some_and(|due| due.unix_timestamp() <= now.unix_timestamp()) {
+                        if defer_date_expiry_once
+                            && !date_expiry_deferred_once
+                            && lifecycle_rule_has_date_expiration(&lc, &event.rule_id)
+                        {
+                            tokio::time::sleep(StdDuration::from_secs(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS as u64)).await;
+                            date_expiry_deferred_once = true;
                         }
+                        apply_existing_object_expiry(api.clone(), object, &event, &src).await;
                     } else {
                         apply_expiry_rule(&event, &src, object).await;
                     }
@@ -1977,9 +2004,10 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
 #[cfg(test)]
 mod tests {
     use super::{
-        StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
-        lifecycle_deleted_object, lifecycle_version_purge_state_from_completed_targets,
-        mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete,
+        DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks,
+        cleanup_stale_multipart_uploads_once_at, lifecycle_deleted_object, lifecycle_rule_has_date_expiration,
+        lifecycle_version_purge_state_from_completed_targets, mark_delete_opts_skip_decommissioned_on_remote_success,
+        merge_stale_multipart_candidate, replication_state_for_delete, should_defer_date_expiry_for_recent_config_update,
         should_reuse_lifecycle_delete_replication_state,
     };
     use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
@@ -1994,6 +2022,7 @@ mod tests {
         BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader,
     };
     use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
+    use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
     use std::collections::HashMap;
@@ -2014,6 +2043,49 @@ mod tests {
         assert!(opts.skip_decommissioned);
     }
 
+    #[test]
+    fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
+        let lc = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    date: Some(Timestamp::from(OffsetDateTime::now_utc())),
+                    ..Default::default()
+                }),
+                id: Some("rule-date".to_string()),
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        assert!(lifecycle_rule_has_date_expiration(&lc, "rule-date"));
+        assert!(!lifecycle_rule_has_date_expiration(&lc, "missing-rule"));
+    }
+
+    #[test]
+    fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
+        let now = OffsetDateTime::now_utc();
+        let recent = BucketLifecycleConfiguration {
+            expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
+            rules: Vec::new(),
+        };
+        let stale = BucketLifecycleConfiguration {
+            expiry_updated_at: Some(Timestamp::from(
+                now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
+            )),
+            rules: Vec::new(),
+        };
+
+        assert!(should_defer_date_expiry_for_recent_config_update(&recent, now));
+        assert!(!should_defer_date_expiry_for_recent_config_update(&stale, now));
+    }
+
     #[test]
     fn mark_delete_opts_skip_decommissioned_on_remote_success_preserves_false_on_failure() {
         let mut opts = ObjectOptions::default();

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -20,15 +20,144 @@
 
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject};
 use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts};
+use crate::client::signer_error::error_chain_contains_signer_header_marker;
 use crate::global::GLOBAL_TierConfigMgr;
+use rustfs_utils::get_env_usize;
 use sha2::{Digest, Sha256};
 use std::any::Any;
+use std::collections::VecDeque;
 use std::io::Write;
+use std::sync::LazyLock;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::{Duration, Instant};
+use tokio::sync::{Mutex, Semaphore};
+use tracing::warn;
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
 static XXHASH_SEED: u64 = 0;
 
+const ENV_REMOTE_DELETE_MAX_CONCURRENCY: &str = "RUSTFS_REMOTE_DELETE_MAX_CONCURRENCY";
+const ENV_REMOTE_DELETE_BREAKER_THRESHOLD: &str = "RUSTFS_REMOTE_DELETE_BREAKER_THRESHOLD";
+const ENV_REMOTE_DELETE_BREAKER_WINDOW_SECS: &str = "RUSTFS_REMOTE_DELETE_BREAKER_WINDOW_SECS";
+const DEFAULT_REMOTE_DELETE_BREAKER_THRESHOLD: usize = 50;
+const DEFAULT_REMOTE_DELETE_BREAKER_WINDOW_SECS: usize = 30;
+const METRIC_DELETE_REMOTE_FAILED_TOTAL: &str = "rustfs_delete_remote_failed_total";
+const METRIC_DELETE_REMOTE_BREAKER_TOTAL: &str = "rustfs_delete_remote_breaker_total";
+const METRIC_DELETE_REMOTE_INFLIGHT: &str = "rustfs_delete_remote_inflight";
+
+static REMOTE_DELETE_INFLIGHT: AtomicUsize = AtomicUsize::new(0);
+
+static REMOTE_DELETE_LIMITER: LazyLock<Semaphore> = LazyLock::new(|| {
+    let default_limit = std::cmp::min(num_cpus::get(), 16).max(1);
+    let concurrency = get_env_usize(ENV_REMOTE_DELETE_MAX_CONCURRENCY, default_limit).max(1);
+    Semaphore::new(concurrency)
+});
+
+static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::new(|| {
+    Mutex::new(RemoteDeleteBreaker::new(
+        get_env_usize(ENV_REMOTE_DELETE_BREAKER_THRESHOLD, DEFAULT_REMOTE_DELETE_BREAKER_THRESHOLD).max(1),
+        Duration::from_secs(
+            get_env_usize(ENV_REMOTE_DELETE_BREAKER_WINDOW_SECS, DEFAULT_REMOTE_DELETE_BREAKER_WINDOW_SECS) as u64,
+        ),
+    ))
+});
+
+#[derive(Debug)]
+struct RemoteDeleteBreaker {
+    threshold: usize,
+    window: Duration,
+    failures: VecDeque<Instant>,
+}
+
+impl RemoteDeleteBreaker {
+    fn new(threshold: usize, window: Duration) -> Self {
+        Self {
+            threshold: threshold.max(1),
+            window: window.max(Duration::from_secs(1)),
+            failures: VecDeque::new(),
+        }
+    }
+
+    fn should_short_circuit(&mut self, now: Instant) -> bool {
+        self.prune(now);
+        self.failures.len() >= self.threshold
+    }
+
+    fn record_signer_failure(&mut self, now: Instant) -> bool {
+        self.prune(now);
+        let was_open = self.failures.len() >= self.threshold;
+        self.failures.push_back(now);
+        !was_open && self.failures.len() >= self.threshold
+    }
+
+    fn prune(&mut self, now: Instant) {
+        while let Some(ts) = self.failures.front().copied() {
+            if now.duration_since(ts) > self.window {
+                self.failures.pop_front();
+            } else {
+                break;
+            }
+        }
+    }
+}
+
+struct RemoteDeleteInflightGuard;
+
+impl RemoteDeleteInflightGuard {
+    fn new() -> Self {
+        let inflight = REMOTE_DELETE_INFLIGHT.fetch_add(1, Ordering::Relaxed) + 1;
+        metrics::gauge!(METRIC_DELETE_REMOTE_INFLIGHT).set(inflight as f64);
+        Self
+    }
+}
+
+impl Drop for RemoteDeleteInflightGuard {
+    fn drop(&mut self) {
+        let inflight = REMOTE_DELETE_INFLIGHT.fetch_sub(1, Ordering::Relaxed) - 1;
+        metrics::gauge!(METRIC_DELETE_REMOTE_INFLIGHT).set(inflight as f64);
+    }
+}
+
+fn is_signer_header_error(err: &std::io::Error) -> bool {
+    if err.kind() != std::io::ErrorKind::InvalidInput {
+        return false;
+    }
+
+    if let Some(source) = err.get_ref() {
+        if error_chain_contains_signer_header_marker(source) {
+            return true;
+        }
+    }
+
+    let message = err.to_string().to_ascii_lowercase();
+    message.contains("invalid utf-8 header value")
+        || message.contains("invalidheadervalue")
+        || (message.contains("sign v4") && message.contains("header value"))
+}
+
+async fn remote_delete_breaker_is_open(now: Instant) -> bool {
+    let mut breaker = REMOTE_DELETE_BREAKER.lock().await;
+    breaker.should_short_circuit(now)
+}
+
+async fn record_remote_delete_failure(err: &std::io::Error, now: Instant) {
+    metrics::counter!(METRIC_DELETE_REMOTE_FAILED_TOTAL).increment(1);
+
+    if !is_signer_header_error(err) {
+        return;
+    }
+
+    let mut breaker = REMOTE_DELETE_BREAKER.lock().await;
+    if breaker.record_signer_failure(now) {
+        warn!(
+            threshold = breaker.threshold,
+            window_secs = breaker.window.as_secs(),
+            "remote tier delete breaker opened by signer/header failures"
+        );
+    }
+}
+
 #[derive(Default)]
 #[allow(dead_code)]
 struct ObjSweeper {
@@ -148,12 +277,31 @@ impl ExpiryOp for Jentry {
 }
 
 pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
+    if remote_delete_breaker_is_open(Instant::now()).await {
+        metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
+        return Err(std::io::Error::other("remote tier delete breaker is open due to signer/header failures"));
+    }
+
+    let _permit = REMOTE_DELETE_LIMITER
+        .acquire()
+        .await
+        .map_err(|_| std::io::Error::other("remote tier delete limiter is closed"))?;
+    let _inflight = RemoteDeleteInflightGuard::new();
+
     let mut config_mgr = GLOBAL_TierConfigMgr.write().await;
     let w = match config_mgr.get_driver(tier_name).await {
         Ok(w) => w,
-        Err(e) => return Err(std::io::Error::other(e)),
+        Err(e) => {
+            let err = std::io::Error::other(e);
+            record_remote_delete_failure(&err, Instant::now()).await;
+            return Err(err);
+        }
     };
-    w.remove(obj_name, rv_id).await
+    let result = w.remove(obj_name, rv_id).await;
+    if let Err(err) = &result {
+        record_remote_delete_failure(err, Instant::now()).await;
+    }
+    result
 }
 
 pub fn transitioned_delete_journal_entry(
@@ -189,4 +337,44 @@ pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject
 }
 
 #[cfg(test)]
-mod test {}
+mod test {
+    use crate::client::signer_error::invalid_utf8_header_error;
+
+    use super::{RemoteDeleteBreaker, is_signer_header_error};
+    use std::io::{Error, ErrorKind};
+    use std::time::{Duration, Instant};
+
+    #[test]
+    fn signer_header_error_detection_matches_utf8_failures() {
+        let err = Error::new(
+            ErrorKind::InvalidInput,
+            "failed to sign v4 request: invalid UTF-8 header value for `x-amz-meta-invalid`",
+        );
+        assert!(is_signer_header_error(&err));
+    }
+
+    #[test]
+    fn signer_header_error_detection_rejects_unrelated_errors() {
+        let err = Error::other("dial tcp: i/o timeout");
+        assert!(!is_signer_header_error(&err));
+    }
+
+    #[test]
+    fn signer_header_error_detection_matches_structured_marker() {
+        let err = invalid_utf8_header_error("failed to sign v4 request", "x-amz-meta-invalid");
+        assert!(is_signer_header_error(&err));
+    }
+
+    #[test]
+    fn breaker_opens_at_threshold_and_recovers_after_window() {
+        let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
+        let start = Instant::now();
+
+        assert!(!breaker.should_short_circuit(start));
+        assert!(!breaker.record_signer_failure(start));
+        assert!(!breaker.record_signer_failure(start + Duration::from_secs(1)));
+        assert!(breaker.record_signer_failure(start + Duration::from_secs(2)));
+        assert!(breaker.should_short_circuit(start + Duration::from_secs(3)));
+        assert!(!breaker.should_short_circuit(start + Duration::from_secs(40)));
+    }
+}

crates/ecstore/src/bucket/replication/config.rs

@@ -224,19 +224,98 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
                 continue;
             }
 
-            if !self.role.is_empty() {
-                arns.push(self.role.clone()); // Use the legacy RoleArn when present
-                return arns;
-            }
-
-            if !targets_map.contains(&rule.destination.bucket) {
+            if !rule.destination.bucket.is_empty() && !targets_map.contains(&rule.destination.bucket) {
                 targets_map.insert(rule.destination.bucket.clone());
             }
         }
 
+        if targets_map.is_empty() && !self.role.is_empty() {
+            arns.push(self.role.clone());
+            return arns;
+        }
+
         for arn in targets_map {
             arns.push(arn);
         }
         arns
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use s3s::dto::{DeleteMarkerReplication, Destination, ExistingObjectReplication, ReplicationRule};
+
+    fn replication_rule(id: &str, arn: &str) -> ReplicationRule {
+        ReplicationRule {
+            delete_marker_replication: Some(DeleteMarkerReplication::default()),
+            delete_replication: None,
+            destination: Destination {
+                bucket: arn.to_string(),
+                ..Default::default()
+            },
+            existing_object_replication: Some(ExistingObjectReplication {
+                status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
+            }),
+            filter: None,
+            id: Some(id.to_string()),
+            prefix: Some(String::new()),
+            priority: Some(1),
+            source_selection_criteria: None,
+            status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+        }
+    }
+
+    #[test]
+    fn filter_target_arns_keeps_multiple_destinations_when_role_is_present() {
+        let config = ReplicationConfiguration {
+            role: "arn:legacy:target".to_string(),
+            rules: vec![
+                replication_rule("rule-1", "arn:target:a"),
+                replication_rule("rule-2", "arn:target:b"),
+            ],
+        };
+
+        let arns = config.filter_target_arns(&ObjectOpts {
+            name: "object".to_string(),
+            op_type: ReplicationType::Object,
+            ..Default::default()
+        });
+
+        assert_eq!(arns.len(), 2);
+        assert!(arns.iter().any(|arn| arn == "arn:target:a"));
+        assert!(arns.iter().any(|arn| arn == "arn:target:b"));
+    }
+
+    #[test]
+    fn filter_target_arns_falls_back_to_role_when_destination_is_empty() {
+        let config = ReplicationConfiguration {
+            role: "arn:legacy:target".to_string(),
+            rules: vec![ReplicationRule {
+                delete_marker_replication: Some(DeleteMarkerReplication::default()),
+                delete_replication: None,
+                destination: Destination {
+                    bucket: String::new(),
+                    ..Default::default()
+                },
+                existing_object_replication: Some(ExistingObjectReplication {
+                    status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
+                }),
+                filter: None,
+                id: Some("rule-1".to_string()),
+                prefix: Some(String::new()),
+                priority: Some(1),
+                source_selection_criteria: None,
+                status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
+            }],
+        };
+
+        let arns = config.filter_target_arns(&ObjectOpts {
+            name: "object".to_string(),
+            op_type: ReplicationType::Object,
+            ..Default::default()
+        });
+
+        assert_eq!(arns, vec!["arn:legacy:target".to_string()]);
+    }
+}

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2578,6 +2578,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             }
         };
 
+        let has_tagging_replication = !put_opts.user_tags.is_empty();
         if let Some(err) = if is_multipart {
             drop(gr);
             let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -2593,6 +2594,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             })
             .await;
             record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            if has_tagging_replication {
+                record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+            }
             result.err()
         } else {
             gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
@@ -2602,6 +2606,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 .await
                 .map_err(|e| std::io::Error::other(e.to_string()));
             record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            if has_tagging_replication {
+                record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+            }
             result.err()
         } {
             rinfo.replication_status = ReplicationStatusType::Failed;
@@ -2867,6 +2874,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 }
             };
 
+            let has_tagging_replication = !put_opts.user_tags.is_empty();
             if let Some(err) = if is_multipart {
                 drop(gr);
                 let result = replicate_object_with_multipart(MultipartReplicationContext {
@@ -2882,6 +2890,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 })
                 .await;
                 record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                if has_tagging_replication {
+                    record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+                }
                 result.err()
             } else {
                 gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
@@ -2891,6 +2902,9 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     .await
                     .map_err(|e| std::io::Error::other(e.to_string()));
                 record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                if has_tagging_replication {
+                    record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
+                }
                 result.err()
             } {
                 rinfo.replication_status = ReplicationStatusType::Failed;

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -489,8 +489,14 @@ impl QueueCache {
 pub struct ProxyMetric {
     pub get_total: i64,
     pub get_failed: i64,
+    pub get_tag_total: i64,
+    pub get_tag_failed: i64,
     pub put_total: i64,
     pub put_failed: i64,
+    pub put_tag_total: i64,
+    pub put_tag_failed: i64,
+    pub delete_tag_total: i64,
+    pub delete_tag_failed: i64,
     pub head_total: i64,
     pub head_failed: i64,
 }
@@ -499,8 +505,14 @@ impl ProxyMetric {
     pub fn add(&mut self, other: &ProxyMetric) {
         self.get_total += other.get_total;
         self.get_failed += other.get_failed;
+        self.get_tag_total += other.get_tag_total;
+        self.get_tag_failed += other.get_tag_failed;
         self.put_total += other.put_total;
         self.put_failed += other.put_failed;
+        self.put_tag_total += other.put_tag_total;
+        self.put_tag_failed += other.put_tag_failed;
+        self.delete_tag_total += other.delete_tag_total;
+        self.delete_tag_failed += other.delete_tag_failed;
         self.head_total += other.head_total;
         self.head_failed += other.head_failed;
     }
@@ -527,18 +539,36 @@ impl ProxyStatsCache {
                     metric.get_failed += 1;
                 }
             }
+            "GetObjectTagging" => {
+                metric.get_tag_total += 1;
+                if is_err {
+                    metric.get_tag_failed += 1;
+                }
+            }
             "PutObject" => {
                 metric.put_total += 1;
                 if is_err {
                     metric.put_failed += 1;
                 }
             }
+            "PutObjectTagging" => {
+                metric.put_tag_total += 1;
+                if is_err {
+                    metric.put_tag_failed += 1;
+                }
+            }
             "HeadObject" => {
                 metric.head_total += 1;
                 if is_err {
                     metric.head_failed += 1;
                 }
             }
+            "DeleteObjectTagging" => {
+                metric.delete_tag_total += 1;
+                if is_err {
+                    metric.delete_tag_failed += 1;
+                }
+            }
             _ => {}
         }
     }

crates/ecstore/src/client/bucket_cache.rs

@@ -22,6 +22,7 @@ use super::constants::UNSIGNED_PAYLOAD;
 use super::credentials::SignatureType;
 use crate::client::{
     api_error_response::http_resp_to_error_response,
+    signer_error,
     transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
 };
 use http::Request;
@@ -35,6 +36,10 @@ use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
 use s3s::S3ErrorCode;
 use std::collections::HashMap;
 
+fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> std::io::Error {
+    signer_error::signer_error_to_io_error(scope, error)
+}
+
 #[derive(Debug, Clone)]
 pub struct BucketLocationCache {
     items: HashMap<String, String>,
@@ -179,10 +184,15 @@ impl TransitionClient {
             content_sha256 = UNSIGNED_PAYLOAD.to_string();
         }
 
-        if let Ok(content_sha256_value) = content_sha256.parse() {
-            req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
-        }
-        let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
+        let content_sha256_value = content_sha256.parse().map_err(|err| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                format!("invalid X-Amz-Content-Sha256 header value: {err}"),
+            )
+        })?;
+        req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
+        let req = rustfs_signer::try_sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1")
+            .map_err(|err| signer_error_to_io_error("failed to sign bucket location request", err))?;
         Ok(req)
     }
 }

crates/ecstore/src/client/mod.rs

@@ -35,5 +35,6 @@ pub mod constants;
 pub mod credentials;
 pub mod object_api_utils;
 pub mod object_handlers_common;
+pub mod signer_error;
 pub mod transition_api;
 pub mod utils;

crates/ecstore/src/client/signer_error.rs

@@ -0,0 +1,105 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::error::Error as StdError;
+use std::fmt::{Display, Formatter};
+use std::io::{Error, ErrorKind};
+
+pub(crate) const SIGNER_HEADER_ERROR_MARKER: &str = "rustfs_signer_header_error";
+
+#[derive(Debug)]
+struct SignerHeaderError {
+    scope: String,
+    header_name: String,
+}
+
+impl SignerHeaderError {
+    fn new(scope: &str, header_name: &str) -> Self {
+        Self {
+            scope: scope.to_string(),
+            header_name: header_name.to_string(),
+        }
+    }
+}
+
+impl Display for SignerHeaderError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}: invalid UTF-8 header value for `{}` [{}]",
+            self.scope, self.header_name, SIGNER_HEADER_ERROR_MARKER
+        )
+    }
+}
+
+impl StdError for SignerHeaderError {}
+
+pub(crate) fn invalid_utf8_header_error(scope: &str, header_name: &str) -> Error {
+    Error::new(ErrorKind::InvalidInput, SignerHeaderError::new(scope, header_name))
+}
+
+pub(crate) fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> Error {
+    match error {
+        rustfs_signer::SignV4Error::InvalidHeaderValue { name } => invalid_utf8_header_error(scope, &name),
+        other => Error::other(format!("{scope}: {other}")),
+    }
+}
+
+pub(crate) fn error_chain_contains_signer_header_marker(err: &(dyn StdError + 'static)) -> bool {
+    let mut current = Some(err);
+    while let Some(source) = current {
+        if source.downcast_ref::<SignerHeaderError>().is_some() {
+            return true;
+        }
+
+        if source.to_string().contains(SIGNER_HEADER_ERROR_MARKER) {
+            return true;
+        }
+
+        current = source.source();
+    }
+
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{error_chain_contains_signer_header_marker, invalid_utf8_header_error, signer_error_to_io_error};
+
+    #[test]
+    fn invalid_utf8_header_error_is_detected_through_error_chain() {
+        let err = invalid_utf8_header_error("failed to sign request", "x-amz-meta-invalid");
+
+        assert!(error_chain_contains_signer_header_marker(&err));
+    }
+
+    #[test]
+    fn mapped_signer_header_error_is_detected_through_error_chain() {
+        let err = signer_error_to_io_error(
+            "failed to sign request",
+            rustfs_signer::SignV4Error::InvalidHeaderValue {
+                name: "x-amz-meta-invalid".to_string(),
+            },
+        );
+
+        assert!(error_chain_contains_signer_header_marker(&err));
+    }
+
+    #[test]
+    fn generic_io_errors_do_not_match_signer_header_marker() {
+        let err = std::io::Error::other("unrelated failure");
+
+        assert!(!error_chain_contains_signer_header_marker(&err));
+    }
+}

crates/ecstore/src/client/transition_api.rs

@@ -31,6 +31,7 @@ use crate::client::{
     },
     constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
     credentials::{CredContext, Credentials, SignatureType, Static},
+    signer_error,
 };
 use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
 use futures::{Future, StreamExt};
@@ -85,6 +86,21 @@ const C_UNKNOWN: i32 = -1;
 const C_OFFLINE: i32 = 0;
 const C_ONLINE: i32 = 1;
 
+fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
+    signer_error::invalid_utf8_header_error(scope, header_name)
+}
+
+fn validate_header_values(headers: &HeaderMap, scope: &str) -> Result<(), std::io::Error> {
+    for (name, value) in headers {
+        value.to_str().map_err(|_| invalid_utf8_header_error(scope, name.as_str()))?;
+    }
+    Ok(())
+}
+
+fn signer_error_to_io_error(scope: &str, error: rustfs_signer::SignV4Error) -> std::io::Error {
+    signer_error::signer_error_to_io_error(scope, error)
+}
+
 //pub type ReaderImpl = Box<dyn Reader + Send + Sync + 'static>;
 pub enum ReaderImpl {
     Body(Bytes),
@@ -560,8 +576,9 @@ impl TransitionClient {
                         "extra signed headers for presign with signature v2 is not supported.",
                     )));
                 }
-                let headers = req.headers_mut();
                 if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
+                    validate_header_values(extra_headers, "presign extra header")?;
+                    let headers = req.headers_mut();
                     for (k, v) in extra_headers {
                         headers.insert(k, v.clone());
                     }
@@ -570,7 +587,7 @@ impl TransitionClient {
             if signer_type == SignatureType::SignatureV2 {
                 req = rustfs_signer::pre_sign_v2(req, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host);
             } else if signer_type == SignatureType::SignatureV4 {
-                req = rustfs_signer::pre_sign_v4(
+                req = rustfs_signer::try_pre_sign_v4(
                     req,
                     &access_key_id,
                     &secret_access_key,
@@ -578,12 +595,14 @@ impl TransitionClient {
                     &location,
                     metadata.expires,
                     OffsetDateTime::now_utc(),
-                );
+                )
+                .map_err(|err| signer_error_to_io_error("failed to presign v4 request", err))?;
             }
             return Ok(req);
         }
 
         self.set_user_agent(&mut req);
+        validate_header_values(&metadata.custom_header, "request custom header")?;
 
         for (k, v) in metadata.custom_header.clone() {
             if let Some(key) = k {
@@ -593,15 +612,15 @@ impl TransitionClient {
 
         //req.content_length = metadata.content_length;
         if metadata.content_length <= -1 {
-            if let Ok(chunked_value) = HeaderValue::from_str(&vec!["chunked"].join(",")) {
-                req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
-            }
+            req.headers_mut()
+                .insert(http::header::TRANSFER_ENCODING, HeaderValue::from_static("chunked"));
         }
 
-        if metadata.content_md5_base64.len() > 0 {
-            if let Ok(md5_value) = HeaderValue::from_str(&metadata.content_md5_base64) {
-                req.headers_mut().insert("Content-Md5", md5_value);
-            }
+        if !metadata.content_md5_base64.is_empty() {
+            let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).map_err(|err| {
+                std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("invalid Content-Md5 header value: {err}"))
+            })?;
+            req.headers_mut().insert("Content-Md5", md5_value);
         }
 
         if signer_type == SignatureType::SignatureAnonymous {
@@ -634,14 +653,15 @@ impl TransitionClient {
                 .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
             req.headers_mut().insert(header_name, header_value);
 
-            req = rustfs_signer::sign_v4_trailer(
+            req = rustfs_signer::try_sign_v4_trailer(
                 req,
                 &access_key_id,
                 &secret_access_key,
                 &session_token,
                 &location,
                 metadata.trailer.clone(),
-            );
+            )
+            .map_err(|err| signer_error_to_io_error("failed to sign v4 request", err))?;
         }
 
         if metadata.content_length > 0 {
@@ -1354,7 +1374,10 @@ pub struct CreateBucketConfiguration {
 
 #[cfg(test)]
 mod tests {
-    use super::{build_tls_config, load_root_store_from_tls_path, with_rustls_init_guard};
+    use super::{
+        build_tls_config, load_root_store_from_tls_path, signer_error_to_io_error, validate_header_values, with_rustls_init_guard,
+    };
+    use http::{HeaderMap, HeaderValue};
 
     #[test]
     fn rustls_guard_converts_panics_to_io_errors() {
@@ -1404,4 +1427,29 @@ mod tests {
         });
         assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
     }
+
+    #[test]
+    fn validate_header_values_returns_header_name_for_non_utf8_values() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            "x-amz-meta-invalid",
+            HeaderValue::from_bytes(&[0xFF]).expect("invalid utf8 bytes should be accepted by HeaderValue"),
+        );
+
+        let err =
+            validate_header_values(&headers, "request custom header").expect_err("invalid header value should fail validation");
+        assert!(err.to_string().contains("x-amz-meta-invalid"));
+    }
+
+    #[test]
+    fn signer_error_mapping_preserves_header_name() {
+        let err = signer_error_to_io_error(
+            "failed to sign v4 request",
+            rustfs_signer::SignV4Error::InvalidHeaderValue {
+                name: "x-amz-meta-invalid".to_string(),
+            },
+        );
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(err.to_string().contains("x-amz-meta-invalid"));
+    }
 }

crates/ecstore/src/disk/disk_store.rs

@@ -46,9 +46,7 @@ const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
-pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
 
 lazy_static::lazy_static! {
     static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]);
@@ -104,6 +102,20 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
     )
 }
 
+pub fn get_drive_active_check_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_active_check_timeout() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
+    ))
+}
+
 /// DiskHealthTracker tracks the health status of a disk.
 /// Similar to Go's diskHealthTracker.
 #[derive(Debug)]
@@ -231,9 +243,26 @@ impl DiskHealthTracker {
             RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
         };
 
-        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        let became_offline = next == RuntimeDriveHealthState::Offline && current != RuntimeDriveHealthState::Offline;
+        if next == RuntimeDriveHealthState::Offline {
+            self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        } else {
+            self.status.store(DISK_HEALTH_OK, Ordering::Release);
+        }
         self.transition_state(endpoint, current, next, reason);
-        current == RuntimeDriveHealthState::Online
+        became_offline
+    }
+
+    pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        if current == RuntimeDriveHealthState::Offline {
+            return false;
+        }
+
+        self.consecutive_successes.store(0, Ordering::Release);
+        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        self.transition_state(endpoint, current, RuntimeDriveHealthState::Offline, reason);
+        true
     }
 
     pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
@@ -268,6 +297,14 @@ impl DiskHealthTracker {
         became_online
     }
 
+    pub fn record_operation_success(&self, endpoint: &Endpoint, reason: &'static str) {
+        if self.runtime_state() == RuntimeDriveHealthState::Online {
+            self.log_success();
+        } else {
+            self.mark_recovery_success(endpoint, reason);
+        }
+    }
+
     fn transition_state(
         &self,
         endpoint: &Endpoint,
@@ -449,7 +486,7 @@ impl LocalDiskWrapper {
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         // TODO: config interval
 
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());
 
         loop {
             tokio::select! {
@@ -482,7 +519,16 @@ impl LocalDiskWrapper {
 
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                    if Self::perform_health_check(
+                        disk.clone(),
+                        &TEST_BUCKET,
+                        &test_obj,
+                        &TEST_DATA,
+                        true,
+                        get_drive_active_check_timeout(),
+                    )
+                    .await
+                    .is_err()
                         && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
                     {
                         // Health check failed, disk is considered faulty
@@ -588,7 +634,16 @@ impl LocalDiskWrapper {
                     }
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
+                    match Self::perform_health_check(
+                        disk.clone(),
+                        &TEST_BUCKET,
+                        &test_obj,
+                        &TEST_DATA,
+                        false,
+                        get_drive_active_check_timeout(),
+                    )
+                    .await
+                    {
                         Ok(_) => {
                             let state_before = health.runtime_state();
                             let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
@@ -707,7 +762,7 @@ impl LocalDiskWrapper {
             let result = operation().await;
             self.health.decrement_waiting();
             if result.is_ok() {
-                self.health.log_success();
+                self.health.record_operation_success(&self.endpoint(), "operation_success");
             }
             return result;
         }
@@ -718,7 +773,7 @@ impl LocalDiskWrapper {
             Ok(operation_result) => {
                 // Log success and decrement waiting counter
                 if operation_result.is_ok() {
-                    self.health.log_success();
+                    self.health.record_operation_success(&self.endpoint(), "operation_success");
                 }
                 self.health.decrement_waiting();
                 operation_result
@@ -919,7 +974,7 @@ impl DiskAPI for LocalDiskWrapper {
         let has_err = result.iter().any(|e| e.is_some());
         if !has_err {
             // Log success and decrement waiting counter
-            self.health.log_success();
+            self.health.record_operation_success(&self.endpoint(), "operation_success");
         }
 
         result
@@ -1112,37 +1167,92 @@ mod tests {
         });
     }
 
+    #[test]
+    fn drive_active_check_interval_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, || {
+            assert_eq!(
+                get_drive_active_check_interval(),
+                Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS)
+            );
+        });
+    }
+
+    #[test]
+    fn drive_active_check_interval_reads_env_override() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, Some("3"), || {
+            assert_eq!(get_drive_active_check_interval(), Duration::from_secs(3));
+        });
+    }
+
+    #[test]
+    fn drive_active_check_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, || {
+            assert_eq!(
+                get_drive_active_check_timeout(),
+                Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS)
+            );
+        });
+    }
+
+    #[test]
+    fn drive_active_check_timeout_reads_env_override() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1"), || {
+            assert_eq!(get_drive_active_check_timeout(), Duration::from_secs(1));
+        });
+    }
+
     #[test]
     fn runtime_state_transitions_from_online_to_suspect_then_offline() {
-        let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
-        let health = DiskHealthTracker::new();
+        temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || {
+            let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
+            let health = DiskHealthTracker::new();
 
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
-        assert!(health.mark_failure(&endpoint, "timeout"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+            assert!(!health.mark_failure(&endpoint, "timeout"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+            assert!(!health.is_faulty());
 
-        assert!(!health.mark_failure(&endpoint, "timeout"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
-        assert!(health.offline_duration().is_some());
+            assert!(health.mark_failure(&endpoint, "timeout"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+            assert!(health.is_faulty());
+            assert!(health.offline_duration().is_some());
+        });
     }
 
     #[test]
     fn runtime_state_transitions_back_online_after_recovery_threshold() {
-        let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+        temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || {
+            let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+            let health = DiskHealthTracker::new();
+
+            health.mark_failure(&endpoint, "timeout");
+            health.mark_failure(&endpoint, "timeout");
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+            assert!(!health.mark_recovery_success(&endpoint, "probe"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+            assert!(!health.mark_recovery_success(&endpoint, "probe"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+            assert!(health.mark_recovery_success(&endpoint, "probe"));
+            assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+            assert!(health.offline_duration().is_none());
+        });
+    }
+
+    #[test]
+    fn operation_success_recovers_suspect_drive_without_faulting() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-suspect-success").expect("endpoint should parse");
         let health = DiskHealthTracker::new();
 
-        health.mark_failure(&endpoint, "timeout");
-        health.mark_failure(&endpoint, "timeout");
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+        assert!(!health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+        assert!(!health.is_faulty());
 
-        assert!(!health.mark_recovery_success(&endpoint, "probe"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
-
-        assert!(!health.mark_recovery_success(&endpoint, "probe"));
-        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
-
-        assert!(health.mark_recovery_success(&endpoint, "probe"));
+        health.record_operation_success(&endpoint, "operation_success");
         assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(!health.is_faulty());
         assert!(health.offline_duration().is_none());
     }
 }

crates/ecstore/src/disk/local.rs

@@ -31,6 +31,7 @@ use crate::disk::{
 use crate::erasure_coding::bitrot_verify;
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
 use bytes::Bytes;
+use metrics::counter;
 use parking_lot::RwLock as ParkingLotRwLock;
 use rustfs_filemeta::{
     Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
@@ -79,6 +80,238 @@ pub enum InternalBuf<'a> {
     Owned(Bytes),
 }
 
+struct FileCacheReclaimWriter {
+    inner: File,
+    reclaim_len: usize,
+    reclaim_on_shutdown: bool,
+    reclaimed: bool,
+}
+
+struct FileCacheReclaimReader {
+    inner: File,
+    reclaim_offset: u64,
+    reclaim_len: usize,
+    reclaim_on_drop: bool,
+    reclaimed: bool,
+}
+
+fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
+    counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "ok".to_string()).increment(1);
+    counter!("rustfs_page_cache_reclaim_bytes_total", "kind" => kind.to_string()).increment(reclaim_len as u64);
+    metrics::histogram!("rustfs_page_cache_reclaim_duration_seconds", "kind" => kind.to_string())
+        .record(started.elapsed().as_secs_f64());
+}
+
+fn record_file_cache_reclaim_error(kind: &'static str) {
+    counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "err".to_string()).increment(1);
+}
+
+impl FileCacheReclaimReader {
+    fn new(inner: File, reclaim_offset: u64, reclaim_len: usize, reclaim_on_drop: bool) -> Self {
+        #[cfg(target_os = "macos")]
+        if reclaim_on_drop {
+            let _ = set_fd_nocache(&inner);
+        }
+
+        Self {
+            inner,
+            reclaim_offset,
+            reclaim_len,
+            reclaim_on_drop,
+            reclaimed: false,
+        }
+    }
+
+    #[cfg(target_os = "linux")]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        use core::num::NonZeroU64;
+        use rustix::fs::{Advice, fadvise};
+
+        if !self.reclaim_on_drop || self.reclaimed || self.reclaim_len == 0 {
+            return Ok(());
+        }
+
+        let started = std::time::Instant::now();
+        let reclaim_len =
+            NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return");
+        fadvise(&self.inner, self.reclaim_offset, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?;
+
+        self.reclaimed = true;
+        record_file_cache_reclaim_success("read", self.reclaim_len, started);
+        Ok(())
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg(target_os = "macos")]
+#[allow(unsafe_code)]
+fn set_fd_nocache(file: &File) -> std::io::Result<()> {
+    use std::os::fd::AsRawFd;
+
+    // SAFETY: `fcntl` is called on a valid file descriptor owned by `file`.
+    let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) };
+    if ret == -1 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(())
+}
+
+#[cfg(target_os = "macos")]
+#[allow(unsafe_code)]
+fn set_std_fd_nocache(file: &std::fs::File) -> std::io::Result<()> {
+    use std::os::fd::AsRawFd;
+
+    // SAFETY: `fcntl` is called on a valid file descriptor owned by `file`.
+    let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) };
+    if ret == -1 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(())
+}
+
+impl Drop for FileCacheReclaimReader {
+    fn drop(&mut self) {
+        if let Err(err) = self.reclaim_file_cache() {
+            record_file_cache_reclaim_error("read");
+            debug!(error = ?err, reclaim_offset = self.reclaim_offset, reclaim_len = self.reclaim_len, "failed to reclaim file cache after read");
+        }
+    }
+}
+
+impl tokio::io::AsyncRead for FileCacheReclaimReader {
+    fn poll_read(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
+    }
+}
+
+impl FileCacheReclaimWriter {
+    fn new(inner: File, reclaim_len: usize, reclaim_on_shutdown: bool) -> Self {
+        #[cfg(target_os = "macos")]
+        if reclaim_on_shutdown {
+            let _ = set_fd_nocache(&inner);
+        }
+
+        Self {
+            inner,
+            reclaim_len,
+            reclaim_on_shutdown,
+            reclaimed: false,
+        }
+    }
+
+    #[cfg(target_os = "linux")]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        use core::num::NonZeroU64;
+        use rustix::fs::{Advice, fadvise};
+
+        if !self.reclaim_on_shutdown || self.reclaimed || self.reclaim_len == 0 {
+            return Ok(());
+        }
+
+        let started = std::time::Instant::now();
+        let reclaim_len =
+            NonZeroU64::new(self.reclaim_len as u64).expect("reclaim_len is guaranteed non-zero by the early return");
+        fadvise(&self.inner, 0, Some(reclaim_len), Advice::DontNeed).map_err(std::io::Error::from)?;
+
+        self.reclaimed = true;
+        record_file_cache_reclaim_success("write", self.reclaim_len, started);
+        Ok(())
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn reclaim_file_cache(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+impl AsyncWrite for FileCacheReclaimWriter {
+    fn poll_write(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        std::pin::Pin::new(&mut self.inner).poll_write(cx, buf)
+    }
+
+    fn poll_flush(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
+        std::pin::Pin::new(&mut self.inner).poll_flush(cx)
+    }
+
+    fn poll_shutdown(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        match std::pin::Pin::new(&mut self.inner).poll_shutdown(cx) {
+            std::task::Poll::Ready(Ok(())) => {
+                if let Err(err) = self.reclaim_file_cache() {
+                    record_file_cache_reclaim_error("write");
+                    debug!(error = ?err, reclaim_len = self.reclaim_len, "failed to reclaim file cache after write");
+                }
+                std::task::Poll::Ready(Ok(()))
+            }
+            other => other,
+        }
+    }
+
+    fn poll_write_vectored(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        bufs: &[std::io::IoSlice<'_>],
+    ) -> std::task::Poll<std::io::Result<usize>> {
+        std::pin::Pin::new(&mut self.inner).poll_write_vectored(cx, bufs)
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.is_write_vectored()
+    }
+}
+
+fn should_reclaim_file_cache_after_write(file_size: i64) -> bool {
+    if file_size <= 0 {
+        return false;
+    }
+
+    if !rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE,
+    ) {
+        return false;
+    }
+
+    let threshold = rustfs_utils::get_env_usize(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+    );
+    file_size as usize >= threshold
+}
+
+fn should_reclaim_file_cache_after_read(length: usize) -> bool {
+    if length == 0 {
+        return false;
+    }
+
+    if !rustfs_utils::get_env_bool(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE,
+    ) {
+        return false;
+    }
+
+    let threshold = rustfs_utils::get_env_usize(
+        rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+        rustfs_config::DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD,
+    );
+    length >= threshold
+}
+
 pub struct LocalDisk {
     pub root: PathBuf,
     pub format_path: PathBuf,
@@ -1847,8 +2080,9 @@ impl DiskAPI for LocalDisk {
         let f = super::fs::open_file(&file_path, O_CREATE | O_WRONLY)
             .await
             .map_err(to_file_error)?;
+        let reclaim_on_shutdown = should_reclaim_file_cache_after_write(_file_size);
 
-        Ok(Box::new(f))
+        Ok(Box::new(FileCacheReclaimWriter::new(f, _file_size.max(0) as usize, reclaim_on_shutdown)))
 
         // Ok(())
     }
@@ -1920,7 +2154,8 @@ impl DiskAPI for LocalDisk {
             f.seek(SeekFrom::Start(offset as u64)).await?;
         }
 
-        Ok(Box::new(f))
+        let reclaim_on_drop = should_reclaim_file_cache_after_read(length);
+        Ok(Box::new(FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop)))
     }
 
     /// Zero-copy file read using memory mapping (Unix) or efficient read (non-Unix).
@@ -1965,9 +2200,15 @@ impl DiskAPI for LocalDisk {
             use memmap2::MmapOptions;
             let file_path_clone = file_path.clone();
 
+            let should_reclaim_after_read = should_reclaim_file_cache_after_read(length);
             let bytes = tokio::task::spawn_blocking(move || {
                 let file = std::fs::File::open(&file_path_clone).map_err(DiskError::from)?;
 
+                #[cfg(target_os = "macos")]
+                if should_reclaim_after_read {
+                    let _ = set_std_fd_nocache(&file);
+                }
+
                 // mmap offsets on Unix must be page-size aligned. Align the
                 // mapping down to the nearest page boundary, then slice out the
                 // originally requested logical range.
@@ -1995,7 +2236,21 @@ impl DiskAPI for LocalDisk {
                 let end = logical_offset
                     .checked_add(length)
                     .ok_or_else(|| DiskError::other("mmap slice length overflow"))?;
-                Ok::<Bytes, DiskError>(Bytes::copy_from_slice(&mmap[logical_offset..end]))
+                let bytes = Bytes::copy_from_slice(&mmap[logical_offset..end]);
+
+                #[cfg(target_os = "linux")]
+                if should_reclaim_after_read {
+                    use core::num::NonZeroU64;
+                    use rustix::fs::{Advice, fadvise};
+
+                    let reclaim_len =
+                        NonZeroU64::new(map_len as u64).ok_or_else(|| DiskError::other("mmap reclaim length overflow"))?;
+                    fadvise(&file, aligned_offset, Some(reclaim_len), Advice::DontNeed)
+                        .map_err(std::io::Error::from)
+                        .map_err(DiskError::from)?;
+                }
+
+                Ok::<Bytes, DiskError>(bytes)
             })
             .await
             .map_err(DiskError::from)??;
@@ -3372,4 +3627,32 @@ mod test {
             assert_eq!(normalize_path_components("C:\\a\\..\\b"), PathBuf::from("C:\\b"));
         }
     }
+
+    #[test]
+    fn should_reclaim_file_cache_after_write_respects_env_and_threshold() {
+        temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, || {
+            assert!(!should_reclaim_file_cache_after_write(8 * 1024 * 1024));
+        });
+
+        temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE, Some("true"), || {
+            temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || {
+                assert!(should_reclaim_file_cache_after_write(8 * 1024 * 1024));
+                assert!(!should_reclaim_file_cache_after_write(1024));
+            });
+        });
+    }
+
+    #[test]
+    fn should_reclaim_file_cache_after_read_respects_env_and_threshold() {
+        temp_env::with_var_unset(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, || {
+            assert!(!should_reclaim_file_cache_after_read(8 * 1024 * 1024));
+        });
+
+        temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE, Some("true"), || {
+            temp_env::with_var(rustfs_config::ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD, Some("4194304"), || {
+                assert!(should_reclaim_file_cache_after_read(8 * 1024 * 1024));
+                assert!(!should_reclaim_file_cache_after_read(1024));
+            });
+        });
+    }
 }

crates/ecstore/src/erasure_coding/encode.rs

@@ -26,6 +26,30 @@ use tokio::io::AsyncRead;
 use tokio::sync::mpsc;
 use tracing::error;
 
+const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
+const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 8 * 1024 * 1024;
+const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 8;
+
+fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
+    if expanded_block_bytes == 0 {
+        return 1;
+    }
+
+    max_inflight_bytes
+        .saturating_div(expanded_block_bytes)
+        .clamp(1, DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS)
+}
+
+fn queued_block_bytes(block: &[Bytes]) -> usize {
+    block.iter().map(Bytes::len).sum()
+}
+
+async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
+    while let Some(block) = rx.recv().await {
+        rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
+    }
+}
+
 pub(crate) struct MultiWriter<'a> {
     writers: &'a mut [Option<BitrotWriterWrapper>],
     write_quorum: usize,
@@ -185,7 +209,14 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
-        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
+        // Bound queued encoded blocks by memory budget to avoid per-request spikes.
+        let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
+        let max_inflight_bytes = rustfs_utils::get_env_usize(
+            ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
+            DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES,
+        );
+        let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
+        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
 
         let task = tokio::spawn(async move {
             let block_size = self.block_size;
@@ -196,7 +227,10 @@ impl Erasure {
                     Ok(n) if n > 0 => {
                         total += n;
                         let res = self.encode_data(&buf[..n])?;
+                        let queued_bytes = queued_block_bytes(&res);
+                        rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
                         if let Err(err) = tx.send(res).await {
+                            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
                             return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                         }
                     }
@@ -229,6 +263,8 @@ impl Erasure {
             if block.is_empty() {
                 break;
             }
+            let queued_bytes = queued_block_bytes(&block);
+            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
             if let Err(err) = writers.write(block).await {
                 write_err = Some(err);
                 break;
@@ -238,6 +274,7 @@ impl Erasure {
         if let Some(err) = write_err {
             task.abort();
             let _ = task.await;
+            drain_queued_inflight_bytes(&mut rx).await;
             if let Err(shutdown_err) = writers.shutdown().await {
                 error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
             }
@@ -310,4 +347,18 @@ mod tests {
         assert_eq!(written, b"small payload".len());
         assert!(!committed.lock().unwrap().is_empty());
     }
+
+    #[test]
+    fn encode_channel_capacity_never_returns_zero() {
+        assert_eq!(encode_channel_capacity(0, 1024), 1);
+        assert_eq!(encode_channel_capacity(4096, 0), 1);
+        assert_eq!(encode_channel_capacity(4096, 1024), 1);
+    }
+
+    #[test]
+    fn encode_channel_capacity_respects_budget_and_hard_cap() {
+        assert_eq!(encode_channel_capacity(4 * 1024 * 1024, 32 * 1024 * 1024), 8);
+        assert_eq!(encode_channel_capacity(16 * 1024 * 1024, 32 * 1024 * 1024), 2);
+        assert_eq!(encode_channel_capacity(1, usize::MAX), DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS);
+    }
 }

crates/ecstore/src/pools.rs

@@ -52,7 +52,7 @@ use rmp_serde::Serializer;
 use rustfs_common::defer;
 use rustfs_common::heal_channel::HealOpts;
 use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
-use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join};
+use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
 use rustfs_workers::workers::Workers;
 use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
 use serde::{Deserialize, Serialize};
@@ -986,25 +986,11 @@ impl PoolMeta {
 }
 
 pub fn path2_bucket_object(name: &str) -> (String, String) {
-    path2_bucket_object_with_base_path("", name)
+    path_to_bucket_object(name)
 }
 
 pub fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
-    // Trim the base path and leading slash
-    let trimmed_path = path
-        .strip_prefix(base_path)
-        .unwrap_or(path)
-        .strip_prefix(SLASH_SEPARATOR)
-        .unwrap_or(path);
-    // Find the position of the first '/'
-    let Some(pos) = trimmed_path.find(SLASH_SEPARATOR) else {
-        return (trimmed_path.to_string(), "".to_string());
-    };
-    // Split into bucket and prefix
-    let bucket = &trimmed_path[0..pos];
-    let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists
-
-    (bucket.to_string(), prefix.to_string())
+    path_to_bucket_object_with_base_path(base_path, path)
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -3746,4 +3732,13 @@ mod pools_tests {
                 .contains("failed to start decommission routines: scheduled 1 of 2 expected workers")
         );
     }
+
+    #[test]
+    #[cfg(windows)]
+    fn test_path2_bucket_object_with_base_path_supports_windows_separators() {
+        let (bucket, object) = super::path2_bucket_object_with_base_path("C:\\data", "C:\\data\\my-bucket\\nested\\object.txt");
+
+        assert_eq!(bucket, "my-bucket");
+        assert_eq!(object, "nested/object.txt");
+    }
 }

crates/ecstore/src/rpc/client.rs

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::error::{DiskError, Error as DiskErrorType};
 use crate::rpc::{TONIC_RPC_PREFIX, gen_signature_headers};
 use http::Method;
 use rustfs_common::GLOBAL_CONN_MAP;
 use rustfs_protos::{create_new_channel, proto_gen::node_service::node_service_client::NodeServiceClient};
-use std::error::Error;
+use std::{error::Error, io::ErrorKind};
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
 use tracing::debug;
 
@@ -51,6 +52,46 @@ pub async fn node_service_time_out_client_no_auth(
     node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
 }
 
+pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
+    match err {
+        DiskError::Timeout => true,
+        DiskError::Io(io_err) => {
+            if matches!(
+                io_err.kind(),
+                ErrorKind::TimedOut
+                    | ErrorKind::ConnectionRefused
+                    | ErrorKind::ConnectionReset
+                    | ErrorKind::BrokenPipe
+                    | ErrorKind::NotConnected
+                    | ErrorKind::ConnectionAborted
+                    | ErrorKind::UnexpectedEof
+            ) {
+                return true;
+            }
+
+            let message = io_err.to_string().to_ascii_lowercase();
+            [
+                "transport error",
+                "unavailable",
+                "error trying to connect",
+                "connection refused",
+                "connection reset",
+                "broken pipe",
+                "not connected",
+                "unexpected eof",
+                "timed out",
+                "deadline has elapsed",
+                "connection closed",
+                "connection aborted",
+                "tcp connect error",
+            ]
+            .iter()
+            .any(|needle| message.contains(needle))
+        }
+        _ => false,
+    }
+}
+
 pub struct TonicSignatureInterceptor;
 
 impl tonic::service::Interceptor for TonicSignatureInterceptor {

crates/ecstore/src/rpc/peer_s3_client.rs

@@ -18,13 +18,15 @@ use crate::disk::error::{Error, Result};
 use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
 use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration};
 use crate::global::GLOBAL_LOCAL_DISK_MAP;
-use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::rpc::client::{
+    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+};
 use crate::store::all_local_disk;
 use crate::store_utils::is_reserved_or_invalid_bucket;
 use crate::{
     disk::{
         self, VolumeInfo,
-        disk_store::{CHECK_EVERY, CHECK_TIMEOUT_DURATION, DiskHealthTracker},
+        disk_store::{DiskHealthTracker, get_drive_active_check_interval, get_drive_active_check_timeout},
     },
     endpoints::{EndpointServerPools, Node},
     store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
@@ -613,7 +615,7 @@ impl RemotePeerS3Client {
 
     /// Monitor remote peer health periodically
     async fn monitor_remote_peer_health(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());
 
         loop {
             tokio::select! {
@@ -682,7 +684,7 @@ impl RemotePeerS3Client {
         let port = url.port_or_known_default().unwrap_or(80);
 
         // Try to establish TCP connection
-        match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
+        match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
             Ok(Ok(_)) => Ok(()),
             _ => Err(Error::other(format!("Cannot connect to {host}:{port}"))),
         }
@@ -717,16 +719,39 @@ impl RemotePeerS3Client {
                     self.health.log_success();
                 }
                 self.health.decrement_waiting();
+                if let Err(err) = &operation_result
+                    && is_network_like_disk_error(err)
+                {
+                    self.mark_faulty_and_start_recovery("operation_network_error").await;
+                }
                 operation_result
             }
             Err(_) => {
                 // Timeout occurred, mark peer as potentially faulty
                 self.health.decrement_waiting();
+                self.mark_faulty_and_start_recovery("operation_timeout").await;
                 warn!("Remote peer operation timeout after {:?}", timeout_duration);
                 Err(Error::other(format!("Remote peer operation timeout after {timeout_duration:?}")))
             }
         }
     }
+
+    async fn mark_faulty_and_start_recovery(&self, reason: &'static str) {
+        if self.health.swap_ok_to_faulty() {
+            warn!(
+                addr = %self.addr,
+                reason,
+                "Remote peer marked faulty after network failure"
+            );
+
+            let health = Arc::clone(&self.health);
+            let cancel_token = self.cancel_token.clone();
+            let addr = self.addr.clone();
+            tokio::spawn(async move {
+                Self::monitor_remote_peer_recovery(addr, health, cancel_token).await;
+            });
+        }
+    }
 }
 
 #[async_trait]
@@ -1001,3 +1026,69 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResu
 async fn clone_drives() -> Vec<Option<DiskStore>> {
     GLOBAL_LOCAL_DISK_MAP.read().await.values().cloned().collect::<Vec<_>>()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn test_remote_peer(addr: &str) -> RemotePeerS3Client {
+        let node = Node {
+            url: url::Url::parse(addr).expect("test peer URL should parse"),
+            pools: vec![0],
+            is_local: false,
+            grid_host: addr.to_string(),
+        };
+
+        RemotePeerS3Client {
+            node: Some(node),
+            pools: Some(vec![0]),
+            addr: addr.to_string(),
+            health: Arc::new(DiskHealthTracker::new()),
+            cancel_token: CancellationToken::new(),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
+        let client = test_remote_peer("http://peer-network-error:9000");
+
+        let err = client
+            .execute_with_timeout(
+                || async {
+                    Err::<(), Error>(DiskError::Io(std::io::Error::new(
+                        std::io::ErrorKind::ConnectionRefused,
+                        "connection refused",
+                    )))
+                },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("network-like error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io network error, got {other:?}"),
+            },
+            std::io::ErrorKind::ConnectionRefused
+        );
+        assert!(client.health.is_faulty(), "network-like errors should mark remote peer faulty");
+
+        client.cancel_token.cancel();
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_keeps_remote_peer_online_for_business_error() {
+        let client = test_remote_peer("http://peer-business-error:9000");
+
+        let err = client
+            .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
+            .await
+            .expect_err("business error should fail");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(!client.health.is_faulty(), "business errors should not mark remote peer faulty");
+
+        client.cancel_token.cancel();
+    }
+}

crates/ecstore/src/rpc/remote_disk.rs

@@ -16,15 +16,17 @@ use crate::disk::{
     CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader,
     FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
     disk_store::{
-        CHECK_EVERY, CHECK_TIMEOUT_DURATION, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING,
-        SKIP_IF_SUCCESS_BEFORE, get_drive_disk_info_timeout, get_drive_list_dir_timeout, get_drive_metadata_timeout,
-        get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
+        DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
+        get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
+        get_drive_metadata_timeout, get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
     },
     endpoint::Endpoint,
     health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
-use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::rpc::client::{
+    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+};
 use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
 use crate::{
     disk::error::{Error, Result},
@@ -47,7 +49,7 @@ use rustfs_protos::proto_gen::node_service::{
 use rustfs_rio::{HttpReader, HttpWriter};
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
-    io::{Cursor, ErrorKind},
+    io::Cursor,
     path::PathBuf,
     sync::{
         Arc,
@@ -176,10 +178,10 @@ impl RemoteDisk {
         health: Arc<DiskHealthTracker>,
         cancel_token: CancellationToken,
     ) {
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());
 
         // Perform basic connectivity check
-        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
+        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
             warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
             // Start recovery monitoring
@@ -222,7 +224,7 @@ impl RemoteDisk {
                     }
 
                     // Perform basic connectivity check
-                    if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
+                    if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
                         warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
                         // Start recovery monitoring
@@ -263,7 +265,7 @@ impl RemoteDisk {
                             return;
                         }
                     } else {
-                        health.mark_failure(&endpoint, "connectivity_probe_failed");
+                        health.mark_offline(&endpoint, "connectivity_probe_failed");
                     }
                 }
             }
@@ -281,7 +283,7 @@ impl RemoteDisk {
         let port = url.port_or_known_default().unwrap_or(80);
 
         // Try to establish TCP connection
-        match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
+        match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
             Ok(Ok(stream)) => {
                 drop(stream);
                 Ok(())
@@ -334,10 +336,10 @@ impl RemoteDisk {
                 }
                 self.health.decrement_waiting();
                 if let Err(err) = &operation_result
-                    && Self::is_timeout_like_error(err)
+                    && is_network_like_disk_error(err)
                 {
                     counter!(
-                        "rustfs_drive_op_timeout_total",
+                        "rustfs_drive_op_network_error_total",
                         "endpoint" => self.endpoint.to_string(),
                         "op" => op.to_string()
                     )
@@ -347,9 +349,9 @@ impl RemoteDisk {
                         addr = %self.addr,
                         op,
                         timeout_ms = timeout_duration.as_millis(),
-                        "Remote disk operation returned a timeout-like error"
+                        "Remote disk operation returned a network-like error"
                     );
-                    self.mark_faulty_and_evict("operation_timeout_error").await;
+                    self.mark_faulty_and_evict("operation_network_error").await;
                 }
                 operation_result
             }
@@ -375,12 +377,8 @@ impl RemoteDisk {
         }
     }
 
-    fn is_timeout_like_error(err: &Error) -> bool {
-        matches!(err, DiskError::Timeout) || matches!(err, DiskError::Io(io_err) if io_err.kind() == ErrorKind::TimedOut)
-    }
-
     async fn mark_faulty_and_evict(&self, reason: &'static str) {
-        if self.health.mark_failure(&self.endpoint, reason) {
+        if self.health.mark_offline(&self.endpoint, reason) {
             self.spawn_recovery_monitor_if_needed();
             counter!(
                 "rustfs_drive_faulty_mark_total",
@@ -2073,6 +2071,96 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_faulty_on_network_like_error() {
+        let addr = "http://127.0.0.1:59993".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(
+                || async {
+                    Err::<(), Error>(DiskError::Io(std::io::Error::new(
+                        std::io::ErrorKind::ConnectionRefused,
+                        "connection refused",
+                    )))
+                },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("network-like operation error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io network error, got {other:?}"),
+            },
+            std::io::ErrorKind::ConnectionRefused
+        );
+        assert!(!remote_disk.is_online().await, "network-like errors should mark remote disk faulty");
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "network-like errors should evict cached connection"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_keeps_remote_disk_online_for_business_error() {
+        let addr = "http://127.0.0.1:59994".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
+            .await
+            .expect_err("business error should still fail the operation");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(remote_disk.is_online().await, "business errors should not mark remote disk faulty");
+        assert!(
+            GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "business errors should not evict cached connection"
+        );
+    }
+
     #[test]
     fn test_remote_disk_sync_properties() {
         let url = url::Url::parse("https://secure-remote:9000/data").unwrap();

crates/ecstore/src/set_disk/multipart.rs

@@ -13,6 +13,87 @@
 // limitations under the License.
 
 use super::*;
+use std::future::Future;
+use std::time::Duration;
+use tokio::task::JoinSet;
+
+async fn collect_list_parts_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> disk::error::Result<(Vec<Option<DiskError>>, Vec<Vec<String>>)>
+where
+    F: Future<Output = disk::error::Result<Vec<String>>> + Send + 'static,
+{
+    let mut errs = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut object_parts = vec![Vec::new(); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(parts))) => {
+                errs[index] = None;
+                object_parts[index] = parts;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errs[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(DiskError::ErasureReadQuorum);
+        }
+    }
+
+    Ok((errs, object_parts))
+}
+
+fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize) -> Vec<usize> {
+    let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
+
+    for drive_parts in object_parts {
+        let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
+
+        // part files can be either part.N or part.N.meta
+        for part_path in drive_parts {
+            if let Some(num_str) = part_path.strip_prefix("part.") {
+                if let Some(meta_idx) = num_str.find(".meta") {
+                    if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
+                        *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
+                    }
+                } else if let Ok(part_num) = num_str.parse::<usize>() {
+                    *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
+                }
+            }
+        }
+
+        // Include only part.N.meta files with corresponding part.N
+        for (&part_num, &cnt) in &parts_with_meta_count {
+            if cnt >= 2 {
+                *part_quorum_map.entry(part_num).or_insert(0) += 1;
+            }
+        }
+    }
+
+    let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
+    for (part_num, count) in part_quorum_map {
+        if count >= read_quorum {
+            part_numbers.push(part_num);
+        }
+    }
+
+    part_numbers.sort();
+    part_numbers
+}
 
 impl SetDisks {
     pub(super) async fn list_parts(
@@ -21,10 +102,13 @@ impl SetDisks {
         read_quorum: usize,
     ) -> disk::error::Result<Vec<usize>> {
         let mut futures = Vec::with_capacity(disks.len());
-        for (i, disk) in disks.iter().enumerate() {
+        let part_path = part_path.to_string();
+        for disk in disks.iter() {
+            let disk = disk.clone();
+            let part_path = part_path.clone();
             futures.push(async move {
                 if let Some(disk) = disk {
-                    disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
+                    disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
                         .await
                 } else {
                     Err(DiskError::DiskNotFound)
@@ -35,60 +119,15 @@ impl SetDisks {
         let mut errs = Vec::with_capacity(disks.len());
         let mut object_parts = Vec::with_capacity(disks.len());
 
-        let results = join_all(futures).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    errs.push(None);
-                    object_parts.push(res);
-                }
-                Err(e) => {
-                    errs.push(Some(e));
-                    object_parts.push(vec![]);
-                }
-            }
-        }
+        let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?;
+        errs.extend(collected_errs);
+        object_parts.extend(collected_parts);
 
         if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
             return Err(err);
         }
 
-        let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
-
-        for drive_parts in object_parts {
-            let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
-
-            // part files can be either part.N or part.N.meta
-            for part_path in drive_parts {
-                if let Some(num_str) = part_path.strip_prefix("part.") {
-                    if let Some(meta_idx) = num_str.find(".meta") {
-                        if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
-                            *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
-                        }
-                    } else if let Ok(part_num) = num_str.parse::<usize>() {
-                        *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
-                    }
-                }
-            }
-
-            // Include only part.N.meta files with corresponding part.N
-            for (&part_num, &cnt) in &parts_with_meta_count {
-                if cnt >= 2 {
-                    *part_quorum_map.entry(part_num).or_insert(0) += 1;
-                }
-            }
-        }
-
-        let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
-        for (part_num, count) in part_quorum_map {
-            if count >= read_quorum {
-                part_numbers.push(part_num);
-            }
-        }
-
-        part_numbers.sort();
-
-        Ok(part_numbers)
+        Ok(reduce_quorum_part_numbers(object_parts, read_quorum))
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
@@ -145,3 +184,78 @@ impl SetDisks {
         Ok((fi, parts_metadata))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn collect_list_parts_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let err = collect_list_parts_results(tasks, 2)
+            .await
+            .expect_err("quorum should become impossible before slow tail completes");
+
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+        assert!(started.elapsed() < Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_list_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| async move {
+                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                if should_panic {
+                    panic!("simulated task panic");
+                }
+                Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])
+            })
+            .collect();
+
+        let (errs, object_parts) = collect_list_parts_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(errs.iter().filter(|err| err.is_none()).count(), 2);
+        assert_eq!(object_parts.iter().filter(|parts| !parts.is_empty()).count(), 2);
+    }
+
+    #[test]
+    fn reduce_quorum_part_numbers_only_keeps_parts_present_on_quorum_of_drives() {
+        let object_parts = vec![
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.2".to_string(),
+                "part.2.meta".to_string(),
+            ],
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.3".to_string(),
+                "part.3.meta".to_string(),
+            ],
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.2".to_string(),
+                "part.2.meta".to_string(),
+            ],
+        ];
+
+        let parts = reduce_quorum_part_numbers(object_parts, 2);
+        assert_eq!(parts, vec![1, 2]);
+    }
+}

crates/ecstore/src/set_disk/read.rs

@@ -14,6 +14,88 @@
 
 use super::*;
 use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
+use std::future::Future;
+use tokio::task::JoinSet;
+
+async fn collect_read_multiple_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> std::result::Result<(Vec<Option<Vec<ReadMultipleResp>>>, Vec<Option<DiskError>>), ()>
+where
+    F: Future<Output = disk::error::Result<Vec<ReadMultipleResp>>> + Send + 'static,
+{
+    let mut responses = vec![None; tasks.len()];
+    let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(resp))) => {
+                responses[index] = Some(resp);
+                errors[index] = None;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errors[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(());
+        }
+    }
+
+    Ok((responses, errors))
+}
+
+async fn collect_read_parts_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> std::result::Result<(Vec<Option<Vec<ObjectPartInfo>>>, Vec<Option<DiskError>>), ()>
+where
+    F: Future<Output = disk::error::Result<Vec<ObjectPartInfo>>> + Send + 'static,
+{
+    let mut responses = vec![None; tasks.len()];
+    let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(resp))) => {
+                responses[index] = Some(resp);
+                errors[index] = None;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errors[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(());
+        }
+    }
+
+    Ok((responses, errors))
+}
 
 impl SetDisks {
     pub(super) async fn read_parts(
@@ -25,9 +107,6 @@ impl SetDisks {
     ) -> disk::error::Result<Vec<ObjectPartInfo>> {
         let mut errs = Vec::with_capacity(disks.len());
         let mut object_parts = Vec::with_capacity(disks.len());
-
-        // Use batch processor for better performance
-        let processor = get_global_processors().read_processor();
         let bucket = bucket.to_string();
         let part_meta_paths = part_meta_paths.to_vec();
 
@@ -48,19 +127,13 @@ impl SetDisks {
             })
             .collect();
 
-        let results = processor.execute_batch(tasks).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    errs.push(None);
-                    object_parts.push(res);
-                }
-                Err(e) => {
-                    errs.push(Some(e));
-                    object_parts.push(vec![]);
-                }
-            }
-        }
+        let (responses, collected_errors) = match collect_read_parts_results(tasks, read_quorum).await {
+            Ok(collected) => collected,
+            Err(()) => return Err(DiskError::ErasureReadQuorum),
+        };
+
+        errs.extend(collected_errors);
+        object_parts.extend(responses.into_iter().map(|resp| resp.unwrap_or_default()));
 
         if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
             return Err(err);
@@ -384,10 +457,23 @@ impl SetDisks {
         read_quorum: usize,
     ) -> Vec<ReadMultipleResp> {
         let mut futures = Vec::with_capacity(disks.len());
-        let mut ress = Vec::with_capacity(disks.len());
-        let mut errors = Vec::with_capacity(disks.len());
+        let empty_quorum_result = || {
+            req.files
+                .iter()
+                .map(|want| ReadMultipleResp {
+                    bucket: req.bucket.clone(),
+                    prefix: req.prefix.clone(),
+                    file: want.clone(),
+                    exists: false,
+                    error: Error::ErasureReadQuorum.to_string(),
+                    data: Vec::new(),
+                    mod_time: None,
+                })
+                .collect::<Vec<_>>()
+        };
 
         for disk in disks.iter() {
+            let disk = disk.clone();
             let req = req.clone();
             futures.push(async move {
                 if let Some(disk) = disk {
@@ -398,19 +484,10 @@ impl SetDisks {
             });
         }
 
-        let results = join_all(futures).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    ress.push(Some(res));
-                    errors.push(None);
-                }
-                Err(e) => {
-                    ress.push(None);
-                    errors.push(Some(e));
-                }
-            }
-        }
+        let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
+            Ok(collected) => collected,
+            Err(()) => return empty_quorum_result(),
+        };
 
         // debug!("ReadMultipleResp ress {:?}", ress);
         // debug!("ReadMultipleResp errors {:?}", errors);
@@ -839,3 +916,181 @@ impl SetDisks {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let result = collect_read_multiple_results(tasks, 2).await;
+        assert!(result.is_err(), "quorum should become impossible before slow tail completes");
+        assert!(started.elapsed() < std::time::Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_returns_collected_responses_on_quorum() {
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1, 2, 3],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
+            (15, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
+            (250, Err(DiskError::DiskNotFound)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let (responses, errors) = collect_read_multiple_results(tasks, 2).await.expect("quorum should succeed");
+
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1, 2, 3],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| {
+                let resp = resp.clone();
+                async move {
+                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+                    if should_panic {
+                        panic!("simulated task panic");
+                    }
+                    Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])
+                }
+            })
+            .collect();
+
+        let (responses, errors) = collect_read_multiple_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let result = collect_read_parts_results(tasks, 2).await;
+        assert!(result.is_err(), "quorum should become impossible before slow tail completes");
+        assert!(started.elapsed() < std::time::Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_returns_collected_responses_on_quorum() {
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
+            (15, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
+            (250, Err(DiskError::DiskNotFound)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let (responses, errors) = collect_read_parts_results(tasks, 2).await.expect("quorum should succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| {
+                let part = part.clone();
+                async move {
+                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+                    if should_panic {
+                        panic!("simulated task panic");
+                    }
+                    Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])
+                }
+            })
+            .collect();
+
+        let (responses, errors) = collect_read_parts_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+}

crates/ecstore/src/store_list_objects.rs

@@ -829,6 +829,7 @@ impl ECStore {
         let (merge_tx, mut merge_rx) = mpsc::channel::<MetaCacheEntry>(100);
 
         let bucket = bucket.to_owned();
+        let bucket_clone = bucket.clone();
 
         let vcf = match get_versioning_config(&bucket).await {
             Ok((res, _)) => Some(res),
@@ -839,7 +840,7 @@ impl ECStore {
             let mut sent_err = false;
             while let Some(entry) = merge_rx.recv().await {
                 if opts.latest_only {
-                    let fi = match entry.to_fileinfo(&bucket) {
+                    let fi = match entry.to_fileinfo(&bucket_clone) {
                         Ok(res) => res,
                         Err(err) => {
                             if !sent_err {
@@ -864,7 +865,7 @@ impl ECStore {
                     if let Some(filter) = opts.filter {
                         if filter(&fi) {
                             let item = ObjectInfoOrErr {
-                                item: Some(ObjectInfo::from_file_info(&fi, &bucket, &fi.name, {
+                                item: Some(ObjectInfo::from_file_info(&fi, &bucket_clone, &fi.name, {
                                     if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                                 })),
                                 err: None,
@@ -876,7 +877,7 @@ impl ECStore {
                         }
                     } else {
                         let item = ObjectInfoOrErr {
-                            item: Some(ObjectInfo::from_file_info(&fi, &bucket, &fi.name, {
+                            item: Some(ObjectInfo::from_file_info(&fi, &bucket_clone, &fi.name, {
                                 if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                             })),
                             err: None,
@@ -889,7 +890,7 @@ impl ECStore {
                     continue;
                 }
 
-                let fvs = match entry.file_info_versions(&bucket) {
+                let fvs = match entry.file_info_versions(&bucket_clone) {
                     Ok(res) => res,
                     Err(err) => {
                         let item = ObjectInfoOrErr {
@@ -912,7 +913,7 @@ impl ECStore {
                     if let Some(filter) = opts.filter {
                         if filter(fi) {
                             let item = ObjectInfoOrErr {
-                                item: Some(ObjectInfo::from_file_info(fi, &bucket, &fi.name, {
+                                item: Some(ObjectInfo::from_file_info(fi, &bucket_clone, &fi.name, {
                                     if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                                 })),
                                 err: None,
@@ -924,7 +925,7 @@ impl ECStore {
                         }
                     } else {
                         let item = ObjectInfoOrErr {
-                            item: Some(ObjectInfo::from_file_info(fi, &bucket, &fi.name, {
+                            item: Some(ObjectInfo::from_file_info(fi, &bucket_clone, &fi.name, {
                                 if let Some(v) = &vcf { v.versioned(&fi.name) } else { false }
                             })),
                             err: None,
@@ -940,7 +941,18 @@ impl ECStore {
 
         tokio::spawn(async move { merge_entry_channels(rx, inputs, merge_tx, 1).await });
 
-        join_all(futures).await;
+        let walk_results = join_all(futures).await;
+        for (idx, walk_result) in walk_results.into_iter().enumerate() {
+            if let Err(err) = walk_result {
+                error!(
+                    bucket = %bucket,
+                    prefix = %prefix,
+                    set_task_index = idx,
+                    error = ?err,
+                    "walk_internal list_path_raw task failed"
+                );
+            }
+        }
 
         Ok(())
     }

crates/heal/src/heal/manager.rs

@@ -237,6 +237,14 @@ impl PriorityHealQueue {
     }
 }
 
+fn publish_active_heal_count(active_heals: &HashMap<String, Arc<HealTask>>) {
+    crate::set_heal_active_tasks(active_heals.len());
+}
+
+fn publish_heal_queue_length(queue: &PriorityHealQueue) {
+    crate::set_heal_queue_length(queue.len());
+}
+
 /// Heal config
 #[derive(Debug, Clone)]
 pub struct HealConfig {
@@ -420,6 +428,8 @@ impl HealManager {
             }
         }
         active_heals.clear();
+        publish_active_heal_count(&active_heals);
+        crate::set_heal_queue_length(0);
 
         // update state
         let mut state = self.state.write().await;
@@ -435,6 +445,7 @@ impl HealManager {
         let mut queue = self.heal_queue.lock().await;
 
         let queue_len = queue.len();
+        publish_heal_queue_length(&queue);
         let queue_capacity = config.queue_size;
 
         if queue.contains_key(&request) {
@@ -505,6 +516,7 @@ impl HealManager {
 
         let push_outcome = queue.push(request);
         debug_assert_eq!(push_outcome, QueuePushOutcome::Accepted);
+        publish_heal_queue_length(&queue);
 
         // Log queue statistics periodically (when adding high/urgent priority items)
         if matches!(priority, HealPriority::High | HealPriority::Urgent) {
@@ -544,7 +556,9 @@ impl HealManager {
 
     /// Get task progress
     pub async fn get_active_tasks_count(&self) -> usize {
-        self.active_heals.lock().await.len()
+        let active_heals = self.active_heals.lock().await;
+        publish_active_heal_count(&active_heals);
+        active_heals.len()
     }
 
     pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
@@ -564,6 +578,7 @@ impl HealManager {
         if let Some(task) = active_heals.get(task_id) {
             task.cancel().await?;
             active_heals.remove(task_id);
+            publish_active_heal_count(&active_heals);
             info!("Cancelled heal task: {}", task_id);
             Ok(())
         } else {
@@ -581,12 +596,14 @@ impl HealManager {
     /// Get active task count
     pub async fn get_active_task_count(&self) -> usize {
         let active_heals = self.active_heals.lock().await;
+        publish_active_heal_count(&active_heals);
         active_heals.len()
     }
 
     /// Get queue length
     pub async fn get_queue_length(&self) -> usize {
         let queue = self.heal_queue.lock().await;
+        publish_heal_queue_length(&queue);
         queue.len()
     }
 
@@ -731,6 +748,7 @@ impl HealManager {
                             );
                             let mut queue = heal_queue.lock().await;
                             if matches!(queue.push(req), QueuePushOutcome::Accepted) {
+                                publish_heal_queue_length(&queue);
                                 let config = config.read().await;
                                 if config.event_driven_scheduler_enable {
                                     notify.notify_one();
@@ -757,6 +775,7 @@ impl HealManager {
     ) {
         let config = config.read().await;
         let mut active_heals_guard = active_heals.lock().await;
+        publish_active_heal_count(&active_heals_guard);
 
         // Check if new heal tasks can be started
         let active_count = active_heals_guard.len();
@@ -769,6 +788,7 @@ impl HealManager {
 
         let mut queue = heal_queue.lock().await;
         let queue_len = queue.len();
+        publish_heal_queue_length(&queue);
 
         if queue_len == 0 {
             return;
@@ -804,6 +824,7 @@ impl HealManager {
                 let task = Arc::new(HealTask::from_request(request, storage.clone()));
                 let task_id = task.id.clone();
                 active_heals_guard.insert(task_id.clone(), task.clone());
+                publish_active_heal_count(&active_heals_guard);
                 update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
                 let active_heals_clone = active_heals.clone();
                 let statistics_clone = statistics.clone();
@@ -828,6 +849,7 @@ impl HealManager {
                     }
                     let mut active_heals_guard = active_heals_clone.lock().await;
                     if let Some(completed_task) = active_heals_guard.remove(&task_id) {
+                        publish_active_heal_count(&active_heals_guard);
                         update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
                         // update statistics
                         let mut stats = statistics_clone.write().await;
@@ -853,6 +875,8 @@ impl HealManager {
         let mut stats = statistics.write().await;
         stats.total_tasks += tasks_started as u64;
         stats.update_running_tasks(active_heals_guard.len() as u64);
+        publish_active_heal_count(&active_heals_guard);
+        publish_heal_queue_length(&queue);
 
         // Log queue status if items remain
         if !queue.is_empty() {

crates/heal/src/lib.rs

@@ -17,6 +17,7 @@ pub mod heal;
 
 pub use error::{Error, Result};
 pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType, channel::HealChannelProcessor};
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, OnceLock};
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info};
@@ -55,6 +56,8 @@ static GLOBAL_HEAL_MANAGER: OnceLock<Arc<HealManager>> = OnceLock::new();
 
 /// Global heal channel processor instance
 static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock<Arc<tokio::sync::Mutex<HealChannelProcessor>>> = OnceLock::new();
+static GLOBAL_HEAL_ACTIVE_TASKS: AtomicU64 = AtomicU64::new(0);
+static GLOBAL_HEAL_QUEUE_LENGTH: AtomicU64 = AtomicU64::new(0);
 
 /// Initialize and start heal manager with channel processor
 pub async fn init_heal_manager(
@@ -107,3 +110,19 @@ pub fn get_heal_manager() -> Option<&'static Arc<HealManager>> {
 pub fn get_heal_channel_processor() -> Option<&'static Arc<tokio::sync::Mutex<HealChannelProcessor>>> {
     GLOBAL_HEAL_CHANNEL_PROCESSOR.get()
 }
+
+pub fn current_heal_active_tasks() -> u64 {
+    GLOBAL_HEAL_ACTIVE_TASKS.load(Ordering::Relaxed)
+}
+
+pub fn current_heal_queue_length() -> u64 {
+    GLOBAL_HEAL_QUEUE_LENGTH.load(Ordering::Relaxed)
+}
+
+pub(crate) fn set_heal_active_tasks(count: usize) {
+    GLOBAL_HEAL_ACTIVE_TASKS.store(count as u64, Ordering::Relaxed);
+}
+
+pub(crate) fn set_heal_queue_length(count: usize) {
+    GLOBAL_HEAL_QUEUE_LENGTH.store(count as u64, Ordering::Relaxed);
+}

crates/iam/src/manager.rs

@@ -36,7 +36,7 @@ use rustfs_policy::{
 use rustfs_utils::{get_env_opt_str, path::path_join_buf};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use std::sync::atomic::AtomicU8;
+use std::sync::atomic::{AtomicU8, AtomicU64};
 use std::{
     collections::{HashMap, HashSet},
     sync::{
@@ -93,6 +93,17 @@ pub struct IamCache<T> {
     pub roles: HashMap<ARN, Vec<String>>,
     pub send_chan: Sender<i64>,
     pub last_timestamp: AtomicI64,
+    pub sync_failures: AtomicU64,
+    pub sync_successes: AtomicU64,
+    pub last_sync_duration_millis: AtomicU64,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct IamSyncMetricsSnapshot {
+    pub last_sync_duration_millis: u64,
+    pub since_last_sync_millis: u64,
+    pub sync_failures: u64,
+    pub sync_successes: u64,
 }
 
 impl<T> IamCache<T>
@@ -116,6 +127,9 @@ where
             send_chan: sender,
             roles: HashMap::new(),
             last_timestamp: AtomicI64::new(0),
+            sync_failures: AtomicU64::new(0),
+            sync_successes: AtomicU64::new(0),
+            last_sync_duration_millis: AtomicU64::new(0),
         });
 
         sys.clone().init(receiver).await.unwrap();
@@ -200,11 +214,38 @@ where
     }
 
     async fn load(self: Arc<Self>) -> Result<()> {
-        // debug!("load iam to cache");
-        self.api.load_all(&self.cache).await?;
-        self.last_timestamp
-            .store(OffsetDateTime::now_utc().unix_timestamp(), Ordering::Relaxed);
-        Ok(())
+        let started_at = std::time::Instant::now();
+        match self.api.load_all(&self.cache).await {
+            Ok(()) => {
+                self.last_timestamp
+                    .store(OffsetDateTime::now_utc().unix_timestamp(), Ordering::Relaxed);
+                self.sync_successes.fetch_add(1, Ordering::Relaxed);
+                self.last_sync_duration_millis
+                    .store(started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, Ordering::Relaxed);
+                Ok(())
+            }
+            Err(err) => {
+                self.sync_failures.fetch_add(1, Ordering::Relaxed);
+                Err(err)
+            }
+        }
+    }
+
+    pub fn sync_metrics_snapshot(&self) -> IamSyncMetricsSnapshot {
+        let now_secs = OffsetDateTime::now_utc().unix_timestamp();
+        let last_sync_secs = self.last_timestamp.load(Ordering::Relaxed);
+        let since_last_sync_millis = if last_sync_secs > 0 && now_secs >= last_sync_secs {
+            ((now_secs - last_sync_secs) as u64).saturating_mul(1000)
+        } else {
+            0
+        };
+
+        IamSyncMetricsSnapshot {
+            last_sync_duration_millis: self.last_sync_duration_millis.load(Ordering::Relaxed),
+            since_last_sync_millis,
+            sync_failures: self.sync_failures.load(Ordering::Relaxed),
+            sync_successes: self.sync_successes.load(Ordering::Relaxed),
+        }
     }
 
     pub async fn load_user(&self, access_key: &str) -> Result<()> {
@@ -214,18 +255,30 @@ where
         let mut sts_policy_map = HashMap::new();
         let mut policy_docs_map = HashMap::new();
 
-        let _ = self.api.load_user(access_key, UserType::Svc, &mut users_map).await;
+        match self.api.load_user(access_key, UserType::Svc, &mut users_map).await {
+            Ok(()) => {}
+            Err(err) if is_err_no_such_user(&err) => {}
+            Err(err) => return Err(err),
+        }
 
         let parent_user = users_map.get(access_key).map(|svc| svc.credentials.parent_user.clone());
 
         if let Some(parent_user) = parent_user {
-            let _ = self.api.load_user(&parent_user, UserType::Reg, &mut users_map).await;
+            match self.api.load_user(&parent_user, UserType::Reg, &mut users_map).await {
+                Ok(()) => {}
+                Err(err) if is_err_no_such_user(&err) => {}
+                Err(err) => return Err(err),
+            }
             let _ = self
                 .api
                 .load_mapped_policy(&parent_user, UserType::Reg, false, &mut user_policy_map)
                 .await;
         } else {
-            let _ = self.api.load_user(access_key, UserType::Reg, &mut users_map).await;
+            match self.api.load_user(access_key, UserType::Reg, &mut users_map).await {
+                Ok(()) => {}
+                Err(err) if is_err_no_such_user(&err) => {}
+                Err(err) => return Err(err),
+            }
             if users_map.contains_key(access_key) {
                 let _ = self
                     .api
@@ -233,7 +286,11 @@ where
                     .await;
             }
 
-            let _ = self.api.load_user(access_key, UserType::Sts, &mut sts_users_map).await;
+            match self.api.load_user(access_key, UserType::Sts, &mut sts_users_map).await {
+                Ok(()) => {}
+                Err(err) if is_err_no_such_user(&err) => {}
+                Err(err) => return Err(err),
+            }
 
             let has_sts_user = sts_users_map.get(access_key);
 
@@ -588,6 +645,20 @@ where
             }
         }
 
+        let sts_accounts = self.cache.sts_accounts.load();
+        for (_, v) in sts_accounts.iter() {
+            if v.credentials.parent_user == access_key {
+                user_exists = true;
+                if v.credentials.is_temp() && !v.credentials.is_service_account() {
+                    let mut u = v.clone();
+                    u.credentials.secret_key = String::new();
+                    u.credentials.session_token = String::new();
+
+                    ret.push(u);
+                }
+            }
+        }
+
         if !user_exists {
             return Err(Error::NoSuchUser(access_key.to_string()));
         }
@@ -596,8 +667,8 @@ where
     }
 
     pub async fn list_sts_accounts(&self, access_key: &str) -> Result<Vec<Credentials>> {
-        let users = self.cache.users.load();
-        Ok(users
+        let sts_accounts = self.cache.sts_accounts.load();
+        Ok(sts_accounts
             .values()
             .filter_map(|x| {
                 if !access_key.is_empty()
@@ -1364,14 +1435,13 @@ where
     }
 
     pub async fn is_temp_user(&self, access_key: &str) -> Result<(bool, String)> {
-        let users = self.cache.users.load();
-        let u = match users.get(access_key) {
+        let u = match self.get_user(access_key).await {
             Some(u) => u,
             None => return Err(Error::NoSuchUser(access_key.to_string())),
         };
 
         if u.credentials.is_temp() {
-            Ok((true, u.credentials.parent_user.clone()))
+            Ok((true, u.credentials.parent_user))
         } else {
             Ok((false, String::new()))
         }

crates/iam/src/oidc.rs

@@ -31,11 +31,11 @@ use rustfs_ecstore::config::{Config as ServerConfig, KVS, get_global_server_conf
 use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::future::Future;
 use std::net::IpAddr;
 use std::pin::Pin;
-use std::sync::RwLock;
+use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
 use std::time::{Duration as StdDuration, Instant};
 use tokio::time::sleep;
 use tracing::{error, info, warn};
@@ -44,6 +44,127 @@ use url::Url;
 const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
 const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
 const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
+const OIDC_PLUGIN_AUTHN_WINDOW: StdDuration = StdDuration::from_secs(60);
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct OidcPluginAuthnMetricsSnapshot {
+    pub failed_requests_minute: u64,
+    pub last_fail_seconds: u64,
+    pub last_succ_seconds: u64,
+    pub succ_avg_rtt_ms_minute: u64,
+    pub succ_max_rtt_ms_minute: u64,
+    pub total_requests_minute: u64,
+}
+
+#[derive(Debug, Clone)]
+struct OidcPluginAuthnSample {
+    observed_at: Instant,
+    succeeded: bool,
+    rtt_ms: u64,
+}
+
+#[derive(Debug, Default)]
+struct OidcPluginAuthnMetrics {
+    samples: Mutex<VecDeque<OidcPluginAuthnSample>>,
+    last_fail_at: Mutex<Option<Instant>>,
+    last_succ_at: Mutex<Option<Instant>>,
+}
+
+fn lock_oidc_plugin_authn_metrics<'a, T>(mutex: &'a Mutex<T>, metric: &'static str) -> MutexGuard<'a, T> {
+    match mutex.lock() {
+        Ok(guard) => guard,
+        Err(err) => {
+            warn!("recovering poisoned OIDC plugin authn metrics lock: {}", metric);
+            err.into_inner()
+        }
+    }
+}
+
+fn seconds_since(now: Instant, observed_at: Option<Instant>) -> u64 {
+    observed_at
+        .map(|instant| now.duration_since(instant).as_secs())
+        .unwrap_or_default()
+}
+
+impl OidcPluginAuthnMetrics {
+    fn record(&self, rtt_ms: u64, succeeded: bool) {
+        let now = Instant::now();
+        let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples");
+        samples.push_back(OidcPluginAuthnSample {
+            observed_at: now,
+            succeeded,
+            rtt_ms,
+        });
+        while samples
+            .front()
+            .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW)
+        {
+            samples.pop_front();
+        }
+        drop(samples);
+
+        if succeeded {
+            *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at") = Some(now);
+        } else {
+            *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at") = Some(now);
+        }
+    }
+
+    fn snapshot(&self) -> OidcPluginAuthnMetricsSnapshot {
+        let now = Instant::now();
+        let (total_requests_minute, failed_requests_minute, succ_avg_rtt_ms_minute, succ_max_rtt_ms_minute) = {
+            let mut samples = lock_oidc_plugin_authn_metrics(&self.samples, "samples");
+            while samples
+                .front()
+                .is_some_and(|sample| now.duration_since(sample.observed_at) > OIDC_PLUGIN_AUTHN_WINDOW)
+            {
+                samples.pop_front();
+            }
+
+            let mut failed_requests_minute = 0u64;
+            let mut successful_requests = 0u64;
+            let mut successful_rtt_sum = 0u64;
+            let mut succ_max_rtt_ms_minute = 0u64;
+
+            for sample in samples.iter() {
+                if sample.succeeded {
+                    successful_requests += 1;
+                    successful_rtt_sum += sample.rtt_ms;
+                    succ_max_rtt_ms_minute = succ_max_rtt_ms_minute.max(sample.rtt_ms);
+                } else {
+                    failed_requests_minute += 1;
+                }
+            }
+
+            let succ_avg_rtt_ms_minute = successful_rtt_sum.checked_div(successful_requests).unwrap_or_default();
+
+            (
+                samples.len() as u64,
+                failed_requests_minute,
+                succ_avg_rtt_ms_minute,
+                succ_max_rtt_ms_minute,
+            )
+        };
+
+        let last_fail_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_fail_at, "last_fail_at"));
+        let last_succ_seconds = seconds_since(now, *lock_oidc_plugin_authn_metrics(&self.last_succ_at, "last_succ_at"));
+
+        OidcPluginAuthnMetricsSnapshot {
+            failed_requests_minute,
+            last_fail_seconds,
+            last_succ_seconds,
+            succ_avg_rtt_ms_minute,
+            succ_max_rtt_ms_minute,
+            total_requests_minute,
+        }
+    }
+}
+
+static OIDC_PLUGIN_AUTHN_METRICS: LazyLock<OidcPluginAuthnMetrics> = LazyLock::new(OidcPluginAuthnMetrics::default);
+
+pub fn oidc_plugin_authn_metrics_snapshot() -> OidcPluginAuthnMetricsSnapshot {
+    OIDC_PLUGIN_AUTHN_METRICS.snapshot()
+}
 
 // ---- HTTP Client Adapter ----
 
@@ -120,6 +241,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
 
     fn call(&'c self, request: http::Request<Vec<u8>>) -> Self::Future {
         Box::pin(async move {
+            let started_at = Instant::now();
             let (parts, body) = request.into_parts();
             let uri = parts.uri.to_string();
             let client = self.client_for_uri(&uri);
@@ -128,8 +250,13 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
                 .headers(parts.headers)
                 .body(body)
                 .send()
-                .await
-                .map_err(OidcHttpError::Reqwest)?;
+                .await;
+
+            let elapsed_ms = started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64;
+            let succeeded = response.as_ref().is_ok_and(|resp| resp.status().is_success());
+            OIDC_PLUGIN_AUTHN_METRICS.record(elapsed_ms, succeeded);
+
+            let response = response.map_err(OidcHttpError::Reqwest)?;
 
             let status = response.status();
             let headers = response.headers().clone();
@@ -1472,7 +1599,9 @@ mod tests {
         use std::time::{Duration, Instant};
 
         // After the last completed response, exit if no new connection arrives within this window.
-        const IDLE_SHUTDOWN: Duration = Duration::from_millis(500);
+        // Keep the mock server alive long enough for slower CI/macOS test environments to finish
+        // discovery + JWKS requests without racing the shutdown timer.
+        const IDLE_SHUTDOWN: Duration = Duration::from_secs(1);
         const ABSOLUTE_CAP: Duration = Duration::from_secs(5);
 
         let listener = TcpListener::bind("127.0.0.1:0").unwrap();
@@ -1574,6 +1703,17 @@ mod tests {
         err.contains(base) && err.contains(&format!("{base}/")) && err.contains("discovery failed for all issuer variants")
     }
 
+    async fn validate_mocked_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
+        let http_client = ReqwestHttpClient::new()?;
+        let state = OidcSys::discover_provider(config, &http_client).await?;
+
+        Ok(OidcProviderValidationResult {
+            issuer: state.metadata.issuer().to_string(),
+            authorization_endpoint: state.metadata.authorization_endpoint().to_string(),
+            token_endpoint: state.metadata.token_endpoint().map(ToString::to_string),
+        })
+    }
+
     #[tokio::test]
     async fn test_validate_oidc_provider_config_retries_with_issuer_candidates() {
         // Discovery document must advertise the canonical issuer path. The first candidate has no
@@ -1582,7 +1722,7 @@ mod tests {
         let config_url = format!("{base}/application/o/rustfs");
         let config = build_mocked_oidc_provider_config("default", &config_url);
 
-        let result = validate_oidc_provider_config(&config).await;
+        let result = validate_mocked_oidc_provider_config(&config).await;
 
         let validation_result = result.expect("OIDC provider validation should succeed");
         assert_eq!(validation_result.issuer, format!("{base}/application/o/rustfs/"));
@@ -1595,7 +1735,7 @@ mod tests {
         let config_url = format!("{base}/application/o/rustfs");
         let config = build_mocked_oidc_provider_config("default", &config_url);
 
-        let err = validate_oidc_provider_config(&config)
+        let err = validate_mocked_oidc_provider_config(&config)
             .await
             .expect_err("OIDC provider validation should fail");
         assert!(discovery_error_contains_all_variants(&err, &base));

crates/iam/src/store/object.rs

@@ -227,20 +227,13 @@ impl ObjectStore {
         Ok(encrypted)
     }
 
-    fn encrypt_data(data: &[u8]) -> Result<Vec<u8>> {
+    fn prepare_data_for_storage(data: &[u8]) -> Result<Vec<u8>> {
         if keyring::encrypt_key().is_some() {
             let encrypted = Self::encrypt_data_with_master_key(data)?;
             return Ok(encrypted);
         }
 
-        let cred = get_global_action_cred().unwrap_or_default();
-        let password = if !cred.access_key.is_empty() && !cred.secret_key.is_empty() {
-            format!("{}:{}", cred.access_key, cred.secret_key).into_bytes()
-        } else {
-            cred.secret_key.into_bytes()
-        };
-        let en = rustfs_crypto::encrypt_stream_io(&password, data)?;
-        Ok(en)
+        Ok(data.to_vec())
     }
 
     fn should_lazy_rewrite(source: DecryptSource) -> bool {
@@ -343,8 +336,8 @@ impl ObjectStore {
     }
 
     #[cfg(test)]
-    fn encrypt_data_for_test(data: &[u8]) -> Result<Vec<u8>> {
-        Self::encrypt_data(data)
+    fn prepare_data_for_storage_for_test(data: &[u8]) -> Result<Vec<u8>> {
+        Self::prepare_data_for_storage(data)
     }
 
     async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef<str> + Send) -> Result<(Vec<u8>, ObjectInfo)> {
@@ -369,9 +362,17 @@ impl ObjectStore {
 
         let path = prefix.to_owned();
         tokio::spawn(async move {
-            store
+            if let Err(err) = store
                 .walk(ctx.clone(), Self::BUCKET_NAME, &path, tx, WalkOptions::default())
                 .await
+            {
+                error!(
+                    bucket = Self::BUCKET_NAME,
+                    prefix = %path,
+                    error = ?err,
+                    "list_iam_config_items walk task failed"
+                );
+            }
         });
 
         let prefix = prefix.to_owned();
@@ -648,7 +649,7 @@ impl Store for ObjectStore {
     #[tracing::instrument(skip(self, item, path))]
     async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()> {
         let mut data = serde_json::to_vec(&item)?;
-        data = Self::encrypt_data(&data)?;
+        data = Self::prepare_data_for_storage(&data)?;
 
         let mut attempts = 0;
         let max_attempts = 5;
@@ -1450,28 +1451,29 @@ mod tests {
     }
 
     #[test]
-    fn test_encrypt_data_produces_stream_io_format() {
-        let _ = test_cred();
+    #[serial]
+    fn test_prepare_data_defaults_to_plaintext_without_iam_master_key() {
         let plain = br#"{"Version":1,"policy":"readonly"}"#;
-        let encrypted = ObjectStore::encrypt_data_for_test(plain).expect("encrypt should succeed");
-        // stream_io header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes
-        const STREAM_IO_HEADER_LEN: usize = 41;
-        assert!(
-            encrypted.len() >= STREAM_IO_HEADER_LEN,
-            "encrypted should have at least 41-byte stream_io header"
+
+        with_vars(
+            [
+                (keyring::ENV_IAM_MASTER_KEY, None::<&str>),
+                (keyring::ENV_IAM_MASTER_KEY_OLD_KEYS, None::<&str>),
+            ],
+            || {
+                let stored = ObjectStore::prepare_data_for_storage_for_test(plain).expect("store bytes should build");
+                assert_eq!(stored, plain);
+
+                let decrypted = ObjectStore::decrypt_data_with_source(&stored).expect("plaintext should load");
+                assert_eq!(plain, decrypted.plain.as_slice());
+                assert_eq!(decrypted.source, DecryptSource::Plaintext);
+            },
         );
-        assert!(
-            encrypted[32] == 0x00 || encrypted[32] == 0x01 || encrypted[32] == 0x02,
-            "alg_id should be 0x00, 0x01, or 0x02"
-        );
-        // Round-trip: encrypt then decrypt
-        let decrypted = ObjectStore::decrypt_data_with_source(&encrypted).expect("decrypt should succeed");
-        assert_eq!(plain, decrypted.plain.as_slice());
     }
 
     #[test]
     #[serial]
-    fn test_encrypt_data_prefers_iam_master_key_roundtrip() {
+    fn test_prepare_data_uses_iam_master_key_roundtrip() {
         let _ = test_cred();
         let plain = br#"{"Version":1,"policy":"master-key"}"#;
         let master_key = "iam-master-key-roundtrip";
@@ -1482,7 +1484,7 @@ mod tests {
                 (keyring::ENV_IAM_MASTER_KEY_OLD_KEYS, None),
             ],
             || {
-                let encrypted = ObjectStore::encrypt_data_for_test(plain).expect("encrypt with iam master key");
+                let encrypted = ObjectStore::prepare_data_for_storage_for_test(plain).expect("encrypt with iam master key");
 
                 let by_master =
                     rustfs_crypto::decrypt_stream_io(master_key.as_bytes(), &encrypted).expect("decrypt via master key");

crates/iam/src/sys.rs

@@ -16,9 +16,9 @@ use crate::error::Error as IamError;
 use crate::error::is_err_no_such_account;
 use crate::error::is_err_no_such_temp_account;
 use crate::error::{Error, Result};
-use crate::manager::IamCache;
 use crate::manager::extract_jwt_claims;
 use crate::manager::get_default_policyes;
+use crate::manager::{IamCache, IamSyncMetricsSnapshot};
 use crate::store::GroupInfo;
 use crate::store::MappedPolicy;
 use crate::store::Store;
@@ -186,6 +186,10 @@ impl<T: Store> IamSys<T> {
         self.store.api.has_watcher()
     }
 
+    pub fn sync_metrics_snapshot(&self) -> IamSyncMetricsSnapshot {
+        self.store.sync_metrics_snapshot()
+    }
+
     pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
         let policy_plugin_client = get_policy_plugin_client();
         let mut guard = policy_plugin_client.write().await;
@@ -751,7 +755,7 @@ impl<T: Store> IamSys<T> {
                 Ok((Some(res), ok))
             }
             None => {
-                let _ = self.store.load_user(access_key).await;
+                self.store.load_user(access_key).await?;
 
                 if let Some(res) = self.store.get_user(access_key).await {
                     let ok = res.credentials.is_valid();
@@ -863,16 +867,6 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        let Ok((is_temp, parent_user)) = self.is_temp_user(args.account).await else {
-            return PreparedIamAuth {
-                needs_existing_object_tag: false,
-                mode: PreparedIamMode::Deny,
-            };
-        };
-        if is_temp {
-            return self.prepare_sts_auth(args, &parent_user).await;
-        }
-
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
             return PreparedIamAuth {
                 needs_existing_object_tag: false,
@@ -883,6 +877,16 @@ impl<T: Store> IamSys<T> {
             return self.prepare_service_account_auth(args, &parent_user).await;
         }
 
+        let Ok((is_temp, parent_user)) = self.is_temp_user(args.account).await else {
+            return PreparedIamAuth {
+                needs_existing_object_tag: false,
+                mode: PreparedIamMode::Deny,
+            };
+        };
+        if is_temp {
+            return self.prepare_sts_auth(args, &parent_user).await;
+        }
+
         self.prepare_regular_auth(args).await
     }
 
@@ -1291,7 +1295,7 @@ mod tests {
     use crate::manager::get_default_policyes;
     use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
     use rustfs_credentials::{Credentials, get_global_action_cred, init_global_action_credentials};
-    use rustfs_policy::auth::UserIdentity;
+    use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
     use rustfs_policy::policy::Args;
     use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
     use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
@@ -1368,6 +1372,10 @@ mod tests {
         }
 
         async fn load_user(&self, name: &str, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()> {
+            if user_type == UserType::Reg && name == "load-failure-user" {
+                return Err(Error::Io(std::io::Error::other("load user failed")));
+            }
+
             if user_type == UserType::Reg && name == "notify-user" {
                 let user = UserIdentity::from(Credentials {
                     access_key: name.to_string(),
@@ -1669,6 +1677,189 @@ mod tests {
         assert_eq!(claims.get("exp").and_then(|v| v.as_i64()), Some(updated_expiration.unix_timestamp()));
     }
 
+    #[tokio::test]
+    async fn test_created_access_token_authorizes_with_parent_policy() {
+        ensure_test_global_credentials();
+
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-fallback-test-parent";
+        let groups = Some(vec!["testgroup".to_string()]);
+        let (cred, _) = iam_sys
+            .new_service_account(
+                parent_user,
+                groups.clone(),
+                NewServiceAccountOpts {
+                    access_key: "ACCESSTOKENTESTUSER".to_string(),
+                    secret_key: "accessTokenTestSecret".to_string(),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("access token should be created");
+
+        let stored = iam_sys
+            .get_user(&cred.access_key)
+            .await
+            .expect("created access token should be cached");
+        assert!(stored.credentials.is_service_account());
+        assert_eq!(stored.credentials.parent_user, parent_user);
+
+        let claims = stored
+            .credentials
+            .claims
+            .as_ref()
+            .expect("created access token should have decoded JWT claims");
+        assert_eq!(claims.get("parent").and_then(Value::as_str), Some(parent_user));
+        assert_eq!(
+            claims.get(&iam_policy_claim_name_sa()).and_then(Value::as_str),
+            Some(INHERITED_POLICY_TYPE)
+        );
+
+        let (is_service_account, resolved_parent) = iam_sys
+            .is_service_account(&cred.access_key)
+            .await
+            .expect("created access token should be recognized as a service account");
+        assert!(is_service_account);
+        assert_eq!(resolved_parent, parent_user);
+
+        let (redacted, policy) = iam_sys
+            .get_service_account(&cred.access_key)
+            .await
+            .expect("created access token should be readable");
+        assert_eq!(redacted.access_key, cred.access_key);
+        assert_eq!(redacted.parent_user, parent_user);
+        assert!(redacted.secret_key.is_empty());
+        assert!(redacted.session_token.is_empty());
+        assert!(policy.is_none());
+
+        let args = Args {
+            account: &cred.access_key,
+            groups: &groups,
+            action: Action::S3Action(S3Action::ListBucketAction),
+            bucket: "mybucket",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims,
+            deny_only: false,
+        };
+
+        let prepared = iam_sys.prepare_auth(&args).await;
+        assert!(
+            matches!(prepared.mode, PreparedIamMode::ServiceAccount { .. }),
+            "created access token must use service-account authorization path"
+        );
+        assert!(
+            iam_sys.eval_prepared(&prepared, &args).await,
+            "created access token should be allowed through the parent's group policy"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_created_sts_credentials_authorize_with_session_token_claims() {
+        ensure_test_global_credentials();
+
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let parent_user = "sts-fallback-test-parent";
+        let token_secret = get_global_action_cred()
+            .expect("global action credentials should be initialized")
+            .secret_key;
+        let mut claims = HashMap::new();
+        claims.insert("parent".to_string(), Value::String(parent_user.to_string()));
+        claims.insert(
+            "exp".to_string(),
+            Value::Number(serde_json::Number::from(
+                (OffsetDateTime::now_utc() + time::Duration::hours(1)).unix_timestamp(),
+            )),
+        );
+
+        let mut cred = get_new_credentials_with_metadata(&claims, &token_secret).expect("STS credentials should be created");
+        cred.parent_user = parent_user.to_string();
+
+        iam_sys
+            .set_temp_user(&cred.access_key, &cred, None)
+            .await
+            .expect("STS credentials should be persisted in the temp-user cache");
+
+        let stored = iam_sys
+            .get_user(&cred.access_key)
+            .await
+            .expect("created STS credentials should be cached");
+        assert!(stored.credentials.is_temp());
+        assert!(!stored.credentials.is_service_account());
+        assert_eq!(stored.credentials.parent_user, parent_user);
+
+        let (is_temp, resolved_parent) = iam_sys
+            .is_temp_user(&cred.access_key)
+            .await
+            .expect("created STS credentials should be recognized as temp");
+        assert!(is_temp);
+        assert_eq!(resolved_parent, parent_user);
+
+        let listed = iam_sys
+            .list_sts_accounts(parent_user)
+            .await
+            .expect("created STS credentials should be listable by parent");
+        assert_eq!(listed.len(), 1);
+        assert_eq!(listed[0].access_key, cred.access_key);
+        assert_eq!(listed[0].parent_user, parent_user);
+        assert!(listed[0].secret_key.is_empty());
+        assert!(listed[0].session_token.is_empty());
+
+        let temp_accounts = iam_sys
+            .list_temp_accounts(parent_user)
+            .await
+            .expect("created STS credentials should be listable as temp accounts");
+        assert_eq!(temp_accounts.len(), 1);
+        assert_eq!(temp_accounts[0].credentials.access_key, cred.access_key);
+        assert_eq!(temp_accounts[0].credentials.parent_user, parent_user);
+        assert!(temp_accounts[0].credentials.secret_key.is_empty());
+        assert!(temp_accounts[0].credentials.session_token.is_empty());
+
+        let (redacted, policy) = iam_sys
+            .get_temporary_account(&cred.access_key)
+            .await
+            .expect("created STS credentials should be readable");
+        assert_eq!(redacted.access_key, cred.access_key);
+        assert_eq!(redacted.parent_user, parent_user);
+        assert!(redacted.secret_key.is_empty());
+        assert!(redacted.session_token.is_empty());
+        assert!(policy.is_none());
+
+        let decoded_claims = get_claims_from_token_with_secret(&cred.session_token, &token_secret)
+            .expect("created STS session token should decode with the active signing key");
+        assert_eq!(decoded_claims.get("parent").and_then(Value::as_str), Some(parent_user));
+
+        let groups: Option<Vec<String>> = None;
+        let args = Args {
+            account: &cred.access_key,
+            groups: &groups,
+            action: Action::S3Action(S3Action::ListBucketAction),
+            bucket: "mybucket",
+            conditions: &HashMap::new(),
+            is_owner: false,
+            object: "",
+            claims: &decoded_claims,
+            deny_only: false,
+        };
+
+        let prepared = iam_sys.prepare_auth(&args).await;
+        assert!(
+            matches!(prepared.mode, PreparedIamMode::Sts { .. }),
+            "created STS credentials must use STS authorization path"
+        );
+        assert!(
+            iam_sys.eval_prepared(&prepared, &args).await,
+            "created STS credentials should inherit the parent user's group policy"
+        );
+    }
+
     /// Regression test: temp credentials without groups in args still receive group-attached
     /// policies via the parent user (groups fallback). Without the fallback, policy_db_get
     /// would get None for groups and the user would have no group policies, so the action
@@ -1809,6 +2000,17 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_check_key_propagates_cache_miss_load_failure() {
+        let store = StsTestMockStore { empty_policies: false };
+        let cache_manager = IamCache::new(store).await;
+        let iam_sys = IamSys::new(cache_manager);
+
+        let result = iam_sys.check_key("load-failure-user").await;
+
+        assert!(matches!(result, Err(Error::Io(_))));
+    }
+
     #[tokio::test]
     async fn test_prepare_auth_eval_matches_prepare_sts_auth_for_parent_policy_fallback() {
         let store = StsTestMockStore { empty_policies: false };

crates/io-metrics/src/adaptive_ttl.rs

@@ -31,14 +31,14 @@ use std::time::{Duration, Instant};
 pub fn record_ttl_adjustment(_key: &str, base_ttl: u64, adjusted_ttl: u64) {
     use metrics::{counter, gauge};
 
-    counter!("rustfs.cache.ttl.adjustments").increment(1);
-    gauge!("rustfs.cache.ttl.base").set(base_ttl as f64);
-    gauge!("rustfs.cache.ttl.adjusted").set(adjusted_ttl as f64);
+    counter!("rustfs_cache_ttl_adjustments").increment(1);
+    gauge!("rustfs_cache_ttl_base").set(base_ttl as f64);
+    gauge!("rustfs_cache_ttl_adjusted").set(adjusted_ttl as f64);
 
     if adjusted_ttl > base_ttl {
-        counter!("rustfs.cache.ttl.extensions").increment(1);
+        counter!("rustfs_cache_ttl_extensions").increment(1);
     } else if adjusted_ttl < base_ttl {
-        counter!("rustfs.cache.ttl.reductions").increment(1);
+        counter!("rustfs_cache_ttl_reductions").increment(1);
     }
 }
 
@@ -46,7 +46,7 @@ pub fn record_ttl_adjustment(_key: &str, base_ttl: u64, adjusted_ttl: u64) {
 #[inline(always)]
 pub fn record_ttl_expiration() {
     use metrics::counter;
-    counter!("rustfs.cache.ttl.expirations").increment(1);
+    counter!("rustfs_cache_ttl_expirations").increment(1);
 }
 
 /// Record early eviction.
@@ -57,7 +57,7 @@ pub fn record_ttl_expiration() {
 #[inline(always)]
 pub fn record_early_eviction(reason: &str) {
     use metrics::counter;
-    counter!("rustfs.cache.evictions.early", "reason" => reason.to_string()).increment(1);
+    counter!("rustfs_cache_evictions_early", "reason" => reason.to_string()).increment(1);
 }
 
 /// Record access pattern change.
@@ -69,7 +69,7 @@ pub fn record_early_eviction(reason: &str) {
 #[inline(always)]
 pub fn record_access_pattern_change(from: &str, to: &str) {
     use metrics::counter;
-    counter!("rustfs.cache.access_pattern.changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
+    counter!("rustfs_cache_access_pattern_changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
 }
 
 /// Adaptive TTL statistics.

crates/io-metrics/src/backpressure_metrics.rs

@@ -18,35 +18,35 @@
 #[inline(always)]
 pub fn record_backpressure_state_change(from: &str, to: &str) {
     use metrics::counter;
-    counter!("rustfs.backpressure.state.changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
+    counter!("rustfs_backpressure_state_changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
 }
 
 /// Record backpressure rejection.
 #[inline(always)]
 pub fn record_backpressure_rejection() {
     use metrics::counter;
-    counter!("rustfs.backpressure.rejections").increment(1);
+    counter!("rustfs_backpressure_rejections").increment(1);
 }
 
 /// Record concurrent operations count.
 #[inline(always)]
 pub fn record_concurrent_operations(count: usize) {
     use metrics::gauge;
-    gauge!("rustfs.backpressure.concurrent").set(count as f64);
+    gauge!("rustfs_backpressure_concurrent").set(count as f64);
 }
 
 /// Record backpressure activation.
 #[inline(always)]
 pub fn record_backpressure_activation() {
     use metrics::counter;
-    counter!("rustfs.backpressure.activations").increment(1);
+    counter!("rustfs_backpressure_activations").increment(1);
 }
 
 /// Record backpressure deactivation.
 #[inline(always)]
 pub fn record_backpressure_deactivation() {
     use metrics::counter;
-    counter!("rustfs.backpressure.deactivations").increment(1);
+    counter!("rustfs_backpressure_deactivations").increment(1);
 }
 
 #[cfg(test)]

crates/io-metrics/src/capacity_metrics.rs

@@ -20,35 +20,35 @@ use std::time::Duration;
 /// Record capacity cache hit.
 #[inline(always)]
 pub fn record_capacity_cache_hit() {
-    counter!("rustfs.capacity.cache.hits").increment(1);
+    counter!("rustfs_capacity_cache_hits").increment(1);
 }
 
 /// Record capacity cache miss.
 #[inline(always)]
 pub fn record_capacity_cache_miss() {
-    counter!("rustfs.capacity.cache.misses").increment(1);
+    counter!("rustfs_capacity_cache_misses").increment(1);
 }
 
 /// Record how capacity cache was served to the caller.
 #[inline(always)]
 pub fn record_capacity_cache_served(state: &'static str) {
-    counter!("rustfs.capacity.cache.served.total", "state" => state).increment(1);
+    counter!("rustfs_capacity_cache_served_total", "state" => state).increment(1);
 }
 
 /// Record current capacity gauge.
 #[inline(always)]
 pub fn record_capacity_current_bytes(used_bytes: u64) {
-    gauge!("rustfs.capacity.current").set(used_bytes as f64);
+    gauge!("rustfs_capacity_current_bytes").set(used_bytes as f64);
 }
 
 /// Record capacity update completion.
 #[inline(always)]
 pub fn record_capacity_update_completed(source: &'static str, duration: Duration, used_bytes: u64, is_estimated: bool) {
-    counter!("rustfs.capacity.update.total", "source" => source).increment(1);
-    histogram!("rustfs.capacity.update.duration.seconds", "source" => source).record(duration.as_secs_f64());
-    histogram!("rustfs.capacity.update.bytes", "source" => source).record(used_bytes as f64);
+    counter!("rustfs_capacity_update_total", "source" => source).increment(1);
+    histogram!("rustfs_capacity_update_duration_seconds", "source" => source).record(duration.as_secs_f64());
+    histogram!("rustfs_capacity_update_bytes", "source" => source).record(used_bytes as f64);
     counter!(
-        "rustfs.capacity.update.estimated.total",
+        "rustfs_capacity_update_estimated_total",
         "source" => source,
         "estimated" => if is_estimated { "true" } else { "false" }
     )
@@ -58,86 +58,86 @@ pub fn record_capacity_update_completed(source: &'static str, duration: Duration
 /// Record failed capacity update.
 #[inline(always)]
 pub fn record_capacity_update_failed(source: &'static str) {
-    counter!("rustfs.capacity.update.failures", "source" => source).increment(1);
+    counter!("rustfs_capacity_update_failures", "source" => source).increment(1);
 }
 
 /// Record a capacity refresh request.
 #[inline(always)]
 pub fn record_capacity_refresh_request(mode: &'static str, source: &'static str) {
-    counter!("rustfs.capacity.refresh.requests.total", "mode" => mode, "source" => source).increment(1);
+    counter!("rustfs_capacity_refresh_requests_total", "mode" => mode, "source" => source).increment(1);
 }
 
 /// Record a refresh joiner waiting for an inflight refresh.
 #[inline(always)]
 pub fn record_capacity_refresh_joiner(source: &'static str) {
-    counter!("rustfs.capacity.refresh.joiners.total", "source" => source).increment(1);
+    counter!("rustfs_capacity_refresh_joiners_total", "source" => source).increment(1);
 }
 
 /// Record the number of inflight capacity refreshes.
 #[inline(always)]
 pub fn record_capacity_refresh_inflight(count: usize) {
-    gauge!("rustfs.capacity.refresh.inflight").set(count as f64);
+    gauge!("rustfs_capacity_refresh_inflight").set(count as f64);
 }
 
 /// Record the final result of a capacity refresh.
 #[inline(always)]
 pub fn record_capacity_refresh_result(source: &'static str, result: &'static str, duration: Duration) {
-    counter!("rustfs.capacity.refresh.result.total", "source" => source, "result" => result).increment(1);
-    histogram!("rustfs.capacity.refresh.duration.seconds", "source" => source, "result" => result).record(duration.as_secs_f64());
+    counter!("rustfs_capacity_refresh_result_total", "source" => source, "result" => result).increment(1);
+    histogram!("rustfs_capacity_refresh_duration_seconds", "source" => source, "result" => result).record(duration.as_secs_f64());
 }
 
 /// Record the refresh scope selected for a capacity refresh.
 #[inline(always)]
 pub fn record_capacity_refresh_scope(scope: &'static str, disk_count: usize) {
-    counter!("rustfs.capacity.refresh.scope.total", "scope" => scope).increment(1);
-    histogram!("rustfs.capacity.refresh.scope.disks", "scope" => scope).record(disk_count as f64);
+    counter!("rustfs_capacity_refresh_scope_total", "scope" => scope).increment(1);
+    histogram!("rustfs_capacity_refresh_scope_disks", "scope" => scope).record(disk_count as f64);
 }
 
 /// Record the current number of dirty disks tracked by capacity management.
 #[inline(always)]
 pub fn record_capacity_dirty_disk_count(count: usize) {
-    gauge!("rustfs.capacity.dirty.disks").set(count as f64);
+    gauge!("rustfs_capacity_dirty_disks").set(count as f64);
 }
 
 /// Record capacity write activity.
 #[inline(always)]
 pub fn record_capacity_write_operation(write_frequency: usize) {
-    counter!("rustfs.capacity.write.operations").increment(1);
-    gauge!("rustfs.capacity.write.frequency").set(write_frequency as f64);
+    counter!("rustfs_capacity_write_operations").increment(1);
+    gauge!("rustfs_capacity_write_frequency").set(write_frequency as f64);
 }
 
 /// Record symlink accounting.
 #[inline(always)]
 pub fn record_capacity_symlink(size_bytes: u64) {
-    counter!("rustfs.capacity.symlinks.encountered").increment(1);
-    histogram!("rustfs.capacity.symlinks.size.bytes").record(size_bytes as f64);
+    counter!("rustfs_capacity_symlinks_encountered").increment(1);
+    histogram!("rustfs_capacity_symlinks_size_bytes").record(size_bytes as f64);
 }
 
 /// Record timeout fallback event.
 #[inline(always)]
 pub fn record_capacity_timeout_fallback() {
-    counter!("rustfs.capacity.timeout.fallback").increment(1);
+    counter!("rustfs_capacity_timeout_fallback").increment(1);
 }
 
 /// Record stall detection event.
 #[inline(always)]
 pub fn record_capacity_stall_detected() {
-    counter!("rustfs.capacity.timeout.stall").increment(1);
+    counter!("rustfs_capacity_timeout_stall").increment(1);
 }
 
 /// Record dynamic timeout usage.
 #[inline(always)]
 pub fn record_capacity_dynamic_timeout(timeout: Duration) {
-    counter!("rustfs.capacity.timeout.dynamic").increment(1);
-    histogram!("rustfs.capacity.timeout.dynamic.seconds").record(timeout.as_secs_f64());
+    counter!("rustfs_capacity_timeout_dynamic").increment(1);
+    histogram!("rustfs_capacity_timeout_dynamic_seconds").record(timeout.as_secs_f64());
 }
 
 /// Record scan sampling outcome.
 #[inline(always)]
 pub fn record_capacity_scan_sampling(sampled_count: usize, estimated: bool) {
-    histogram!("rustfs.capacity.scan.sampled.count").record(sampled_count as f64);
+    histogram!("rustfs_capacity_scan_sampled_count").record(sampled_count as f64);
     counter!(
-        "rustfs.capacity.scan.estimated.total",
+        "rustfs_capacity_scan_estimated_total",
         "estimated" => if estimated { "true" } else { "false" }
     )
     .increment(1);
@@ -146,7 +146,7 @@ pub fn record_capacity_scan_sampling(sampled_count: usize, estimated: bool) {
 /// Record the scan mode used for a capacity result.
 #[inline(always)]
 pub fn record_capacity_scan_mode(mode: &'static str) {
-    counter!("rustfs.capacity.scan.mode.total", "mode" => mode).increment(1);
+    counter!("rustfs_capacity_scan_mode_total", "mode" => mode).increment(1);
 }
 
 /// Record per-disk capacity scan statistics.
@@ -159,16 +159,16 @@ pub fn record_capacity_scan_disk(
     estimated: bool,
     partial_errors: bool,
 ) {
-    histogram!("rustfs.capacity.scan.disk.duration.seconds", "disk" => disk.to_owned()).record(duration.as_secs_f64());
-    histogram!("rustfs.capacity.scan.disk.files", "disk" => disk.to_owned()).record(file_count as f64);
-    histogram!("rustfs.capacity.scan.disk.sampled", "disk" => disk.to_owned()).record(sampled_count as f64);
+    histogram!("rustfs_capacity_scan_disk_duration_seconds", "disk" => disk.to_owned()).record(duration.as_secs_f64());
+    histogram!("rustfs_capacity_scan_disk_files", "disk" => disk.to_owned()).record(file_count as f64);
+    histogram!("rustfs_capacity_scan_disk_sampled", "disk" => disk.to_owned()).record(sampled_count as f64);
     counter!(
-        "rustfs.capacity.scan.disk.estimated.total",
+        "rustfs_capacity_scan_disk_estimated_total",
         "disk" => disk.to_owned(),
         "estimated" => if estimated { "true" } else { "false" }
     )
     .increment(1);
     if partial_errors {
-        counter!("rustfs.capacity.scan.disk.partial_errors.total", "disk" => disk.to_owned()).increment(1);
+        counter!("rustfs_capacity_scan_disk_partial_errors_total", "disk" => disk.to_owned()).increment(1);
     }
 }

crates/io-metrics/src/deadlock_metrics.rs

@@ -20,52 +20,52 @@ use std::time::Duration;
 #[inline(always)]
 pub fn record_deadlock_detected(cycle_length: usize) {
     use metrics::{counter, histogram};
-    counter!("rustfs.deadlock.detected").increment(1);
-    histogram!("rustfs.deadlock.cycle_length").record(cycle_length as f64);
+    counter!("rustfs_deadlock_detected_total").increment(1);
+    histogram!("rustfs_deadlock_cycle_length").record(cycle_length as f64);
 }
 
 /// Record long-held lock.
 #[inline(always)]
 pub fn record_long_held_lock(_lock_id: u64, hold_time: Duration) {
     use metrics::{counter, histogram};
-    counter!("rustfs.deadlock.long_held").increment(1);
-    histogram!("rustfs.deadlock.hold_time.secs").record(hold_time.as_secs_f64());
+    counter!("rustfs_deadlock_long_held").increment(1);
+    histogram!("rustfs_deadlock_hold_time_secs").record(hold_time.as_secs_f64());
 }
 
 /// Record lock acquisition.
 #[inline(always)]
 pub fn record_lock_acquisition(lock_type: &str) {
     use metrics::counter;
-    counter!("rustfs.lock.acquisitions", "type" => lock_type.to_string()).increment(1);
+    counter!("rustfs_lock_acquisitions", "type" => lock_type.to_string()).increment(1);
 }
 
 /// Record lock release.
 #[inline(always)]
 pub fn record_lock_release(lock_type: &str, hold_time: Duration) {
     use metrics::{counter, histogram};
-    counter!("rustfs.lock.releases", "type" => lock_type.to_string()).increment(1);
-    histogram!("rustfs.lock.hold_time.secs", "type" => lock_type.to_string()).record(hold_time.as_secs_f64());
+    counter!("rustfs_lock_releases", "type" => lock_type.to_string()).increment(1);
+    histogram!("rustfs_lock_hold_time_secs", "type" => lock_type.to_string()).record(hold_time.as_secs_f64());
 }
 
 /// Record lock contention.
 #[inline(always)]
 pub fn record_lock_contention(lock_type: &str) {
     use metrics::counter;
-    counter!("rustfs.lock.contentions", "type" => lock_type.to_string()).increment(1);
+    counter!("rustfs_lock_contentions", "type" => lock_type.to_string()).increment(1);
 }
 
 /// Record wait graph edge added.
 #[inline(always)]
 pub fn record_wait_edge_added() {
     use metrics::counter;
-    counter!("rustfs.deadlock.wait_edges.added").increment(1);
+    counter!("rustfs_deadlock_wait_edges_added").increment(1);
 }
 
 /// Record wait graph edge removed.
 #[inline(always)]
 pub fn record_wait_edge_removed() {
     use metrics::counter;
-    counter!("rustfs.deadlock.wait_edges.removed").increment(1);
+    counter!("rustfs_deadlock_wait_edges_removed").increment(1);
 }
 
 #[cfg(test)]

crates/io-metrics/src/io_metrics.rs

@@ -27,11 +27,11 @@
 pub fn record_io_scheduler_decision(buffer_size: usize, load_level: &str, strategy: &str) {
     use metrics::{counter, gauge, histogram};
 
-    counter!("rustfs.io.scheduler.decisions").increment(1);
-    gauge!("rustfs.io.scheduler.buffer_size").set(buffer_size as f64);
-    counter!("rustfs.io.scheduler.load", "level" => load_level.to_string()).increment(1);
-    counter!("rustfs.io.scheduler.strategy", "type" => strategy.to_string()).increment(1);
-    histogram!("rustfs.io.scheduler.buffer_size.histogram").record(buffer_size as f64);
+    counter!("rustfs_io_scheduler_decisions").increment(1);
+    gauge!("rustfs_io_scheduler_buffer_size").set(buffer_size as f64);
+    counter!("rustfs_io_scheduler_load", "level" => load_level.to_string()).increment(1);
+    counter!("rustfs_io_scheduler_strategy", "type" => strategy.to_string()).increment(1);
+    histogram!("rustfs_io_scheduler_buffer_size_histogram").record(buffer_size as f64);
 }
 
 /// Record I/O priority decision.
@@ -44,9 +44,9 @@ pub fn record_io_scheduler_decision(buffer_size: usize, load_level: &str, strate
 pub fn record_io_priority_decision(priority: &str, size: usize) {
     use metrics::{counter, histogram};
 
-    counter!("rustfs.io.priority.decisions").increment(1);
-    counter!("rustfs.io.priority.by_level", "priority" => priority.to_string()).increment(1);
-    histogram!("rustfs.io.priority.request_size").record(size as f64);
+    counter!("rustfs_io_priority_decisions").increment(1);
+    counter!("rustfs_io_priority_by_level", "priority" => priority.to_string()).increment(1);
+    histogram!("rustfs_io_priority_request_size").record(size as f64);
 }
 
 /// Record load level change.
@@ -58,7 +58,7 @@ pub fn record_io_priority_decision(priority: &str, size: usize) {
 #[inline(always)]
 pub fn record_load_level_change(from: &str, to: &str) {
     use metrics::counter;
-    counter!("rustfs.io.load.changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
+    counter!("rustfs_io_load_changes", "from" => from.to_string(), "to" => to.to_string()).increment(1);
 }
 
 /// Record bandwidth observation.
@@ -69,8 +69,8 @@ pub fn record_load_level_change(from: &str, to: &str) {
 #[inline(always)]
 pub fn record_bandwidth_observation(bps: u64) {
     use metrics::{gauge, histogram};
-    gauge!("rustfs.io.bandwidth.bps").set(bps as f64);
-    histogram!("rustfs.io.bandwidth.histogram").record(bps as f64);
+    gauge!("rustfs_io_bandwidth_bps").set(bps as f64);
+    histogram!("rustfs_io_bandwidth_histogram").record(bps as f64);
 }
 
 /// Record buffer size adjustment.
@@ -83,9 +83,9 @@ pub fn record_bandwidth_observation(bps: u64) {
 #[inline(always)]
 pub fn record_buffer_size_adjustment(original: usize, adjusted: usize, reason: &str) {
     use metrics::{counter, gauge};
-    counter!("rustfs.io.buffer.adjustments", "reason" => reason.to_string()).increment(1);
-    gauge!("rustfs.io.buffer.original").set(original as f64);
-    gauge!("rustfs.io.buffer.adjusted").set(adjusted as f64);
+    counter!("rustfs_io_buffer_adjustments", "reason" => reason.to_string()).increment(1);
+    gauge!("rustfs_io_buffer_original").set(original as f64);
+    gauge!("rustfs_io_buffer_adjusted").set(adjusted as f64);
 }
 
 /// Record queue operation.
@@ -98,8 +98,8 @@ pub fn record_buffer_size_adjustment(original: usize, adjusted: usize, reason: &
 #[inline(always)]
 pub fn record_queue_operation(operation: &str, priority: &str, queue_size: usize) {
     use metrics::{counter, gauge};
-    counter!("rustfs.io.queue.operations", "operation" => operation.to_string(), "priority" => priority.to_string()).increment(1);
-    gauge!("rustfs.io.queue.size", "priority" => priority.to_string()).set(queue_size as f64);
+    counter!("rustfs_io_queue_operations", "operation" => operation.to_string(), "priority" => priority.to_string()).increment(1);
+    gauge!("rustfs_io_queue_size", "priority" => priority.to_string()).set(queue_size as f64);
 }
 
 /// Record starvation event.
@@ -110,7 +110,7 @@ pub fn record_queue_operation(operation: &str, priority: &str, queue_size: usize
 #[inline(always)]
 pub fn record_starvation_event(priority: &str) {
     use metrics::counter;
-    counter!("rustfs.io.starvation.events", "priority" => priority.to_string()).increment(1);
+    counter!("rustfs_io_starvation_events", "priority" => priority.to_string()).increment(1);
 }
 
 /// I/O scheduler statistics.

crates/io-metrics/src/lib.rs

@@ -49,6 +49,8 @@
 #[macro_use]
 extern crate metrics;
 
+use std::sync::atomic::{AtomicU64, Ordering};
+
 // Public modules
 pub mod adaptive_ttl;
 pub mod autotuner;
@@ -137,6 +139,43 @@ pub use config::{
 pub use collector::MetricsCollector;
 pub use performance::PerformanceMetrics;
 
+static EC_ENCODE_INFLIGHT_BYTES: AtomicU64 = AtomicU64::new(0);
+static GET_OBJECT_BUFFERED_BYTES: AtomicU64 = AtomicU64::new(0);
+
+fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 {
+    let mut current = counter.load(Ordering::Relaxed);
+    loop {
+        let next = current.saturating_sub(bytes);
+        match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => return next,
+            Err(actual) => current = actual,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum TrackedMemoryGauge {
+    GetObjectBufferedBytes,
+}
+
+/// Drop-based guard for tracked in-memory payloads.
+#[derive(Debug)]
+pub struct MemoryGaugeGuard {
+    gauge: TrackedMemoryGauge,
+    bytes: u64,
+}
+
+impl Drop for MemoryGaugeGuard {
+    fn drop(&mut self) {
+        match self.gauge {
+            TrackedMemoryGauge::GetObjectBufferedBytes => {
+                let next = saturating_sub_atomic(&GET_OBJECT_BUFFERED_BYTES, self.bytes);
+                gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64);
+            }
+        }
+    }
+}
+
 /// Record GetObject request start.
 #[inline(always)]
 pub fn record_get_object_request_start(concurrent_requests: usize) {
@@ -217,9 +256,9 @@ pub fn record_get_object_io_state(
 /// * `duration_ms` - Time taken for the read operation in milliseconds
 #[inline(always)]
 pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) {
-    counter!("rustfs.zero_copy.reads.total").increment(1);
-    histogram!("rustfs.zero_copy.read.size.bytes").record(size_bytes as f64);
-    histogram!("rustfs.zero_copy.read.duration.ms").record(duration_ms);
+    counter!("rustfs_zero_copy_reads_total").increment(1);
+    histogram!("rustfs_zero_copy_read_size_bytes").record(size_bytes as f64);
+    histogram!("rustfs_zero_copy_read_duration_ms").record(duration_ms);
 }
 
 /// Record memory copies avoided by using zero-copy.
@@ -229,7 +268,7 @@ pub fn record_zero_copy_read(size_bytes: usize, duration_ms: f64) {
 /// * `bytes_saved` - Number of bytes that would have been copied without zero-copy
 #[inline(always)]
 pub fn record_memory_copy_saved(bytes_saved: usize) {
-    counter!("rustfs.zero_copy.memory.saved.bytes").increment(bytes_saved as u64);
+    counter!("rustfs_zero_copy_memory_saved_bytes_total").increment(bytes_saved as u64);
 }
 
 /// Record a fallback from zero-copy to regular read.
@@ -242,7 +281,7 @@ pub fn record_memory_copy_saved(bytes_saved: usize) {
 /// * `reason` - Reason for the fallback (e.g., "mmap_unavailable", "file_too_large")
 #[inline(always)]
 pub fn record_zero_copy_fallback(reason: &str) {
-    counter!("rustfs.zero_copy.fallback.total", "reason" => reason.to_string()).increment(1);
+    counter!("rustfs_zero_copy_fallback_total", "reason" => reason.to_string()).increment(1);
 }
 
 // ============================================================================
@@ -258,13 +297,13 @@ pub fn record_zero_copy_fallback(reason: &str) {
 /// * `from_pool` - Whether buffer was reused from pool
 #[inline(always)]
 pub fn record_bytes_pool_acquire(tier: &str, size: usize, from_pool: bool) {
-    counter!("rustfs.bytes.pool.acquisitions.total", "tier" => tier.to_string()).increment(1);
-    gauge!("rustfs.bytes.pool.size.bytes", "tier" => tier.to_string()).set(size as f64);
+    counter!("rustfs_bytes_pool_acquisitions_total", "tier" => tier.to_string()).increment(1);
+    gauge!("rustfs_bytes_pool_size_bytes", "tier" => tier.to_string()).set(size as f64);
 
     if from_pool {
-        counter!("rustfs.bytes.pool.hits.total", "tier" => tier.to_string()).increment(1);
+        counter!("rustfs_bytes_pool_hits_total", "tier" => tier.to_string()).increment(1);
     } else {
-        counter!("rustfs.bytes.pool.misses.total", "tier" => tier.to_string()).increment(1);
+        counter!("rustfs_bytes_pool_misses_total", "tier" => tier.to_string()).increment(1);
     }
 }
 
@@ -275,7 +314,7 @@ pub fn record_bytes_pool_acquire(tier: &str, size: usize, from_pool: bool) {
 /// * `tier` - Pool tier ("small", "medium", "large", "xlarge")
 #[inline(always)]
 pub fn record_bytes_pool_return(tier: &str) {
-    counter!("rustfs.bytes.pool.returns.total", "tier" => tier.to_string()).increment(1);
+    counter!("rustfs_bytes_pool_returns_total", "tier" => tier.to_string()).increment(1);
 }
 
 /// Record current BytesPool allocated bytes.
@@ -286,7 +325,7 @@ pub fn record_bytes_pool_return(tier: &str) {
 /// * `bytes` - Currently allocated bytes
 #[inline(always)]
 pub fn record_bytes_pool_allocated(tier: &str, bytes: u64) {
-    gauge!("rustfs.bytes.pool.allocated.bytes", "tier" => tier.to_string()).set(bytes as f64);
+    gauge!("rustfs_bytes_pool_allocated_bytes", "tier" => tier.to_string()).set(bytes as f64);
 }
 
 /// Get BytesPool hit rate as a gauge metric.
@@ -297,7 +336,7 @@ pub fn record_bytes_pool_allocated(tier: &str, bytes: u64) {
 /// * `hit_rate` - Hit rate (0.0 - 1.0)
 #[inline(always)]
 pub fn record_bytes_pool_hit_rate(tier: &str, hit_rate: f64) {
-    gauge!("rustfs.bytes.pool.hit.rate", "tier" => tier.to_string()).set(hit_rate * 100.0);
+    gauge!("rustfs_bytes_pool_hit_rate", "tier" => tier.to_string()).set(hit_rate * 100.0);
 }
 
 /// Record zero-copy write operation.
@@ -308,9 +347,9 @@ pub fn record_bytes_pool_hit_rate(tier: &str, hit_rate: f64) {
 /// * `duration_ms` - Time taken for the write operation in milliseconds
 #[inline(always)]
 pub fn record_zero_copy_write(size_bytes: usize, duration_ms: f64) {
-    counter!("rustfs.zero_copy.write.total").increment(1);
-    histogram!("rustfs.zero_copy.write.size.bytes").record(size_bytes as f64);
-    histogram!("rustfs.zero_copy.write.duration.ms").record(duration_ms);
+    counter!("rustfs_zero_copy_write_total").increment(1);
+    histogram!("rustfs_zero_copy_write_size_bytes").record(size_bytes as f64);
+    histogram!("rustfs_zero_copy_write_duration_ms").record(duration_ms);
 }
 
 /// Record zero-copy write fallback.
@@ -322,7 +361,7 @@ pub fn record_zero_copy_write(size_bytes: usize, duration_ms: f64) {
 /// * `reason` - Reason for the fallback
 #[inline(always)]
 pub fn record_zero_copy_write_fallback(reason: &str) {
-    counter!("rustfs.zero_copy.write.fallback.total", "reason" => reason.to_string()).increment(1);
+    counter!("rustfs_zero_copy_write_fallback_total", "reason" => reason.to_string()).increment(1);
 }
 
 /// Record bytes saved from zero-copy.
@@ -332,7 +371,7 @@ pub fn record_zero_copy_write_fallback(reason: &str) {
 /// * `size_bytes` - Number of bytes saved from zero-copy
 #[inline(always)]
 pub fn record_bytes_saved(size_bytes: usize) {
-    counter!("rustfs.zero_copy.bytes.saved.total").increment(size_bytes as u64);
+    counter!("rustfs_zero_copy_bytes_saved_total").increment(size_bytes as u64);
 }
 
 // ============================================================================
@@ -350,11 +389,11 @@ pub fn record_bytes_saved(size_bytes: usize) {
 /// interpreted as the definitive source of truth for data-plane copy mode.
 #[inline(always)]
 pub fn record_get_object(duration_ms: f64, size_bytes: i64) {
-    counter!("rustfs.s3.get_object.total").increment(1);
-    histogram!("rustfs.s3.get_object.duration.ms").record(duration_ms);
+    counter!("rustfs_s3_get_object_total").increment(1);
+    histogram!("rustfs_s3_get_object_duration_ms").record(duration_ms);
 
     if size_bytes > 0 {
-        histogram!("rustfs.s3.get_object.size.bytes").record(size_bytes as f64);
+        histogram!("rustfs_s3_get_object_size_bytes").record(size_bytes as f64);
     }
 }
 
@@ -367,15 +406,15 @@ pub fn record_get_object(duration_ms: f64, size_bytes: i64) {
 /// * `zero_copy_enabled` - Whether zero-copy was enabled for this operation
 #[inline(always)]
 pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_enabled: bool) {
-    counter!("rustfs.s3.put_object.total").increment(1);
-    histogram!("rustfs.s3.put_object.duration.ms").record(duration_ms);
+    counter!("rustfs_s3_put_object_total").increment(1);
+    histogram!("rustfs_s3_put_object_duration_ms").record(duration_ms);
 
     if size_bytes > 0 {
-        histogram!("rustfs.s3.put_object.size.bytes").record(size_bytes as f64);
+        histogram!("rustfs_s3_put_object_size_bytes").record(size_bytes as f64);
     }
 
     if zero_copy_enabled {
-        counter!("rustfs.s3.put_object.zero_copy.enabled.total").increment(1);
+        counter!("rustfs_s3_put_object_zero_copy_enabled_total").increment(1);
     }
 }
 
@@ -388,12 +427,12 @@ pub fn record_put_object(duration_ms: f64, size_bytes: i64, zero_copy_enabled: b
 /// * `is_truncated` - Whether the response was truncated
 #[inline(always)]
 pub fn record_list_objects(duration_ms: f64, objects_count: u64, is_truncated: bool) {
-    counter!("rustfs.s3.list_objects.total").increment(1);
-    histogram!("rustfs.s3.list_objects.duration.ms").record(duration_ms);
-    histogram!("rustfs.s3.list_objects.count").record(objects_count as f64);
+    counter!("rustfs_s3_list_objects_total").increment(1);
+    histogram!("rustfs_s3_list_objects_duration_ms").record(duration_ms);
+    histogram!("rustfs_s3_list_objects_count").record(objects_count as f64);
 
     if is_truncated {
-        counter!("rustfs.s3.list_objects.truncated.total").increment(1);
+        counter!("rustfs_s3_list_objects_truncated_total").increment(1);
     }
 }
 
@@ -405,11 +444,11 @@ pub fn record_list_objects(duration_ms: f64, objects_count: u64, is_truncated: b
 /// * `version_deleted` - Whether a specific version was deleted
 #[inline(always)]
 pub fn record_delete_object(duration_ms: f64, version_deleted: bool) {
-    counter!("rustfs.s3.delete_object.total").increment(1);
-    histogram!("rustfs.s3.delete_object.duration.ms").record(duration_ms);
+    counter!("rustfs_s3_delete_object_total").increment(1);
+    histogram!("rustfs_s3_delete_object_duration_ms").record(duration_ms);
 
     if version_deleted {
-        counter!("rustfs.s3.delete_object.version.total").increment(1);
+        counter!("rustfs_s3_delete_object_version_total").increment(1);
     }
 }
 
@@ -427,18 +466,18 @@ pub fn record_delete_object(duration_ms: f64, version_deleted: bool) {
 /// * `concurrent_requests` - Number of concurrent requests
 #[inline(always)]
 pub fn record_io_strategy(storage_media: &str, access_pattern: &str, buffer_size: usize, concurrent_requests: u64) {
-    counter!("rustfs.io.strategy.total",
+    counter!("rustfs_io_strategy_total",
         "storage_media" => storage_media.to_string(),
         "access_pattern" => access_pattern.to_string(),
     )
     .increment(1);
 
-    gauge!("rustfs.io.buffer.size.bytes",
+    gauge!("rustfs_io_buffer_size_bytes",
         "storage_media" => storage_media.to_string(),
     )
     .set(buffer_size as f64);
 
-    gauge!("rustfs.io.concurrent.requests").set(concurrent_requests as f64);
+    gauge!("rustfs_io_concurrent_requests").set(concurrent_requests as f64);
 }
 
 /// Record disk permit wait time (load tracking).
@@ -448,7 +487,7 @@ pub fn record_io_strategy(storage_media: &str, access_pattern: &str, buffer_size
 /// * `duration_ms` - Time spent waiting for disk permit
 #[inline(always)]
 pub fn record_permit_wait(duration_ms: f64) {
-    histogram!("rustfs.io.permit.wait.duration.ms").record(duration_ms);
+    histogram!("rustfs_io_permit_wait_duration_ms").record(duration_ms);
 }
 
 /// Record I/O load level.
@@ -459,12 +498,12 @@ pub fn record_permit_wait(duration_ms: f64) {
 /// * `concurrent_requests` - Number of concurrent requests
 #[inline(always)]
 pub fn record_io_load_level(load_level: &str, concurrent_requests: u64) {
-    counter!("rustfs.io.load.level",
+    counter!("rustfs_io_load_level",
         "level" => load_level.to_string(),
     )
     .increment(1);
 
-    gauge!("rustfs.io.concurrent.requests").set(concurrent_requests as f64);
+    gauge!("rustfs_io_concurrent_requests").set(concurrent_requests as f64);
 }
 
 /// Record cache size and entry count.
@@ -476,12 +515,12 @@ pub fn record_io_load_level(load_level: &str, concurrent_requests: u64) {
 /// * `entries` - Number of entries in the cache
 #[inline(always)]
 pub fn record_cache_size(tier: &str, size_bytes: usize, entries: u64) {
-    gauge!("rustfs.cache.size.bytes",
+    gauge!("rustfs_cache_size_bytes",
         "tier" => tier.to_string(),
     )
     .set(size_bytes as f64);
 
-    gauge!("rustfs.cache.entries",
+    gauge!("rustfs_cache_entries",
         "tier" => tier.to_string(),
     )
     .set(entries as f64);
@@ -499,13 +538,11 @@ pub fn record_cache_size(tier: &str, size_bytes: usize, entries: u64) {
 /// * `tier` - Bandwidth tier ("low", "medium", "high", "unknown")
 #[inline(always)]
 pub fn record_bandwidth(bytes_per_second: u64, tier: &str) {
-    gauge!("rustfs.bandwidth.current.bps").set(bytes_per_second as f64);
-    gauge!("rustfs.bandwidth.current.bps",
-        "tier" => tier.to_string(),
-    )
-    .set(bytes_per_second as f64);
+    let tier_label = if tier.is_empty() { "unknown" } else { tier };
+    gauge!("rustfs_bandwidth_current_bps", "tier" => "all").set(bytes_per_second as f64);
+    gauge!("rustfs_bandwidth_current_bps", "tier" => tier_label.to_string()).set(bytes_per_second as f64);
 
-    histogram!("rustfs.bandwidth.observed.bps").record(bytes_per_second as f64);
+    histogram!("rustfs_bandwidth_observed_bps").record(bytes_per_second as f64);
 }
 
 /// Record data transfer for bandwidth calculation.
@@ -516,12 +553,12 @@ pub fn record_bandwidth(bytes_per_second: u64, tier: &str) {
 /// * `duration_ms` - Duration of the transfer in milliseconds
 #[inline(always)]
 pub fn record_data_transfer(bytes: u64, duration_ms: f64) {
-    counter!("rustfs.io.transfer.bytes").increment(bytes);
-    histogram!("rustfs.io.transfer.duration.ms").record(duration_ms);
+    counter!("rustfs_io_transfer_bytes_total").increment(bytes);
+    histogram!("rustfs_io_transfer_duration_ms").record(duration_ms);
 
     if duration_ms > 0.0 {
         let bps = (bytes as f64 * 1000.0) / duration_ms;
-        histogram!("rustfs.io.transfer.bandwidth.bps").record(bps);
+        histogram!("rustfs_io_transfer_bandwidth_bps").record(bps);
     }
 }
 
@@ -537,15 +574,94 @@ pub fn record_data_transfer(bytes: u64, duration_ms: f64) {
 /// * `total_bytes` - Total memory in bytes
 #[inline(always)]
 pub fn record_memory_usage(used_bytes: u64, total_bytes: u64) {
-    gauge!("rustfs.memory.used.bytes").set(used_bytes as f64);
-    gauge!("rustfs.memory.total.bytes").set(total_bytes as f64);
+    gauge!("rustfs_memory_used_bytes").set(used_bytes as f64);
+    gauge!("rustfs_memory_total_bytes").set(total_bytes as f64);
 
     if total_bytes > 0 {
         let usage_percent = (used_bytes as f64 / total_bytes as f64) * 100.0;
-        gauge!("rustfs.memory.usage.percent").set(usage_percent);
+        gauge!("rustfs_memory_usage_percent").set(usage_percent);
     }
 }
 
+/// Record process-level memory split metrics.
+#[inline(always)]
+pub fn record_process_memory_split(resident_bytes: u64, virtual_bytes: u64) {
+    gauge!("rustfs_memory_process_resident_bytes").set(resident_bytes as f64);
+    gauge!("rustfs_memory_process_virtual_bytes").set(virtual_bytes as f64);
+}
+
+/// Record cgroup memory split metrics when available.
+#[inline(always)]
+pub fn record_cgroup_memory_split(
+    current_bytes: Option<u64>,
+    limit_bytes: Option<u64>,
+    anon_bytes: Option<u64>,
+    file_bytes: Option<u64>,
+    active_file_bytes: Option<u64>,
+    inactive_file_bytes: Option<u64>,
+) {
+    if let Some(current_bytes) = current_bytes {
+        gauge!("rustfs_memory_cgroup_current_bytes").set(current_bytes as f64);
+    }
+    if let Some(limit_bytes) = limit_bytes {
+        gauge!("rustfs_memory_cgroup_limit_bytes").set(limit_bytes as f64);
+    }
+    if let Some(anon_bytes) = anon_bytes {
+        gauge!("rustfs_memory_cgroup_anon_bytes").set(anon_bytes as f64);
+    }
+    if let Some(file_bytes) = file_bytes {
+        gauge!("rustfs_memory_cgroup_file_bytes").set(file_bytes as f64);
+    }
+    if let Some(active_file_bytes) = active_file_bytes {
+        gauge!("rustfs_memory_cgroup_active_file_bytes").set(active_file_bytes as f64);
+    }
+    if let Some(inactive_file_bytes) = inactive_file_bytes {
+        gauge!("rustfs_memory_cgroup_inactive_file_bytes").set(inactive_file_bytes as f64);
+    }
+}
+
+/// Track encoded bytes currently queued between erasure encode and disk writers.
+#[inline(always)]
+pub fn add_ec_encode_inflight_bytes(bytes: usize) {
+    let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
+    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
+}
+
+/// Remove encoded bytes from the tracked erasure encode in-flight gauge.
+#[inline(always)]
+pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
+    let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
+    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
+}
+
+/// Return the current tracked EC encode in-flight bytes.
+#[inline(always)]
+pub fn current_ec_encode_inflight_bytes() -> u64 {
+    EC_ENCODE_INFLIGHT_BYTES.load(Ordering::Relaxed)
+}
+
+/// Track whole-object buffering on the GET path.
+#[inline(always)]
+pub fn track_get_object_buffered_bytes(bytes: usize) -> Option<MemoryGaugeGuard> {
+    if bytes == 0 {
+        return None;
+    }
+
+    let next = GET_OBJECT_BUFFERED_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
+    gauge!("rustfs_get_object_buffered_bytes_current").set(next as f64);
+
+    Some(MemoryGaugeGuard {
+        gauge: TrackedMemoryGauge::GetObjectBufferedBytes,
+        bytes: bytes as u64,
+    })
+}
+
+/// Return the current tracked GET whole-buffered bytes.
+#[inline(always)]
+pub fn current_get_object_buffered_bytes() -> u64 {
+    GET_OBJECT_BUFFERED_BYTES.load(Ordering::Relaxed)
+}
+
 /// Record CPU usage.
 ///
 /// # Arguments
@@ -553,7 +669,7 @@ pub fn record_memory_usage(used_bytes: u64, total_bytes: u64) {
 /// * `percent` - CPU usage percentage (0.0 - 100.0)
 #[inline(always)]
 pub fn record_cpu_usage(percent: f64) {
-    gauge!("rustfs.cpu.usage.percent").set(percent);
+    gauge!("rustfs_cpu_usage_percent").set(percent);
 }
 
 /// Record disk I/O statistics.
@@ -566,13 +682,10 @@ pub fn record_cpu_usage(percent: f64) {
 /// * `write_ops` - Number of write operations
 #[inline(always)]
 pub fn record_disk_io(read_bytes: u64, write_bytes: u64, read_ops: u64, write_ops: u64) {
-    counter!("rustfs.disk.read.bytes").increment(read_bytes);
-    counter!("rustfs.disk.write.bytes").increment(write_bytes);
-    counter!("rustfs.disk.read.ops").increment(read_ops);
-    counter!("rustfs.disk.write.ops").increment(write_ops);
-
-    gauge!("rustfs.disk.read.bytes_total").set(read_bytes as f64);
-    gauge!("rustfs.disk.write.bytes_total").set(write_bytes as f64);
+    counter!("rustfs_disk_read_bytes_total").increment(read_bytes);
+    counter!("rustfs_disk_write_bytes_total").increment(write_bytes);
+    counter!("rustfs_disk_read_ops_total").increment(read_ops);
+    counter!("rustfs_disk_write_ops_total").increment(write_ops);
 }
 
 // ============================================================================
@@ -587,7 +700,7 @@ pub fn record_disk_io(read_bytes: u64, write_bytes: u64, read_ops: u64, write_op
 /// * `error_type` - Error type (e.g., "timeout", "disk_error", "network")
 #[inline(always)]
 pub fn record_error(operation: &str, error_type: &str) {
-    counter!("rustfs.errors.total",
+    counter!("rustfs_errors_total",
         "operation" => operation.to_string(),
         "type" => error_type.to_string(),
     )
@@ -602,12 +715,12 @@ pub fn record_error(operation: &str, error_type: &str) {
 /// * `duration_ms` - Duration before timeout
 #[inline(always)]
 pub fn record_timeout(operation: &str, duration_ms: f64) {
-    counter!("rustfs.timeouts.total",
+    counter!("rustfs_timeouts_total",
         "operation" => operation.to_string(),
     )
     .increment(1);
 
-    histogram!("rustfs.timeouts.duration.ms",
+    histogram!("rustfs_timeouts_duration_ms",
         "operation" => operation.to_string(),
     )
     .record(duration_ms);
@@ -621,12 +734,12 @@ pub fn record_timeout(operation: &str, duration_ms: f64) {
 /// * `attempt_number` - Attempt number (1-based)
 #[inline(always)]
 pub fn record_retry(operation: &str, attempt_number: u32) {
-    counter!("rustfs.retries.total",
+    counter!("rustfs_retries_total",
         "operation" => operation.to_string(),
     )
     .increment(1);
 
-    histogram!("rustfs.retries.attempt",
+    histogram!("rustfs_retries_attempt",
         "operation" => operation.to_string(),
     )
     .record(attempt_number as f64);
@@ -643,7 +756,7 @@ pub fn record_retry(operation: &str, attempt_number: u32) {
 /// * `latency_ms` - I/O latency in milliseconds
 #[inline(always)]
 pub fn record_io_latency(latency_ms: f64) {
-    histogram!("rustfs.io.latency.ms").record(latency_ms);
+    histogram!("rustfs_io_latency_ms").record(latency_ms);
 }
 
 /// Record I/O latency P95 in milliseconds.
@@ -653,7 +766,7 @@ pub fn record_io_latency(latency_ms: f64) {
 /// * `latency_ms` - P95 I/O latency in milliseconds
 #[inline(always)]
 pub fn record_io_latency_p95(latency_ms: f64) {
-    gauge!("rustfs.io.latency.p95.ms").set(latency_ms);
+    gauge!("rustfs_io_latency_p95_ms").set(latency_ms);
 }
 
 /// Record I/O latency P99 in milliseconds.
@@ -663,7 +776,7 @@ pub fn record_io_latency_p95(latency_ms: f64) {
 /// * `latency_ms` - P99 I/O latency in milliseconds
 #[inline(always)]
 pub fn record_io_latency_p99(latency_ms: f64) {
-    gauge!("rustfs.io.latency.p99.ms").set(latency_ms);
+    gauge!("rustfs_io_latency_p99_ms").set(latency_ms);
 }
 
 #[cfg(test)]
@@ -763,6 +876,48 @@ mod tests {
         record_memory_usage(2 * 1024 * 1024 * 1024, 8 * 1024 * 1024 * 1024);
     }
 
+    #[test]
+    fn test_record_process_memory_split() {
+        record_process_memory_split(1024, 2048);
+        record_process_memory_split(4096, 8192);
+    }
+
+    #[test]
+    fn test_record_cgroup_memory_split() {
+        record_cgroup_memory_split(Some(1), Some(2), Some(3), Some(4), Some(5), Some(6));
+        record_cgroup_memory_split(None, None, None, None, None, None);
+    }
+
+    #[test]
+    fn test_ec_encode_inflight_bytes_tracking() {
+        EC_ENCODE_INFLIGHT_BYTES.store(0, Ordering::Relaxed);
+        add_ec_encode_inflight_bytes(1024);
+        add_ec_encode_inflight_bytes(2048);
+        remove_ec_encode_inflight_bytes(1024);
+        remove_ec_encode_inflight_bytes(2048);
+        remove_ec_encode_inflight_bytes(4096);
+        assert_eq!(current_ec_encode_inflight_bytes(), 0);
+    }
+
+    #[test]
+    fn test_get_object_buffered_bytes_guard() {
+        GET_OBJECT_BUFFERED_BYTES.store(0, Ordering::Relaxed);
+        drop(track_get_object_buffered_bytes(1024));
+        let guard = track_get_object_buffered_bytes(2048);
+        drop(guard);
+        assert_eq!(current_get_object_buffered_bytes(), 0);
+    }
+
+    #[test]
+    fn test_get_object_buffered_bytes_guard_saturates_on_underflow() {
+        GET_OBJECT_BUFFERED_BYTES.store(1024, Ordering::Relaxed);
+        drop(MemoryGaugeGuard {
+            gauge: TrackedMemoryGauge::GetObjectBufferedBytes,
+            bytes: 2048,
+        });
+        assert_eq!(current_get_object_buffered_bytes(), 0);
+    }
+
     #[test]
     fn test_record_cpu_usage() {
         record_cpu_usage(25.5);

crates/io-metrics/src/lock_metrics.rs

@@ -20,7 +20,7 @@ use std::time::Duration;
 #[inline(always)]
 pub fn record_lock_optimization_enabled(enabled: bool) {
     use metrics::gauge;
-    gauge!("rustfs.lock.optimization.enabled").set(if enabled { 1.0 } else { 0.0 });
+    gauge!("rustfs_lock_optimization_enabled").set(if enabled { 1.0 } else { 0.0 });
 }
 
 /// Record spin attempt.
@@ -28,9 +28,9 @@ pub fn record_lock_optimization_enabled(enabled: bool) {
 pub fn record_spin_attempt(success: bool) {
     use metrics::counter;
     if success {
-        counter!("rustfs.lock.spin.successes").increment(1);
+        counter!("rustfs_lock_spin_successes").increment(1);
     } else {
-        counter!("rustfs.lock.spin.failures").increment(1);
+        counter!("rustfs_lock_spin_failures").increment(1);
     }
 }
 
@@ -38,28 +38,28 @@ pub fn record_spin_attempt(success: bool) {
 #[inline(always)]
 pub fn record_spin_count_change(new_count: usize) {
     use metrics::gauge;
-    gauge!("rustfs.lock.spin.count").set(new_count as f64);
+    gauge!("rustfs_lock_spin_count").set(new_count as f64);
 }
 
 /// Record lock hold time.
 #[inline(always)]
 pub fn record_lock_hold_time(hold_time: Duration) {
     use metrics::histogram;
-    histogram!("rustfs.lock.hold_time.secs").record(hold_time.as_secs_f64());
+    histogram!("rustfs_lock_hold_time_secs").record(hold_time.as_secs_f64());
 }
 
 /// Record early release.
 #[inline(always)]
 pub fn record_early_release() {
     use metrics::counter;
-    counter!("rustfs.lock.early_releases").increment(1);
+    counter!("rustfs_lock_early_releases").increment(1);
 }
 
 /// Record contention event.
 #[inline(always)]
 pub fn record_contention_event() {
     use metrics::counter;
-    counter!("rustfs.lock.contentions").increment(1);
+    counter!("rustfs_lock_contentions").increment(1);
 }
 
 /// Lock statistics summary.

crates/io-metrics/src/metric_names.rs

@@ -49,6 +49,6 @@ pub mod zero_copy {
     /// Throughput in MB/s
     pub const THROUGHPUT_MBPS: &str = "rustfs_zero_copy_throughput_mbps";
 
-    /// Memory saved by zero-copy in bytes
-    pub const MEMORY_SAVED_BYTES: &str = "rustfs_zero_copy_memory_saved_bytes";
+    /// Current memory saved estimate by zero-copy in bytes
+    pub const MEMORY_SAVED_BYTES: &str = "rustfs_zero_copy_memory_saved_bytes_current";
 }

crates/io-metrics/src/timeout_metrics.rs

@@ -34,23 +34,23 @@ pub fn record_operation_duration(operation: &str, duration: Duration) {
 #[inline(always)]
 pub fn record_dynamic_timeout(size_bytes: u64, timeout: Duration) {
     use metrics::{gauge, histogram};
-    gauge!("rustfs.timeout.dynamic.size").set(size_bytes as f64);
-    gauge!("rustfs.timeout.dynamic.secs").set(timeout.as_secs_f64());
-    histogram!("rustfs.timeout.dynamic.size.histogram").record(size_bytes as f64);
+    gauge!("rustfs_timeout_dynamic_size").set(size_bytes as f64);
+    gauge!("rustfs_timeout_dynamic_secs").set(timeout.as_secs_f64());
+    histogram!("rustfs_timeout_dynamic_size_histogram").record(size_bytes as f64);
 }
 
 /// Record operation progress.
 #[inline(always)]
 pub fn record_operation_progress(operation: &str, percent: f64) {
     use metrics::gauge;
-    gauge!("rustfs.operation.progress", "operation" => operation.to_string()).set(percent);
+    gauge!("rustfs_operation_progress", "operation" => operation.to_string()).set(percent);
 }
 
 /// Record stalled operation.
 #[inline(always)]
 pub fn record_stalled_operation(operation: &str) {
     use metrics::counter;
-    counter!("rustfs.operation.stalled", "operation" => operation.to_string()).increment(1);
+    counter!("rustfs_operation_stalled", "operation" => operation.to_string()).increment(1);
 }
 
 /// Record operation completion.
@@ -58,7 +58,7 @@ pub fn record_stalled_operation(operation: &str) {
 pub fn record_operation_completion(operation: &str, success: bool) {
     use metrics::counter;
     let status = if success { "success" } else { "failure" };
-    counter!("rustfs.operation.completions", "operation" => operation.to_string(), "status" => status).increment(1);
+    counter!("rustfs_operation_completions", "operation" => operation.to_string(), "status" => status).increment(1);
 }
 
 /// Timeout statistics summary.

crates/obs/Cargo.toml

@@ -34,11 +34,14 @@ workspace = true
 
 [dependencies]
 rustfs-audit = { workspace = true }
+rustfs-common = { workspace = true }
 rustfs-config = { workspace = true, features = ["constants", "observability"] }
 rustfs-ecstore = { workspace = true }
+rustfs-iam = { workspace = true }
 rustfs-io-metrics = { workspace = true }
 rustfs-notify = { workspace = true }
 rustfs-utils = { workspace = true, features = ["ip"] }
+chrono = { workspace = true }
 flate2 = { workspace = true }
 glob = { workspace = true }
 jiff = { workspace = true }

crates/obs/README.md

@@ -80,7 +80,9 @@ The library selects a backend automatically based on configuration:
 
 ```
 1. Any OTLP endpoint set?
-   └─ YES → Full OTLP/HTTP pipeline (traces + metrics + logs + profiling)
+   └─ YES → Full OTLP/HTTP pipeline (traces + metrics + logs)
+            + Profiling (Pyroscope) only if:
+              - RUSTFS_OBS_PROFILING_EXPORT_ENABLED=true (explicit opt-in, default: false)
 
 2. RUSTFS_OBS_LOG_DIRECTORY set to a non-empty path?
    └─ YES → Rolling-file JSON logging
@@ -117,7 +119,7 @@ All configuration is read from environment variables at startup.
 | `RUSTFS_OBS_TRACES_EXPORT_ENABLED`    | `true`    | Toggle trace export                                        |
 | `RUSTFS_OBS_METRICS_EXPORT_ENABLED`   | `true`    | Toggle metrics export                                      |
 | `RUSTFS_OBS_LOGS_EXPORT_ENABLED`      | `true`    | Toggle OTLP log export                                     |
-| `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `true`    | Toggle profiling export                                    |
+| `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `false`   | Toggle profiling export                                    |
 | `RUSTFS_OBS_USE_STDOUT`               | `false`   | Mirror all signals to stdout alongside OTLP                |
 | `RUSTFS_OBS_SAMPLE_RATIO`             | `0.1`     | Trace sampling ratio `0.0`–`1.0`                           |
 | `RUSTFS_OBS_METER_INTERVAL`           | `15`      | Metrics export interval (seconds)                          |
@@ -171,16 +173,16 @@ The log rotation and cleanup pipeline emits these metrics (via the `metrics` fac
 
 | Metric | Type | Description |
 |---|---|---|
-| `rustfs.log_cleaner.deleted_files_total` | counter | Number of files deleted per cleanup pass |
-| `rustfs.log_cleaner.freed_bytes_total` | counter | Bytes reclaimed by deletion |
-| `rustfs.log_cleaner.compress_duration_seconds` | histogram | Compression stage duration |
-| `rustfs.log_cleaner.steal_success_rate` | gauge | Work-stealing success ratio in parallel mode |
-| `rustfs.log_cleaner.runs_total` | counter | Successful cleanup loop runs |
-| `rustfs.log_cleaner.run_failures_total` | counter | Failed or panicked cleanup loop runs |
-| `rustfs.log_cleaner.rotation_total` | counter | Successful file rotations |
-| `rustfs.log_cleaner.rotation_failures_total` | counter | Failed file rotations |
-| `rustfs.log_cleaner.rotation_duration_seconds` | histogram | Rotation latency |
-| `rustfs.log_cleaner.active_file_size_bytes` | gauge | Current active log file size |
+| `rustfs_log_cleaner_deleted_files_total` | counter | Number of files deleted per cleanup pass |
+| `rustfs_log_cleaner_freed_bytes_total` | counter | Bytes reclaimed by deletion |
+| `rustfs_log_cleaner_compress_duration_seconds` | histogram | Compression stage duration |
+| `rustfs_log_cleaner_steal_success_rate` | gauge | Work-stealing success ratio in parallel mode |
+| `rustfs_log_cleaner_runs_total` | counter | Successful cleanup loop runs |
+| `rustfs_log_cleaner_run_failures_total` | counter | Failed or panicked cleanup loop runs |
+| `rustfs_log_cleaner_rotation_total` | counter | Successful file rotations |
+| `rustfs_log_cleaner_rotation_failures_total` | counter | Failed file rotations |
+| `rustfs_log_cleaner_rotation_duration_seconds` | histogram | Rotation latency |
+| `rustfs_log_cleaner_active_file_size_bytes` | gauge | Current active log file size |
 
 These metrics cover compression, cleanup, and file rotation end-to-end.
 
@@ -195,8 +197,8 @@ These metrics cover compression, cleanup, and file rotation end-to-end.
 ### Grafana Dashboard JSON Draft (Ready to Import)
 
 > Save this as `rustfs-log-cleaner-dashboard.json`, then import from Grafana UI.
-> For Prometheus datasources, metric names are usually normalized to underscores,
-> so `rustfs.log_cleaner.deleted_files_total` becomes `rustfs_log_cleaner_deleted_files_total`.
+> The canonical metric names use underscore notation, for example
+> `rustfs_log_cleaner_deleted_files_total`.
 >
 > The same panels are now checked in at:
 > `.docker/observability/grafana/dashboards/rustfs.json`

crates/obs/src/cleaner/README.md

@@ -58,14 +58,14 @@ This strategy keeps local cache affinity while still balancing stragglers.
 
 The cleaner emits tracing events and runtime metrics:
 
-- `rustfs.log_cleaner.deleted_files_total` (counter)
-- `rustfs.log_cleaner.freed_bytes_total` (counter)
-- `rustfs.log_cleaner.compress_duration_seconds` (histogram)
-- `rustfs.log_cleaner.steal_success_rate` (gauge)
-- `rustfs.log_cleaner.rotation_total` (counter)
-- `rustfs.log_cleaner.rotation_failures_total` (counter)
-- `rustfs.log_cleaner.rotation_duration_seconds` (histogram)
-- `rustfs.log_cleaner.active_file_size_bytes` (gauge)
+- `rustfs_log_cleaner_deleted_files_total` (counter)
+- `rustfs_log_cleaner_freed_bytes_total` (counter)
+- `rustfs_log_cleaner_compress_duration_seconds` (histogram)
+- `rustfs_log_cleaner_steal_success_rate` (gauge)
+- `rustfs_log_cleaner_rotation_total` (counter)
+- `rustfs_log_cleaner_rotation_failures_total` (counter)
+- `rustfs_log_cleaner_rotation_duration_seconds` (histogram)
+- `rustfs_log_cleaner_active_file_size_bytes` (gauge)
 
 These values can be wired into dashboards and alert rules for cleanup health.
 
@@ -127,4 +127,3 @@ let _ = cleaner.cleanup();
 - Prefer `FileMatchMode::Suffix` when rotations prepend timestamps to the filename.
 - Prefer `FileMatchMode::Prefix` when rotations append counters or timestamps after a stable base name.
 - Keep `parallel_workers` modest when `zstd_workers` is greater than `1`, because each compression task may already use internal codec threads.
-

crates/obs/src/config.rs

@@ -48,10 +48,15 @@ use rustfs_config::{
     DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO,
     SERVICE_VERSION, USE_STDOUT,
 };
-use rustfs_utils::{get_env_bool, get_env_f64, get_env_opt_str, get_env_opt_u64, get_env_str, get_env_u64, get_env_usize};
+use rustfs_utils::{
+    get_env_bool, get_env_bool_with_aliases, get_env_f64, get_env_opt_str, get_env_opt_u64, get_env_str, get_env_u64,
+    get_env_usize,
+};
 use serde::{Deserialize, Serialize};
 use std::env;
 
+const LEGACY_ENV_OBS_PROFILING_ENABLED: &str = "RUSTFS_OBS_PROFILING_ENABLED";
+
 /// Full observability configuration used by all telemetry backends.
 ///
 /// Fields are grouped into three logical sections:
@@ -134,7 +139,7 @@ pub struct OtelConfig {
     pub metrics_export_enabled: Option<bool>,
     /// Whether to export logs via OTLP (default: `true`).
     pub logs_export_enabled: Option<bool>,
-    /// Whether to export profiles via pyroscope (default: `true`).
+    /// Whether to export profiles via pyroscope (default: `false`).
     pub profiling_export_enabled: Option<bool>,
     /// **[OTLP-only]** Mirror all signals to stdout in addition to OTLP export.
     /// Only applies when an OTLP endpoint is configured.
@@ -282,7 +287,11 @@ impl OtelConfig {
             traces_export_enabled: Some(get_env_bool(ENV_OBS_TRACES_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED)),
             metrics_export_enabled: Some(get_env_bool(ENV_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED)),
             logs_export_enabled: Some(get_env_bool(ENV_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED)),
-            profiling_export_enabled: Some(get_env_bool(ENV_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_PROFILING_EXPORT_ENABLED)),
+            profiling_export_enabled: Some(get_env_bool_with_aliases(
+                ENV_OBS_PROFILING_EXPORT_ENABLED,
+                &[LEGACY_ENV_OBS_PROFILING_ENABLED],
+                DEFAULT_OBS_PROFILING_EXPORT_ENABLED,
+            )),
             use_stdout: Some(use_stdout),
             sample_ratio: Some(get_env_f64(ENV_OBS_SAMPLE_RATIO, SAMPLE_RATIO)),
             meter_interval: Some(get_env_u64(ENV_OBS_METER_INTERVAL, METER_INTERVAL)),
@@ -415,3 +424,38 @@ impl Default for AppConfig {
 pub fn is_production_environment() -> bool {
     get_env_str(ENV_OBS_ENVIRONMENT, ENVIRONMENT).eq_ignore_ascii_case(DEFAULT_OBS_ENVIRONMENT_PRODUCTION)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn profiling_export_defaults_to_disabled_when_unset() {
+        temp_env::with_var_unset(ENV_OBS_PROFILING_EXPORT_ENABLED, || {
+            temp_env::with_var_unset(LEGACY_ENV_OBS_PROFILING_ENABLED, || {
+                let config = OtelConfig::extract_otel_config_from_env(None);
+                assert_eq!(config.profiling_export_enabled, Some(false));
+            });
+        });
+    }
+
+    #[test]
+    fn profiling_export_accepts_legacy_env_alias() {
+        temp_env::with_var_unset(ENV_OBS_PROFILING_EXPORT_ENABLED, || {
+            temp_env::with_var(LEGACY_ENV_OBS_PROFILING_ENABLED, Some("true"), || {
+                let config = OtelConfig::extract_otel_config_from_env(None);
+                assert_eq!(config.profiling_export_enabled, Some(true));
+            });
+        });
+    }
+
+    #[test]
+    fn canonical_profiling_toggle_has_priority_over_legacy_alias() {
+        temp_env::with_var(LEGACY_ENV_OBS_PROFILING_ENABLED, Some("true"), || {
+            temp_env::with_var(ENV_OBS_PROFILING_EXPORT_ENABLED, Some("false"), || {
+                let config = OtelConfig::extract_otel_config_from_env(None);
+                assert_eq!(config.profiling_export_enabled, Some(false));
+            });
+        });
+    }
+}

crates/obs/src/metrics/collectors/cluster_iam.rs

@@ -25,6 +25,10 @@ use crate::metrics::schema::cluster_iam::*;
 /// IAM statistics.
 #[derive(Debug, Clone, Default)]
 pub struct IamStats {
+    /// Last successful IAM data sync duration in milliseconds
+    pub last_sync_duration_millis: u64,
+    /// Failed requests count in the last full minute
+    pub plugin_authn_service_failed_requests_minute: u64,
     /// Time in seconds since last failed authn service request
     pub plugin_authn_service_last_fail_seconds: u64,
     /// Time in seconds since last successful authn service request
@@ -48,6 +52,11 @@ pub struct IamStats {
 /// Returns a vector of Prometheus metrics for IAM statistics.
 pub fn collect_iam_metrics(stats: &IamStats) -> Vec<PrometheusMetric> {
     vec![
+        PrometheusMetric::from_descriptor(&LAST_SYNC_DURATION_MILLIS_MD, stats.last_sync_duration_millis as f64),
+        PrometheusMetric::from_descriptor(
+            &PLUGIN_AUTHN_SERVICE_FAILED_REQUESTS_MINUTE_MD,
+            stats.plugin_authn_service_failed_requests_minute as f64,
+        ),
         PrometheusMetric::from_descriptor(
             &PLUGIN_AUTHN_SERVICE_LAST_FAIL_SECONDS_MD,
             stats.plugin_authn_service_last_fail_seconds as f64,
@@ -82,6 +91,8 @@ mod tests {
     #[test]
     fn test_collect_iam_metrics() {
         let stats = IamStats {
+            last_sync_duration_millis: 250,
+            plugin_authn_service_failed_requests_minute: 7,
             plugin_authn_service_last_fail_seconds: 3600,
             plugin_authn_service_last_succ_seconds: 10,
             plugin_authn_service_succ_avg_rtt_ms_minute: 50,
@@ -95,7 +106,7 @@ mod tests {
         let metrics = collect_iam_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 8);
+        assert_eq!(metrics.len(), 10);
 
         let sync_successes_name = SYNC_SUCCESSES_MD.get_full_metric_name();
         let sync_successes = metrics.iter().find(|m| m.name == sync_successes_name);
@@ -108,7 +119,7 @@ mod tests {
         let stats = IamStats::default();
         let metrics = collect_iam_metrics(&stats);
 
-        assert_eq!(metrics.len(), 8);
+        assert_eq!(metrics.len(), 10);
         for metric in &metrics {
             assert_eq!(metric.value, 0.0);
             assert!(metric.labels.is_empty());

crates/obs/src/metrics/scheduler.rs

@@ -36,13 +36,20 @@ use crate::metrics::collectors::{
     collect_bucket_metrics,
     collect_bucket_replication_bandwidth_metrics,
     collect_bucket_replication_metrics,
+    collect_bucket_usage_metrics,
+    collect_cluster_config_metrics,
     collect_cluster_health_metrics,
     collect_cluster_metrics,
+    collect_cluster_usage_metrics,
     collect_cpu_metrics,
     collect_drive_count_metrics,
     collect_drive_detailed_metrics,
+    collect_erasure_set_metrics,
     collect_host_network_metrics,
+    collect_iam_metrics,
+    collect_ilm_metrics,
     collect_memory_metrics,
+    collect_network_metrics,
     collect_node_metrics,
     collect_notification_metrics,
     collect_notification_target_metrics,
@@ -53,6 +60,7 @@ use crate::metrics::collectors::{
     collect_process_metrics,
     collect_replication_metrics,
     collect_resource_metrics,
+    collect_scanner_metrics,
 };
 use crate::metrics::config::{
     DEFAULT_AUDIT_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
@@ -64,8 +72,10 @@ use crate::metrics::config::{
 use crate::metrics::report::{PrometheusMetric, report_metrics};
 use crate::metrics::stats_collector::{
     ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
-    collect_bucket_stats, collect_cluster_and_health_stats, collect_disk_and_system_drive_stats, collect_host_network_stats,
-    collect_process_metric_bundle, collect_replication_stats, collect_system_cpu_and_memory_stats_with,
+    collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats,
+    collect_disk_and_system_drive_stats, collect_erasure_set_stats, collect_host_network_stats, collect_iam_stats,
+    collect_ilm_metric_stats, collect_internode_network_stats, collect_process_metric_bundle, collect_replication_stats,
+    collect_scanner_metric_stats, collect_system_cpu_and_memory_stats_with,
 };
 use rustfs_audit::audit_target_metrics;
 use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
@@ -176,6 +186,46 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         }
     });
 
+    // Spawn task for supplementary cluster metrics that are defined in schema/collector
+    // but filled by later task-specific runtime sources.
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(cluster_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let mut metrics = Vec::new();
+
+                    if let Some(stats) = collect_cluster_config_stats().await {
+                        metrics.extend(collect_cluster_config_metrics(&stats));
+                    }
+
+                    let erasure_sets = collect_erasure_set_stats().await;
+                    if !erasure_sets.is_empty() {
+                        metrics.extend(collect_erasure_set_metrics(&erasure_sets));
+                    }
+
+                    if let Some(stats) = collect_iam_stats().await {
+                        metrics.extend(collect_iam_metrics(&stats));
+                    }
+
+                    if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
+                        metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
+                        metrics.extend(collect_bucket_usage_metrics(&bucket_usage));
+                    }
+
+                    if !metrics.is_empty() {
+                        report_metrics(&metrics);
+                    }
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for supplementary cluster stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
     // Spawn task for bucket metrics
     let token_clone = token.clone();
     tokio::spawn(async move {
@@ -302,6 +352,35 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         }
     });
 
+    // Spawn task for background workflow metrics such as ILM and scanner.
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(cluster_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let mut metrics = Vec::new();
+
+                    if let Some(stats) = collect_ilm_metric_stats().await {
+                        metrics.extend(collect_ilm_metrics(&stats));
+                    }
+
+                    if let Some(stats) = collect_scanner_metric_stats().await {
+                        metrics.extend(collect_scanner_metrics(&stats));
+                    }
+
+                    if !metrics.is_empty() {
+                        report_metrics(&metrics);
+                    }
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for background workflow stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
     // Spawn task for system monitoring metrics (migrated from rustfs-obs::system)
     let system_interval = get_env_opt_u64(ENV_SYSTEM_METRICS_INTERVAL)
         .or_else(|| get_env_opt_u64(LEGACY_SYSTEM_METRICS_INTERVAL).map(|ms| ms / 1000)) // Convert ms to seconds
@@ -310,7 +389,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         .map(Duration::from_secs)
         .unwrap_or(DEFAULT_SYSTEM_METRICS_INTERVAL);
 
-    let token_clone = token;
+    let token_clone = token.clone();
     tokio::spawn(async move {
         let labels = current_process_metric_labels();
         let mut host_system = System::new_all();
@@ -378,6 +457,28 @@ pub fn init_metrics_runtime(token: CancellationToken) {
             }
         }
     });
+
+    // Spawn task for internode/system network metrics.
+    let token_clone = token;
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(system_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    if let Some(stats) = collect_internode_network_stats() {
+                        let metrics = collect_network_metrics(&stats);
+                        if !metrics.is_empty() {
+                            report_metrics(&metrics);
+                        }
+                    }
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for internode network stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
 }
 
 /// Backward-compatible alias kept during migration.

crates/obs/src/metrics/stats_collector.rs

@@ -21,10 +21,14 @@
 //! and convert them to the Stats structs used by collectors.
 
 use crate::metrics::collectors::{
-    BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, ClusterHealthStats,
-    ClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, HostNetworkStats, MemoryStats, ProcessStats,
-    ProcessStatusType, ReplicationStats, ResourceStats,
+    BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats,
+    ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats, CpuStats, DiskStats, DriveCountStats,
+    DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats,
+    ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
 };
+use chrono::Utc;
+use rustfs_common::{internode_metrics::global_internode_metrics, metrics::global_metrics};
+use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
 use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
 use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
 use rustfs_ecstore::data_usage::load_data_usage_from_backend;
@@ -32,7 +36,9 @@ use rustfs_ecstore::global::get_global_bucket_monitor;
 use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
 use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
 use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
+use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot};
 use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
+use std::collections::HashMap;
 use std::time::Duration;
 use sysinfo::{Networks, System};
 use tracing::{instrument, warn};
@@ -42,6 +48,15 @@ const DRIVE_STATE_ONLINE: &str = "online";
 const DRIVE_STATE_UNFORMATTED: &str = "unformatted";
 const DRIVE_RUNTIME_STATE_RETURNING: &str = "returning";
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct ErasureSetQuorumShape {
+    data_shards: u32,
+    read_quorum: u32,
+    write_quorum: u32,
+    read_tolerance: u32,
+    write_tolerance: u32,
+}
+
 fn disk_is_online_for_metrics(state: &str, runtime_state: Option<&str>) -> bool {
     let state_is_acceptable = state.eq_ignore_ascii_case(DRIVE_STATE_OK)
         || state.eq_ignore_ascii_case(DRIVE_STATE_ONLINE)
@@ -56,6 +71,30 @@ fn disk_is_online_for_metrics(state: &str, runtime_state: Option<&str>) -> bool
     state_is_acceptable
 }
 
+fn derive_erasure_set_quorum_shape(set_drive_count: usize, parity: usize) -> ErasureSetQuorumShape {
+    let data_shards = set_drive_count.saturating_sub(parity);
+    let read_quorum = data_shards.max(1);
+    let mut write_quorum = read_quorum;
+    if data_shards == parity {
+        write_quorum += 1;
+    }
+
+    ErasureSetQuorumShape {
+        data_shards: data_shards as u32,
+        read_quorum: read_quorum as u32,
+        write_quorum: write_quorum as u32,
+        read_tolerance: parity as u32,
+        write_tolerance: set_drive_count.saturating_sub(write_quorum) as u32,
+    }
+}
+
+fn apply_erasure_set_health(entry: &mut ErasureSetStats) {
+    let online = entry.online_drives_count;
+    entry.read_health = u8::from(online >= entry.read_quorum);
+    entry.write_health = u8::from(online >= entry.write_quorum);
+    entry.health = u8::from(entry.write_health == 1);
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ProcessMetricBundle {
     pub resource: ResourceStats,
@@ -285,14 +324,12 @@ pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationS
             proxied_get_requests_failures: proxy.get_failed.max(0) as u64,
             proxied_head_requests_total: proxy.head_total.max(0) as u64,
             proxied_head_requests_failures: proxy.head_failed.max(0) as u64,
-            // Proxy cache currently tracks generic PutObject requests, not tagging-specific APIs.
-            // Keep tagging counters zero until PutObjectTagging stats are tracked separately.
-            proxied_put_tagging_requests_total: 0,
-            proxied_put_tagging_requests_failures: 0,
-            proxied_get_tagging_requests_total: 0,
-            proxied_get_tagging_requests_failures: 0,
-            proxied_delete_tagging_requests_total: 0,
-            proxied_delete_tagging_requests_failures: 0,
+            proxied_put_tagging_requests_total: proxy.put_tag_total.max(0) as u64,
+            proxied_put_tagging_requests_failures: proxy.put_tag_failed.max(0) as u64,
+            proxied_get_tagging_requests_total: proxy.get_tag_total.max(0) as u64,
+            proxied_get_tagging_requests_failures: proxy.get_tag_failed.max(0) as u64,
+            proxied_delete_tagging_requests_total: proxy.delete_tag_total.max(0) as u64,
+            proxied_delete_tagging_requests_failures: proxy.delete_tag_failed.max(0) as u64,
             targets,
         });
     }
@@ -585,6 +622,226 @@ pub fn collect_host_network_stats() -> HostNetworkStats {
     }
 }
 
+/// Collect internode network metrics from the global internode metrics snapshot.
+///
+/// The returned values come directly from `global_internode_metrics().snapshot()`
+/// and currently include only the counters and dial timing data tracked by the
+/// internode metrics runtime.
+pub fn collect_internode_network_stats() -> Option<NetworkStats> {
+    let snapshot = global_internode_metrics().snapshot();
+
+    Some(NetworkStats {
+        internode_errors_total: snapshot.errors_total,
+        internode_dial_errors_total: snapshot.dial_errors_total,
+        internode_dial_avg_time_nanos: snapshot.dial_avg_time_nanos,
+        internode_sent_bytes_total: snapshot.sent_bytes_total,
+        internode_recv_bytes_total: snapshot.recv_bytes_total,
+    })
+}
+
+/// Collect cluster config metrics from backend parity configuration.
+pub async fn collect_cluster_config_stats() -> Option<ClusterConfigStats> {
+    let store = new_object_layer_fn()?;
+    let backend = store.backend_info().await;
+
+    Some(ClusterConfigStats {
+        rrs_parity: backend.rr_sc_parity.unwrap_or_default() as u32,
+        standard_parity: backend.standard_sc_parity.unwrap_or_default() as u32,
+    })
+}
+
+/// Collect cluster erasure set metrics from storage and backend topology info.
+pub async fn collect_erasure_set_stats() -> Vec<ErasureSetStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    let storage_info = store.storage_info().await;
+    let backend = store.backend_info().await;
+    let mut grouped: HashMap<(usize, usize), ErasureSetStats> = HashMap::new();
+
+    for disk in &storage_info.disks {
+        let pool_idx = disk.pool_index.max(0) as usize;
+        let set_idx = disk.set_index.max(0) as usize;
+        let set_drive_count = backend.drives_per_set.get(pool_idx).copied().unwrap_or_default();
+        let parity = backend
+            .standard_sc_parities
+            .get(pool_idx)
+            .copied()
+            .or(backend.standard_sc_parity)
+            .unwrap_or(set_drive_count / 2);
+        let quorum_shape = derive_erasure_set_quorum_shape(set_drive_count, parity);
+
+        let entry = grouped.entry((pool_idx, set_idx)).or_insert_with(|| ErasureSetStats {
+            pool_id: pool_idx as u32,
+            set_id: set_idx as u32,
+            size: set_drive_count as u32,
+            parity: parity as u32,
+            data_shards: quorum_shape.data_shards,
+            read_quorum: quorum_shape.read_quorum,
+            write_quorum: quorum_shape.write_quorum,
+            online_drives_count: 0,
+            healing_drives_count: 0,
+            health: 0,
+            read_tolerance: quorum_shape.read_tolerance,
+            write_tolerance: quorum_shape.write_tolerance,
+            read_health: 0,
+            write_health: 0,
+        });
+
+        if disk_is_online_for_metrics(disk.state.as_str(), disk.runtime_state.as_deref()) {
+            entry.online_drives_count += 1;
+        }
+        if disk.healing {
+            entry.healing_drives_count += 1;
+        }
+    }
+
+    for entry in grouped.values_mut() {
+        apply_erasure_set_health(entry);
+    }
+
+    let mut stats = grouped.into_values().collect::<Vec<_>>();
+    stats.sort_by_key(|stat| (stat.pool_id, stat.set_id));
+    stats
+}
+
+pub async fn collect_iam_stats() -> Option<IamStats> {
+    let iam_sys = get_global_iam_sys()?;
+    let sync = iam_sys.sync_metrics_snapshot();
+    let oidc = oidc_plugin_authn_metrics_snapshot();
+
+    Some(IamStats {
+        last_sync_duration_millis: sync.last_sync_duration_millis,
+        plugin_authn_service_failed_requests_minute: oidc.failed_requests_minute,
+        plugin_authn_service_last_fail_seconds: oidc.last_fail_seconds,
+        plugin_authn_service_last_succ_seconds: oidc.last_succ_seconds,
+        plugin_authn_service_succ_avg_rtt_ms_minute: oidc.succ_avg_rtt_ms_minute,
+        plugin_authn_service_succ_max_rtt_ms_minute: oidc.succ_max_rtt_ms_minute,
+        plugin_authn_service_total_requests_minute: oidc.total_requests_minute,
+        since_last_sync_millis: sync.since_last_sync_millis,
+        sync_failures: sync.sync_failures,
+        sync_successes: sync.sync_successes,
+    })
+}
+
+/// Collect cluster and per-bucket usage metrics from backend usage snapshots.
+///
+/// This reads persisted usage data via `load_data_usage_from_backend()` and
+/// builds cluster totals plus per-bucket distributions from the returned
+/// histograms. It does not trigger an inline object-data rescan.
+pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Vec<BucketUsageStats>)> {
+    let store = new_object_layer_fn()?;
+    let data_usage = load_data_usage_from_backend(store.clone()).await.ok()?;
+    let mut buckets = Vec::with_capacity(data_usage.buckets_usage.len());
+
+    for (bucket_name, usage) in &data_usage.buckets_usage {
+        if bucket_name.starts_with('.') {
+            continue;
+        }
+
+        let quota_bytes = match get_quota_config(bucket_name).await {
+            Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
+            Err(_) => 0,
+        };
+
+        buckets.push(BucketUsageStats {
+            bucket: bucket_name.clone(),
+            total_bytes: usage.size,
+            objects_count: usage.objects_count,
+            versions_count: usage.versions_count,
+            delete_markers_count: usage.delete_markers_count,
+            quota_bytes,
+            object_size_distribution: usage
+                .object_size_histogram
+                .iter()
+                .map(|(range, count)| (range.clone(), *count))
+                .collect(),
+            version_count_distribution: usage
+                .object_versions_histogram
+                .iter()
+                .map(|(range, count)| (range.clone(), *count))
+                .collect(),
+        });
+    }
+
+    buckets.sort_by(|a, b| a.bucket.cmp(&b.bucket));
+
+    Some((
+        ClusterUsageStats {
+            total_bytes: data_usage.objects_total_size,
+            objects_count: data_usage.objects_total_count,
+            versions_count: data_usage.versions_total_count,
+            delete_markers_count: data_usage.delete_markers_total_count,
+            object_size_distribution: data_usage
+                .buckets_usage
+                .values()
+                .flat_map(|usage| usage.object_size_histogram.iter())
+                .fold(HashMap::<String, u64>::new(), |mut acc, (range, count)| {
+                    *acc.entry(range.clone()).or_default() += *count;
+                    acc
+                })
+                .into_iter()
+                .collect(),
+            versions_distribution: data_usage
+                .buckets_usage
+                .values()
+                .flat_map(|usage| usage.object_versions_histogram.iter())
+                .fold(HashMap::<String, u64>::new(), |mut acc, (range, count)| {
+                    *acc.entry(range.clone()).or_default() += *count;
+                    acc
+                })
+                .into_iter()
+                .collect(),
+        },
+        buckets,
+    ))
+}
+
+/// Collect ILM metrics from the current lifecycle runtime state.
+pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
+    let expiry_pending_tasks = GLOBAL_ExpiryState.read().await.pending_tasks().await as u64;
+    let transition_active_tasks = GLOBAL_TransitionState.active_tasks().max(0) as u64;
+    let transition_pending_tasks = GLOBAL_TransitionState.pending_tasks() as u64;
+    let transition_missed_immediate_tasks = GLOBAL_TransitionState.missed_immediate_tasks().max(0) as u64;
+    let metrics = global_metrics().report().await;
+    let versions_scanned = metrics.life_time_ilm.values().copied().sum();
+
+    Some(IlmStats {
+        expiry_pending_tasks,
+        transition_active_tasks,
+        transition_pending_tasks,
+        transition_missed_immediate_tasks,
+        versions_scanned,
+    })
+}
+
+/// Collect scanner metrics from a runtime source.
+///
+/// Task 5 maps scanner runtime snapshots from `global_metrics()` into the
+/// rustfs-obs scanner collector shape.
+pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
+    let metrics = global_metrics().report().await;
+    let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
+    let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
+    let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
+    let versions_scanned = metrics.life_time_ilm.values().copied().sum();
+    let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
+    let last_activity_seconds = Utc::now().signed_duration_since(reference_time).num_seconds().max(0) as u64;
+
+    Some(ScannerStats {
+        bucket_scans_finished,
+        // `global_metrics()` currently tracks completed bucket-drive scans, not a
+        // separate started counter. Mirror the finished count until Task 5/Task 10
+        // expands the scanner runtime source shape.
+        bucket_scans_started: bucket_scans_finished,
+        directories_scanned,
+        objects_scanned,
+        versions_scanned,
+        last_activity_seconds,
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -598,4 +855,57 @@ mod tests {
     fn disk_is_online_for_metrics_rejects_offline_runtime_state() {
         assert!(!disk_is_online_for_metrics(DRIVE_STATE_OK, Some("offline")));
     }
+
+    #[test]
+    fn derive_erasure_set_quorum_shape_handles_standard_layout() {
+        let shape = derive_erasure_set_quorum_shape(16, 4);
+
+        assert_eq!(
+            shape,
+            ErasureSetQuorumShape {
+                data_shards: 12,
+                read_quorum: 12,
+                write_quorum: 12,
+                read_tolerance: 4,
+                write_tolerance: 4,
+            }
+        );
+    }
+
+    #[test]
+    fn derive_erasure_set_quorum_shape_handles_equal_data_and_parity() {
+        let shape = derive_erasure_set_quorum_shape(4, 2);
+
+        assert_eq!(
+            shape,
+            ErasureSetQuorumShape {
+                data_shards: 2,
+                read_quorum: 2,
+                write_quorum: 3,
+                read_tolerance: 2,
+                write_tolerance: 1,
+            }
+        );
+    }
+
+    #[test]
+    fn apply_erasure_set_health_marks_read_and_write_health_from_online_count() {
+        let mut stats = ErasureSetStats {
+            read_quorum: 3,
+            write_quorum: 4,
+            online_drives_count: 3,
+            ..Default::default()
+        };
+
+        apply_erasure_set_health(&mut stats);
+        assert_eq!(stats.read_health, 1);
+        assert_eq!(stats.write_health, 0);
+        assert_eq!(stats.health, 0);
+
+        stats.online_drives_count = 4;
+        apply_erasure_set_health(&mut stats);
+        assert_eq!(stats.read_health, 1);
+        assert_eq!(stats.write_health, 1);
+        assert_eq!(stats.health, 1);
+    }
 }

crates/obs/src/telemetry/local.rs

@@ -148,7 +148,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
         .init();
 
     set_observability_metric_enabled(false);
-    counter!("rustfs.start.total").increment(1);
+    counter!("rustfs_start_total").increment(1);
     info!("Init stdout logging (level: {})", logger_level);
 
     OtelGuard {

crates/obs/src/telemetry/otel.rs

@@ -304,7 +304,7 @@ pub(super) fn init_observability_http(
         .with(metrics_layer)
         .init();
 
-    counter!("rustfs.start.total").increment(1);
+    counter!("rustfs_start_total").increment(1);
     info!(
         "Init observability (HTTP): trace='{}', metric='{}', log='{}'",
         trace_ep, metric_ep, log_ep

crates/policy/src/policy/function.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::policy::function::condition::Condition;
+use crate::policy::function::{condition::Condition, key_name::KeyName};
 use crate::policy::variables::PolicyVariableResolver;
 use serde::ser::SerializeMap;
 use serde::{Deserialize, Serialize, Serializer, de};
@@ -71,6 +71,14 @@ impl Functions {
     pub fn is_empty(&self) -> bool {
         self.for_all_values.is_empty() && self.for_any_value.is_empty() && self.for_normal.is_empty()
     }
+
+    pub fn references_key_name(&self, key_name: &KeyName) -> bool {
+        self.for_any_value
+            .iter()
+            .chain(self.for_all_values.iter())
+            .chain(self.for_normal.iter())
+            .any(|condition| condition.references_key_name(key_name))
+    }
 }
 
 impl Serialize for Functions {

crates/policy/src/policy/function/condition.rs

@@ -19,6 +19,7 @@ use serde::ser::SerializeMap;
 use std::collections::HashMap;
 use time::OffsetDateTime;
 
+use super::key_name::KeyName;
 use super::{addr::AddrFunc, binary::BinaryFunc, bool_null::BoolFunc, date::DateFunc, number::NumberFunc, string::StringFunc};
 
 #[derive(Clone, Deserialize, Debug)]
@@ -171,6 +172,39 @@ impl Condition {
         }
     }
 
+    pub fn references_key_name(&self, key_name: &KeyName) -> bool {
+        use Condition::*;
+        match self {
+            StringEquals(s)
+            | StringNotEquals(s)
+            | StringEqualsIgnoreCase(s)
+            | StringNotEqualsIgnoreCase(s)
+            | StringLike(s)
+            | StringNotLike(s)
+            | ArnLike(s)
+            | ArnNotLike(s)
+            | ArnEquals(s)
+            | ArnNotEquals(s) => s.contains_key_name(key_name),
+            BinaryEquals(s) => s.contains_key_name(key_name),
+            IpAddress(s) | NotIpAddress(s) => s.contains_key_name(key_name),
+            Null(s) | Bool(s) => s.contains_key_name(key_name),
+            NumericEquals(s)
+            | NumericNotEquals(s)
+            | NumericLessThan(s)
+            | NumericLessThanEquals(s)
+            | NumericGreaterThan(s)
+            | NumericGreaterThanIfExists(s)
+            | NumericGreaterThanEquals(s) => s.contains_key_name(key_name),
+            DateEquals(s)
+            | DateNotEquals(s)
+            | DateLessThan(s)
+            | DateLessThanEquals(s)
+            | DateGreaterThan(s)
+            | DateGreaterThanEquals(s) => s.contains_key_name(key_name),
+            IfExists(inner) => inner.references_key_name(key_name),
+        }
+    }
+
     pub fn evaluate_with_resolver<'a>(
         &'a self,
         for_all: bool,

crates/policy/src/policy/function/func.rs

@@ -19,7 +19,7 @@ use serde::{
     de::{self, Visitor},
 };
 
-use super::key::Key;
+use super::{key::Key, key_name::KeyName};
 
 #[derive(PartialEq, Eq, Debug)]
 pub struct InnerFunc<T>(pub(crate) Vec<FuncKeyValue<T>>);
@@ -49,6 +49,10 @@ impl<T> InnerFunc<T> {
     pub fn key_names(&self) -> impl Iterator<Item = String> + '_ {
         self.0.iter().map(|kv| kv.key.name())
     }
+
+    pub fn contains_key_name(&self, key_name: &KeyName) -> bool {
+        self.0.iter().any(|kv| kv.key.is(key_name))
+    }
 }
 
 impl<T: Serialize> Serialize for InnerFunc<T> {

crates/policy/src/policy/policy.rs

@@ -357,7 +357,7 @@ pub mod default {
 
     use crate::policy::{
         ActionSet, DEFAULT_VERSION, Effect, Functions, ResourceSet, Statement,
-        action::{Action, AdminAction, KmsAction, S3Action},
+        action::{Action, AdminAction, KmsAction, S3Action, StsAction},
         resource::Resource,
     };
 
@@ -377,6 +377,7 @@ pub mod default {
                         actions: ActionSet({
                             let mut hash_set = HashSet::new();
                             hash_set.insert(Action::S3Action(S3Action::AllActions));
+                            hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                             hash_set
                         }),
                         not_actions: ActionSet(Default::default()),
@@ -403,6 +404,7 @@ pub mod default {
                             hash_set.insert(Action::S3Action(S3Action::GetBucketLocationAction));
                             hash_set.insert(Action::S3Action(S3Action::GetObjectAction));
                             hash_set.insert(Action::S3Action(S3Action::GetBucketQuotaAction));
+                            hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                             hash_set
                         }),
                         not_actions: ActionSet(Default::default()),
@@ -427,6 +429,7 @@ pub mod default {
                         actions: ActionSet({
                             let mut hash_set = HashSet::new();
                             hash_set.insert(Action::S3Action(S3Action::PutObjectAction));
+                            hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                             hash_set
                         }),
                         not_actions: ActionSet(Default::default()),
@@ -451,6 +454,7 @@ pub mod default {
                         actions: ActionSet({
                             let mut hash_set = HashSet::new();
                             hash_set.insert(Action::S3Action(S3Action::PutObjectAction));
+                            hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                             hash_set
                         }),
                         not_actions: ActionSet(Default::default()),
@@ -482,6 +486,7 @@ pub mod default {
                             hash_set.insert(Action::AdminAction(AdminAction::HealthInfoAdminAction));
                             hash_set.insert(Action::AdminAction(AdminAction::PrometheusAdminAction));
                             hash_set.insert(Action::AdminAction(AdminAction::BandwidthMonitorAction));
+                            hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                             hash_set
                         }),
                         not_actions: ActionSet(Default::default()),
@@ -507,6 +512,7 @@ pub mod default {
                             actions: ActionSet({
                                 let mut hash_set = HashSet::new();
                                 hash_set.insert(Action::AdminAction(AdminAction::AllAdminActions));
+                                hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                                 hash_set
                             }),
                             not_actions: ActionSet(Default::default()),
@@ -520,6 +526,7 @@ pub mod default {
                             actions: ActionSet({
                                 let mut hash_set = HashSet::new();
                                 hash_set.insert(Action::KmsAction(KmsAction::AllActions));
+                                hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                                 hash_set
                             }),
                             not_actions: ActionSet(Default::default()),
@@ -533,6 +540,7 @@ pub mod default {
                             actions: ActionSet({
                                 let mut hash_set = HashSet::new();
                                 hash_set.insert(Action::S3Action(S3Action::AllActions));
+                                hash_set.insert(Action::StsAction(StsAction::AssumeRoleAction));
                                 hash_set
                             }),
                             not_actions: ActionSet(Default::default()),
@@ -684,6 +692,27 @@ mod test {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_default_policies_allow_sts_assume_role() {
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let args = Args {
+            account: "testuser",
+            groups: &None,
+            action: Action::StsAction(crate::policy::action::StsAction::AssumeRoleAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: false,
+        };
+
+        for (name, policy) in default::DEFAULT_POLICIES.iter() {
+            assert!(policy.is_allowed(&args).await, "default policy {name} should allow sts:AssumeRole");
+        }
+    }
+
     #[tokio::test]
     async fn test_deny_only_checks_only_deny_statements() -> Result<()> {
         let data = r#"
@@ -798,6 +827,176 @@ mod test {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_list_bucket_prefix_condition_uses_bucket_resource() -> Result<()> {
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:ListBucket"],
+      "Resource": ["arn:aws:s3:::polaris-test-bucket"],
+      "Condition": {
+        "StringLike": {
+          "s3:prefix": [
+            "polaris_test/snowflake_catalog/db1/schema/iceberg_table/*"
+          ]
+        }
+      }
+    }
+  ]
+}"#,
+        )?;
+
+        let mut conditions = HashMap::new();
+        conditions.insert(
+            "prefix".to_string(),
+            vec!["polaris_test/snowflake_catalog/db1/schema/iceberg_table/metadata/".to_string()],
+        );
+        let claims = HashMap::new();
+        let args = Args {
+            account: "polaris-session",
+            groups: &None,
+            action: Action::S3Action(crate::policy::action::S3Action::ListBucketAction),
+            bucket: "polaris-test-bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "polaris_test/snowflake_catalog/db1/schema/iceberg_table/metadata/",
+            claims: &claims,
+            deny_only: false,
+        };
+
+        assert!(
+            policy.is_allowed(&args).await,
+            "ListBucket should match the bucket resource and apply the prefix through the condition, not by converting the prefix into an object resource"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_list_bucket_versions_prefix_condition_uses_bucket_resource() -> Result<()> {
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:ListBucketVersions"],
+      "Resource": ["arn:aws:s3:::polaris-test-bucket"],
+      "Condition": {
+        "StringLike": {
+          "s3:prefix": [
+            "polaris_test/snowflake_catalog/db1/schema/iceberg_table/*"
+          ]
+        }
+      }
+    }
+  ]
+}"#,
+        )?;
+
+        let mut conditions = HashMap::new();
+        conditions.insert(
+            "prefix".to_string(),
+            vec!["polaris_test/snowflake_catalog/db1/schema/iceberg_table/metadata/".to_string()],
+        );
+        let claims = HashMap::new();
+        let args = Args {
+            account: "polaris-session",
+            groups: &None,
+            action: Action::S3Action(crate::policy::action::S3Action::ListBucketVersionsAction),
+            bucket: "polaris-test-bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "polaris_test/snowflake_catalog/db1/schema/iceberg_table/metadata/",
+            claims: &claims,
+            deny_only: false,
+        };
+
+        assert!(
+            policy.is_allowed(&args).await,
+            "ListBucketVersions should match the bucket resource and apply the prefix through the condition"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_list_bucket_gateway_prefix_uses_object_resource_when_condition_missing() -> Result<()> {
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:ListBucket"],
+      "Resource": ["arn:aws:s3:::polaris-test-bucket/home/alice/*"]
+    }
+  ]
+}"#,
+        )?;
+
+        let mut conditions = HashMap::new();
+        conditions.insert("prefix".to_string(), vec!["home/alice/projects/".to_string()]);
+        let claims = HashMap::new();
+        let args = Args {
+            account: "polaris-session",
+            groups: &None,
+            action: Action::S3Action(crate::policy::action::S3Action::ListBucketAction),
+            bucket: "polaris-test-bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "home/alice/projects/",
+            claims: &claims,
+            deny_only: false,
+        };
+
+        assert!(
+            policy.is_allowed(&args).await,
+            "Gateway ListBucket auth without an s3:prefix condition should continue matching prefix-scoped resources via args.object"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_bucket_policy_gateway_prefix_uses_object_resource_when_condition_missing() -> Result<()> {
+        let bucket_policy: BucketPolicy = serde_json::from_str(
+            r#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {"AWS": "*"},
+      "Action": ["s3:ListBucket"],
+      "Resource": ["arn:aws:s3:::polaris-test-bucket/home/alice/*"]
+    }
+  ]
+}"#,
+        )?;
+
+        let mut conditions = HashMap::new();
+        conditions.insert("prefix".to_string(), vec!["home/alice/projects/".to_string()]);
+        let args = BucketPolicyArgs {
+            account: "polaris-session",
+            groups: &None,
+            action: Action::S3Action(crate::policy::action::S3Action::ListBucketAction),
+            bucket: "polaris-test-bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "home/alice/projects/",
+        };
+
+        assert!(
+            bucket_policy.is_allowed(&args).await,
+            "Bucket policy ListBucket without an s3:prefix condition should continue matching prefix-scoped resources via args.object"
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_aws_username_policy_variable() -> Result<()> {
         let data = r#"
@@ -1408,6 +1607,52 @@ mod test {
         assert_eq!(arr[0].as_str().unwrap(), "s3:ListBucket");
     }
 
+    #[tokio::test]
+    async fn test_bucket_policy_list_bucket_prefix_condition_uses_bucket_resource() -> Result<()> {
+        let bucket_policy: BucketPolicy = serde_json::from_str(
+            r#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {"AWS": "*"},
+      "Action": ["s3:ListBucket"],
+      "Resource": ["arn:aws:s3:::polaris-test-bucket"],
+      "Condition": {
+        "StringLike": {
+          "s3:prefix": [
+            "polaris_test/snowflake_catalog/db1/schema/iceberg_table/*"
+          ]
+        }
+      }
+    }
+  ]
+}"#,
+        )?;
+
+        let mut conditions = HashMap::new();
+        conditions.insert(
+            "prefix".to_string(),
+            vec!["polaris_test/snowflake_catalog/db1/schema/iceberg_table/metadata/".to_string()],
+        );
+        let args = BucketPolicyArgs {
+            account: "polaris-session",
+            groups: &None,
+            action: Action::S3Action(crate::policy::action::S3Action::ListBucketAction),
+            bucket: "polaris-test-bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "polaris_test/snowflake_catalog/db1/schema/iceberg_table/metadata/",
+        };
+
+        assert!(
+            bucket_policy.is_allowed(&args).await,
+            "Bucket policy ListBucket should match the bucket resource and apply the prefix through the condition"
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_bucket_policy_deny_with_string_not_equals() -> Result<()> {
         let data = r#"

crates/policy/src/policy/statement.rs

@@ -14,7 +14,8 @@
 
 use super::{
     ActionSet, Args, BucketPolicyArgs, Effect, Error as IamError, Functions, ID, Principal, ResourceSet, Validator,
-    action::Action,
+    action::{Action, S3Action},
+    function::key_name::{KeyName, S3KeyName},
     variables::{VariableContext, VariableResolver},
 };
 use crate::error::{Error, Result};
@@ -56,6 +57,27 @@ pub(crate) fn variable_resolver_for_policy_args(args: &Args<'_>) -> VariableReso
     VariableResolver::new(context)
 }
 
+fn build_resource(action: &Action, bucket: &str, object: &str, bucket_resource_only: bool) -> String {
+    let bucket_resource_only = matches!(
+        action,
+        Action::S3Action(
+            S3Action::ListBucketAction | S3Action::ListBucketVersionsAction | S3Action::ListBucketMultipartUploadsAction
+        )
+    ) && bucket_resource_only;
+
+    let mut resource = String::from(bucket);
+    if bucket_resource_only || object.is_empty() {
+        resource.push('/');
+        return resource;
+    }
+
+    if !object.starts_with('/') {
+        resource.push('/');
+    }
+    resource.push_str(object);
+    resource
+}
+
 impl Statement {
     fn is_kms(&self) -> bool {
         for act in self.actions.iter() {
@@ -94,16 +116,12 @@ impl Statement {
             return false;
         }
 
-        let mut resource = String::from(args.bucket);
-        if !args.object.is_empty() {
-            if !args.object.starts_with('/') {
-                resource.push('/');
-            }
-
-            resource.push_str(args.object);
-        } else {
-            resource.push('/');
-        }
+        let resource = build_resource(
+            &args.action,
+            args.bucket,
+            args.object,
+            self.conditions.references_key_name(&KeyName::S3(S3KeyName::S3Prefix)),
+        );
 
         if self.is_kms() && (resource == "/" || self.resources.is_empty()) {
             return true;
@@ -230,16 +248,12 @@ impl BPStatement {
             return false;
         }
 
-        let mut resource = String::from(args.bucket);
-        if !args.object.is_empty() {
-            if !args.object.starts_with('/') {
-                resource.push('/');
-            }
-
-            resource.push_str(args.object);
-        } else {
-            resource.push('/');
-        }
+        let resource = build_resource(
+            &args.action,
+            args.bucket,
+            args.object,
+            self.conditions.references_key_name(&KeyName::S3(S3KeyName::S3Prefix)),
+        );
 
         if !self.resources.is_empty() && !self.resources.is_match(&resource, args.conditions).await {
             return false;

crates/protocols/Cargo.toml

@@ -53,7 +53,7 @@ swift = [
     "dep:base64",
     "dep:async-compression",
 ]
-webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls"]
+webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding"]
 
 [dependencies]
 # Core RustFS dependencies

crates/protocols/src/webdav/driver.rs

@@ -21,6 +21,7 @@ use dav_server::fs::{
     DavDirEntry, DavFile, DavFileSystem, DavMetaData, FsError, FsFuture, FsResult, FsStream, OpenOptions, ReadDirMeta,
 };
 use futures_util::{FutureExt, StreamExt, stream};
+use percent_encoding::percent_decode_str;
 use rustfs_utils::path;
 use s3s::dto::*;
 use std::fmt::Debug;
@@ -432,7 +433,10 @@ where
     /// Parse WebDAV path to bucket and object key
     fn parse_path(&self, path: &DavPath) -> Result<(String, Option<String>), FsError> {
         let path_str = path.as_url_string();
-        let cleaned_path = path::clean(&path_str);
+        let decoded_path = percent_decode_str(&path_str)
+            .decode_utf8()
+            .map_err(|_| FsError::GeneralFailure)?;
+        let cleaned_path = path::clean(&decoded_path);
         let (bucket, object) = path::path_to_bucket_object(&cleaned_path);
 
         if bucket.is_empty() {

crates/protos/Cargo.toml

@@ -34,6 +34,8 @@ path = "src/main.rs"
 
 [dependencies]
 rustfs-common.workspace = true
+rustfs-config.workspace = true
+rustfs-utils.workspace = true
 flatbuffers = { workspace = true }
 prost = { workspace = true }
 tonic = { workspace = true, features = ["transport"] }

crates/protos/src/lib.rs

@@ -40,51 +40,78 @@ pub use generated::*;
 // Default 100 MB
 pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024;
 
-/// Timeout for connection establishment - reduced for faster failure detection
-const CONNECT_TIMEOUT_SECS: u64 = 3;
-
-/// TCP keepalive interval - how often to probe the connection
-const TCP_KEEPALIVE_SECS: u64 = 10;
-
-/// HTTP/2 keepalive interval - application-layer heartbeat
-const HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5;
-
-/// HTTP/2 keepalive timeout - how long to wait for PING ACK
-const HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
-
-/// Overall RPC timeout - maximum time for any single RPC operation
-const RPC_TIMEOUT_SECS: u64 = 30;
-
 /// Default HTTPS prefix for rustfs
 /// This is the default HTTPS prefix for rustfs.
 /// It is used to identify HTTPS URLs.
 /// Default value: https://
 const RUSTFS_HTTPS_PREFIX: &str = "https://";
 
+fn internode_connect_timeout() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_INTERNODE_CONNECT_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS,
+    ))
+}
+
+fn internode_tcp_keepalive() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_INTERNODE_TCP_KEEPALIVE_SECS,
+        rustfs_config::DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS,
+    ))
+}
+
+fn internode_http2_keep_alive_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
+        rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
+    ))
+}
+
+fn internode_http2_keep_alive_timeout() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
+    ))
+}
+
+fn internode_rpc_timeout() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_INTERNODE_RPC_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_INTERNODE_RPC_TIMEOUT_SECS,
+    ))
+}
+
 /// Creates a new gRPC channel with optimized keepalive settings for cluster resilience.
 ///
-/// This function is designed to detect dead peers quickly:
-/// - Fast connection timeout (3s instead of default 30s+)
-/// - Aggressive TCP keepalive (10s)
-/// - HTTP/2 PING every 5s, timeout at 3s
-/// - Overall RPC timeout of 30s (reduced from 60s)
+/// This function is designed to detect dead peers quickly using env-configurable
+/// internode transport settings. Defaults come from `rustfs_config` constants:
+/// - Connect timeout: `DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS` (3s)
+/// - TCP keepalive: `DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS` (10s)
+/// - HTTP/2 keepalive interval: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS` (5s)
+/// - HTTP/2 keepalive timeout: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS` (3s)
+/// - RPC timeout: `DEFAULT_INTERNODE_RPC_TIMEOUT_SECS` (10s)
 pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
     debug!("Creating new gRPC channel to: {}", addr);
     let dial_started_at = Instant::now();
+    let connect_timeout = internode_connect_timeout();
+    let tcp_keepalive = internode_tcp_keepalive();
+    let http2_keepalive_interval = internode_http2_keep_alive_interval();
+    let http2_keepalive_timeout = internode_http2_keep_alive_timeout();
+    let rpc_timeout = internode_rpc_timeout();
 
     let mut connector = Endpoint::from_shared(addr.to_string())?
         // Fast connection timeout for dead peer detection
-        .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
+        .connect_timeout(connect_timeout)
         // TCP-level keepalive - OS will probe connection
-        .tcp_keepalive(Some(Duration::from_secs(TCP_KEEPALIVE_SECS)))
+        .tcp_keepalive(Some(tcp_keepalive))
         // HTTP/2 PING frames for application-layer health check
-        .http2_keep_alive_interval(Duration::from_secs(HTTP2_KEEPALIVE_INTERVAL_SECS))
+        .http2_keep_alive_interval(http2_keepalive_interval)
         // How long to wait for PING ACK before considering connection dead
-        .keep_alive_timeout(Duration::from_secs(HTTP2_KEEPALIVE_TIMEOUT_SECS))
+        .keep_alive_timeout(http2_keepalive_timeout)
         // Send PINGs even when no active streams (critical for idle connections)
         .keep_alive_while_idle(true)
         // Overall timeout for any RPC - fail fast on unresponsive peers
-        .timeout(Duration::from_secs(RPC_TIMEOUT_SECS));
+        .timeout(rpc_timeout);
 
     let root_cert = GLOBAL_ROOT_CERT.read().await;
     if addr.starts_with(RUSTFS_HTTPS_PREFIX) {

crates/s3select-api/src/query/execution.rs

@@ -52,7 +52,7 @@ impl Drop for PhaseTimer {
 
         if !std::thread::panicking() {
             metrics::histogram!(
-                "rustfs.s3select.phase.duration.seconds",
+                "rustfs_s3select_phase_duration_seconds",
                 "phase" => self.phase_name
             )
             .record(duration.as_secs_f64());
@@ -171,7 +171,7 @@ impl QueryStateMachine {
     fn record_phase_timestamp(&self, phase: &'static str, event: &'static str) {
         let elapsed = self.start.elapsed();
         metrics::histogram!(
-            "rustfs.s3select.phase.timestamp.seconds",
+            "rustfs_s3select_phase_timestamp_seconds",
             "phase" => phase,
             "event" => event
         )

crates/scanner/src/lib.rs

@@ -32,3 +32,26 @@ pub use data_usage_define::*;
 pub use error::ScannerError;
 pub use scanner::init_data_scanner;
 pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
+
+pub fn current_scanner_activity() -> u64 {
+    SCANNER_ACTIVE_WORK_UNITS.load(Ordering::Relaxed)
+}
+
+pub(crate) struct ScannerActivityGuard;
+
+impl ScannerActivityGuard {
+    pub(crate) fn new() -> Self {
+        SCANNER_ACTIVE_WORK_UNITS.fetch_add(1, Ordering::Relaxed);
+        Self
+    }
+}
+
+impl Drop for ScannerActivityGuard {
+    fn drop(&mut self) {
+        let _ = SCANNER_ACTIVE_WORK_UNITS
+            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)));
+    }
+}

crates/scanner/src/scanner.rs

@@ -18,7 +18,7 @@ use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_
 use crate::scanner_folder::data_usage_update_dir_cycles;
 use crate::scanner_io::ScannerIO;
 use crate::sleeper::SCANNER_SLEEPER;
-use crate::{DataUsageInfo, ScannerError};
+use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError};
 use chrono::{DateTime, Utc};
 use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics};
@@ -183,6 +183,7 @@ fn get_lock_acquire_timeout() -> Duration {
 
 #[instrument(skip_all)]
 async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
+    let _activity_guard = ScannerActivityGuard::new();
     SCANNER_SLEEPER.refresh_from_env();
     info!("Start run data scanner cycle");
     cycle_info.current = cycle_info.next;
@@ -324,6 +325,7 @@ pub async fn store_data_usage_in_backend(
     let mut attempts = 1u32;
 
     while let Some(data_usage_info) = receiver.recv().await {
+        let _activity_guard = ScannerActivityGuard::new();
         if ctx.is_cancelled() {
             break;
         }

crates/scanner/src/scanner_io.rs

@@ -40,9 +40,10 @@ use rustfs_ecstore::set_disk::SetDisks;
 use rustfs_ecstore::store_api::{BucketInfo, BucketOperations, BucketOptions, ObjectInfo};
 use rustfs_ecstore::{StorageAPI, error::Result, store::ECStore};
 use rustfs_filemeta::FileMeta;
-use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
+use rustfs_utils::path::path_join_buf;
 use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
 use std::collections::HashMap;
+use std::path::Path;
 use std::time::SystemTime;
 use std::{fmt::Debug, sync::Arc};
 use time::OffsetDateTime;
@@ -69,6 +70,13 @@ fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error
     Ok(())
 }
 
+fn is_xl_meta_path(path: &str) -> bool {
+    Path::new(path)
+        .file_name()
+        .and_then(|name| name.to_str())
+        .is_some_and(|name| name == STORAGE_FORMAT_FILE)
+}
+
 async fn persist_and_publish_cache_snapshot<S: StorageAPI>(
     store: Arc<S>,
     updates: &mpsc::Sender<DataUsageCache>,
@@ -540,7 +548,7 @@ impl ScannerIODisk for Disk {
     async fn get_size(&self, mut item: ScannerItem) -> Result<SizeSummary> {
         let done_object = Metrics::time(Metric::ScanObject);
 
-        if !item.path.ends_with(&format!("{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}")) {
+        if !is_xl_meta_path(&item.path) {
             return Err(StorageError::other("skip file".to_string()));
         }
 
@@ -741,4 +749,15 @@ mod tests {
             .expect_err("all failed sets should bubble first error");
         assert!(err.to_string().contains("set failed"));
     }
+
+    #[test]
+    #[cfg(windows)]
+    fn is_xl_meta_path_accepts_windows_separator() {
+        assert!(is_xl_meta_path("D:\\data\\bucket\\object\\xl.meta"));
+    }
+
+    #[test]
+    fn is_xl_meta_path_accepts_forward_separator() {
+        assert!(is_xl_meta_path("/data/bucket/object/xl.meta"));
+    }
 }

crates/scanner/tests/lifecycle_integration_test.rs

@@ -850,6 +850,7 @@ mod serial_tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
+    #[ignore = "requires isolated global object layer state"]
     async fn test_transition_and_restore_flows() {
         let (_disk_paths, ecstore) = setup_test_env().await;
 
@@ -1218,6 +1219,7 @@ mod serial_tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
+    #[ignore = "requires isolated global object layer state"]
     async fn test_scanner_expires_zero_day_current_version() {
         let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
 
@@ -1244,6 +1246,7 @@ mod serial_tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
+    #[ignore = "requires isolated global object layer state"]
     async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
         let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
 
@@ -1281,6 +1284,7 @@ mod serial_tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
+    #[ignore = "requires isolated global object layer state"]
     async fn test_scanner_expires_zero_day_noncurrent_version() {
         let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
 
@@ -1347,6 +1351,7 @@ mod serial_tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
+    #[ignore = "requires isolated global object layer state"]
     async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
         let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
 
@@ -1431,6 +1436,7 @@ mod serial_tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
+    #[ignore = "requires isolated global object layer state"]
     async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
         let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
 

crates/signer/Cargo.toml

@@ -35,6 +35,7 @@ serde_urlencoded.workspace = true
 rustfs-utils = { workspace = true, features = ["full"] }
 s3s.workspace = true
 base64-simd.workspace = true
+thiserror.workspace = true
 
 [lints]
 workspace = true

crates/signer/src/lib.rs

@@ -22,6 +22,10 @@ pub mod utils;
 pub use request_signature_streaming::streaming_sign_v4;
 pub use request_signature_v2::pre_sign_v2;
 pub use request_signature_v2::sign_v2;
+pub use request_signature_v4::SignV4Error;
 pub use request_signature_v4::pre_sign_v4;
 pub use request_signature_v4::sign_v4;
 pub use request_signature_v4::sign_v4_trailer;
+pub use request_signature_v4::try_pre_sign_v4;
+pub use request_signature_v4::try_sign_v4;
+pub use request_signature_v4::try_sign_v4_trailer;

crates/signer/src/request_signature_v4.rs

@@ -20,11 +20,11 @@ use std::collections::HashMap;
 use std::fmt::Write;
 use std::sync::LazyLock;
 use time::{OffsetDateTime, macros::format_description};
-use tracing::debug;
+use tracing::{debug, warn};
 
 use super::constants::UNSIGNED_PAYLOAD;
 use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4;
-use super::utils::{get_host_addr, sign_v4_trim_all};
+use super::utils::{HostAddrError, sign_v4_trim_all, try_get_host_addr};
 use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256};
 use s3s::Body;
 
@@ -32,6 +32,36 @@ pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
 pub const SERVICE_TYPE_S3: &str = "s3";
 pub const SERVICE_TYPE_STS: &str = "sts";
 
+#[derive(Debug, thiserror::Error)]
+pub enum SignV4Error {
+    #[error("invalid UTF-8 header value for `{name}`")]
+    InvalidHeaderValue { name: String },
+    #[error("failed to format signing timestamp: {reason}")]
+    TimeFormat { reason: String },
+    #[error("failed to build signing timestamp: {reason}")]
+    TimeComponent { reason: String },
+    #[error("failed to encode query parameters: {reason}")]
+    QueryEncode { reason: String },
+    #[error("failed to parse uri: {reason}")]
+    InvalidUri { reason: String },
+    #[error("failed to build uri from parts: {reason}")]
+    InvalidUriParts { reason: String },
+    #[error("failed to convert canonical headers to UTF-8: {reason}")]
+    CanonicalUtf8 { reason: String },
+    #[error("failed to parse header value for `{name}`: {reason}")]
+    HeaderValueParse { name: String, reason: String },
+}
+
+pub type SignResult<T> = std::result::Result<T, SignV4Error>;
+
+#[derive(Debug)]
+struct SignFailure {
+    request: request::Request<Body>,
+    error: SignV4Error,
+}
+
+type SignOutcome = std::result::Result<request::Request<Body>, Box<SignFailure>>;
+
 #[allow(non_upper_case_globals)] // FIXME
 static v4_ignored_headers: LazyLock<HashMap<String, bool>> = LazyLock::new(|| {
     let mut m = <HashMap<String, bool>>::new();
@@ -41,11 +71,28 @@ static v4_ignored_headers: LazyLock<HashMap<String, bool>> = LazyLock::new(|| {
     m
 });
 
+fn fail(request: request::Request<Body>, error: SignV4Error) -> SignOutcome {
+    Err(Box::new(SignFailure { request, error }))
+}
+
+fn format_yyyymmdd(t: OffsetDateTime) -> String {
+    let mut value = String::with_capacity(8);
+    // Build YYYYMMDD directly from date components to avoid formatter fallbacks.
+    let _ = write!(value, "{:04}{:02}{:02}", t.year(), u8::from(t.month()), t.day());
+    value
+}
+
+fn format_amz_datetime(t: OffsetDateTime) -> SignResult<String> {
+    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
+    t.format(&format)
+        .map_err(|err| SignV4Error::TimeFormat { reason: err.to_string() })
+}
+
 pub fn get_signing_key(secret: &str, loc: &str, t: OffsetDateTime, service_type: &str) -> [u8; 32] {
     let mut s = "AWS4".to_string();
     s.push_str(secret);
-    let format = format_description!("[year][month][day]");
-    let date = hmac_sha256(s.into_bytes(), t.format(&format).unwrap().into_bytes());
+    let date_value = format_yyyymmdd(t);
+    let date = hmac_sha256(s.into_bytes(), date_value.into_bytes());
     let location = hmac_sha256(date, loc);
     let service = hmac_sha256(location, service_type);
 
@@ -57,9 +104,8 @@ pub fn get_signature(signing_key: [u8; 32], string_to_sign: &str) -> String {
 }
 
 pub fn get_scope(location: &str, t: OffsetDateTime, service_type: &str) -> String {
-    let format = format_description!("[year][month][day]");
     let mut ans = String::from("");
-    ans.push_str(&t.format(&format).unwrap());
+    ans.push_str(format_yyyymmdd(t).as_str());
     ans.push('/');
     ans.push_str(location);
     ans.push('/');
@@ -76,19 +122,21 @@ fn get_credential(access_key_id: &str, location: &str, t: OffsetDateTime, servic
     s
 }
 
-fn get_hashed_payload(req: &request::Request<Body>) -> String {
+fn try_get_hashed_payload(req: &request::Request<Body>) -> SignResult<String> {
     let headers = req.headers();
     let mut hashed_payload = "";
     if let Some(payload) = headers.get("X-Amz-Content-Sha256") {
-        hashed_payload = payload.to_str().unwrap();
+        hashed_payload = payload.to_str().map_err(|_| SignV4Error::InvalidHeaderValue {
+            name: "x-amz-content-sha256".to_string(),
+        })?;
     }
     if hashed_payload.is_empty() {
         hashed_payload = UNSIGNED_PAYLOAD;
     }
-    hashed_payload.to_string()
+    Ok(hashed_payload.to_string())
 }
 
-fn get_canonical_headers(req: &request::Request<Body>, ignored_headers: &HashMap<String, bool>) -> String {
+fn try_get_canonical_headers(req: &request::Request<Body>, ignored_headers: &HashMap<String, bool>) -> SignResult<String> {
     let mut headers = Vec::<String>::new();
     let mut vals = HashMap::<String, Vec<String>>::new();
     for k in req.headers().keys() {
@@ -100,8 +148,14 @@ fn get_canonical_headers(req: &request::Request<Body>, ignored_headers: &HashMap
             .headers()
             .get_all(k)
             .iter()
-            .map(|e| e.to_str().unwrap().to_string())
-            .collect();
+            .map(|e| {
+                e.to_str()
+                    .map(|v| v.to_string())
+                    .map_err(|_| SignV4Error::InvalidHeaderValue {
+                        name: k.as_str().to_lowercase(),
+                    })
+            })
+            .collect::<SignResult<Vec<String>>>()?;
         vals.insert(k.as_str().to_lowercase(), vv);
     }
     if !header_exists("host", &headers) {
@@ -119,11 +173,22 @@ fn get_canonical_headers(req: &request::Request<Body>, ignored_headers: &HashMap
         let k: &str = &k;
         match k {
             "host" => {
-                let _ = buf.write_str(&get_host_addr(req));
+                let host_addr = try_get_host_addr(req).map_err(|err| match err {
+                    HostAddrError::InvalidHostHeader => SignV4Error::InvalidHeaderValue {
+                        name: "host".to_string(),
+                    },
+                    HostAddrError::MissingUriHost => SignV4Error::InvalidUri {
+                        reason: "request uri has no host".to_string(),
+                    },
+                })?;
+                let _ = buf.write_str(&host_addr);
                 let _ = buf.write_char('\n');
             }
             _ => {
-                for (idx, v) in vals[k].iter().enumerate() {
+                let Some(values) = vals.get(k) else {
+                    continue;
+                };
+                for (idx, v) in values.iter().enumerate() {
                     if idx > 0 {
                         let _ = buf.write_char(',');
                     }
@@ -133,7 +198,7 @@ fn get_canonical_headers(req: &request::Request<Body>, ignored_headers: &HashMap
             }
         }
     }
-    String::from_utf8(buf.to_vec()).unwrap()
+    String::from_utf8(buf.to_vec()).map_err(|err| SignV4Error::CanonicalUtf8 { reason: err.to_string() })
 }
 
 fn header_exists(key: &str, headers: &[String]) -> bool {
@@ -162,7 +227,11 @@ fn get_signed_headers(req: &request::Request<Body>, ignored_headers: &HashMap<St
     headers.join(";")
 }
 
-fn get_canonical_request(req: &request::Request<Body>, ignored_headers: &HashMap<String, bool>, hashed_payload: &str) -> String {
+fn try_get_canonical_request(
+    req: &request::Request<Body>,
+    ignored_headers: &HashMap<String, bool>,
+    hashed_payload: &str,
+) -> SignResult<String> {
     let mut canonical_query_string = "".to_string();
     if let Some(q) = req.uri().query() {
         // Parse query string into key-value pairs
@@ -192,26 +261,30 @@ fn get_canonical_request(req: &request::Request<Body>, ignored_headers: &HashMap
         req.method().to_string(),
         req.uri().path().to_string(),
         canonical_query_string,
-        get_canonical_headers(req, ignored_headers),
+        try_get_canonical_headers(req, ignored_headers)?,
         get_signed_headers(req, ignored_headers),
         hashed_payload.to_string(),
     ];
-    canonical_request.join("\n")
+    Ok(canonical_request.join("\n"))
 }
 
-fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: &str, service_type: &str) -> String {
+fn try_get_string_to_sign_v4(
+    t: OffsetDateTime,
+    location: &str,
+    canonical_request: &str,
+    service_type: &str,
+) -> SignResult<String> {
     let mut string_to_sign = SIGN_V4_ALGORITHM.to_string();
     string_to_sign.push('\n');
-    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
-    string_to_sign.push_str(&t.format(&format).unwrap());
+    string_to_sign.push_str(format_amz_datetime(t)?.as_str());
     string_to_sign.push('\n');
     string_to_sign.push_str(&get_scope(location, t, service_type));
     string_to_sign.push('\n');
     string_to_sign.push_str(&hex_sha256(canonical_request.as_bytes(), |s| s.to_string()));
-    string_to_sign
+    Ok(string_to_sign)
 }
 
-pub fn pre_sign_v4(
+fn pre_sign_v4_inner(
     req: request::Request<Body>,
     access_key_id: &str,
     secret_access_key: &str,
@@ -219,9 +292,9 @@ pub fn pre_sign_v4(
     location: &str,
     expires: i64,
     t: OffsetDateTime,
-) -> request::Request<Body> {
+) -> SignOutcome {
     if access_key_id.is_empty() || secret_access_key.is_empty() {
-        return req;
+        return Ok(req);
     }
 
     let credential = get_credential(access_key_id, location, t, SERVICE_TYPE_S3);
@@ -233,8 +306,11 @@ pub fn pre_sign_v4(
         query = result.unwrap_or_default();
     }
     query.push(("X-Amz-Algorithm".to_string(), SIGN_V4_ALGORITHM.to_string()));
-    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
-    query.push(("X-Amz-Date".to_string(), t.format(&format).unwrap()));
+    let amz_date = match format_amz_datetime(t) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
+    query.push(("X-Amz-Date".to_string(), amz_date));
     query.push(("X-Amz-Expires".to_string(), format!("{expires:010}")));
     query.push(("X-Amz-SignedHeaders".to_string(), signed_headers));
     query.push(("X-Amz-Credential".to_string(), credential));
@@ -244,16 +320,38 @@ pub fn pre_sign_v4(
 
     let uri = req.uri().clone();
     let mut parts = req.uri().clone().into_parts();
-    parts.path_and_query = Some(
-        format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap())
-            .parse()
-            .unwrap(),
-    );
+    let query_str = match serde_urlencoded::to_string(&query) {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(req, SignV4Error::QueryEncode { reason: err.to_string() });
+        }
+    };
+    parts.path_and_query = Some(match format!("{}?{}", uri.path(), query_str).parse() {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(req, SignV4Error::InvalidUri { reason: err.to_string() });
+        }
+    });
     let mut req = req;
-    *req.uri_mut() = Uri::from_parts(parts).unwrap();
+    *req.uri_mut() = match Uri::from_parts(parts) {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(req, SignV4Error::InvalidUriParts { reason: err.to_string() });
+        }
+    };
 
-    let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
-    let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, SERVICE_TYPE_S3);
+    let hashed_payload = match try_get_hashed_payload(&req) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
+    let canonical_request = match try_get_canonical_request(&req, &v4_ignored_headers, &hashed_payload) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
+    let string_to_sign = match try_get_string_to_sign_v4(t, location, &canonical_request, SERVICE_TYPE_S3) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
     //println!("canonical_request: \n{}\n", canonical_request);
     //println!("string_to_sign: \n{}\n", string_to_sign);
     let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3);
@@ -261,20 +359,57 @@ pub fn pre_sign_v4(
 
     let uri = req.uri().clone();
     let mut parts = req.uri().clone().into_parts();
-    parts.path_and_query = Some(
-        format!(
-            "{}?{}&X-Amz-Signature={}",
-            uri.path(),
-            serde_urlencoded::to_string(&query).unwrap(),
-            signature
-        )
-        .parse()
-        .unwrap(),
-    );
+    let query_str = match serde_urlencoded::to_string(&query) {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(req, SignV4Error::QueryEncode { reason: err.to_string() });
+        }
+    };
+    parts.path_and_query = Some(match format!("{}?{}&X-Amz-Signature={}", uri.path(), query_str, signature).parse() {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(req, SignV4Error::InvalidUri { reason: err.to_string() });
+        }
+    });
 
-    *req.uri_mut() = Uri::from_parts(parts).unwrap();
+    *req.uri_mut() = match Uri::from_parts(parts) {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(req, SignV4Error::InvalidUriParts { reason: err.to_string() });
+        }
+    };
 
-    req
+    Ok(req)
+}
+
+pub fn try_pre_sign_v4(
+    req: request::Request<Body>,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+    expires: i64,
+    t: OffsetDateTime,
+) -> SignResult<request::Request<Body>> {
+    pre_sign_v4_inner(req, access_key_id, secret_access_key, session_token, location, expires, t).map_err(|f| f.error)
+}
+
+pub fn pre_sign_v4(
+    req: request::Request<Body>,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+    expires: i64,
+    t: OffsetDateTime,
+) -> request::Request<Body> {
+    match pre_sign_v4_inner(req, access_key_id, secret_access_key, session_token, location, expires, t) {
+        Ok(request) => request,
+        Err(failure) => {
+            warn!(error = %failure.error, "failed to presign v4 request");
+            failure.request
+        }
+    }
 }
 
 fn _post_pre_sign_signature_v4(policy_base64: &str, t: OffsetDateTime, secret_access_key: &str, location: &str) -> String {
@@ -289,7 +424,13 @@ fn _sign_v4_sts(
     secret_access_key: &str,
     location: &str,
 ) -> request::Request<Body> {
-    sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new())
+    match sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) {
+        Ok(request) => request,
+        Err(failure) => {
+            warn!(error = %failure.error, "failed to sign v4 sts request");
+            failure.request
+        }
+    }
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -302,38 +443,119 @@ fn sign_v4_inner(
     location: &str,
     service_type: &str,
     trailer: HeaderMap,
-) -> request::Request<Body> {
+) -> SignOutcome {
     if access_key_id.is_empty() || secret_access_key.is_empty() {
-        return req;
+        return Ok(req);
     }
 
     let t = OffsetDateTime::now_utc();
-    let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
+    let t2 = match time::Time::from_hms(0, 0, 0) {
+        Ok(midnight) => t.replace_time(midnight),
+        Err(err) => {
+            return fail(req, SignV4Error::TimeComponent { reason: err.to_string() });
+        }
+    };
 
-    let headers = req.headers_mut();
-    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
-    headers.insert("X-Amz-Date", t.format(&format).unwrap().parse().unwrap());
+    let amz_date = match format_amz_datetime(t) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
+    let amz_date_value = match amz_date.parse::<http::HeaderValue>() {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(
+                req,
+                SignV4Error::HeaderValueParse {
+                    name: "X-Amz-Date".to_string(),
+                    reason: err.to_string(),
+                },
+            );
+        }
+    };
+    req.headers_mut().insert("X-Amz-Date", amz_date_value);
 
     if !session_token.is_empty() {
-        headers.insert("X-Amz-Security-Token", session_token.parse().unwrap());
+        let token_value = match session_token.parse::<http::HeaderValue>() {
+            Ok(value) => value,
+            Err(err) => {
+                return fail(
+                    req,
+                    SignV4Error::HeaderValueParse {
+                        name: "X-Amz-Security-Token".to_string(),
+                        reason: err.to_string(),
+                    },
+                );
+            }
+        };
+        req.headers_mut().insert("X-Amz-Security-Token", token_value);
     }
 
     if !trailer.is_empty() {
+        let mut trailer_values = Vec::new();
         for (k, _) in &trailer {
-            headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap());
+            let parsed = match k.as_str().to_lowercase().parse::<http::HeaderValue>() {
+                Ok(value) => value,
+                Err(err) => {
+                    return fail(
+                        req,
+                        SignV4Error::HeaderValueParse {
+                            name: "X-Amz-Trailer".to_string(),
+                            reason: err.to_string(),
+                        },
+                    );
+                }
+            };
+            trailer_values.push(parsed);
+        }
+        let content_encoding = match "aws-chunked".parse::<http::HeaderValue>() {
+            Ok(value) => value,
+            Err(err) => {
+                return fail(
+                    req,
+                    SignV4Error::HeaderValueParse {
+                        name: "Content-Encoding".to_string(),
+                        reason: err.to_string(),
+                    },
+                );
+            }
+        };
+        let decoded_len = match format!("{content_len:010}").parse::<http::HeaderValue>() {
+            Ok(value) => value,
+            Err(err) => {
+                return fail(
+                    req,
+                    SignV4Error::HeaderValueParse {
+                        name: "x-amz-decoded-content-length".to_string(),
+                        reason: err.to_string(),
+                    },
+                );
+            }
+        };
+        let headers = req.headers_mut();
+        for value in trailer_values {
+            headers.append("X-Amz-Trailer", value);
         }
 
-        headers.insert("Content-Encoding", "aws-chunked".parse().unwrap());
-        headers.insert("x-amz-decoded-content-length", format!("{content_len:010}").parse().unwrap());
+        headers.insert("Content-Encoding", content_encoding);
+        headers.insert("x-amz-decoded-content-length", decoded_len);
     }
 
     if service_type == SERVICE_TYPE_STS {
-        headers.remove("X-Amz-Content-Sha256");
+        req.headers_mut().remove("X-Amz-Content-Sha256");
     }
 
-    let hashed_payload = get_hashed_payload(&req);
-    let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &hashed_payload);
-    let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, service_type);
+    let hashed_payload = match try_get_hashed_payload(&req) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
+    let canonical_request = match try_get_canonical_request(&req, &v4_ignored_headers, &hashed_payload) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
+    let string_to_sign = match try_get_string_to_sign_v4(t, location, &canonical_request, service_type) {
+        Ok(value) => value,
+        Err(err) => return fail(req, err),
+    };
     let signing_key = get_signing_key(secret_access_key, location, t, service_type);
     let credential = get_credential(access_key_id, location, t2, service_type);
     let signed_headers = get_signed_headers(&req, &v4_ignored_headers);
@@ -343,42 +565,28 @@ fn sign_v4_inner(
     let headers = req.headers_mut();
 
     let auth = format!("{SIGN_V4_ALGORITHM} Credential={credential}, SignedHeaders={signed_headers}, Signature={signature}");
-    headers.insert("Authorization", auth.parse().unwrap());
+    let auth_value = match auth.parse::<http::HeaderValue>() {
+        Ok(value) => value,
+        Err(err) => {
+            return fail(
+                req,
+                SignV4Error::HeaderValueParse {
+                    name: "Authorization".to_string(),
+                    reason: err.to_string(),
+                },
+            );
+        }
+    };
+    headers.insert("Authorization", auth_value);
 
     if !trailer.is_empty() {
         //req.Trailer = trailer;
         for (_, v) in &trailer {
             headers.append(http::header::TRAILER, v.clone());
         }
-        return streaming_unsigned_v4(req, session_token, content_len, t);
+        return Ok(streaming_unsigned_v4(req, session_token, content_len, t));
     }
-    req
-}
-
-fn _unsigned_trailer(mut req: request::Request<Body>, content_len: i64, trailer: HeaderMap) {
-    if !trailer.is_empty() {
-        return;
-    }
-    let t = OffsetDateTime::now_utc();
-    let t = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap());
-
-    let headers = req.headers_mut();
-    let format = format_description!("[year][month][day]T[hour][minute][second]Z");
-    headers.insert("X-Amz-Date", t.format(&format).unwrap().parse().unwrap());
-
-    for (k, _) in &trailer {
-        headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap());
-    }
-
-    headers.insert("Content-Encoding", "aws-chunked".parse().unwrap());
-    headers.insert("x-amz-decoded-content-length", format!("{content_len:010}").parse().unwrap());
-
-    if !trailer.is_empty() {
-        for (_, v) in &trailer {
-            headers.append(http::header::TRAILER, v.clone());
-        }
-    }
-    streaming_unsigned_v4(req, "", content_len, t);
+    Ok(req)
 }
 
 pub fn sign_v4(
@@ -389,6 +597,32 @@ pub fn sign_v4(
     session_token: &str,
     location: &str,
 ) -> request::Request<Body> {
+    match sign_v4_inner(
+        req,
+        content_len,
+        access_key_id,
+        secret_access_key,
+        session_token,
+        location,
+        SERVICE_TYPE_S3,
+        HeaderMap::new(),
+    ) {
+        Ok(request) => request,
+        Err(failure) => {
+            warn!(error = %failure.error, "failed to sign v4 request");
+            failure.request
+        }
+    }
+}
+
+pub fn try_sign_v4(
+    req: request::Request<Body>,
+    content_len: i64,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+) -> SignResult<request::Request<Body>> {
     sign_v4_inner(
         req,
         content_len,
@@ -399,6 +633,7 @@ pub fn sign_v4(
         SERVICE_TYPE_S3,
         HeaderMap::new(),
     )
+    .map_err(|failure| failure.error)
 }
 
 pub fn sign_v4_trailer(
@@ -409,6 +644,32 @@ pub fn sign_v4_trailer(
     location: &str,
     trailer: HeaderMap,
 ) -> request::Request<Body> {
+    match sign_v4_inner(
+        req,
+        0,
+        access_key_id,
+        secret_access_key,
+        session_token,
+        location,
+        SERVICE_TYPE_S3,
+        trailer,
+    ) {
+        Ok(request) => request,
+        Err(failure) => {
+            warn!(error = %failure.error, "failed to sign v4 trailer request");
+            failure.request
+        }
+    }
+}
+
+pub fn try_sign_v4_trailer(
+    req: request::Request<Body>,
+    access_key_id: &str,
+    secret_access_key: &str,
+    session_token: &str,
+    location: &str,
+    trailer: HeaderMap,
+) -> SignResult<request::Request<Body>> {
     sign_v4_inner(
         req,
         0,
@@ -419,11 +680,13 @@ pub fn sign_v4_trailer(
         SERVICE_TYPE_S3,
         trailer,
     )
+    .map_err(|failure| failure.error)
 }
 
 #[cfg(test)]
 #[allow(unused_variables, unused_mut)]
 mod tests {
+    use http::HeaderValue;
     use http::request;
     use time::macros::datetime;
 
@@ -468,7 +731,9 @@ mod tests {
         );
         *req.uri_mut() = Uri::from_parts(parts).unwrap();
 
-        let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+        let hashed_payload = try_get_hashed_payload(&req).expect("example request should have valid payload header");
+        let canonical_request =
+            try_get_canonical_request(&req, &v4_ignored_headers, &hashed_payload).expect("example request should canonicalize");
         assert_eq!(
             canonical_request,
             concat!(
@@ -486,7 +751,8 @@ mod tests {
             )
         );
 
-        let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
+        let string_to_sign = try_get_string_to_sign_v4(t, region, &canonical_request, service)
+            .expect("example request should build string-to-sign");
         assert_eq!(
             string_to_sign,
             concat!(
@@ -542,7 +808,9 @@ mod tests {
         //println!("parts.path_and_query: {:?}", parts.path_and_query);
         *req.uri_mut() = Uri::from_parts(parts).unwrap();
 
-        let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+        let hashed_payload = try_get_hashed_payload(&req).expect("example request should have valid payload header");
+        let canonical_request =
+            try_get_canonical_request(&req, &v4_ignored_headers, &hashed_payload).expect("example request should canonicalize");
         println!("canonical_request: \n{canonical_request}\n");
         assert_eq!(
             canonical_request,
@@ -561,7 +829,8 @@ mod tests {
             )
         );
 
-        let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
+        let string_to_sign = try_get_string_to_sign_v4(t, region, &canonical_request, service)
+            .expect("example request should build string-to-sign");
         println!("string_to_sign: \n{string_to_sign}\n");
         assert_eq!(
             string_to_sign,
@@ -607,7 +876,9 @@ mod tests {
         headers.insert("x-amz-date", timestamp.parse().unwrap());
 
         println!("{:?}", req.uri().query());
-        let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+        let hashed_payload = try_get_hashed_payload(&req).expect("example request should have valid payload header");
+        let canonical_request =
+            try_get_canonical_request(&req, &v4_ignored_headers, &hashed_payload).expect("example request should canonicalize");
         println!("canonical_request: \n{canonical_request}\n");
         assert_eq!(
             canonical_request,
@@ -626,7 +897,8 @@ mod tests {
             )
         );
 
-        let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
+        let string_to_sign = try_get_string_to_sign_v4(t, region, &canonical_request, service)
+            .expect("example request should build string-to-sign");
         println!("string_to_sign: \n{string_to_sign}\n");
         assert_eq!(
             string_to_sign,
@@ -672,7 +944,9 @@ mod tests {
         headers.insert("x-amz-date", timestamp.parse().unwrap());
 
         println!("{:?}", req.uri().query());
-        let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req));
+        let hashed_payload = try_get_hashed_payload(&req).expect("example request should have valid payload header");
+        let canonical_request =
+            try_get_canonical_request(&req, &v4_ignored_headers, &hashed_payload).expect("example request should canonicalize");
         println!("canonical_request: \n{canonical_request}\n");
         assert_eq!(
             canonical_request,
@@ -691,7 +965,8 @@ mod tests {
             )
         );
 
-        let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service);
+        let string_to_sign = try_get_string_to_sign_v4(t, region, &canonical_request, service)
+            .expect("example request should build string-to-sign");
         println!("string_to_sign: \n{string_to_sign}\n");
         assert_eq!(
             string_to_sign,
@@ -739,11 +1014,19 @@ mod tests {
         canonical_request.push('\n');
         canonical_request.push_str(req.uri().query().unwrap());
         canonical_request.push('\n');
-        canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers));
+        canonical_request.push_str(
+            try_get_canonical_headers(&req, &v4_ignored_headers)
+                .expect("presigned request should canonicalize headers")
+                .as_str(),
+        );
         canonical_request.push('\n');
         canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers));
         canonical_request.push('\n');
-        canonical_request.push_str(&get_hashed_payload(&req));
+        canonical_request.push_str(
+            try_get_hashed_payload(&req)
+                .expect("presigned request should include payload hash")
+                .as_str(),
+        );
         //println!("canonical_request: \n{}\n", canonical_request);
         assert_eq!(
             canonical_request,
@@ -787,11 +1070,19 @@ mod tests {
         canonical_request.push('\n');
         canonical_request.push_str(req.uri().query().unwrap());
         canonical_request.push('\n');
-        canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers));
+        canonical_request.push_str(
+            try_get_canonical_headers(&req, &v4_ignored_headers)
+                .expect("presigned request should canonicalize headers")
+                .as_str(),
+        );
         canonical_request.push('\n');
         canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers));
         canonical_request.push('\n');
-        canonical_request.push_str(&get_hashed_payload(&req));
+        canonical_request.push_str(
+            try_get_hashed_payload(&req)
+                .expect("presigned request should include payload hash")
+                .as_str(),
+        );
         //println!("canonical_request: \n{}\n", canonical_request);
         assert_eq!(
             canonical_request,
@@ -806,4 +1097,87 @@ mod tests {
             )
         );
     }
+
+    fn build_request_with_invalid_header_value(uri: &str) -> request::Request<Body> {
+        let mut req = request::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .body(Body::empty())
+            .unwrap();
+        let headers = req.headers_mut();
+        headers.insert("host", HeaderValue::from_static("examplebucket.s3.amazonaws.com"));
+        headers.insert("x-amz-content-sha256", HeaderValue::from_static(UNSIGNED_PAYLOAD));
+        headers.insert("x-amz-meta-invalid", HeaderValue::from_bytes(&[0xFF]).unwrap());
+        req
+    }
+
+    #[test]
+    fn try_sign_v4_returns_error_for_non_utf8_header_value() {
+        let req = build_request_with_invalid_header_value("http://examplebucket.s3.amazonaws.com/object");
+        let err = try_sign_v4(req, 0, "rustfsadmin", "rustfsadmin", "", "us-east-1").unwrap_err();
+        assert!(matches!(
+            err,
+            SignV4Error::InvalidHeaderValue { name } if name == "x-amz-meta-invalid"
+        ));
+    }
+
+    #[test]
+    fn try_sign_v4_returns_invalid_uri_error_when_uri_has_no_host() {
+        let mut req = request::Request::builder()
+            .method(http::Method::GET)
+            .uri("/object")
+            .body(Body::empty())
+            .unwrap();
+        let headers = req.headers_mut();
+        headers.insert("host", HeaderValue::from_static("examplebucket.s3.amazonaws.com"));
+        headers.insert("x-amz-content-sha256", HeaderValue::from_static(UNSIGNED_PAYLOAD));
+
+        let err = try_sign_v4(req, 0, "rustfsadmin", "rustfsadmin", "", "us-east-1").unwrap_err();
+        assert!(matches!(
+            err,
+            SignV4Error::InvalidUri { reason } if reason.contains("no host")
+        ));
+    }
+
+    #[test]
+    fn legacy_sign_apis_do_not_panic_on_non_utf8_header_value() {
+        let signed = sign_v4(
+            build_request_with_invalid_header_value("http://examplebucket.s3.amazonaws.com/object"),
+            0,
+            "rustfsadmin",
+            "rustfsadmin",
+            "",
+            "us-east-1",
+        );
+        assert!(signed.headers().get(http::header::AUTHORIZATION).is_none());
+
+        let presigned = pre_sign_v4(
+            build_request_with_invalid_header_value("http://examplebucket.s3.amazonaws.com/object"),
+            "rustfsadmin",
+            "rustfsadmin",
+            "",
+            "us-east-1",
+            60,
+            datetime!(2026-04-27 00:00:00 UTC),
+        );
+        let query = presigned.uri().query().unwrap_or_default();
+        assert!(!query.contains("X-Amz-Signature="));
+    }
+
+    #[test]
+    fn sign_v4_sts_returns_original_request_on_non_utf8_header_value() {
+        let signed = _sign_v4_sts(
+            build_request_with_invalid_header_value("http://examplebucket.s3.amazonaws.com/object"),
+            "rustfsadmin",
+            "rustfsadmin",
+            "us-east-1",
+        );
+        assert!(signed.headers().get(http::header::AUTHORIZATION).is_none());
+    }
+
+    #[test]
+    fn format_yyyymmdd_is_zero_padded() {
+        let t = datetime!(0001-01-02 03:04:05 UTC);
+        assert_eq!(format_yyyymmdd(t), "00010102");
+    }
 }

crates/signer/src/utils.rs

@@ -16,24 +16,37 @@ use http::request;
 
 use s3s::Body;
 
-pub fn get_host_addr(req: &request::Request<Body>) -> String {
+#[derive(Debug, thiserror::Error)]
+pub enum HostAddrError {
+    #[error("invalid UTF-8 header value for `host`")]
+    InvalidHostHeader,
+    #[error("request uri has no host")]
+    MissingUriHost,
+}
+
+pub fn try_get_host_addr(req: &request::Request<Body>) -> Result<String, HostAddrError> {
     let host = req.headers().get("host");
     let uri = req.uri();
-    let req_host;
-    if let Some(port) = uri.port() {
-        req_host = format!("{}:{}", uri.host().unwrap(), port);
+    let uri_host = uri.host().ok_or(HostAddrError::MissingUriHost)?;
+
+    let req_host = if let Some(port) = uri.port() {
+        format!("{uri_host}:{port}")
     } else {
-        req_host = uri.host().unwrap().to_string();
+        uri_host.to_string()
+    };
+
+    if let Some(host) = host {
+        let host = host.to_str().map_err(|_| HostAddrError::InvalidHostHeader)?;
+        if req_host != host {
+            return Ok(host.to_string());
+        }
     }
-    if let Some(host) = host
-        && req_host != *host.to_str().unwrap()
-    {
-        return (*host.to_str().unwrap()).to_string();
-    }
-    /*if req.uri_ref().unwrap().host().is_some() {
-        return req.uri_ref().unwrap().host().unwrap();
-    }*/
-    req_host
+
+    Ok(req_host)
+}
+
+pub fn get_host_addr(req: &request::Request<Body>) -> String {
+    try_get_host_addr(req).unwrap()
 }
 
 pub fn sign_v4_trim_all(input: &str) -> String {
@@ -47,3 +60,56 @@ where
 {
     v.sort_by(|lhs, rhs| lhs.0.cmp(&rhs.0));
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{HostAddrError, try_get_host_addr};
+    use http::HeaderValue;
+    use http::request;
+    use s3s::Body;
+
+    #[test]
+    fn try_get_host_addr_prefers_explicit_host_header_when_it_differs_from_uri() {
+        let mut req = request::Request::builder()
+            .method(http::Method::GET)
+            .uri("https://bucket.example.com/object")
+            .body(Body::empty())
+            .expect("request should build");
+        req.headers_mut()
+            .insert("host", HeaderValue::from_static("proxy.internal:9443"));
+
+        let host = try_get_host_addr(&req).expect("host lookup should succeed");
+
+        assert_eq!(host, "proxy.internal:9443");
+    }
+
+    #[test]
+    fn try_get_host_addr_rejects_non_utf8_host_header_value() {
+        let mut req = request::Request::builder()
+            .method(http::Method::GET)
+            .uri("https://bucket.example.com/object")
+            .body(Body::empty())
+            .expect("request should build");
+        req.headers_mut().insert(
+            "host",
+            HeaderValue::from_bytes(&[0xFF]).expect("invalid utf8 bytes should be accepted by HeaderValue"),
+        );
+
+        let err = try_get_host_addr(&req).expect_err("invalid host header should fail");
+
+        assert!(matches!(err, HostAddrError::InvalidHostHeader));
+    }
+
+    #[test]
+    fn try_get_host_addr_rejects_relative_uri_without_host() {
+        let req = request::Request::builder()
+            .method(http::Method::GET)
+            .uri("/object")
+            .body(Body::empty())
+            .expect("request should build");
+
+        let err = try_get_host_addr(&req).expect_err("relative uri should fail");
+
+        assert!(matches!(err, HostAddrError::MissingUriHost));
+    }
+}

crates/utils/src/envs.rs

@@ -663,7 +663,9 @@ pub fn apply_external_env_compat() -> ExternalEnvCompatReport {
 
 #[cfg(test)]
 mod tests {
-    use super::{apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_str};
+    use super::{
+        apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_bool_with_aliases, get_env_str,
+    };
 
     fn source_key(suffix: &str) -> String {
         let mut key = super::external_env_prefix().to_string();
@@ -750,6 +752,15 @@ mod tests {
         });
     }
 
+    #[test]
+    fn rustfs_bool_env_takes_precedence_over_minio_alias() {
+        temp_env::with_var("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", Some("false"), || {
+            temp_env::with_var("MINIO_CI", Some("1"), || {
+                assert!(!get_env_bool_with_aliases("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", &["MINIO_CI"], true,));
+            });
+        });
+    }
+
     #[test]
     fn apply_external_env_compat_copies_missing_rustfs_keys() {
         temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || {

crates/utils/src/path.rs

@@ -90,7 +90,12 @@ pub fn retain_slash(s: &str) -> String {
 
 /// Checks if string `s` starts with `prefix` using case-insensitive comparison.
 pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
-    s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix))
+    if s.starts_with(prefix) {
+        return true;
+    }
+
+    s.get(..prefix.len())
+        .is_some_and(|s_prefix| s_prefix.eq_ignore_ascii_case(prefix))
 }
 
 /// Checks if string `s` starts with `prefix`.
@@ -875,4 +880,16 @@ mod tests {
         assert_eq!(bucket, "bucket");
         assert_eq!(object, "object");
     }
+
+    #[test]
+    #[cfg(target_os = "windows")]
+    fn test_path_to_bucket_object_with_base_path_handles_unicode_without_panicking() {
+        let (bucket, object) = path_to_bucket_object_with_base_path(
+            "D:\\Github\\rustfs\\target\\volumes\\test1",
+            "s3-test-bucket/中文/日本語/한글-9cd5599a-f8eb-4e24-9df7-32ecd8d8ad1f",
+        );
+
+        assert_eq!(bucket, "s3-test-bucket");
+        assert_eq!(object, "中文/日本語/한글-9cd5599a-f8eb-4e24-9df7-32ecd8d8ad1f");
+    }
 }

crates/utils/src/string.rs

@@ -506,13 +506,12 @@ pub fn parse_ellipses_range(pattern: &str) -> Result<Vec<String>> {
 /// ```
 ///
 pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
-    if s.len() < prefix.len() {
-        return false;
+    if s.starts_with(prefix) {
+        return true;
     }
 
-    let s_prefix = &s[..prefix.len()];
-    // Test match with case first, then case-insensitive
-    s_prefix == prefix || s_prefix.to_lowercase() == prefix.to_lowercase()
+    s.get(..prefix.len())
+        .is_some_and(|s_prefix| s_prefix.eq_ignore_ascii_case(prefix))
 }
 
 #[cfg(test)]
@@ -872,4 +871,13 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    #[cfg(target_os = "windows")]
+    fn test_strings_has_prefix_fold_handles_unicode_without_panicking() {
+        assert!(!strings_has_prefix_fold(
+            "s3-test-bucket/中文/日本語/한글-9cd5599a-f8eb-4e24-9df7-32ecd8d8ad1f",
+            "D:\\Github\\rustfs\\target\\volumes\\test1",
+        ));
+    }
 }

docker-compose-simple.yml

@@ -35,6 +35,9 @@ services:
       - RUSTFS_SECRET_KEY=rustfsadmin # CHANGEME
       - RUSTFS_OBS_LOGGER_LEVEL=info
       - RUSTFS_TLS_PATH=/opt/tls
+      # Keep strict disk topology checks enabled by default.
+      # For local testing only, set `RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true` explicitly.
+      - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-false}
 
     volumes:
       - rustfs_data_0:/data/rustfs0

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1776329215,
-        "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=",
+        "lastModified": 1776949667,
+        "narHash": "sha256-GMSVw35Q+294GlrTUKlx087E31z7KurReQ1YHSKp5iw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b86751bc4085f48661017fa226dee99fab6c651b",
+        "rev": "01fbdeef22b76df85ea168fbfe1bfd9e63681b30",
         "type": "github"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1776481912,
-        "narHash": "sha256-Xq7p+Ex3YHFAd+fFFLOYw2Wv67582X7SAmrEDtIDZQ4=",
+        "lastModified": 1777086717,
+        "narHash": "sha256-vEl3cGHRxEFdVNuP9PbrhAWnmU98aPOLGy9/1JXzSuM=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "e611106c527e8ab0adbb641183cda284411d575c",
+        "rev": "3be56bd430bfd65d3c468a50626c3a601c7dee03",
         "type": "github"
       },
       "original": {

helm/rustfs/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: rustfs
 description: RustFS helm chart to deploy RustFS on kubernetes cluster.
 type: application
-version: 0.0.93
-appVersion: "1.0.0-alpha.93"
+version: "v1.0.0-beta.1"
+appVersion: "v1.0.0-beta.1"
 home: https://rustfs.com
 icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
 maintainers:

helm/rustfs/templates/configmap.yaml

@@ -65,9 +65,10 @@ data:
     {{- end }}
     {{- if .profiling.enabled }}
   RUSTFS_OBS_PROFILING_ENDPOINT: {{ .profiling.endpoint | quote }}
+  RUSTFS_OBS_PROFILING_EXPORT_ENABLED: "true"
     {{- else }}
   RUSTFS_OBS_PROFILING_ENDPOINT: ""
-  RUSTFS_OBS_PROFILING_ENABLED: "false"
+  RUSTFS_OBS_PROFILING_EXPORT_ENABLED: "false"
     {{- end }}
   {{- end }}
   {{- end }}

helm/rustfs/templates/deployment.yaml

@@ -14,8 +14,13 @@ metadata:
 spec:
   replicas: 1
   {{- with .Values.mode.standalone.strategy }}
+  {{- $type := default "RollingUpdate" .type }}
   strategy:
-    {{- toYaml . | nindent 4 }}
+    type: {{ $type }}
+    {{- if and (eq $type "RollingUpdate") .rollingUpdate }}
+    rollingUpdate:
+      {{- toYaml .rollingUpdate | nindent 6 }}
+    {{- end }}
   {{- end }}
   selector:
     matchLabels:

rustfs/Cargo.toml

@@ -171,6 +171,7 @@ libsystemd.workspace = true
 
 [target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
 mimalloc = { workspace = true }
+libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
 
 
 

rustfs/src/admin/handlers/audit.rs

@@ -17,14 +17,16 @@ use crate::admin::{
     handlers::target_descriptor::{
         AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
         collect_validated_key_values as shared_collect_validated_key_values,
-        merge_target_endpoints as shared_merge_target_endpoints,
+        merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
         target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
         validate_target_request,
     },
     router::{AdminOperation, Operation, S3Router},
 };
 use crate::auth::{check_key_valid, get_session_token};
-use crate::server::{ADMIN_PREFIX, RemoteAddr};
+use crate::server::{
+    ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, refresh_audit_module_enabled, refresh_persisted_module_switches_from_store,
+};
 use futures::stream::{FuturesUnordered, StreamExt};
 use http::{HeaderMap, StatusCode};
 use hyper::Method;
@@ -168,6 +170,17 @@ fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target
     )
 }
 
+async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
+    if let Err(err) = refresh_persisted_module_switches_from_store().await {
+        warn!(
+            error = %err,
+            "failed to reload persisted module switches before checking audit target operation gating"
+        );
+    }
+    refresh_audit_module_enabled();
+    target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action)
+}
+
 fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
     shared_merge_target_endpoints(&audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
         .into_iter()
@@ -272,6 +285,9 @@ impl Operation for AuditTargetConfig {
         let (target_type, target_name) = extract_target_params(&params)?;
 
         authorize_audit_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
+        if let Some(reason) = audit_target_operation_block_reason("managing audit targets from the console").await {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
         let config_snapshot = load_server_config_from_store().await?;
         if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
             return Err(s3_error!(InvalidRequest, "{reason}"));
@@ -373,6 +389,9 @@ impl Operation for RemoveAuditTarget {
         let (target_type, target_name) = extract_target_params(&params)?;
 
         authorize_audit_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
+        if let Some(reason) = audit_target_operation_block_reason("managing audit targets from the console").await {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
         let config_snapshot = load_server_config_from_store().await?;
         if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
             return Err(s3_error!(InvalidRequest, "{reason}"));
@@ -534,6 +553,26 @@ mod tests {
         );
     }
 
+    #[test]
+    fn audit_target_operation_block_reason_requires_audit_module_enable() {
+        with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("false"), || {
+            let reason =
+                futures::executor::block_on(audit_target_operation_block_reason("managing audit targets from the console"));
+            assert!(reason.is_some());
+            assert!(reason.unwrap().contains("set RUSTFS_AUDIT_ENABLE=true"));
+        });
+    }
+
+    #[test]
+    fn audit_target_operation_block_reason_allows_when_audit_module_enabled() {
+        with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("true"), || {
+            assert!(
+                futures::executor::block_on(audit_target_operation_block_reason("managing audit targets from the console"))
+                    .is_none()
+            );
+        });
+    }
+
     #[test]
     fn audit_target_mutation_block_reason_rejects_mixed_target() {
         with_var("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
@@ -727,6 +766,10 @@ mod tests {
             put_block.contains("authorize_audit_admin_request(&req, AdminAction::SetBucketTargetAction).await?;"),
             "audit target writes should require SetBucketTargetAction"
         );
+        assert!(
+            put_block.contains("audit_target_operation_block_reason(\"managing audit targets from the console\")"),
+            "audit target writes should reject requests when the audit module is disabled"
+        );
         assert!(
             list_block.contains("authorize_audit_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
             "audit target list should require GetBucketTargetAction"
@@ -735,6 +778,10 @@ mod tests {
             delete_block.contains("authorize_audit_admin_request(&req, AdminAction::SetBucketTargetAction).await?;"),
             "audit target deletion should require SetBucketTargetAction"
         );
+        assert!(
+            delete_block.contains("audit_target_operation_block_reason(\"managing audit targets from the console\")"),
+            "audit target deletion should reject requests when the audit module is disabled"
+        );
     }
 
     fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {

rustfs/src/admin/handlers/event.rs

@@ -17,14 +17,17 @@ use crate::admin::{
     handlers::target_descriptor::{
         AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
         collect_validated_key_values as shared_collect_validated_key_values,
-        merge_target_endpoints as shared_merge_target_endpoints,
+        merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
         target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
         validate_target_request,
     },
     router::{AdminOperation, Operation, S3Router},
 };
 use crate::auth::{check_key_valid, get_session_token};
-use crate::server::{ADMIN_PREFIX, RemoteAddr};
+use crate::server::{
+    ADMIN_PREFIX, RemoteAddr, is_notify_module_enabled, refresh_notify_module_enabled,
+    refresh_persisted_module_switches_from_store,
+};
 use futures::stream::{FuturesUnordered, StreamExt};
 use http::{HeaderMap, StatusCode};
 use hyper::Method;
@@ -167,6 +170,17 @@ fn target_mutation_block_reason(config: &Config, target_type: &str, target_name:
     )
 }
 
+async fn notification_target_operation_block_reason(action: &str) -> Option<String> {
+    if let Err(err) = refresh_persisted_module_switches_from_store().await {
+        warn!(
+            error = %err,
+            "failed to reload persisted module switches before checking notification target operation gating"
+        );
+    }
+    refresh_notify_module_enabled();
+    target_module_disabled_reason("notify", rustfs_config::ENV_NOTIFY_ENABLE, is_notify_module_enabled(), action)
+}
+
 fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
     shared_merge_target_endpoints(&notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
         .into_iter()
@@ -197,6 +211,9 @@ impl Operation for NotificationTarget {
         let (target_type, target_name) = extract_target_params(&params)?;
 
         authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
+        if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
         let ns = get_notification_system()?;
         let config_snapshot = ns.config.read().await.clone();
         if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
@@ -291,6 +308,13 @@ impl Operation for ListTargetsArns {
         let span = Span::current();
         let _enter = span.enter();
         authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;
+        if let Some(reason) = notification_target_operation_block_reason(
+            "querying notification target ARNs for bucket associations from the console",
+        )
+        .await
+        {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
         let ns = get_notification_system()?;
 
         let targets = ns.get_target_values().await;
@@ -336,6 +360,9 @@ impl Operation for RemoveNotificationTarget {
         let (target_type, target_name) = extract_target_params(&params)?;
 
         authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
+        if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
         let ns = get_notification_system()?;
         let config_snapshot = ns.config.read().await.clone();
         if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
@@ -543,6 +570,29 @@ mod tests {
         );
     }
 
+    #[test]
+    fn notification_target_operation_block_reason_requires_notify_module_enable() {
+        with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("false"), || {
+            let reason = futures::executor::block_on(notification_target_operation_block_reason(
+                "managing notification targets from the console",
+            ));
+            assert!(reason.is_some());
+            assert!(reason.unwrap().contains("set RUSTFS_NOTIFY_ENABLE=true"));
+        });
+    }
+
+    #[test]
+    fn notification_target_operation_block_reason_allows_when_notify_module_enabled() {
+        with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"), || {
+            assert!(
+                futures::executor::block_on(notification_target_operation_block_reason(
+                    "managing notification targets from the console"
+                ))
+                .is_none()
+            );
+        });
+    }
+
     #[test]
     fn target_mutation_block_reason_rejects_mixed_target() {
         with_var("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
@@ -705,6 +755,11 @@ mod tests {
             put_block.contains("authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;"),
             "notification target writes should require SetBucketTargetAction"
         );
+        assert!(
+            put_block.contains("notification_target_operation_block_reason(")
+                && put_block.contains("\"managing notification targets from the console\""),
+            "notification target writes should reject requests when the notify module is disabled"
+        );
         assert!(
             list_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
             "notification target list should require GetBucketTargetAction"
@@ -713,10 +768,20 @@ mod tests {
             arns_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
             "notification target arn listing should require GetBucketTargetAction"
         );
+        assert!(
+            arns_block.contains("notification_target_operation_block_reason(")
+                && arns_block.contains("\"querying notification target ARNs for bucket associations from the console\""),
+            "notification target arn listing should reject requests when the notify module is disabled"
+        );
         assert!(
             delete_block.contains("authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;"),
             "notification target deletion should require SetBucketTargetAction"
         );
+        assert!(
+            delete_block.contains("notification_target_operation_block_reason(")
+                && delete_block.contains("\"managing notification targets from the console\""),
+            "notification target deletion should reject requests when the notify module is disabled"
+        );
     }
 
     #[test]

rustfs/src/admin/handlers/group.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use super::iam_error::iam_error_to_s3_error;
 use crate::{
     admin::{
         auth::validate_admin_request,
@@ -155,7 +156,7 @@ impl Operation for GetGroup {
 
         let g = iam_store.get_group_description(&query.group).await.map_err(|e| {
             warn!("get group failed, e: {:?}", e);
-            S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+            iam_error_to_s3_error(e)
         })?;
 
         let body = serde_json::to_vec(&g).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
@@ -222,7 +223,7 @@ impl Operation for DeleteGroup {
                 }
                 _ => {
                     if is_err_no_such_group(&e) {
-                        s3_error!(NoSuchKey, "group '{group}' does not exist")
+                        iam_error_to_s3_error(e)
                     } else {
                         s3_error!(InternalError, "{e}")
                     }
@@ -327,11 +328,11 @@ impl Operation for SetGroupStatus {
             match status {
                 "enabled" => iam_store.set_group_status(&query.group, true).await.map_err(|e| {
                     warn!("enable group failed, e: {:?}", e);
-                    S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+                    iam_error_to_s3_error(e)
                 })?,
                 "disabled" => iam_store.set_group_status(&query.group, false).await.map_err(|e| {
                     warn!("enable group failed, e: {:?}", e);
-                    S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+                    iam_error_to_s3_error(e)
                 })?,
                 _ => {
                     return Err(s3_error!(InvalidArgument, "invalid status"));
@@ -437,7 +438,7 @@ impl Operation for UpdateGroupMembers {
                 }
                 Err(e) => {
                     if !is_err_no_such_user(&e) {
-                        return Err(S3Error::with_message(S3ErrorCode::InternalError, e.to_string()));
+                        return Err(iam_error_to_s3_error(e));
                     }
                 }
             }
@@ -450,7 +451,7 @@ impl Operation for UpdateGroupMembers {
                 .await
                 .map_err(|e| {
                     warn!("remove group members failed, e: {:?}", e);
-                    S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+                    iam_error_to_s3_error(e)
                 })?
         } else {
             warn!("add group members");
@@ -467,7 +468,7 @@ impl Operation for UpdateGroupMembers {
                 .await
                 .map_err(|e| {
                     warn!("add group members failed, e: {:?}", e);
-                    S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
+                    iam_error_to_s3_error(e)
                 })?
         };
 

rustfs/src/admin/handlers/iam_error.rs

@@ -0,0 +1,62 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_iam::error::Error as IamError;
+use s3s::{S3Error, S3ErrorCode};
+
+pub(super) fn iam_error_to_s3_error(err: IamError) -> S3Error {
+    let code = match &err {
+        IamError::NoSuchUser(_)
+        | IamError::NoSuchAccount(_)
+        | IamError::NoSuchServiceAccount(_)
+        | IamError::NoSuchTempAccount(_)
+        | IamError::NoSuchGroup(_)
+        | IamError::NoSuchPolicy => S3ErrorCode::NoSuchResource,
+        _ => S3ErrorCode::InternalError,
+    };
+
+    let message = err.to_string();
+    let mut s3_error = S3Error::with_message(code, message);
+    s3_error.set_source(Box::new(err));
+    s3_error
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn iam_not_found_errors_map_to_not_found_status_class() {
+        let errors = [
+            IamError::NoSuchUser("user".to_string()),
+            IamError::NoSuchAccount("account".to_string()),
+            IamError::NoSuchServiceAccount("service".to_string()),
+            IamError::NoSuchTempAccount("temp".to_string()),
+            IamError::NoSuchGroup("group".to_string()),
+            IamError::NoSuchPolicy,
+        ];
+
+        for err in errors {
+            let s3_error = iam_error_to_s3_error(err);
+            assert_eq!(s3_error.code(), &S3ErrorCode::NoSuchResource);
+        }
+    }
+
+    #[test]
+    fn non_not_found_iam_errors_remain_internal_errors() {
+        let s3_error = iam_error_to_s3_error(IamError::IamSysNotInitialized);
+
+        assert_eq!(s3_error.code(), &S3ErrorCode::InternalError);
+    }
+}