Fix remaining review follow-ups for #2361 (#2421)

Signed-off-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: jeadie <jack@spice.ai>
Co-authored-by: Jack Eadie <jack.eadie0@gmail.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

.agents/skills/pr-creation-checker/SKILL.md

@@ -26,6 +26,8 @@ Use this skill before `gh pr create`, before `gh pr edit`, or when reviewing whe
 - Review the diff and summarize what changed.
 - Call out unrelated edits, generated artifacts, logs, or secrets as blockers.
 - Mark risky areas explicitly: auth, storage, config, network, migrations, breaking changes.
+- Scan the diff for newly added string literals and confirm whether they duplicate values already defined as constants/enums/typed wrappers in the same module or shared modules.
+- Treat introducing a new hardcoded literal where a project constant already exists as a likely regression risk; require either a refactor to reuse the constant or an explicit exception explanation in the PR body.
 
 3. Verify readiness requirements
 - Require `make pre-commit` before marking the PR ready.
@@ -82,6 +84,7 @@ Use this skill before `gh pr create`, before `gh pr edit`, or when reviewing whe
 - Return `BLOCKED` if required template sections are missing.
 - Return `BLOCKED` if the title/body is not in English.
 - Return `BLOCKED` if the title does not follow the repository's Conventional Commit rule.
+- Return `BLOCKED` if the diff introduces string literals that should use existing constants but did not.
 
 ## Reference
 

.agents/skills/pr-creation-checker/references/pr-readiness-checklist.md

@@ -12,3 +12,4 @@
 - Confirm non-applicable sections are filled with `N/A`.
 - Confirm the PR body does not include local absolute paths unless explicitly required.
 - Confirm multiline GitHub CLI commands use `--body-file`.
+- Confirm new hardcoded string literals were not introduced for values already represented by existing constants/enums (including protocol labels, error identifiers, headers, and metric names), or record a justified exception.

.cargo/config.toml

@@ -0,0 +1,26 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# RustFS Cargo configuration
+
+# Enable tokio_unstable cfg for dial9-tokio-telemetry support
+# This allows dial9 to hook into Tokio's internal runtime events
+[build]
+# Enable Tokio unstable features required by dial9-tokio-telemetry for runtime tracing.
+# See: https://docs.rs/tokio/latest/tokio/#unstable-features
+rustflags = ["--cfg", "tokio_unstable"]
+
+# Enable frame pointers for CPU profiling (Linux only, optional but recommended)
+# Uncomment the following line for better CPU profiling data
+# rustflags = ["--cfg", "tokio_unstable", "-C", "force-frame-pointers=yes"]

.docker/observability/docker-compose.yml

@@ -15,10 +15,11 @@
 services:
 
   # --- Tracing ---
-
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.10.3
     container_name: tempo
+    depends_on:
+      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -37,6 +38,38 @@ services:
       timeout: 5s
       retries: 3
       start_period: 15s
+  redpanda:
+    image: redpandadata/redpanda:latest
+    ports:
+      - "9092:9092" # Kafka API for clients
+    command: >
+      redpanda start --overprovisioned
+      --mode=dev-container
+      --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
+
+  redpanda-console:
+    image: docker.redpanda.com/redpandadata/console:latest
+    environment:
+      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
+    volumes:
+      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
+    ports:
+      - "8080:8080"
+    depends_on:
+      - redpanda
+
+  vulture:
+    image: grafana/tempo-vulture:latest
+    restart: always
+    command:
+      [
+        "-prometheus-listen-address=:8080",
+        "-tempo-query-url=http://tempo:3200",
+        "-tempo-push-url=http://tempo:4317",
+      ]
+    depends_on:
+      - tempo
 
   jaeger:
     image: jaegertracing/jaeger:latest

.docker/observability/grafana/dashboards/rustfs.json

@@ -91,7 +91,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_process_uptime_seconds{job=~\"$job\"}",
+          "expr": "rustfs_system_process_uptime_seconds{job=~\"$job\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -2473,7 +2473,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (process_cpu_usage_percent{job=~\"$job\"})",
+          "expr": "sum by (job) (rustfs_system_process_cpu_usage{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "A"
@@ -2572,7 +2572,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rustfs_process_memory_bytes{job=~\"$job\"})",
+          "expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "B"
@@ -2670,7 +2670,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"received\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
           "legendFormat": "RX - {{job}}",
           "range": true,
           "refId": "C"
@@ -2681,7 +2681,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
           "legendFormat": "TX - {{job}}",
           "range": true,
           "refId": "D"
@@ -3339,7 +3339,7 @@
                   "index": 0,
                   "text": "INACTIVE"
                 },
-                "to": 1e-9
+                "to": 1e-09
               },
               "type": "range"
             }
@@ -3905,6 +3905,879 @@
       ],
       "title": "Active File Size",
       "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 60
+      },
+      "id": 100,
+      "panels": [],
+      "title": "Performance Monitoring (S3 & Zero-Copy)",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 61
+      },
+      "id": 101,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_get_object_total{job=~\"$job\"}[5m])",
+          "legendFormat": "GetObject - {{tier}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_put_object_total{job=~\"$job\"}[5m])",
+          "legendFormat": "PutObject - {{zero_copy_enabled}}",
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Operations Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 100
+              },
+              {
+                "color": "red",
+                "value": 500
+              }
+            ]
+          },
+          "unit": "ms"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 61
+      },
+      "id": 102,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "histogram_quantile(0.95, rate(rustfs_s3_get_object_duration_ms_bucket{job=~\"$job\"}[5m]))",
+          "legendFormat": "GetObject P95",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "histogram_quantile(0.95, rate(rustfs_s3_put_object_duration_ms_bucket{job=~\"$job\"}[5m]))",
+          "legendFormat": "PutObject P95",
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Operation Latency (P95)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 69
+      },
+      "id": 103,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_get_object_size_bytes_sum{job=~\"$job\"}[5m]) / rate(rustfs_s3_get_object_size_bytes_count{job=~\"$job\"}[5m])",
+          "legendFormat": "GetObject Avg Size",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_put_object_size_bytes_sum{job=~\"$job\"}[5m]) / rate(rustfs_s3_put_object_size_bytes_count{job=~\"$job\"}[5m])",
+          "legendFormat": "PutObject Avg Size",
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Operation Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 69
+      },
+      "id": 104,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_zero_copy_memory_saved_bytes{job=~\"$job\"}",
+          "legendFormat": "Memory Saved ({{operation}})",
+          "refId": "A"
+        }
+      ],
+      "title": "Zero-Copy Memory Savings",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 77
+      },
+      "id": 105,
+      "options": {
+        "legend": {
+          "calcs": ["mean"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_hit_rate{job=~\"$job\",tier=\"small\"}",
+          "legendFormat": "Hit Rate (Small)",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_hit_rate{job=~\"$job\",tier=\"medium\"}",
+          "legendFormat": "Hit Rate (Medium)",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_hit_rate{job=~\"$job\",tier=\"large\"}",
+          "legendFormat": "Hit Rate (Large)",
+          "refId": "C"
+        }
+      ],
+      "title": "BytesPool Hit Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 77
+      },
+      "id": 106,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_allocated_bytes{job=~\"$job\",tier=\"small\"}",
+          "legendFormat": "Allocated (Small)",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_allocated_bytes{job=~\"$job\",tier=\"medium\"}",
+          "legendFormat": "Allocated (Medium)",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_allocated_bytes{job=~\"$job\",tier=\"large\"}",
+          "legendFormat": "Allocated (Large)",
+          "refId": "C"
+        }
+      ],
+      "title": "BytesPool Allocated Memory",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 85
+      },
+      "id": 107,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_io_buffer_size_bytes{job=~\"$job\"}",
+          "legendFormat": "Buffer Size ({{storage_media}})",
+          "refId": "A"
+        }
+      ],
+      "title": "I/O Buffer Size (Adaptive)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "MB/s"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 85
+      },
+      "id": 108,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_io_bandwidth_bytes_sum{job=~\"$job\"}[5m]) / 1024 / 1024",
+          "legendFormat": "Bandwidth",
+          "refId": "A"
+        }
+      ],
+      "title": "I/O Bandwidth",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 93
+      },
+      "id": 109,
+      "options": {
+        "legend": {
+          "calcs": ["mean"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_cache_hits_total{job=~\"$job\"}[5m]) / (rate(rustfs_cache_hits_total{job=~\"$job\"}[5m]) + rate(rustfs_cache_misses_total{job=~\"$job\"}[5m])) * 100",
+          "legendFormat": "Cache Hit Rate ({{cache}})",
+          "refId": "A"
+        }
+      ],
+      "title": "Cache Hit Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 93
+      },
+      "id": 110,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_cache_size_bytes{job=~\"$job\",cache=\"l1\"}",
+          "legendFormat": "L1 Cache Size",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_cache_size_bytes{job=~\"$job\",cache=\"l2\"}",
+          "legendFormat": "L2 Cache Size",
+          "refId": "B"
+        }
+      ],
+      "title": "Cache Size",
+      "type": "timeseries"
     }
   ],
   "preload": false,
@@ -4048,5 +4921,5 @@
   "timezone": "browser",
   "title": "RustFS",
   "uid": "rustfs-s3",
-  "version": 11
+  "version": 12
 }

.docker/observability/prometheus.yml

@@ -50,6 +50,11 @@ scrape_configs:
     static_configs:
       - targets: [ 'localhost:9090' ]
 
+  - job_name: 'vulture'
+    static_configs:
+      - targets:
+          - 'vulture:8080'
+
 otlp:
   promote_resource_attributes:
     - service.instance.id

.docker/observability/redpanda-console.yaml

@@ -0,0 +1,3 @@
+kafka:
+  brokers:
+    - redpanda:9092

.docker/observability/tempo.yaml

@@ -25,32 +25,18 @@ memberlist:
   join_members:
     - tempo:7946
 
-# Distributor configuration - receives traces and writes directly to ingesters
 distributor:
-  ingester_write_path_enabled: true
-  kafka_write_path_enabled: false
   receivers:
     otlp:
       protocols:
         grpc:
-          endpoint: "tempo:4317"
+          endpoint: "0.0.0.0:4317"
         http:
-          endpoint: "tempo:4318"
-  ring:
-    kvstore:
-      store: memberlist
-
-# Ingester configuration - consumes from Kafka and stores traces
-ingester:
-  lifecycler:
-    ring:
-      kvstore:
-        store: memberlist
-      replication_factor: 1
-    tokens_file_path: /var/tempo/tokens.json
-  trace_idle_period: 10s
-  max_block_bytes: 1_000_000
-  max_block_duration: 5m
+          endpoint: "0.0.0.0:4318"
+  #log_received_spans:
+  #    enabled: true
+  #  log_discarded_spans:
+  #    enabled: true
 
 backend_scheduler:
   provider:
@@ -67,8 +53,7 @@ backend_worker:
       store: memberlist
 
 querier:
-  frontend_worker:
-    frontend_address: tempo:3200
+  query_live_store: true
 
 metrics_generator:
   registry:
@@ -90,9 +75,9 @@ storage:
   trace:
     backend: local
     wal:
-      path: /var/tempo/wal
+      path: /var/tempo/wal # where to store the wal locally
     local:
-      path: /var/tempo/blocks
+      path: /var/tempo/blocks # where to store the traces locally
 
 overrides:
   defaults:
@@ -101,24 +86,14 @@ overrides:
       generate_native_histograms: both
 
 ingest:
-  enabled: false
-  # Disabled because using direct ingester write path
-  # If you want Kafka path, enable this and set:
-  # kafka:
-  #   brokers: [redpanda:9092]
-  #   topic: tempo-ingest
-  #   encoding: protobuf
-  #   consumer_group: tempo-ingest-consumer
+  enabled: true
+  kafka:
+    address: redpanda:9092
+    topic: tempo-ingest
 
 block_builder:
   consume_cycle_duration: 30s
 
-compactor:
-  compaction:
-    block_retention: 168h  # 7 days
-  ring:
-    kvstore:
-      store: memberlist
-
 usage_report:
   reporting_enabled: false
+

.github/cla.yml

@@ -0,0 +1,17 @@
+enabled: true
+
+document:
+  version: v1
+  url: https://github.com/rustfs/cla/blob/main/cla/v1.md
+
+signing:
+  mode: comment
+  comment_pattern: I have read and agree to the CLA.
+
+registry:
+  type: json-repo
+  repository: rustfs/cla
+  path_prefix: signatures
+
+status:
+  check_name: CLA Check

.github/dependabot.yml

@@ -26,11 +26,19 @@ updates:
       day: "monday"
       timezone: "Asia/Shanghai"
       time: "08:00"
+    assignees:
+      - "heihutu"
+    reviewers:
+      - "houseme"
+      - "overtrue"
+      - "majinghe"
     ignore:
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"
         versions: [ "0.23.x" ]
+      - dependency-name: "ratelimit"
+        versions: [ "1.x" ]
     groups:
       s3s:
         update-types:

.github/pull_request_template.md

@@ -34,4 +34,4 @@ Pull Request Template for RustFS
 
 ---
 
-Thank you for your contribution! Please ensure your PR follows the community standards ([CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)) and sign the CLA if this is your first contribution.
+Thank you for your contribution! Please ensure your PR follows the community standards ([CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)). If this is your first contribution, review the [CLA document](https://github.com/rustfs/cla/blob/main/cla/v1.md) and sign it by commenting `I have read and agree to the CLA.` on the PR.

.github/workflows/build.yml

@@ -204,7 +204,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     env:
-      RUSTFLAGS: ${{ matrix.rustflags }}
+      # Always enable Tokio unstable features (required by dial9-tokio-telemetry).
+      # The RUSTFLAGS env var takes precedence over .cargo/config.toml [build] rustflags,
+      # so we must include --cfg tokio_unstable here explicitly; otherwise an empty
+      # RUSTFLAGS value would shadow the config-file flag and silently break tracing.
+      RUSTFLAGS: "--cfg tokio_unstable ${{ matrix.rustflags }}"
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.prepare-platform-matrix.outputs.matrix) }}

.github/workflows/cla.yml

@@ -0,0 +1,48 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CLA Check
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+  issue_comment:
+    types: [created, edited]
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+  checks: write
+
+jobs:
+  cla:
+    if: ${{ github.event_name != 'issue_comment' || github.event.issue.pull_request }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create token for rustfs/cla
+        id: registry-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ vars.CLA_BOT_APP_ID }}
+          private-key: ${{ secrets.CLA_BOT_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: cla
+          permission-contents: write
+
+      - name: Run CLA Bot
+        uses: overtrue/cla-bot@v0.0.9
+        with:
+          github-token: ${{ github.token }}
+          registry-token: ${{ steps.registry-token.outputs.token }}

.gitignore

@@ -20,7 +20,8 @@ deploy/data/*
 *jsonl
 .env
 .rustfs.sys
-.cargo
+.cargo/
+!.cargo/config.toml
 profile.json
 .docker/openobserve-otel/data
 *.zst
@@ -39,7 +40,6 @@ artifacts/
 *.audit
 *.snappy
 PR_DESCRIPTION.md
-IMPLEMENTATION_PLAN.md
 scripts/s3-tests/selected_tests.txt
 docs
 
@@ -48,4 +48,8 @@ result*
 *.gz
 rustfs-webdav.code-workspace
 
-.aiexclude
+.aiexclude
+*.bak
+# Local test/benchmark artifacts
+benchmarks.logs
+tmp/

.vscode/launch.json

@@ -197,19 +197,27 @@
                 // "__RUSTFS_SSE_SIMPLE_CMK": "2dfNXGHlsEflGVCxb+5DIdGEl1sIvtwX+QfmYasi5QM=",
 
                 // 2. kms local backend test key
-                "RUSTFS_KMS_ENABLE": "true",
-                "RUSTFS_KMS_BACKEND": "local",
-                "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
-                "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
-                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+                // "RUSTFS_KMS_ENABLE": "true",
+                // "RUSTFS_KMS_BACKEND": "local",
+                // "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
+                // "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
+                // "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
 
                 // 3. kms vault backend test key
                 // "RUSTFS_KMS_ENABLE": "true",
-                // "RUSTFS_KMS_BACKEND": "vault",
+                // "RUSTFS_KMS_BACKEND": "vault-kv2",
                 // "RUSTFS_KMS_VAULT_ADDRESS": "http://127.0.0.1:8200",
                 // "RUSTFS_KMS_VAULT_TOKEN": "Dev Token",
                 // "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
 
+                // 4. kms vault transit backend test key
+                "RUSTFS_KMS_ENABLE": "true",
+                "RUSTFS_KMS_BACKEND": "vault-transit",
+                "RUSTFS_KMS_VAULT_ADDRESS": "http://127.0.0.1:8200",
+                "RUSTFS_KMS_VAULT_TOKEN": "Dev Token",
+                "RUSTFS_KMS_VAULT_MOUNT_PATH": "transit",
+                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+
             },
             "sourceLanguages": [
                 "rust"

AGENTS.md

@@ -16,6 +16,22 @@ If repo-level instructions conflict, follow the nearest file and keep behavior a
 - Respond in the same language used by the requester.
 - Keep source code, comments, commit messages, and PR title/body in English.
 
+## Change Style for Existing Logic
+
+- Prefer direct, local code over extracting one-off helpers.
+- Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow.
+- Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths.
+- Do not refactor existing code only to make it easier to unit test.
+- Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths.
+- Keep code elegant, concise, and direct. Prefer minimal, readable implementations over over-engineering and excessive abstraction. Use comments to clarify non-obvious intent and invariants, not to compensate for unclear code.
+
+## Constant and String Usage
+
+- Before introducing new string literals, search for existing constants/enums that already represent the same semantic value.
+- Reuse existing constants for protocol labels, error identifiers, header keys, event names, metric names, command tags, and similar fixed tokens.
+- If a new string is truly unique, define a local constant near related logic and avoid scattering the literal across multiple sites.
+- When changing existing behavior, keep naming and format consistency by aligning with established project constants.
+
 ## Sources of Truth
 
 - Workspace layout and crate membership: `Cargo.toml` (`[workspace].members`)
@@ -26,16 +42,23 @@ If repo-level instructions conflict, follow the nearest file and keep behavior a
 Avoid duplicating long crate lists or command matrices in instruction files.
 Reference the source files above instead.
 
-## Mandatory Before Commit
+## Verification Before PR
 
-Run and pass:
+For code changes, run and pass the following before opening a PR:
 
 ```bash
 make pre-commit
 ```
 
+Before pushing code changes, make sure formatting is clean:
+
+- Run `cargo fmt --all`.
+- Run `cargo fmt --all --check` and ensure no files are modified unexpectedly.
+
 If `make` is unavailable, run the equivalent checks defined under `.config/make/`.
-Do not commit when required checks fail.
+Documentation-only or instruction-only changes are exempt from the verification commands above (including the `.config/make/` equivalents), though any installed git pre-commit hooks (for example, from `make setup-hooks`) may still run on commit unless explicitly skipped.
+After build-based verification completes, clean generated build artifacts before wrapping up to avoid unnecessary disk usage.
+Do not open a PR with code changes when the required checks fail.
 
 ## Git and PR Baseline
 
@@ -46,6 +69,11 @@ Do not commit when required checks fail.
 - Use `N/A` for non-applicable template sections.
 - Include verification commands in the PR description.
 - When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
+- After fixing code review comments or CI findings, always mark corresponding review
+  comments/threads as resolved before returning to the user.
+- In handling review comments, confirm the underlying issue before changing code.
+  If a suggested change is not appropriate for behavior or risk, reply with a
+  concise rationale instead of blindly applying it.
 
 ## Security Baseline
 

Cargo.toml

@@ -39,6 +39,7 @@ members = [
     "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
     "crates/protos", # Protocol buffer definitions
     "crates/rio", # Rust I/O utilities and abstractions
+    "crates/concurrency", # Rust I/O utilities and abstractions
     "crates/s3-common", # Common utilities and data structures for S3 compatibility
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -48,6 +49,9 @@ members = [
     "crates/trusted-proxies", # Trusted proxies management
     "crates/utils", # Utility functions and helpers
     "crates/workers", # Worker thread pools and task scheduling
+    "crates/io-metrics", # Zero-copy metrics collection for performance analysis
+    "crates/io-core", # Zero-copy core reader and writer implementations
+    "crates/object-io", # Object I/O policy and zero-copy helper primitives
     "crates/zip", # ZIP file handling and compression
 ]
 resolver = "3"
@@ -79,6 +83,7 @@ rustfs-audit = { path = "crates/audit", version = "0.0.5" }
 rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
 rustfs-common = { path = "crates/common", version = "0.0.5" }
 rustfs-config = { path = "./crates/config", version = "0.0.5" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "0.0.5" }
 rustfs-credentials = { path = "crates/credentials", version = "0.0.5" }
 rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
 rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
@@ -91,6 +96,9 @@ rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
 rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
 rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
 rustfs-notify = { path = "crates/notify", version = "0.0.5" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
+rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
+rustfs-object-io = { path = "crates/object-io", version = "0.0.5" }
 rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
 rustfs-protos = { path = "crates/protos", version = "0.0.5" }
@@ -117,7 +125,7 @@ futures = "0.3.32"
 futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
-hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
+hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 http = "1.4.0"
@@ -125,7 +133,7 @@ http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 socket2 = { version = "0.6.3", features = ["all"] }
-tokio = { version = "1.50.0", features = ["fs", "rt-multi-thread"] }
+tokio = { version = "1.51.0", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
@@ -144,31 +152,29 @@ flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
 quick-xml = "0.39.2"
-rmcp = { version = "1.2.0" }
+rmcp = { version = "1.3.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_bytes = "0.11"
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
-argon2 = { version = "0.6.0-rc.7" }
+argon2 = { version = "0.6.0-rc.8" }
 blake2 = "0.11.0-rc.5"
-blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
-hmac = { version = "0.13.0-rc.5" }
+hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
-pbkdf2 = "0.13.0-rc.9"
+pbkdf2 = "0.13.0-rc.10"
 rsa = { version = "0.10.0-rc.17" }
 rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = "1.14.0"
-sha1 = "0.11.0-rc.5"
-sha2 = "0.11.0-rc.5"
+sha1 = "0.11.0"
+sha2 = "0.11.0"
 subtle = "2.6"
 zeroize = { version = "1.8.2", features = ["derive"] }
 
@@ -180,15 +186,15 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros
 
 # Utilities and Tools
 anyhow = "1.0.102"
-arc-swap = "1.8.2"
+arc-swap = "1.9.1"
 astral-tokio-tar = "0.6.0"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.8.15" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.126.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.128.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
-aws-smithy-types = { version = "1.4.6" }
+aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
 base64 = "0.22.1"
 base64-simd = "0.8.0"
@@ -202,33 +208,34 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "52.3.0"
+datafusion = "53.0.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
 glob = "0.3.3"
-google-cloud-storage = "1.9.0"
-google-cloud-auth = "1.7.0"
+google-cloud-storage = "1.10.0"
+google-cloud-auth = "1.8.0"
 hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
 hex = "0.4.3"
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.183"
+libc = "0.2.184"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.10"
+local-ip-address = "0.6.11"
+memmap2 = "0.9.10"
 lz4 = "1.28.1"
 matchit = "0.9.1"
-md-5 = "0.11.0-rc.5"
+md-5 = "0.11.0"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
-moka = { version = "0.12.14", features = ["future"] }
+moka = { version = "0.12.15", features = ["future"] }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.12.0"
-object_store = "0.12.5"
+nvml-wrapper = "0.12.1"
+object_store = "0.13.2"
 parking_lot = "0.12.5"
 path-absolutize = "3.1.1"
 path-clean = "1.0.1"
@@ -236,16 +243,16 @@ percent-encoding = "2.3.2"
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
 rand = { version = "0.10.0", features = ["serde"] }
-ratelimit = "0.10.0"
+ratelimit = "0.10.1"
 rayon = "1.11.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
-rumqttc = { version = "0.25.1" }
+rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
-rustc-hash = { version = "2.1.1" }
-s3s = { git = "https://github.com/rustfs/s3s", rev = "d9556e3c0036bd3f2b330966009cbaa5aebf19a3", features = ["minio"] }
+rustc-hash = { version = "2.1.2" }
+s3s = { git = "https://github.com/rustfs/s3s", rev = "010ae03bdd8c93a9c046108190987bb0e6cb0692", features = ["minio"] }
 serial_test = "3.4.0"
 shadow-rs = { version = "1.7.1", default-features = false }
 siphasher = "1.0.2"
@@ -268,20 +275,21 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.22.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
-vaultrs = { version = "0.7.4" }
+uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "8.2.0"
+zip = "8.5.0"
 zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
+dial9-tokio-telemetry = { version = "0.2", git = "https://github.com/dial9-rs/dial9-tokio-telemetry.git", rev = "60502082601b647c4a51962595721f631b7bbce1" }
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
-opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rustls"] }
+opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }

Dockerfile

@@ -72,7 +72,8 @@ LABEL name="RustFS" \
       url="https://rustfs.com" \
       license="Apache-2.0"
 
-RUN apk add --no-cache ca-certificates coreutils curl
+RUN apk update && \
+    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

README.md

@@ -195,6 +195,19 @@ nix build
 nix run
 ```
 
+### 6\. X-CMD (Option 6)
+
+If you are an [x-cmd](https://www.x-cmd.com/install/rustfs) user:
+
+```bash
+# Run directly without installing
+x rustfs
+
+# Download the binary and install it to the global environment
+x env use rustfs
+rustfs --help
+```
+
 ---
 
 ### Accessing RustFS

README_ZH.md

@@ -163,6 +163,36 @@ make help-docker                      # 显示所有 Docker 相关命令
 
 请按照 [Helm Chart README](https://charts.rustfs.com) 上的说明在 Kubernetes 集群上安装 RustFS。
 
+### 5\. Nix Flake (Option 5)
+
+如果你已经 启用了 [Nix Flakes 功能](https://nixos.wiki/wiki/Flakes#Enable_flakes):
+
+```bash
+# 直接运行，无需安装
+nix run github:rustfs/rustfs
+
+# 编译二进制文件
+nix build github:rustfs/rustfs
+./result/bin/rustfs --help
+
+# 或者从本地检出的代码库运行/编译
+nix build
+nix run
+```
+
+### 6\. X-CMD (Option 6)
+
+如果你是 [x-cmd](https://www.x-cmd.com/install/rustfs) 用户：
+
+```bash
+# 直接运行，无需安装
+x rustfs
+
+# 下载二进制文件并安装到全局环境中
+x env use rustfs
+rustfs --help
+```
+
 ---
 
 ### 访问 RustFS

_typos.toml

@@ -39,6 +39,7 @@ abd = "abd"
 mak = "mak"
 gae = "gae"
 GAE = "GAE"
+writeable = "writeable"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
 consts = "consts"

build-rustfs.sh

@@ -213,7 +213,7 @@ setup_rust_environment() {
     # Set up environment variables for musl targets
     if [[ "$PLATFORM" == *"musl"* ]]; then
         print_message $YELLOW "Setting up environment for musl target..."
-        export RUSTFLAGS="-C target-feature=-crt-static"
+        export RUSTFLAGS="'--cfg tokio_unstable -C target-feature=-crt-static'"
 
         # For cargo-zigbuild, set up additional environment variables
         if command -v cargo-zigbuild &> /dev/null; then
@@ -430,7 +430,7 @@ build_binary() {
         fi
     else
         # Native compilation
-        build_cmd="RUSTFLAGS=-Clink-arg=-lm cargo build"
+        build_cmd="RUSTFLAGS='--cfg tokio_unstable -Clink-arg=-lm' cargo build"
     fi
 
     if [ "$BUILD_TYPE" = "release" ]; then

crates/appauth/Cargo.toml

@@ -33,3 +33,6 @@ rand.workspace = true
 
 [lints]
 workspace = true
+
+[lib]
+doctest = false

crates/audit/Cargo.toml

@@ -44,6 +44,12 @@ tracing = { workspace = true, features = ["std", "attributes"] }
 url = { workspace = true }
 rumqttc = { workspace = true }
 
+[dev-dependencies]
+temp-env = { workspace = true }
 
 [lints]
 workspace = true
+
+[lib]
+test = false
+doctest = false

crates/audit/src/factory.rs

@@ -19,7 +19,8 @@ use rumqttc::QoS;
 use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
 use rustfs_config::{
     AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
+    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
+    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
     WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
     WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
@@ -27,7 +28,10 @@ use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
     error::TargetError,
-    target::{mqtt::MQTTArgs, webhook::WebhookArgs},
+    target::{
+        mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
+        webhook::WebhookArgs,
+    },
 };
 use std::time::Duration;
 use tracing::{debug, warn};
@@ -166,6 +170,14 @@ impl TargetFactory for MQTTTargetFactory {
                 .and_then(|v| v.parse::<u64>().ok())
                 .map(Duration::from_secs)
                 .unwrap_or_else(|| Duration::from_secs(30)),
+            tls: MQTTTlsConfig::from_values(
+                config.lookup(MQTT_TLS_POLICY).as_deref(),
+                config.lookup(MQTT_TLS_CA).as_deref(),
+                config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
+                config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
+                config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
+                config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
+            )?,
             queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
             queue_limit: config
                 .lookup(MQTT_QUEUE_LIMIT)
@@ -185,12 +197,15 @@ impl TargetFactory for MQTTTargetFactory {
         let url = Url::parse(&broker)
             .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
 
-        match url.scheme() {
-            "tcp" | "ssl" | "ws" | "wss" | "mqtt" | "mqtts" => {}
-            _ => {
-                return Err(TargetError::Configuration("Unsupported broker URL scheme".to_string()));
-            }
-        }
+        let tls = MQTTTlsConfig::from_values(
+            config.lookup(MQTT_TLS_POLICY).as_deref(),
+            config.lookup(MQTT_TLS_CA).as_deref(),
+            config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
+            config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
+            config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
+            config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
+        )?;
+        validate_mqtt_broker_url(&url, &tls)?;
 
         if config.lookup(MQTT_TOPIC).is_none() {
             return Err(TargetError::Configuration("Missing MQTT topic".to_string()));

crates/audit/src/registry.rs

@@ -109,9 +109,6 @@ impl AuditRegistry {
         let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
         // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // let final_config = config.clone(); // Clone a configuration for aggregating the final result
-        // Record the defaults for each segment so that the segment can eventually be rebuilt
-        let mut section_defaults: HashMap<String, KVS> = HashMap::new();
         // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
@@ -125,9 +122,6 @@ impl AuditRegistry {
             let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
             debug!(?default_cfg, "Get the default configuration");
 
-            // Save defaults for eventual write back
-            section_defaults.insert(section_name.clone(), default_cfg.clone());
-
             // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
             debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -235,97 +229,28 @@ impl AuditRegistry {
                 if enabled {
                     info!(instance_id = %id, "Target is enabled, ready to create a task");
                     // 5.3. Create asynchronous tasks for enabled instances
-                    let target_type_clone = target_type.clone();
                     let tid = id.clone();
                     let merged_config_arc = Arc::new(merged_config);
                     tasks.push(async move {
                         let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (target_type_clone, tid, result, Arc::clone(&merged_config_arc))
+                        (tid, result)
                     });
                 } else {
-                    info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
-                    // Remove disabled target from final configuration
-                    // final_config.0.entry(section_name.clone()).or_default().remove(&id);
+                    info!(instance_id = %id, "Skip disabled target");
                 }
             }
         }
 
         // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
-        let mut successful_configs = Vec::new();
-        while let Some((target_type, id, result, final_config)) = tasks.next().await {
+        while let Some((id, result)) = tasks.next().await {
             match result {
                 Ok(target) => {
-                    info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
+                    info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully");
                     successful_targets.push(target);
-                    successful_configs.push((target_type, id, final_config));
                 }
                 Err(e) => {
-                    error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
-                }
-            }
-        }
-
-        // 7. Aggregate new configuration and write back to system configuration
-        if !successful_configs.is_empty() || !section_defaults.is_empty() {
-            info!(
-                "Prepare to update {} successfully created target configurations to the system configuration...",
-                successful_configs.len()
-            );
-
-            let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
-
-            for (target_type, id, kvs) in successful_configs {
-                let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
-                successes_by_section
-                    .entry(section_name)
-                    .or_default()
-                    .insert(id.to_lowercase(), (*kvs).clone());
-            }
-
-            let mut new_config = config.clone();
-            // Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
-            let mut sections: HashSet<String> = HashSet::new();
-            sections.extend(section_defaults.keys().cloned());
-            sections.extend(successes_by_section.keys().cloned());
-
-            for section in sections {
-                let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
-                // Add default item
-                if let Some(default_kvs) = section_defaults.get(&section)
-                    && !default_kvs.is_empty()
-                {
-                    section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
-                }
-
-                // Add successful instance item
-                if let Some(instances) = successes_by_section.get(&section) {
-                    for (id, kvs) in instances {
-                        section_map.insert(id.clone(), kvs.clone());
-                    }
-                }
-
-                // Empty breaks are removed and non-empty breaks are replaced entirely.
-                if section_map.is_empty() {
-                    new_config.0.remove(&section);
-                } else {
-                    new_config.0.insert(section, section_map);
-                }
-            }
-
-            let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
-                return Err(AuditError::StorageNotAvailable(
-                    "Failed to save target configuration: server storage not initialized".to_string(),
-                ));
-            };
-
-            match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
-                Ok(_) => {
-                    info!("The new configuration was saved to the system successfully.")
-                }
-                Err(e) => {
-                    error!("Failed to save the new configuration: {}", e);
-                    return Err(AuditError::SaveConfig(Box::new(e)));
+                    error!(instance_id = %id, error = %e, "Failed to create a target");
                 }
             }
         }
@@ -365,6 +290,11 @@ impl AuditRegistry {
         self.targets.get(id).map(|t| t.as_ref())
     }
 
+    /// Lists cloned target values for runtime inspection without exposing mutable registry access.
+    pub fn list_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
+        self.targets.values().map(|target| target.clone_dyn()).collect()
+    }
+
     /// Lists all target IDs
     ///
     /// # Returns

crates/audit/src/system.rs

@@ -13,14 +13,15 @@
 //  limitations under the License.
 
 use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
+use hashbrown::HashMap;
 use rustfs_ecstore::config::Config;
 use rustfs_targets::{
     StoreError, Target, TargetError,
     store::{Key, Store},
-    target::EntityTarget,
+    target::{EntityTarget, QueuedPayload},
 };
 use std::sync::Arc;
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::{Mutex, RwLock, mpsc};
 use tracing::{error, info, warn};
 
 /// State of the audit system
@@ -39,6 +40,8 @@ pub struct AuditSystem {
     registry: Arc<Mutex<AuditRegistry>>,
     state: Arc<RwLock<AuditSystemState>>,
     config: Arc<RwLock<Option<Config>>>,
+    /// Cancellation senders for active audit stream tasks (target_id -> cancel tx)
+    stream_cancellers: Arc<RwLock<HashMap<String, mpsc::Sender<()>>>>,
 }
 
 impl Default for AuditSystem {
@@ -54,6 +57,7 @@ impl AuditSystem {
             registry: Arc::new(Mutex::new(AuditRegistry::new())),
             state: Arc::new(RwLock::new(AuditSystemState::Stopped)),
             config: Arc::new(RwLock::new(None)),
+            stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
         }
     }
 
@@ -110,27 +114,7 @@ impl AuditSystem {
 
                 // Initialize all targets
                 for target in targets {
-                    let target_id = target.id().to_string();
-                    if let Err(e) = target.init().await {
-                        error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
-                    } else {
-                        // After successful initialization, if enabled and there is a store, start the send from storage task
-                        if target.is_enabled() {
-                            if let Some(store) = target.store() {
-                                info!(target_id = %target_id, "Start audit stream processing for target");
-                                let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
-                                    store.boxed_clone();
-                                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
-                                self.start_audit_stream_with_batching(store_clone, target_arc);
-                                info!(target_id = %target_id, "Audit stream processing started");
-                            } else {
-                                info!(target_id = %target_id, "No store configured, skip audit stream processing");
-                            }
-                        } else {
-                            info!(target_id = %target_id, "Target disabled, skip audit stream processing");
-                        }
-                        registry.add_target(target_id, target);
-                    }
+                    self.init_and_register_target(target, &mut registry).await;
                 }
 
                 // Update state to running
@@ -214,6 +198,9 @@ impl AuditSystem {
 
         info!("Stopping audit system");
 
+        // Stop all stream tasks first
+        self.stop_all_streams().await;
+
         // Close all targets
         let mut registry = self.registry.lock().await;
         if let Err(e) = registry.close_all().await {
@@ -258,52 +245,49 @@ impl AuditSystem {
         let state = self.state.read().await;
 
         match *state {
-            AuditSystemState::Running => {
-                // Continue with dispatch
-                info!("Dispatching audit log entry");
-            }
+            AuditSystemState::Running => {}
             AuditSystemState::Paused => {
-                // Skip dispatch when paused
                 return Ok(());
             }
             _ => {
-                // Don't dispatch when not running
                 return Err(AuditError::NotInitialized("Audit system is not running".to_string()));
             }
         }
         drop(state);
 
-        let registry = self.registry.lock().await;
-        let target_keys = registry.list_targets();
+        // Collect cloned targets under lock, then dispatch without holding it
+        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
+            let registry = self.registry.lock().await;
+            let target_keys = registry.list_targets();
 
-        if target_keys.is_empty() {
-            warn!("No audit targets configured for dispatch");
-            return Ok(());
-        }
+            if target_keys.is_empty() {
+                warn!("No audit targets configured for dispatch");
+                return Ok(());
+            }
 
-        // Dispatch to all targets concurrently
+            target_keys
+                .into_iter()
+                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
+                .collect()
+        };
+
+        // Dispatch to all targets concurrently (no lock held)
         let mut tasks = Vec::new();
 
-        for target_key in target_keys {
-            if let Some(target) = registry.get_target(&target_key) {
-                let entry_clone = Arc::clone(&entry);
-                let target_key_clone = target_key.clone();
+        for (target_key, target) in targets {
+            let entity_target = EntityTarget {
+                object_name: entry.api.name.clone().unwrap_or_default(),
+                bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                event_name: entry.event,
+                data: (*entry).clone(),
+            };
 
-                // Create EntityTarget for the audit log entry
-                let entity_target = EntityTarget {
-                    object_name: entry.api.name.clone().unwrap_or_default(),
-                    bucket_name: entry.api.bucket.clone().unwrap_or_default(),
-                    event_name: entry.event, // Default, should be derived from entry
-                    data: (*entry_clone).clone(),
-                };
+            let task = async move {
+                let result = target.save(Arc::new(entity_target)).await;
+                (target_key, result)
+            };
 
-                let task = async move {
-                    let result = target.save(Arc::new(entity_target)).await;
-                    (target_key_clone, result)
-                };
-
-                tasks.push(task);
-            }
+            tasks.push(task);
         }
 
         // Execute all dispatch tasks
@@ -359,39 +343,45 @@ impl AuditSystem {
         }
         drop(state);
 
-        let registry = self.registry.lock().await;
-        let target_keys = registry.list_targets();
+        // Collect targets under lock, then dispatch without holding it
+        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
+            let registry = self.registry.lock().await;
+            let target_keys = registry.list_targets();
 
-        if target_keys.is_empty() {
-            warn!("No audit targets configured for batch dispatch");
-            return Ok(());
-        }
+            if target_keys.is_empty() {
+                warn!("No audit targets configured for batch dispatch");
+                return Ok(());
+            }
+
+            target_keys
+                .into_iter()
+                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
+                .collect()
+        };
 
         let mut tasks = Vec::new();
-        for target_key in target_keys {
-            if let Some(target) = registry.get_target(&target_key) {
-                let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
-                let target_key_clone = target_key.clone();
+        for (target_key, target) in targets {
+            let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
+            let target_key_clone = target_key.clone();
 
-                let task = async move {
-                    let mut success_count = 0;
-                    let mut errors = Vec::new();
-                    for entry in entries_clone {
-                        let entity_target = EntityTarget {
-                            object_name: entry.api.name.clone().unwrap_or_default(),
-                            bucket_name: entry.api.bucket.clone().unwrap_or_default(),
-                            event_name: entry.event,
-                            data: (*entry).clone(),
-                        };
-                        match target.save(Arc::new(entity_target)).await {
-                            Ok(_) => success_count += 1,
-                            Err(e) => errors.push(e),
-                        }
+            let task = async move {
+                let mut success_count = 0;
+                let mut errors = Vec::new();
+                for entry in entries_clone {
+                    let entity_target = EntityTarget {
+                        object_name: entry.api.name.clone().unwrap_or_default(),
+                        bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                        event_name: entry.event,
+                        data: (*entry).clone(),
+                    };
+                    match target.save(Arc::new(entity_target)).await {
+                        Ok(_) => success_count += 1,
+                        Err(e) => errors.push(e),
                     }
-                    (target_key_clone, success_count, errors)
-                };
-                tasks.push(task);
-            }
+                }
+                (target_key_clone, success_count, errors)
+            };
+            tasks.push(task);
         }
 
         let results = futures::future::join_all(tasks).await;
@@ -417,6 +407,60 @@ impl AuditSystem {
         Ok(())
     }
 
+    /// Stops all active audit stream tasks by sending cancellation signals.
+    async fn stop_all_streams(&self) {
+        let mut cancellers = self.stream_cancellers.write().await;
+        for (target_id, cancel_tx) in cancellers.drain() {
+            info!(target_id = %target_id, "Stopping audit stream");
+            let _ = cancel_tx.send(()).await;
+        }
+    }
+
+    /// Initializes a single target: runs init(), starts stream if store is present,
+    /// and adds it to the registry. For store-backed targets, registration and stream
+    /// startup proceed even if init() fails so queued entries can be drained later.
+    async fn init_and_register_target(
+        &self,
+        target: Box<dyn Target<AuditEntry> + Send + Sync>,
+        registry: &mut AuditRegistry,
+    ) -> Option<String> {
+        let target_id = target.id().to_string();
+        let has_store = target.store().is_some();
+
+        if let Err(e) = target.init().await {
+            error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
+            // Non-store targets: init failure is fatal.
+            if !has_store {
+                return None;
+            }
+            // Store-backed targets: still register and start the stream so queued
+            // entries can be drained when connectivity recovers.
+            warn!(
+                target_id = %target_id,
+                "Proceeding with store-backed audit target despite init failure"
+            );
+        }
+
+        if target.is_enabled() {
+            if let Some(store) = target.store() {
+                info!(target_id = %target_id, "Start audit stream processing for target");
+                let store_clone: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send> = store.boxed_clone();
+                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
+                let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc);
+
+                self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx);
+                info!(target_id = %target_id, "Audit stream processing started");
+            } else {
+                info!(target_id = %target_id, "No store configured, skip audit stream processing");
+            }
+        } else {
+            info!(target_id = %target_id, "Target disabled, skip audit stream processing");
+        }
+
+        registry.add_target(target_id.clone(), target);
+        Some(target_id)
+    }
+
     /// Starts the audit stream processing for a target with batching and retry logic
     ///
     /// # Arguments
@@ -427,9 +471,10 @@ impl AuditSystem {
     /// and attempts to send them to the specified target. It implements retry logic with exponential backoff
     fn start_audit_stream_with_batching(
         &self,
-        store: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send>,
+        store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
         target: Arc<dyn Target<AuditEntry> + Send + Sync>,
-    ) {
+    ) -> mpsc::Sender<()> {
+        let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
         let state = self.state.clone();
 
         tokio::spawn(async move {
@@ -442,6 +487,12 @@ impl AuditSystem {
             const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
 
             loop {
+                // Check for cancellation signal
+                if cancel_rx.try_recv().is_ok() {
+                    info!("Audit stream cancelled for target: {}", target.id());
+                    break;
+                }
+
                 match *state.read().await {
                     AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {}
                     _ => {
@@ -452,11 +503,22 @@ impl AuditSystem {
 
                 let keys: Vec<Key> = store.list();
                 if keys.is_empty() {
-                    sleep(Duration::from_millis(500)).await;
+                    tokio::select! {
+                        _ = sleep(Duration::from_millis(500)) => {},
+                        _ = cancel_rx.recv() => {
+                            info!("Audit stream cancelled during idle for target: {}", target.id());
+                            return;
+                        }
+                    }
                     continue;
                 }
 
                 for key in keys {
+                    if cancel_rx.try_recv().is_ok() {
+                        info!("Audit stream cancelled during processing for target: {}", target.id());
+                        return;
+                    }
+
                     let mut retries = 0usize;
                     let mut success = false;
 
@@ -497,6 +559,8 @@ impl AuditSystem {
                 sleep(Duration::from_millis(100)).await;
             }
         });
+
+        cancel_tx
     }
 
     /// Enables a specific target
@@ -594,6 +658,12 @@ impl AuditSystem {
         registry.list_targets()
     }
 
+    /// Returns cloned target values for read-only runtime inspection.
+    pub async fn get_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
+        let registry = self.registry.lock().await;
+        registry.list_target_values()
+    }
+
     /// Gets information about a specific target
     ///
     /// # Arguments
@@ -616,9 +686,11 @@ impl AuditSystem {
     pub async fn reload_config(&self, new_config: Config) -> AuditResult<()> {
         info!("Reloading audit system configuration");
 
-        // Record config reload
         observability::record_config_reload();
 
+        // Stop all existing stream tasks first
+        self.stop_all_streams().await;
+
         // Store new configuration
         {
             let mut config_guard = self.config.write().await;
@@ -636,28 +708,9 @@ impl AuditSystem {
             Ok(targets) => {
                 info!(target_count = targets.len(), "Reloaded audit targets successfully");
 
-                // Initialize all new targets
                 for target in targets {
-                    let target_id = target.id().to_string();
-                    if let Err(e) = target.init().await {
-                        error!(target_id = %target_id, error = %e, "Failed to initialize reloaded audit target");
-                    } else {
-                        // Same starts the storage stream after a heavy load
-                        if target.is_enabled() {
-                            if let Some(store) = target.store() {
-                                info!(target_id = %target_id, "Start audit stream processing for target (reload)");
-                                let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
-                                    store.boxed_clone();
-                                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
-                                self.start_audit_stream_with_batching(store_clone, target_arc);
-                                info!(target_id = %target_id, "Audit stream processing started (reload)");
-                            } else {
-                                info!(target_id = %target_id, "No store configured, skip audit stream processing (reload)");
-                            }
-                        } else {
-                            info!(target_id = %target_id, "Target disabled, skip audit stream processing (reload)");
-                        }
-                        registry.add_target(target.id().to_string(), target);
+                    if let Some(target_id) = self.init_and_register_target(target, &mut registry).await {
+                        info!(target_id = %target_id, "Target initialized (reload)");
                     }
                 }
 

crates/audit/tests/config_parsing_test.rs

@@ -53,6 +53,12 @@ fn test_mqtt_valid_fields() {
         "reconnect_interval",
         "queue_dir",
         "queue_limit",
+        "tls_policy",
+        "tls_ca",
+        "tls_client_cert",
+        "tls_client_key",
+        "tls_trust_leaf_as_ca",
+        "ws_path_allowlist",
     ];
 
     // This tests the MQTT configuration fields we support

crates/audit/tests/integration_test.rs

@@ -15,6 +15,7 @@
 use rustfs_audit::*;
 use rustfs_ecstore::config::{Config, KVS};
 use std::collections::HashMap;
+use temp_env::with_vars;
 
 #[tokio::test]
 async fn test_audit_system_creation() {
@@ -35,34 +36,42 @@ async fn test_config_parsing_webhook() {
     let mut config = Config(HashMap::new());
     let mut audit_webhook_section = HashMap::new();
 
-    // Create default configuration
     let mut default_kvs = KVS::new();
-    default_kvs.insert("enable".to_string(), "on".to_string());
-    default_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
-
+    default_kvs.insert("enable".to_string(), "off".to_string());
+    default_kvs.insert("endpoint".to_string(), "".to_string());
     audit_webhook_section.insert("_".to_string(), default_kvs);
+    let mut instance_kvs = KVS::new();
+    instance_kvs.insert("enable".to_string(), "on".to_string());
+    instance_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
+    audit_webhook_section.insert("primary".to_string(), instance_kvs);
     config.0.insert("audit_webhook".to_string(), audit_webhook_section);
 
     let registry = AuditRegistry::new();
 
-    // This should not fail even if server storage is not initialized
-    // as it's an integration test
     let result = registry.create_audit_targets_from_config(&config).await;
+    assert!(result.is_ok(), "audit target creation should not require server storage");
+}
 
-    // We expect this to fail due to server storage not being initialized
-    // but the parsing should work correctly
-    match result {
-        Err(AuditError::StorageNotAvailable(_)) => {
-            // This is expected in test environment
-        }
-        Err(e) => {
-            // Other errors might indicate parsing issues
-            println!("Unexpected error: {e}");
-        }
-        Ok(_) => {
-            // Unexpected success in test environment without server storage
-        }
-    }
+#[test]
+fn test_env_only_audit_target_does_not_require_server_storage() {
+    with_vars(
+        [
+            ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")),
+            ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("http://localhost:3020/webhook")),
+        ],
+        || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("failed to create tokio runtime");
+            runtime.block_on(async {
+                let config = Config(HashMap::new());
+                let registry = AuditRegistry::new();
+                let result = registry.create_audit_targets_from_config(&config).await;
+                assert!(result.is_ok(), "env-only audit target creation should not require server storage");
+            });
+        },
+    )
 }
 
 #[test]

crates/audit/tests/performance_test.rs

@@ -159,7 +159,8 @@ async fn test_audit_log_dispatch_performance() {
 
     let start = Instant::now();
 
-    // Dispatch audit log (should be fast since no targets are configured)
+    // Dispatch audit log against an unstarted system state. Empty config keeps
+    // the audit system stopped, so dispatch should fail fast without targets.
     let result = system.dispatch(Arc::new(audit_entry)).await;
     let elapsed = start.elapsed();
 
@@ -168,8 +169,10 @@ async fn test_audit_log_dispatch_performance() {
     // Should be very fast (sub-millisecond for no targets)
     assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
 
-    // Should succeed even with no targets
-    assert!(result.is_ok(), "Dispatch should succeed with no targets");
+    assert!(
+        matches!(result, Err(AuditError::NotInitialized(_))),
+        "Dispatch on a stopped system should return NotInitialized, got: {result:?}"
+    );
 
     // Clean up
     let _ = system.close().await;
@@ -186,11 +189,11 @@ async fn test_system_state_transitions() {
     let config = rustfs_ecstore::config::Config(std::collections::HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should be running (or failed due to server storage)
+    // Empty config keeps the audit system stopped even when start() succeeds.
     let state = system.get_state().await;
     match start_result {
         Ok(_) => {
-            assert_eq!(state, rustfs_audit::system::AuditSystemState::Running);
+            assert_eq!(state, rustfs_audit::system::AuditSystemState::Stopped);
         }
         Err(_) => {
             // Expected in test environment due to server storage not being initialized

crates/audit/tests/system_integration_test.rs

@@ -29,27 +29,21 @@ async fn test_complete_audit_system_lifecycle() {
     assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
     assert!(!system.is_running().await);
 
-    // 2. Start with empty config (will fail due to no server storage in test)
+    // 2. Start with empty config. The current implementation returns Ok(())
+    // but keeps the system stopped when no audit targets are enabled.
     let config = Config(HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should fail in test environment but state handling should work
+    // State handling should remain consistent for both empty-config success and
+    // storage-unavailable failure paths.
     match start_result {
         Err(AuditError::StorageNotAvailable(_)) => {
             // Expected in test environment
             assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
         }
         Ok(_) => {
-            // If it somehow succeeds, verify running state
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
-            assert!(system.is_running().await);
-
-            // Test pause/resume
-            system.pause().await.expect("Should pause successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Paused);
-
-            system.resume().await.expect("Should resume successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
+            assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
+            assert!(!system.is_running().await);
         }
         Err(e) => {
             panic!("Unexpected error: {e}");

crates/checksums/Cargo.toml

@@ -36,3 +36,6 @@ sha2 = { workspace = true }
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }
+
+[lib]
+doctest = false

crates/checksums/src/lib.rs

@@ -349,7 +349,7 @@ mod tests {
     fn base64_encoded_checksum_to_hex_string(header_value: &HeaderValue) -> String {
         let decoded_checksum = base64::decode(header_value.to_str().unwrap()).unwrap();
         let decoded_checksum = decoded_checksum.into_iter().fold(String::new(), |mut acc, byte| {
-            write!(acc, "{byte:02X?}").expect("string will always be writeable");
+            write!(acc, "{byte:02X?}").expect("string will always be writable");
             acc
         });
 

crates/common/Cargo.toml

@@ -41,3 +41,6 @@ rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 s3s = { workspace = true }
 tracing = { workspace = true }
+
+[lib]
+doctest = false

crates/common/src/internode_metrics.rs

@@ -0,0 +1,170 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::{counter, gauge};
+use std::sync::{
+    Arc, LazyLock,
+    atomic::{AtomicU64, Ordering},
+};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct InternodeMetricsSnapshot {
+    pub sent_bytes_total: u64,
+    pub recv_bytes_total: u64,
+    pub outgoing_requests_total: u64,
+    pub incoming_requests_total: u64,
+    pub errors_total: u64,
+    pub dial_errors_total: u64,
+    pub dial_avg_time_nanos: u64,
+    pub last_dial_unix_millis: u64,
+}
+
+#[derive(Debug, Default)]
+pub struct InternodeMetrics {
+    sent_bytes_total: AtomicU64,
+    recv_bytes_total: AtomicU64,
+    outgoing_requests_total: AtomicU64,
+    incoming_requests_total: AtomicU64,
+    errors_total: AtomicU64,
+    dial_errors_total: AtomicU64,
+    dial_total_time_nanos: AtomicU64,
+    dial_samples_total: AtomicU64,
+    last_dial_unix_millis: AtomicU64,
+}
+
+impl InternodeMetrics {
+    pub fn record_sent_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.sent.bytes.total").increment(bytes);
+    }
+
+    pub fn record_recv_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.recv.bytes.total").increment(bytes);
+    }
+
+    pub fn record_outgoing_request(&self) {
+        self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.outgoing.total").increment(1);
+    }
+
+    pub fn record_incoming_request(&self) {
+        self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.incoming.total").increment(1);
+    }
+
+    pub fn record_error(&self) {
+        self.errors_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.errors.total").increment(1);
+    }
+
+    pub fn record_dial_result(&self, duration: Duration, success: bool) {
+        let elapsed_nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
+        self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
+        let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
+        let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        gauge!("rustfs.internode.dial.avg_time.nanos").set(total as f64 / samples as f64);
+
+        if !success {
+            self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
+            counter!("rustfs.internode.dial.errors.total").increment(1);
+        }
+
+        let now_ms = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_millis()
+            .min(u128::from(u64::MAX)) as u64;
+        self.last_dial_unix_millis.store(now_ms, Ordering::Relaxed);
+    }
+
+    pub fn snapshot(&self) -> InternodeMetricsSnapshot {
+        let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
+        let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        let dial_avg_time_nanos = if dial_samples_total == 0 {
+            0
+        } else {
+            dial_total_time_nanos / dial_samples_total
+        };
+
+        InternodeMetricsSnapshot {
+            sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),
+            recv_bytes_total: self.recv_bytes_total.load(Ordering::Relaxed),
+            outgoing_requests_total: self.outgoing_requests_total.load(Ordering::Relaxed),
+            incoming_requests_total: self.incoming_requests_total.load(Ordering::Relaxed),
+            errors_total: self.errors_total.load(Ordering::Relaxed),
+            dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
+            dial_avg_time_nanos,
+            last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn reset_for_test(&self) {
+        self.sent_bytes_total.store(0, Ordering::Relaxed);
+        self.recv_bytes_total.store(0, Ordering::Relaxed);
+        self.outgoing_requests_total.store(0, Ordering::Relaxed);
+        self.incoming_requests_total.store(0, Ordering::Relaxed);
+        self.errors_total.store(0, Ordering::Relaxed);
+        self.dial_errors_total.store(0, Ordering::Relaxed);
+        self.dial_total_time_nanos.store(0, Ordering::Relaxed);
+        self.dial_samples_total.store(0, Ordering::Relaxed);
+        self.last_dial_unix_millis.store(0, Ordering::Relaxed);
+    }
+}
+
+pub fn global_internode_metrics() -> &'static Arc<InternodeMetrics> {
+    static GLOBAL_INTERNODE_METRICS: LazyLock<Arc<InternodeMetrics>> = LazyLock::new(|| Arc::new(InternodeMetrics::default()));
+    &GLOBAL_INTERNODE_METRICS
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn snapshot_reports_recorded_values() {
+        let metrics = global_internode_metrics();
+        metrics.reset_for_test();
+
+        metrics.record_sent_bytes(64);
+        metrics.record_recv_bytes(32);
+        metrics.record_outgoing_request();
+        metrics.record_incoming_request();
+        metrics.record_error();
+        metrics.record_dial_result(Duration::from_millis(9), true);
+        metrics.record_dial_result(Duration::from_millis(3), false);
+
+        let snapshot = metrics.snapshot();
+        assert_eq!(snapshot.sent_bytes_total, 64);
+        assert_eq!(snapshot.recv_bytes_total, 32);
+        assert_eq!(snapshot.outgoing_requests_total, 1);
+        assert_eq!(snapshot.incoming_requests_total, 1);
+        assert_eq!(snapshot.errors_total, 1);
+        assert_eq!(snapshot.dial_errors_total, 1);
+        assert_eq!(snapshot.dial_avg_time_nanos, 6_000_000);
+        assert!(snapshot.last_dial_unix_millis > 0);
+
+        metrics.reset_for_test();
+    }
+}

crates/common/src/lib.rs

@@ -17,6 +17,7 @@ pub mod bucket_stats;
 pub mod data_usage;
 pub mod globals;
 pub mod heal_channel;
+pub mod internode_metrics;
 pub mod last_minute;
 pub mod metrics;
 mod readiness;

crates/common/src/metrics.rs

@@ -22,12 +22,12 @@ use std::{
     future::Future,
     pin::Pin,
     sync::{
-        Arc, OnceLock,
+        Arc, Mutex, OnceLock,
         atomic::{AtomicU64, Ordering},
     },
     time::{Duration, SystemTime},
 };
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::RwLock;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum IlmAction {
@@ -110,7 +110,7 @@ pub fn global_metrics() -> &'static Arc<Metrics> {
     GLOBAL_METRICS.get_or_init(|| Arc::new(Metrics::new()))
 }
 
-#[derive(Clone, Debug, PartialEq, PartialOrd)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Metric {
     // START Realtime metrics, that only records
     // last minute latencies and total operation count.
@@ -188,7 +188,6 @@ impl Metric {
         if index >= Self::Last as usize {
             return None;
         }
-        // Safe conversion using match instead of unsafe transmute
         match index {
             0 => Some(Self::ReadMetadata),
             1 => Some(Self::CheckMissing),
@@ -220,16 +219,48 @@ impl Metric {
     }
 }
 
-/// Thread-safe wrapper for LastMinuteLatency with atomic operations
-#[derive(Default)]
+// ---------------------------------------------------------------------------
+// LockedLastMinuteLatency
+// ---------------------------------------------------------------------------
+//
+// Uses std::sync::Mutex instead of tokio::sync::Mutex.
+//
+// Rationale: the critical section is a handful of integer additions inside
+// LastMinuteLatency::add_all / get_total — no I/O, no blocking syscalls, no
+// awaiting.  A std blocking mutex is cheaper (no task-wakeup overhead) and,
+// crucially, lets every caller be *synchronous*.  That eliminates the need to
+// spawn a background task just to record a duration, which was the only reason
+// tokio::spawn appeared in log/time/time_size/time_n/time_ilm.
+//
+// Note: vec![LockedLastMinuteLatency::default(); N] with the old Arc-based
+// Clone made every element share the *same* inner mutex — a latent bug where
+// all metrics slots wrote to one counter.  The new Clone creates a fresh
+// independent Mutex per element, matching the intent.
+
+/// Thread-safe wrapper for LastMinuteLatency backed by a std blocking mutex.
 pub struct LockedLastMinuteLatency {
+    // Arc so Clone is cheap *and* each cloned value stays independent (its own
+    // allocation).  We never hand out the Arc to two Metrics at once; the Arc
+    // is purely a convenience for the Clone impl below.
     latency: Arc<Mutex<LastMinuteLatency>>,
 }
 
+impl Default for LockedLastMinuteLatency {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// Produce a fresh, independent slot — *not* a shared alias.  This is what
+// vec![val; N] and #[derive(Clone)] on the parent struct both need.
 impl Clone for LockedLastMinuteLatency {
     fn clone(&self) -> Self {
+        let inner = match self.latency.lock() {
+            Ok(guard) => guard.clone(),
+            Err(poisoned) => poisoned.into_inner().clone(),
+        };
         Self {
-            latency: Arc::clone(&self.latency),
+            latency: Arc::new(Mutex::new(inner)),
         }
     }
 }
@@ -241,14 +272,13 @@ impl LockedLastMinuteLatency {
         }
     }
 
-    /// Add a duration measurement
-    pub async fn add(&self, duration: Duration) {
-        self.add_size(duration, 0).await;
+    /// Record a duration sample (no size).
+    pub fn add(&self, duration: Duration) {
+        self.add_size(duration, 0);
     }
 
-    /// Add a duration measurement with size
-    pub async fn add_size(&self, duration: Duration, size: u64) {
-        let mut latency = self.latency.lock().await;
+    /// Record a duration sample with an associated byte count.
+    pub fn add_size(&self, duration: Duration, size: u64) {
         let now = SystemTime::now()
             .duration_since(SystemTime::UNIX_EPOCH)
             .unwrap_or_default()
@@ -259,17 +289,27 @@ impl LockedLastMinuteLatency {
             total: duration.as_secs(),
             size,
         };
-        latency.add_all(now, &elem);
+
+        match self.latency.lock() {
+            Ok(mut guard) => guard.add_all(now, &elem),
+            Err(poisoned) => poisoned.into_inner().add_all(now, &elem),
+        }
     }
 
-    /// Get total accumulated metrics for the last minute
-    pub async fn total(&self) -> AccElem {
-        let mut latency = self.latency.lock().await;
-        latency.get_total()
+    /// Return accumulated totals for the last minute window.
+    pub fn total(&self) -> AccElem {
+        match self.latency.lock() {
+            Ok(mut latency) => latency.get_total(),
+            Err(poisoned) => poisoned.into_inner().get_total(),
+        }
     }
 }
 
-/// Current path tracker for monitoring active scan paths
+// ---------------------------------------------------------------------------
+// CurrentPathTracker — unchanged, still uses tokio::sync::RwLock because it
+// lives inside async path-update callbacks.
+// ---------------------------------------------------------------------------
+
 struct CurrentPathTracker {
     current_path: Arc<RwLock<String>>,
 }
@@ -290,17 +330,16 @@ impl CurrentPathTracker {
     }
 }
 
-/// Main scanner metrics structure
+// ---------------------------------------------------------------------------
+// Metrics
+// ---------------------------------------------------------------------------
+
 pub struct Metrics {
-    // All fields must be accessed atomically and aligned.
     operations: Vec<AtomicU64>,
     latency: Vec<LockedLastMinuteLatency>,
     actions: Vec<AtomicU64>,
     actions_latency: Vec<LockedLastMinuteLatency>,
-    // Current paths contains disk -> tracker mappings
     current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
-
-    // Cycle information
     cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
 }
 
@@ -331,8 +370,6 @@ const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
 
-/// Emit an OTEL counter increment for the given scanner metric.
-/// ScanCycle and ScanBucketDrive are handled by dedicated emit functions with labels.
 fn emit_otel_counter(metric: usize, count: u64) {
     match Metric::from_index(metric) {
         Some(Metric::ScanObject) => {
@@ -345,8 +382,6 @@ fn emit_otel_counter(metric: usize, count: u64) {
     }
 }
 
-/// Emit OTel metrics for a completed scan cycle.
-/// Counter with result label + gauge for last successful cycle duration.
 pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
     let result = if success { "success" } else { "error" };
     metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
@@ -355,8 +390,6 @@ pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
     }
 }
 
-/// Emit OTel metrics for a completed bucket-drive scan.
-/// Counter with result/bucket/disk labels + histogram for duration.
 pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
     let result = if success { "success" } else { "error" };
     metrics::counter!(
@@ -376,226 +409,202 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
 
 impl Metrics {
     pub fn new() -> Self {
-        let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
-
-        let latency = (0..Metric::LastRealtime as usize)
-            .map(|_| LockedLastMinuteLatency::new())
-            .collect();
-
         Self {
-            operations,
-            latency,
+            operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
+            // Each slot gets its own fresh LockedLastMinuteLatency so that
+            // different metrics never accidentally share state.
+            latency: (0..Metric::LastRealtime as usize)
+                .map(|_| LockedLastMinuteLatency::new())
+                .collect(),
             actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(),
-            actions_latency: vec![LockedLastMinuteLatency::default(); IlmAction::ActionCount as usize],
+            actions_latency: (0..IlmAction::ActionCount as usize)
+                .map(|_| LockedLastMinuteLatency::new())
+                .collect(),
             current_paths: Arc::new(RwLock::new(HashMap::new())),
             cycle_info: Arc::new(RwLock::new(None)),
         }
     }
 
-    /// Log scanner action with custom metadata - compatible with existing usage
+    // -----------------------------------------------------------------------
+    // Metric recording helpers
+    //
+    // All of these are now pure sync closures.  No tokio::spawn, no heap-
+    // allocated future, no scheduler overhead — just an atomic increment and a
+    // std-mutex lock for fewer than ~10 ns of work.
+    // -----------------------------------------------------------------------
+
+    /// Return a closure that records one observation of `metric` (with
+    /// optional caller-supplied metadata).  Call it once the operation ends.
     pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         move |_custom: &HashMap<String, String>| {
-            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-            // Update operation count
-            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-            emit_otel_counter(metric, 1);
-
-            // Update latency for realtime metrics (spawn async task for this)
-            if (metric) < Metric::LastRealtime as usize {
-                let metric_index = metric;
-                tokio::spawn(async move {
-                    global_metrics().latency[metric_index].add(duration).await;
-                });
-            }
-
-            // Log trace metrics
-            if metric as u8 > Metric::StartTrace as u8 {
-                //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
+            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric_idx, 1);
+            if metric_idx < Metric::LastRealtime as usize {
+                global_metrics().latency[metric_idx].add(duration);
             }
         }
     }
 
-    /// Time scanner action with size - returns function that takes size
+    /// Return a closure that records one observation of `metric` together with
+    /// a byte count.  Call `done(size_bytes)` when the operation ends.
     pub fn time_size(metric: Metric) -> impl Fn(u64) {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         move |size: u64| {
-            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-            // Update operation count
-            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-            emit_otel_counter(metric, 1);
-
-            // Update latency for realtime metrics with size (spawn async task)
-            if (metric) < Metric::LastRealtime as usize {
-                let metric_index = metric;
-                tokio::spawn(async move {
-                    global_metrics().latency[metric_index].add_size(duration, size).await;
-                });
+            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric_idx, 1);
+            if metric_idx < Metric::LastRealtime as usize {
+                global_metrics().latency[metric_idx].add_size(duration, size);
             }
         }
     }
 
-    /// Time a scanner action - returns a closure to call when done
+    /// Return a closure that records one observation of `metric`.
+    /// Call `done()` when the operation ends.
     pub fn time(metric: Metric) -> impl Fn() {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         move || {
-            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-            // Update operation count
-            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-            emit_otel_counter(metric, 1);
-
-            // Update latency for realtime metrics (spawn async task)
-            if (metric) < Metric::LastRealtime as usize {
-                let metric_index = metric;
-                tokio::spawn(async move {
-                    global_metrics().latency[metric_index].add(duration).await;
-                });
+            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric_idx, 1);
+            if metric_idx < Metric::LastRealtime as usize {
+                global_metrics().latency[metric_idx].add(duration);
             }
         }
     }
 
-    /// Time N scanner actions - returns function that takes count, then returns completion function
+    /// Return a two-stage closure: first call takes an item count, second call
+    /// (returned closure) fires when the batch of `count` operations ends.
     pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         Box::new(move |count: usize| {
             Box::new(move || {
-                let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-                // Update operation count
-                global_metrics().operations[metric].fetch_add(count as u64, Ordering::Relaxed);
-                emit_otel_counter(metric, count as u64);
-
-                // Update latency for realtime metrics (spawn async task)
-                if (metric) < Metric::LastRealtime as usize {
-                    let metric_index = metric;
-                    tokio::spawn(async move {
-                        global_metrics().latency[metric_index].add(duration).await;
-                    });
+                let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+                global_metrics().operations[metric_idx].fetch_add(count as u64, Ordering::Relaxed);
+                emit_otel_counter(metric_idx, count as u64);
+                if metric_idx < Metric::LastRealtime as usize {
+                    global_metrics().latency[metric_idx].add(duration);
                 }
             })
         })
     }
 
-    /// Time ILM action with versions - returns function that takes versions, then returns completion function
+    /// Return a two-stage closure for ILM actions: first call takes a version
+    /// count, second call fires when the action completes.
     pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
-        let a_clone = a as usize;
-        if a_clone == IlmAction::NoneAction as usize || a_clone >= IlmAction::ActionCount as usize {
-            return Box::new(move |_: u64| Box::new(move || {}));
+        let a_idx = a as usize;
+        if a_idx == IlmAction::NoneAction as usize || a_idx >= IlmAction::ActionCount as usize {
+            return Box::new(|_| Box::new(|| {}));
         }
         let start = SystemTime::now();
         Box::new(move |versions: u64| {
             Box::new(move || {
-                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
-                tokio::spawn(async move {
-                    global_metrics().actions[a_clone].fetch_add(versions, Ordering::Relaxed);
-                    global_metrics().actions_latency[a_clone].add(duration).await;
-                });
+                let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+                global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
+                global_metrics().actions_latency[a_idx].add(duration);
             })
         })
     }
 
-    /// Increment time with specific duration
-    pub async fn inc_time(metric: Metric, duration: Duration) {
-        let metric = metric as usize;
-        // Update operation count
-        global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-        emit_otel_counter(metric, 1);
-
-        // Update latency for realtime metrics
-        if (metric) < Metric::LastRealtime as usize {
-            global_metrics().latency[metric].add(duration).await;
+    /// Record a single observation of `metric` with a caller-supplied duration.
+    /// No longer async — nothing inside requires it.
+    pub fn inc_time(metric: Metric, duration: Duration) {
+        let metric_idx = metric as usize;
+        global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+        emit_otel_counter(metric_idx, 1);
+        if metric_idx < Metric::LastRealtime as usize {
+            global_metrics().latency[metric_idx].add(duration);
         }
     }
 
-    /// Get lifetime operation count for a metric
+    // -----------------------------------------------------------------------
+    // Read-side helpers
+    // -----------------------------------------------------------------------
+
+    /// Lifetime operation count for `metric`.
     pub fn lifetime(&self, metric: Metric) -> u64 {
-        let metric = metric as usize;
-        if (metric) >= Metric::Last as usize {
+        let idx = metric as usize;
+        if idx >= Metric::Last as usize {
             return 0;
         }
-        self.operations[metric].load(Ordering::Relaxed)
+        self.operations[idx].load(Ordering::Relaxed)
     }
 
-    /// Get last minute statistics for a metric
-    pub async fn last_minute(&self, metric: Metric) -> AccElem {
-        let metric = metric as usize;
-        if (metric) >= Metric::LastRealtime as usize {
+    /// Last-minute accumulated stats for a realtime metric.
+    /// No longer async — LockedLastMinuteLatency::total() is now synchronous.
+    pub fn last_minute(&self, metric: Metric) -> AccElem {
+        let idx = metric as usize;
+        if idx >= Metric::LastRealtime as usize {
             return AccElem::default();
         }
-        self.latency[metric].total().await
+        self.latency[idx].total()
     }
 
-    /// Set current cycle information
+    /// Replace the current cycle record.
     pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) {
         *self.cycle_info.write().await = cycle;
     }
 
-    /// Get current cycle information
+    /// Read the current cycle record.
     pub async fn get_cycle(&self) -> Option<CurrentCycle> {
         self.cycle_info.read().await.clone()
     }
 
-    /// Get current active paths
+    /// Snapshot of every path currently being scanned.
     pub async fn get_current_paths(&self) -> Vec<String> {
-        let mut result = Vec::new();
         let paths = self.current_paths.read().await;
-
+        let mut result = Vec::with_capacity(paths.len());
         for (disk, tracker) in paths.iter() {
-            let path = tracker.get_path().await;
-            result.push(format!("{disk}/{path}"));
+            result.push(format!("{disk}/{}", tracker.get_path().await));
         }
-
         result
     }
 
-    /// Get number of active drives
+    /// Number of drives with an active scan in progress.
     pub async fn active_drives(&self) -> usize {
         self.current_paths.read().await.len()
     }
 
-    /// Generate metrics report
+    /// Build a full metrics report snapshot.
     pub async fn report(&self) -> M_ScannerMetrics {
-        let mut metrics = M_ScannerMetrics::default();
+        let mut m = M_ScannerMetrics::default();
 
-        // Set cycle information
         if let Some(cycle) = self.get_cycle().await {
-            metrics.current_cycle = cycle.current;
-            metrics.cycles_completed_at = cycle.cycle_completed;
-            metrics.current_started = cycle.started;
+            m.current_cycle = cycle.current;
+            m.cycles_completed_at = cycle.cycle_completed;
+            m.current_started = cycle.started;
         }
 
-        // Replace default start time with global init time if it's the placeholder
         if let Some(init_time) = crate::get_global_init_time().await {
-            metrics.current_started = init_time;
+            m.current_started = init_time;
         }
 
-        metrics.collected_at = Utc::now();
-        metrics.active_paths = self.get_current_paths().await;
+        m.collected_at = Utc::now();
+        m.active_paths = self.get_current_paths().await;
 
-        // Lifetime operations
+        // Lifetime operation counts
         for i in 0..Metric::Last as usize {
             let count = self.operations[i].load(Ordering::Relaxed);
             if count > 0
                 && let Some(metric) = Metric::from_index(i)
             {
-                metrics.life_time_ops.insert(metric.as_str().to_string(), count);
+                m.life_time_ops.insert(metric.as_str().to_string(), count);
             }
         }
 
-        // Last minute statistics for realtime metrics
+        // Last-minute stats for realtime metrics — now plain sync calls
         for i in 0..Metric::LastRealtime as usize {
-            let last_min = self.latency[i].total().await;
+            let last_min = self.latency[i].total();
             if last_min.n > 0
                 && let Some(metric) = Metric::from_index(i)
             {
-                metrics.last_minute.actions.insert(
+                m.last_minute.actions.insert(
                     metric.as_str().to_string(),
                     TimedAction {
                         count: last_min.n,
@@ -606,23 +615,23 @@ impl Metrics {
             }
         }
 
-        // Lifetime ILM operations
+        // Lifetime ILM counts
         for i in 0..IlmAction::ActionCount as usize {
             let count = self.actions[i].load(Ordering::Relaxed);
             if count > 0
                 && let Some(action) = IlmAction::from_index(i)
             {
-                metrics.life_time_ilm.insert(action.as_str().to_string(), count);
+                m.life_time_ilm.insert(action.as_str().to_string(), count);
             }
         }
 
-        // Last minute ILM latency
+        // Last-minute ILM latency — plain sync calls
         for i in 0..IlmAction::ActionCount as usize {
-            let last_min = self.actions_latency[i].total().await;
+            let last_min = self.actions_latency[i].total();
             if last_min.n > 0
                 && let Some(action) = IlmAction::from_index(i)
             {
-                metrics.last_minute.ilm.insert(
+                m.last_minute.ilm.insert(
                     action.as_str().to_string(),
                     TimedAction {
                         count: last_min.n,
@@ -633,56 +642,65 @@ impl Metrics {
             }
         }
 
-        metrics
+        m
     }
 }
 
-// Type aliases for compatibility with existing code
-pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
-pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
-
-/// Create a current path updater for tracking scan progress
-pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
-    let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
-    let disk_name = disk.to_string();
-
-    // Store the tracker in global metrics
-    let tracker_clone = Arc::clone(&tracker);
-    let disk_clone = disk_name.clone();
-    tokio::spawn(async move {
-        global_metrics().current_paths.write().await.insert(disk_clone, tracker_clone);
-    });
-
-    let update_fn = {
-        let tracker = Arc::clone(&tracker);
-        Arc::new(move |path: &str| -> Pin<Box<dyn Future<Output = ()> + Send>> {
-            let tracker = Arc::clone(&tracker);
-            let path = path.to_string();
-            Box::pin(async move {
-                tracker.update_path(path).await;
-            })
-        })
-    };
-
-    let done_fn = {
-        let disk_name = disk_name.clone();
-        Arc::new(move || -> Pin<Box<dyn Future<Output = ()> + Send>> {
-            let disk_name = disk_name.clone();
-            Box::pin(async move {
-                global_metrics().current_paths.write().await.remove(&disk_name);
-            })
-        })
-    };
-
-    (update_fn, done_fn)
-}
-
 impl Default for Metrics {
     fn default() -> Self {
         Self::new()
     }
 }
 
+// ---------------------------------------------------------------------------
+// Path tracking helpers
+// ---------------------------------------------------------------------------
+
+pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
+pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
+
+/// Register a new disk in the global path tracker and return two callbacks:
+/// one to update the current path and one to deregister the disk when done.
+pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
+    let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
+    let disk_name = disk.to_string();
+
+    let tracker_clone = Arc::clone(&tracker);
+    let disk_insert = disk_name.clone();
+    tokio::spawn(async move {
+        global_metrics()
+            .current_paths
+            .write()
+            .await
+            .insert(disk_insert, tracker_clone);
+    });
+
+    let update_fn: UpdateCurrentPathFn = {
+        let tracker = Arc::clone(&tracker);
+        Arc::new(move |path: &str| {
+            let tracker = Arc::clone(&tracker);
+            let path = path.to_string();
+            Box::pin(async move { tracker.update_path(path).await })
+        })
+    };
+
+    let done_fn: CloseDiskFn = {
+        let disk = disk_name.clone();
+        Arc::new(move || {
+            let disk = disk.clone();
+            Box::pin(async move {
+                global_metrics().current_paths.write().await.remove(&disk);
+            })
+        })
+    };
+
+    (update_fn, done_fn)
+}
+
+// ---------------------------------------------------------------------------
+// CloseDiskGuard
+// ---------------------------------------------------------------------------
+
 pub struct CloseDiskGuard(CloseDiskFn);
 
 impl CloseDiskGuard {
@@ -697,16 +715,10 @@ impl CloseDiskGuard {
 
 impl Drop for CloseDiskGuard {
     fn drop(&mut self) {
-        // Drop cannot be async, so we spawn the async cleanup task
-        // The task will run in the background and complete asynchronously
         if let Ok(handle) = tokio::runtime::Handle::try_current() {
             let close_fn = self.0.clone();
-            handle.spawn(async move {
-                close_fn().await;
-            });
-        } else {
-            // If we're not in a tokio runtime context, we can't spawn
-            // This is a best-effort cleanup, so we just skip it
+            handle.spawn(async move { close_fn().await });
         }
+        // If there is no runtime we are in a test or shutdown path; skip cleanup.
     }
 }

crates/common/src/readiness.rs

@@ -60,6 +60,23 @@ impl GlobalReadiness {
     pub fn is_ready(&self) -> bool {
         self.status.load(Ordering::SeqCst) == SystemStage::FullReady as u8
     }
+
+    /// Get the current system stage
+    /// # Returns
+    /// The current SystemStage of the service
+    pub fn current_stage(&self) -> SystemStage {
+        match self.status.load(Ordering::SeqCst) {
+            0 => SystemStage::Booting,
+            1 => SystemStage::StorageReady,
+            2 => SystemStage::IamReady,
+            3 => SystemStage::FullReady,
+            invalid => {
+                debug_assert!(false, "GlobalReadiness::current_stage: invalid status value {}", invalid);
+                // Fallback to the most conservative stage on invalid values
+                SystemStage::Booting
+            }
+        }
+    }
 }
 
 #[cfg(test)]

crates/concurrency/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "rustfs-concurrency"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling"
+keywords = ["rustfs", "concurrency", "timeout", "backpressure", "scheduling"]
+categories = ["concurrency", "filesystem"]
+
+[dependencies]
+# Internal crates
+rustfs-io-core = { workspace = true }
+rustfs-io-metrics = { workspace = true }
+
+# Async runtime
+tokio = { workspace = true, features = ["sync", "time", "rt"] }
+tokio-util = { workspace = true }
+
+# Error handling
+thiserror = { workspace = true }
+
+# Logging
+tracing = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["full"] }
+
+[features]
+default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"]
+
+# Feature modules
+timeout = []
+lock = []
+deadlock = []
+backpressure = []
+scheduler = []
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]

crates/concurrency/src/backpressure.rs

@@ -0,0 +1,224 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Backpressure management
+
+use rustfs_io_core::{BackpressureMonitor as CoreBackpressureMonitor, BackpressureState};
+use rustfs_io_metrics::backpressure_metrics;
+use std::sync::Arc;
+use std::time::Instant;
+use tokio::io::{DuplexStream, duplex};
+
+/// Backpressure configuration
+#[derive(Debug, Clone)]
+pub struct BackpressureConfig {
+    /// Buffer size in bytes
+    pub buffer_size: usize,
+    /// High watermark percentage
+    pub high_watermark: u32,
+    /// Low watermark percentage
+    pub low_watermark: u32,
+}
+
+impl Default for BackpressureConfig {
+    fn default() -> Self {
+        Self {
+            buffer_size: 4 * 1024 * 1024, // 4MB
+            high_watermark: 80,
+            low_watermark: 50,
+        }
+    }
+}
+
+impl BackpressureConfig {
+    /// Calculate high watermark threshold in bytes
+    pub fn high_watermark_bytes(&self) -> usize {
+        (self.buffer_size as u64 * self.high_watermark as u64 / 100) as usize
+    }
+
+    /// Calculate low watermark threshold in bytes
+    pub fn low_watermark_bytes(&self) -> usize {
+        (self.buffer_size as u64 * self.low_watermark as u64 / 100) as usize
+    }
+}
+
+/// Backpressure manager
+pub struct BackpressureManager {
+    config: BackpressureConfig,
+    monitor: Arc<CoreBackpressureMonitor>,
+}
+
+impl BackpressureManager {
+    /// Create a new backpressure manager
+    pub fn new(buffer_size: usize, high_watermark: u32, low_watermark: u32) -> Self {
+        let config = BackpressureConfig {
+            buffer_size,
+            high_watermark,
+            low_watermark,
+        };
+
+        let core_config = rustfs_io_core::BackpressureConfig {
+            max_concurrent: 32,
+            high_water_mark: high_watermark as f64 / 100.0,
+            low_water_mark: low_watermark as f64 / 100.0,
+            cooldown: std::time::Duration::from_millis(100),
+            enabled: true,
+        };
+
+        Self {
+            config,
+            monitor: Arc::new(CoreBackpressureMonitor::new(core_config)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &BackpressureConfig {
+        &self.config
+    }
+
+    /// Get the monitor
+    pub fn monitor(&self) -> Arc<CoreBackpressureMonitor> {
+        self.monitor.clone()
+    }
+
+    /// Create a backpressure pipe
+    pub fn create_pipe(&self) -> BackpressurePipe {
+        BackpressurePipe::new(self.config.clone(), self.monitor.clone())
+    }
+
+    /// Get current state
+    pub fn state(&self) -> BackpressureState {
+        self.monitor.state()
+    }
+
+    /// Check if backpressure is active
+    pub fn is_active(&self) -> bool {
+        self.monitor.is_active()
+    }
+}
+
+/// Backpressure pipe wrapping tokio's duplex
+pub struct BackpressurePipe {
+    reader: DuplexStream,
+    writer: DuplexStream,
+    config: BackpressureConfig,
+    monitor: Arc<CoreBackpressureMonitor>,
+    created_at: Instant,
+}
+
+impl BackpressurePipe {
+    fn new(config: BackpressureConfig, monitor: Arc<CoreBackpressureMonitor>) -> Self {
+        let (reader, writer) = duplex(config.buffer_size);
+
+        Self {
+            reader,
+            writer,
+            config,
+            monitor,
+            created_at: Instant::now(),
+        }
+    }
+
+    /// Get the reader end
+    pub fn reader(&mut self) -> &mut DuplexStream {
+        &mut self.reader
+    }
+
+    /// Get the writer end
+    pub fn writer(&mut self) -> &mut DuplexStream {
+        &mut self.writer
+    }
+
+    /// Split into reader and writer
+    pub fn into_split(self) -> (DuplexStream, DuplexStream) {
+        (self.reader, self.writer)
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &BackpressureConfig {
+        &self.config
+    }
+
+    /// Get current state
+    pub fn state(&self) -> BackpressureState {
+        self.monitor.state()
+    }
+
+    /// Get the age of this pipe
+    pub fn age(&self) -> std::time::Duration {
+        self.created_at.elapsed()
+    }
+
+    /// Check if should apply backpressure
+    pub fn should_apply_backpressure(&self) -> bool {
+        let should = self.monitor.should_apply_backpressure();
+        if should {
+            backpressure_metrics::record_backpressure_activation();
+        }
+        should
+    }
+}
+
+/// Backpressure event
+#[allow(dead_code)]
+#[derive(Debug, Clone)]
+pub struct BackpressureEvent {
+    /// Event timestamp
+    pub timestamp: Instant,
+    /// Event type
+    pub event_type: BackpressureEventType,
+    /// Buffer usage
+    pub buffer_usage: usize,
+    /// Buffer capacity
+    pub buffer_capacity: usize,
+}
+
+/// Backpressure event type
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy)]
+pub enum BackpressureEventType {
+    /// High watermark reached
+    HighWatermarkReached,
+    /// High watermark exited
+    HighWatermarkExited,
+    /// Backpressure applied
+    BackpressureApplied,
+    /// Backpressure released
+    BackpressureReleased,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_backpressure_config() {
+        let config = BackpressureConfig::default();
+        assert_eq!(config.buffer_size, 4 * 1024 * 1024);
+        assert!(config.high_watermark > config.low_watermark);
+    }
+
+    #[test]
+    fn test_backpressure_manager() {
+        let manager = BackpressureManager::new(1024, 80, 50);
+        assert_eq!(manager.state(), BackpressureState::Normal);
+    }
+
+    #[test]
+    fn test_backpressure_pipe() {
+        let manager = BackpressureManager::new(1024, 80, 50);
+        let pipe = manager.create_pipe();
+        assert_eq!(pipe.state(), BackpressureState::Normal);
+    }
+}

crates/concurrency/src/config.rs

@@ -0,0 +1,256 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Configuration for concurrency management
+
+use std::time::Duration;
+
+/// Feature flags for concurrency modules
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ConcurrencyFeatures {
+    /// Enable timeout control
+    pub timeout: bool,
+    /// Enable lock optimization
+    pub lock: bool,
+    /// Enable deadlock detection
+    pub deadlock: bool,
+    /// Enable backpressure management
+    pub backpressure: bool,
+    /// Enable I/O scheduling
+    pub scheduler: bool,
+}
+
+impl Default for ConcurrencyFeatures {
+    fn default() -> Self {
+        Self {
+            timeout: cfg!(feature = "timeout"),
+            lock: cfg!(feature = "lock"),
+            deadlock: cfg!(feature = "deadlock"),
+            backpressure: cfg!(feature = "backpressure"),
+            scheduler: cfg!(feature = "scheduler"),
+        }
+    }
+}
+
+impl ConcurrencyFeatures {
+    /// Create with all features enabled
+    pub fn all() -> Self {
+        Self {
+            timeout: true,
+            lock: true,
+            deadlock: true,
+            backpressure: true,
+            scheduler: true,
+        }
+    }
+
+    /// Create with no features enabled
+    pub fn none() -> Self {
+        Self {
+            timeout: false,
+            lock: false,
+            deadlock: false,
+            backpressure: false,
+            scheduler: false,
+        }
+    }
+
+    /// Check if any feature is enabled
+    pub fn any_enabled(&self) -> bool {
+        self.timeout || self.lock || self.deadlock || self.backpressure || self.scheduler
+    }
+}
+
+/// Main configuration for concurrency management
+#[derive(Debug, Clone)]
+pub struct ConcurrencyConfig {
+    /// Feature flags
+    pub features: ConcurrencyFeatures,
+
+    // Timeout configuration
+    /// Default timeout duration
+    pub default_timeout: Duration,
+    /// Maximum timeout duration
+    pub max_timeout: Duration,
+    /// Enable dynamic timeout
+    pub enable_dynamic_timeout: bool,
+
+    // Lock configuration
+    /// Enable lock optimization
+    pub enable_lock_optimization: bool,
+    /// Lock acquisition timeout
+    pub lock_acquire_timeout: Duration,
+
+    // Deadlock configuration
+    /// Enable deadlock detection
+    pub enable_deadlock_detection: bool,
+    /// Deadlock check interval
+    pub deadlock_check_interval: Duration,
+    /// Hang threshold
+    pub hang_threshold: Duration,
+
+    // Backpressure configuration
+    /// Buffer size for backpressure
+    pub backpressure_buffer_size: usize,
+    /// High watermark percentage
+    pub high_watermark: u32,
+    /// Low watermark percentage
+    pub low_watermark: u32,
+
+    // Scheduler configuration
+    /// Base buffer size for I/O
+    pub io_buffer_size: usize,
+    /// Maximum buffer size
+    pub max_buffer_size: usize,
+    /// High priority size threshold
+    pub high_priority_threshold: usize,
+    /// Low priority size threshold
+    pub low_priority_threshold: usize,
+}
+
+impl Default for ConcurrencyConfig {
+    fn default() -> Self {
+        Self {
+            features: ConcurrencyFeatures::default(),
+
+            // Timeout defaults
+            default_timeout: Duration::from_secs(30),
+            max_timeout: Duration::from_secs(300),
+            enable_dynamic_timeout: true,
+
+            // Lock defaults
+            enable_lock_optimization: true,
+            lock_acquire_timeout: Duration::from_secs(5),
+
+            // Deadlock defaults
+            enable_deadlock_detection: false,
+            deadlock_check_interval: Duration::from_secs(10),
+            hang_threshold: Duration::from_secs(60),
+
+            // Backpressure defaults
+            backpressure_buffer_size: 4 * 1024 * 1024, // 4MB
+            high_watermark: 80,
+            low_watermark: 50,
+
+            // Scheduler defaults
+            io_buffer_size: 64 * 1024,                // 64KB
+            max_buffer_size: 4 * 1024 * 1024,         // 4MB
+            high_priority_threshold: 1024 * 1024,     // 1MB
+            low_priority_threshold: 10 * 1024 * 1024, // 10MB
+        }
+    }
+}
+
+impl ConcurrencyConfig {
+    /// Create configuration from environment variables
+    pub fn from_env() -> Self {
+        let mut config = Self::default();
+
+        // Read from environment if available
+        if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_DEFAULT")
+            && let Ok(secs) = val.parse::<u64>()
+        {
+            config.default_timeout = Duration::from_secs(secs);
+        }
+
+        if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_MAX")
+            && let Ok(secs) = val.parse::<u64>()
+        {
+            config.max_timeout = Duration::from_secs(secs);
+        }
+
+        if let Ok(val) = std::env::var("RUSTFS_BACKPRESSURE_BUFFER_SIZE")
+            && let Ok(size) = val.parse::<usize>()
+        {
+            config.backpressure_buffer_size = size;
+        }
+
+        if let Ok(val) = std::env::var("RUSTFS_IO_BUFFER_SIZE")
+            && let Ok(size) = val.parse::<usize>()
+        {
+            config.io_buffer_size = size;
+        }
+
+        config
+    }
+
+    /// Validate configuration
+    pub fn validate(&self) -> Result<(), ConfigError> {
+        if self.default_timeout > self.max_timeout {
+            return Err(ConfigError::InvalidTimeout("default_timeout cannot exceed max_timeout".to_string()));
+        }
+
+        if self.high_watermark <= self.low_watermark || self.high_watermark > 100 {
+            return Err(ConfigError::InvalidBackpressure(
+                "high_watermark must be > low_watermark and <= 100".to_string(),
+            ));
+        }
+
+        if self.io_buffer_size > self.max_buffer_size {
+            return Err(ConfigError::InvalidScheduler("io_buffer_size cannot exceed max_buffer_size".to_string()));
+        }
+
+        Ok(())
+    }
+}
+
+/// Configuration error
+#[allow(clippy::enum_variant_names)]
+#[derive(Debug, Clone, thiserror::Error)]
+pub enum ConfigError {
+    /// Invalid timeout configuration
+    #[error("Invalid timeout config: {0}")]
+    InvalidTimeout(String),
+
+    /// Invalid backpressure configuration
+    #[error("Invalid backpressure config: {0}")]
+    InvalidBackpressure(String),
+
+    /// Invalid scheduler configuration
+    #[error("Invalid scheduler config: {0}")]
+    InvalidScheduler(String),
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_default_config() {
+        let config = ConcurrencyConfig::default();
+        assert!(config.validate().is_ok());
+    }
+
+    #[test]
+    fn test_invalid_timeout() {
+        let config = ConcurrencyConfig {
+            default_timeout: Duration::from_secs(100),
+            max_timeout: Duration::from_secs(50),
+            ..Default::default()
+        };
+        assert!(
+            config.validate().is_err(),
+            "validate() should return an error when default_timeout > max_timeout"
+        );
+    }
+
+    #[test]
+    fn test_features() {
+        let features = ConcurrencyFeatures::all();
+        assert!(features.any_enabled());
+
+        let features = ConcurrencyFeatures::none();
+        assert!(!features.any_enabled());
+    }
+}

crates/concurrency/src/deadlock.rs

@@ -0,0 +1,207 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Deadlock detection management
+
+use rustfs_io_core::{DeadlockDetector as CoreDeadlockDetector, LockType};
+use rustfs_io_metrics::deadlock_metrics;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+/// Deadlock configuration
+#[derive(Debug, Clone)]
+pub struct DeadlockConfig {
+    /// Enable deadlock detection
+    pub enabled: bool,
+    /// Check interval
+    pub check_interval: Duration,
+    /// Hang threshold
+    pub hang_threshold: Duration,
+}
+
+impl Default for DeadlockConfig {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            check_interval: Duration::from_secs(10),
+            hang_threshold: Duration::from_secs(60),
+        }
+    }
+}
+
+/// Deadlock manager
+pub struct DeadlockManager {
+    config: DeadlockConfig,
+    detector: Arc<CoreDeadlockDetector>,
+    running: Arc<tokio::sync::Mutex<bool>>,
+}
+
+impl DeadlockManager {
+    /// Create a new deadlock manager
+    pub fn new(enabled: bool, check_interval: Duration, hang_threshold: Duration) -> Self {
+        let config = DeadlockConfig {
+            enabled,
+            check_interval,
+            hang_threshold,
+        };
+
+        let core_config = rustfs_io_core::DeadlockDetectorConfig {
+            enabled,
+            detection_interval: check_interval,
+            max_hold_time: hang_threshold,
+        };
+
+        Self {
+            config,
+            detector: Arc::new(CoreDeadlockDetector::new(core_config)),
+            running: Arc::new(tokio::sync::Mutex::new(false)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &DeadlockConfig {
+        &self.config
+    }
+
+    /// Get the core detector
+    pub fn detector(&self) -> Arc<CoreDeadlockDetector> {
+        self.detector.clone()
+    }
+
+    /// Start the deadlock detection background task
+    pub async fn start(&self) {
+        if !self.config.enabled {
+            return;
+        }
+
+        let mut running = self.running.lock().await;
+        if *running {
+            return;
+        }
+        *running = true;
+        drop(running);
+
+        tracing::info!("Deadlock detection started");
+    }
+
+    /// Stop the deadlock detection
+    pub async fn stop(&self) {
+        let mut running = self.running.lock().await;
+        *running = false;
+
+        tracing::info!("Deadlock detection stopped");
+    }
+
+    /// Create a request tracker
+    pub fn track_request(&self, request_id: String, description: String) -> RequestTracker {
+        RequestTracker::new(request_id, description, self.detector.clone())
+    }
+
+    /// Register a lock
+    pub fn register_lock(&self, lock_type: LockType) -> u64 {
+        self.detector.register_lock(lock_type)
+    }
+
+    /// Unregister a lock
+    pub fn unregister_lock(&self, lock_id: u64) {
+        self.detector.unregister_lock(lock_id);
+    }
+
+    /// Detect deadlock
+    pub fn detect_deadlock(&self) -> Option<Vec<u64>> {
+        let result = self.detector.detect_deadlock();
+        if let Some(ref cycle) = result {
+            deadlock_metrics::record_deadlock_detected(cycle.len());
+        }
+        result
+    }
+}
+
+/// Request tracker for tracking resources
+pub struct RequestTracker {
+    request_id: String,
+    description: String,
+    start_time: Instant,
+    resources: HashMap<String, Vec<String>>,
+    detector: Arc<CoreDeadlockDetector>,
+}
+
+impl RequestTracker {
+    fn new(request_id: String, description: String, detector: Arc<CoreDeadlockDetector>) -> Self {
+        let start_time = Instant::now();
+        detector.register_request(&request_id, 1); // Use placeholder thread ID
+
+        Self {
+            request_id,
+            description,
+            start_time,
+            resources: HashMap::new(),
+            detector,
+        }
+    }
+
+    /// Get the request ID
+    pub fn request_id(&self) -> &str {
+        &self.request_id
+    }
+
+    /// Get the description
+    pub fn description(&self) -> &str {
+        &self.description
+    }
+
+    /// Get the elapsed time
+    pub fn elapsed(&self) -> Duration {
+        self.start_time.elapsed()
+    }
+
+    /// Record a lock acquisition
+    pub fn record_lock_acquire(&mut self, lock_id: u64, resource: String) {
+        self.resources.entry("locks".to_string()).or_default().push(resource);
+        self.detector.record_acquire(lock_id, 1); // Use placeholder thread ID
+        deadlock_metrics::record_lock_acquisition("read");
+    }
+
+    /// Record a lock release
+    pub fn record_lock_release(&mut self, lock_id: u64) {
+        self.detector.record_release(lock_id);
+    }
+}
+
+impl Drop for RequestTracker {
+    fn drop(&mut self) {
+        self.detector.unregister_request(&self.request_id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_deadlock_manager_creation() {
+        let manager = DeadlockManager::new(false, Duration::from_secs(10), Duration::from_secs(60));
+        assert!(!manager.config().enabled);
+    }
+
+    #[tokio::test]
+    async fn test_request_tracker() {
+        let manager = DeadlockManager::new(true, Duration::from_secs(10), Duration::from_secs(60));
+        let tracker = manager.track_request("req-1".to_string(), "test request".to_string());
+
+        assert_eq!(tracker.request_id(), "req-1");
+        assert_eq!(tracker.description(), "test request");
+    }
+}

crates/concurrency/src/lib.rs

@@ -0,0 +1,171 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! # RustFS Concurrency Management
+//!
+//! This crate provides comprehensive concurrency management for RustFS,
+//! including timeout control, lock optimization, deadlock detection,
+//! backpressure management, and I/O scheduling.
+//!
+//! ## Features
+//!
+//! All features are controlled by feature flags and can be enabled/disabled at compile time:
+//!
+//! - **timeout**: Dynamic timeout calculation based on data size and transfer rate
+//! - **lock**: Early lock release to reduce contention
+//! - **deadlock**: Request tracking and cycle detection
+//! - **backpressure**: Buffer-based flow control
+//! - **scheduler**: Adaptive buffer sizing and priority queuing
+//!
+//! ## Architecture
+//!
+//! ```text
+//! rustfs-concurrency (Business Layer)
+//!     ├── timeout (Timeout Control)
+//!     ├── lock (Lock Optimization)
+//!     ├── deadlock (Deadlock Detection)
+//!     ├── backpressure (Backpressure Management)
+//!     └── scheduler (I/O Scheduling)
+//!          │
+//!          ├── rustfs-io-core (Core Algorithms)
+//!          └── rustfs-io-metrics (Metrics Collection)
+//! ```
+//!
+//! ## Usage
+//!
+//! ```rust,no_run
+//! use rustfs_concurrency::{ConcurrencyConfig, ConcurrencyManager};
+//!
+//! # #[tokio::main]
+//! # async fn main() {
+//! // Create manager with all features enabled
+//! let config = ConcurrencyConfig::default();
+//! let manager = ConcurrencyManager::new(config);
+//!
+//! // Start services
+//! manager.start().await;
+//!
+//! // Use timeout control (if enabled)
+//! if manager.is_timeout_enabled() {
+//!     let timeout_manager = manager.timeout();
+//!     let _ = timeout_manager;
+//! }
+//!
+//! // Use lock optimization (if enabled)
+//! if manager.is_lock_enabled() {
+//!     let lock_manager = manager.lock();
+//!     let _ = lock_manager;
+//! }
+//!
+//! // Stop services
+//! manager.stop().await;
+//! # }
+//! ```
+
+#![deny(missing_docs)]
+#![deny(unsafe_code)]
+#![cfg_attr(docsrs, feature(doc_cfg))]
+
+// Re-export core types from io-core
+pub use rustfs_io_core::{
+    // Backpressure types
+    BackpressureConfig as CoreBackpressureConfig,
+    BackpressureMonitor as CoreBackpressureMonitor,
+    BackpressureState,
+
+    // Deadlock types
+    DeadlockDetector as CoreDeadlockDetector,
+    IoLoadLevel,
+    IoLoadMetrics,
+    IoPriority,
+    // Scheduler types
+    IoScheduler,
+    IoSchedulingContext,
+    LockInfo,
+    LockOptimizer as CoreLockOptimizer,
+
+    // Lock types
+    LockStats as CoreLockStats,
+    LockType,
+    // Timeout types
+    OperationProgress,
+    TimeoutError,
+    TimeoutStats,
+    WaitGraphEdge,
+
+    calculate_adaptive_timeout,
+    estimate_bytes_per_second,
+};
+
+// Module declarations with feature gates
+#[cfg(feature = "timeout")]
+mod timeout;
+
+#[cfg(feature = "lock")]
+mod lock;
+
+#[cfg(feature = "deadlock")]
+mod deadlock;
+
+#[cfg(feature = "backpressure")]
+mod backpressure;
+
+#[cfg(feature = "scheduler")]
+mod scheduler;
+
+// Public module exports with feature gates
+#[cfg(feature = "timeout")]
+pub use timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager};
+
+#[cfg(feature = "lock")]
+pub use lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard};
+
+#[cfg(feature = "deadlock")]
+pub use deadlock::{DeadlockConfig, DeadlockManager, RequestTracker};
+
+#[cfg(feature = "backpressure")]
+pub use backpressure::{BackpressureConfig, BackpressureManager, BackpressurePipe};
+
+#[cfg(feature = "scheduler")]
+pub use scheduler::{IoStrategy, SchedulerConfig, SchedulerManager};
+
+// Configuration
+mod config;
+pub use config::{ConcurrencyConfig, ConcurrencyFeatures};
+
+// Manager
+mod manager;
+pub use manager::{ConcurrencyManager, GetObjectQueueSnapshot};
+
+// Prelude for convenient imports
+pub mod prelude {
+    //! Prelude module for convenient imports
+
+    #[cfg(feature = "timeout")]
+    pub use crate::timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager};
+
+    #[cfg(feature = "lock")]
+    pub use crate::lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard};
+
+    #[cfg(feature = "deadlock")]
+    pub use crate::deadlock::{DeadlockConfig, DeadlockManager, RequestTracker};
+
+    #[cfg(feature = "backpressure")]
+    pub use crate::backpressure::{BackpressureConfig, BackpressureManager, BackpressurePipe};
+
+    #[cfg(feature = "scheduler")]
+    pub use crate::scheduler::{IoStrategy, SchedulerConfig, SchedulerManager};
+
+    pub use crate::{ConcurrencyConfig, ConcurrencyFeatures, ConcurrencyManager};
+}

crates/concurrency/src/lock.rs

@@ -0,0 +1,219 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Lock optimization management
+
+use rustfs_io_core::{LockOptimizer as CoreLockOptimizer, LockStats};
+use rustfs_io_metrics::lock_metrics;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+/// Lock configuration
+#[derive(Debug, Clone)]
+pub struct LockConfig {
+    /// Enable lock optimization
+    pub enabled: bool,
+    /// Lock acquisition timeout
+    pub acquire_timeout: Duration,
+}
+
+impl Default for LockConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            acquire_timeout: Duration::from_secs(5),
+        }
+    }
+}
+
+/// Lock manager
+pub struct LockManager {
+    config: LockConfig,
+    optimizer: Arc<CoreLockOptimizer>,
+}
+
+impl LockManager {
+    /// Create a new lock manager
+    pub fn new(enabled: bool, acquire_timeout: Duration) -> Self {
+        let config = LockConfig {
+            enabled,
+            acquire_timeout,
+        };
+
+        let core_config = rustfs_io_core::LockOptimizeConfig {
+            enabled,
+            acquire_timeout,
+            max_hold_time_warning: Duration::from_millis(100),
+            adaptive_spin: true,
+            max_spin_iterations: 1000,
+        };
+
+        Self {
+            config,
+            optimizer: Arc::new(CoreLockOptimizer::new(core_config)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &LockConfig {
+        &self.config
+    }
+
+    /// Get the core optimizer
+    pub fn optimizer(&self) -> Arc<CoreLockOptimizer> {
+        self.optimizer.clone()
+    }
+
+    /// Get lock statistics
+    pub fn stats(&self) -> &LockStats {
+        self.optimizer.stats()
+    }
+
+    /// Optimize a lock guard
+    pub fn optimize<G>(&self, guard: G, resource: impl Into<String>) -> OptimizedLockGuard<G> {
+        OptimizedLockGuard::new(guard, resource, self.optimizer.clone())
+    }
+
+    /// Check if optimization is enabled
+    pub fn is_enabled(&self) -> bool {
+        self.config.enabled
+    }
+}
+
+/// Optimized lock guard with early release support
+pub struct OptimizedLockGuard<G> {
+    guard: Option<G>,
+    acquire_time: Instant,
+    released: bool,
+    resource: String,
+    optimizer: Arc<CoreLockOptimizer>,
+}
+
+impl<G> OptimizedLockGuard<G> {
+    fn new(guard: G, resource: impl Into<String>, optimizer: Arc<CoreLockOptimizer>) -> Self {
+        optimizer.on_acquire();
+        lock_metrics::record_lock_optimization_enabled(optimizer.config().enabled);
+
+        Self {
+            guard: Some(guard),
+            acquire_time: Instant::now(),
+            released: false,
+            resource: resource.into(),
+            optimizer,
+        }
+    }
+
+    /// Get the lock hold time
+    pub fn hold_time(&self) -> Duration {
+        self.acquire_time.elapsed()
+    }
+
+    /// Check if the lock has been released
+    pub fn is_released(&self) -> bool {
+        self.released
+    }
+
+    /// Release the lock early
+    pub fn early_release(&mut self) {
+        if self.released {
+            return;
+        }
+
+        let hold_time = self.hold_time();
+        self.guard.take();
+        self.released = true;
+
+        self.optimizer.on_release(hold_time);
+        lock_metrics::record_lock_hold_time(hold_time);
+
+        tracing::debug!(
+            resource = %self.resource,
+            hold_time_ms = hold_time.as_millis(),
+            "Lock released early (optimization active)"
+        );
+    }
+
+    /// Get a reference to the underlying guard
+    pub fn as_ref(&self) -> Option<&G> {
+        if self.released { None } else { self.guard.as_ref() }
+    }
+}
+
+impl<G> Drop for OptimizedLockGuard<G> {
+    fn drop(&mut self) {
+        if !self.released {
+            let hold_time = self.hold_time();
+            self.guard.take();
+            self.released = true;
+
+            self.optimizer.on_release(hold_time);
+            lock_metrics::record_lock_hold_time(hold_time);
+
+            tracing::debug!(
+                resource = %self.resource,
+                hold_time_ms = hold_time.as_millis(),
+                "Lock released on drop (normal release)"
+            );
+        }
+    }
+}
+
+/// Lock scope guard for RAII semantics
+pub struct LockScopeGuard<G> {
+    guard: Option<G>,
+}
+
+impl<G> LockScopeGuard<G> {
+    /// Create a new scope guard
+    pub fn new(guard: G) -> Self {
+        Self { guard: Some(guard) }
+    }
+
+    /// Get a reference to the guard
+    pub fn as_ref(&self) -> Option<&G> {
+        self.guard.as_ref()
+    }
+}
+
+impl<G> Drop for LockScopeGuard<G> {
+    fn drop(&mut self) {
+        self.guard.take();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Mutex;
+
+    #[test]
+    fn test_lock_manager_creation() {
+        let manager = LockManager::new(true, Duration::from_secs(5));
+        assert!(manager.is_enabled());
+    }
+
+    #[test]
+    fn test_optimized_lock_guard() {
+        let manager = LockManager::new(true, Duration::from_secs(5));
+        let mutex = Mutex::new(42);
+        let guard = mutex.lock().unwrap();
+
+        let optimized = manager.optimize(guard, "test_resource");
+        assert!(!optimized.is_released());
+
+        let mut optimized = optimized;
+        optimized.early_release();
+        assert!(optimized.is_released());
+    }
+}

crates/concurrency/src/manager.rs

@@ -0,0 +1,315 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Main concurrency manager
+
+use crate::config::ConcurrencyConfig;
+use std::sync::Arc;
+
+/// Snapshot of disk permit queue usage for GetObject orchestration.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct GetObjectQueueSnapshot {
+    /// Total permits configured for disk reads.
+    pub total_permits: usize,
+    /// Permits currently in use.
+    pub permits_in_use: usize,
+}
+
+impl GetObjectQueueSnapshot {
+    /// Create a queue snapshot from total and available permits.
+    pub fn from_available_permits(total_permits: usize, available_permits: usize) -> Self {
+        Self {
+            total_permits,
+            permits_in_use: total_permits.saturating_sub(available_permits),
+        }
+    }
+
+    /// Return currently available permits.
+    pub fn permits_available(&self) -> usize {
+        self.total_permits.saturating_sub(self.permits_in_use)
+    }
+
+    /// Return queue utilization percentage in the 0-100 range.
+    pub fn utilization_percent(&self) -> f64 {
+        if self.total_permits == 0 {
+            0.0
+        } else {
+            (self.permits_in_use as f64 / self.total_permits as f64) * 100.0
+        }
+    }
+
+    /// Return whether the queue is considered congested.
+    pub fn is_congested(&self, threshold_percent: f64) -> bool {
+        self.utilization_percent() > threshold_percent
+    }
+}
+
+/// Main concurrency manager that provides access to all concurrency features
+pub struct ConcurrencyManager {
+    config: ConcurrencyConfig,
+
+    #[cfg(feature = "timeout")]
+    timeout: Arc<crate::timeout::TimeoutManager>,
+
+    #[cfg(feature = "lock")]
+    lock: Arc<crate::lock::LockManager>,
+
+    #[cfg(feature = "deadlock")]
+    deadlock: Arc<crate::deadlock::DeadlockManager>,
+
+    #[cfg(feature = "backpressure")]
+    backpressure: Arc<crate::backpressure::BackpressureManager>,
+
+    #[cfg(feature = "scheduler")]
+    scheduler: Arc<crate::scheduler::SchedulerManager>,
+}
+
+impl ConcurrencyManager {
+    /// Create a new concurrency manager with the given configuration
+    pub fn new(config: ConcurrencyConfig) -> Self {
+        // Validate configuration
+        if let Err(e) = config.validate() {
+            panic!("Invalid concurrency configuration: {}", e);
+        }
+
+        Self {
+            #[cfg(feature = "timeout")]
+            timeout: Arc::new(crate::timeout::TimeoutManager::new(
+                config.default_timeout,
+                config.max_timeout,
+                config.enable_dynamic_timeout,
+            )),
+
+            #[cfg(feature = "lock")]
+            lock: Arc::new(crate::lock::LockManager::new(
+                config.enable_lock_optimization,
+                config.lock_acquire_timeout,
+            )),
+
+            #[cfg(feature = "deadlock")]
+            deadlock: Arc::new(crate::deadlock::DeadlockManager::new(
+                config.enable_deadlock_detection,
+                config.deadlock_check_interval,
+                config.hang_threshold,
+            )),
+
+            #[cfg(feature = "backpressure")]
+            backpressure: Arc::new(crate::backpressure::BackpressureManager::new(
+                config.backpressure_buffer_size,
+                config.high_watermark,
+                config.low_watermark,
+            )),
+
+            #[cfg(feature = "scheduler")]
+            scheduler: Arc::new(crate::scheduler::SchedulerManager::new(
+                config.io_buffer_size,
+                config.max_buffer_size,
+                config.high_priority_threshold,
+                config.low_priority_threshold,
+            )),
+
+            config,
+        }
+    }
+
+    /// Create with default configuration
+    pub fn with_defaults() -> Self {
+        Self::new(ConcurrencyConfig::default())
+    }
+
+    /// Create from environment variables
+    pub fn from_env() -> Self {
+        Self::new(ConcurrencyConfig::from_env())
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &ConcurrencyConfig {
+        &self.config
+    }
+
+    // ============================================
+    // Feature enable checks
+    // ============================================
+
+    /// Check if timeout feature is enabled
+    pub fn is_timeout_enabled(&self) -> bool {
+        #[cfg(feature = "timeout")]
+        {
+            self.config.features.timeout
+        }
+        #[cfg(not(feature = "timeout"))]
+        {
+            false
+        }
+    }
+
+    /// Check if lock feature is enabled
+    pub fn is_lock_enabled(&self) -> bool {
+        #[cfg(feature = "lock")]
+        {
+            self.config.features.lock
+        }
+        #[cfg(not(feature = "lock"))]
+        {
+            false
+        }
+    }
+
+    /// Check if deadlock feature is enabled
+    pub fn is_deadlock_enabled(&self) -> bool {
+        #[cfg(feature = "deadlock")]
+        {
+            self.config.features.deadlock
+        }
+        #[cfg(not(feature = "deadlock"))]
+        {
+            false
+        }
+    }
+
+    /// Check if backpressure feature is enabled
+    pub fn is_backpressure_enabled(&self) -> bool {
+        #[cfg(feature = "backpressure")]
+        {
+            self.config.features.backpressure
+        }
+        #[cfg(not(feature = "backpressure"))]
+        {
+            false
+        }
+    }
+
+    /// Check if scheduler feature is enabled
+    pub fn is_scheduler_enabled(&self) -> bool {
+        #[cfg(feature = "scheduler")]
+        {
+            self.config.features.scheduler
+        }
+        #[cfg(not(feature = "scheduler"))]
+        {
+            false
+        }
+    }
+
+    // ============================================
+    // Feature accessors
+    // ============================================
+
+    /// Get timeout manager
+    #[cfg(feature = "timeout")]
+    pub fn timeout(&self) -> Arc<crate::timeout::TimeoutManager> {
+        self.timeout.clone()
+    }
+
+    /// Get lock manager
+    #[cfg(feature = "lock")]
+    pub fn lock(&self) -> Arc<crate::lock::LockManager> {
+        self.lock.clone()
+    }
+
+    /// Get deadlock manager
+    #[cfg(feature = "deadlock")]
+    pub fn deadlock(&self) -> Arc<crate::deadlock::DeadlockManager> {
+        self.deadlock.clone()
+    }
+
+    /// Get backpressure manager
+    #[cfg(feature = "backpressure")]
+    pub fn backpressure(&self) -> Arc<crate::backpressure::BackpressureManager> {
+        self.backpressure.clone()
+    }
+
+    /// Get scheduler manager
+    #[cfg(feature = "scheduler")]
+    pub fn scheduler(&self) -> Arc<crate::scheduler::SchedulerManager> {
+        self.scheduler.clone()
+    }
+
+    // ============================================
+    // Lifecycle management
+    // ============================================
+
+    /// Start all enabled services (e.g., deadlock detection background task)
+    pub async fn start(&self) {
+        #[cfg(feature = "deadlock")]
+        {
+            if self.config.enable_deadlock_detection {
+                self.deadlock.start().await;
+            }
+        }
+
+        tracing::info!(
+            "Concurrency manager started (timeout={}, lock={}, deadlock={}, backpressure={}, scheduler={})",
+            self.is_timeout_enabled(),
+            self.is_lock_enabled(),
+            self.is_deadlock_enabled(),
+            self.is_backpressure_enabled(),
+            self.is_scheduler_enabled()
+        );
+    }
+
+    /// Stop all services
+    pub async fn stop(&self) {
+        #[cfg(feature = "deadlock")]
+        {
+            self.deadlock.stop().await;
+        }
+
+        tracing::info!("Concurrency manager stopped");
+    }
+}
+
+impl Default for ConcurrencyManager {
+    fn default() -> Self {
+        Self::with_defaults()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_queue_snapshot() {
+        let snapshot = GetObjectQueueSnapshot::from_available_permits(64, 16);
+        assert_eq!(snapshot.permits_in_use, 48);
+        assert_eq!(snapshot.permits_available(), 16);
+        assert!(snapshot.is_congested(70.0));
+    }
+
+    #[test]
+    fn test_manager_creation() {
+        let manager = ConcurrencyManager::with_defaults();
+        assert!(manager.config().validate().is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_manager_lifecycle() {
+        let manager = ConcurrencyManager::with_defaults();
+        manager.start().await;
+        manager.stop().await;
+    }
+
+    #[test]
+    fn test_feature_checks() {
+        let manager = ConcurrencyManager::with_defaults();
+
+        // These should return the feature flag status
+        let _ = manager.is_timeout_enabled();
+        let _ = manager.is_lock_enabled();
+        let _ = manager.is_deadlock_enabled();
+        let _ = manager.is_backpressure_enabled();
+        let _ = manager.is_scheduler_enabled();
+    }
+}

crates/concurrency/src/scheduler.rs

@@ -0,0 +1,225 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! I/O scheduler management
+
+use rustfs_io_core::{
+    IoLoadLevel, IoPriority, IoScheduler as CoreIoScheduler, IoSchedulingContext,
+    io_profile::{AccessPattern, StorageMedia},
+};
+use rustfs_io_metrics::io_metrics;
+use std::sync::Arc;
+use std::time::Duration;
+
+/// Scheduler configuration
+#[derive(Debug, Clone)]
+pub struct SchedulerConfig {
+    /// Base buffer size
+    pub base_buffer_size: usize,
+    /// Maximum buffer size
+    pub max_buffer_size: usize,
+    /// High priority threshold
+    pub high_priority_threshold: usize,
+    /// Low priority threshold
+    pub low_priority_threshold: usize,
+}
+
+impl Default for SchedulerConfig {
+    fn default() -> Self {
+        Self {
+            base_buffer_size: 64 * 1024,              // 64KB
+            max_buffer_size: 4 * 1024 * 1024,         // 4MB
+            high_priority_threshold: 1024 * 1024,     // 1MB
+            low_priority_threshold: 10 * 1024 * 1024, // 10MB
+        }
+    }
+}
+
+/// Scheduler manager
+pub struct SchedulerManager {
+    config: SchedulerConfig,
+    scheduler: Arc<CoreIoScheduler>,
+}
+
+impl SchedulerManager {
+    /// Create a new scheduler manager
+    pub fn new(
+        base_buffer_size: usize,
+        max_buffer_size: usize,
+        high_priority_threshold: usize,
+        low_priority_threshold: usize,
+    ) -> Self {
+        let config = SchedulerConfig {
+            base_buffer_size,
+            max_buffer_size,
+            high_priority_threshold,
+            low_priority_threshold,
+        };
+
+        let core_config = rustfs_io_core::IoSchedulerConfig::default();
+
+        Self {
+            config,
+            scheduler: Arc::new(CoreIoScheduler::new(core_config)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &SchedulerConfig {
+        &self.config
+    }
+
+    /// Get the scheduler
+    pub fn scheduler(&self) -> Arc<CoreIoScheduler> {
+        self.scheduler.clone()
+    }
+
+    /// Create an I/O strategy
+    pub fn create_strategy(&self) -> IoStrategy {
+        IoStrategy::new(self.config.clone(), self.scheduler.clone())
+    }
+
+    /// Calculate buffer size
+    pub fn calculate_buffer_size(
+        &self,
+        file_size: i64,
+        media: StorageMedia,
+        pattern: AccessPattern,
+        load: IoLoadLevel,
+        concurrent: usize,
+    ) -> usize {
+        let strategy = self.create_strategy();
+        strategy.calculate_buffer_size(file_size, media, pattern, load, concurrent)
+    }
+
+    /// Get I/O priority
+    pub fn get_priority(&self, size: i64) -> IoPriority {
+        IoPriority::from_size(size, self.config.high_priority_threshold, self.config.low_priority_threshold)
+    }
+}
+
+/// I/O strategy
+pub struct IoStrategy {
+    config: SchedulerConfig,
+    scheduler: Arc<CoreIoScheduler>,
+}
+
+impl IoStrategy {
+    fn new(config: SchedulerConfig, scheduler: Arc<CoreIoScheduler>) -> Self {
+        Self { config, scheduler }
+    }
+
+    /// Calculate buffer size with multi-factor strategy
+    pub fn calculate_buffer_size(
+        &self,
+        file_size: i64,
+        media: StorageMedia,
+        pattern: AccessPattern,
+        load: IoLoadLevel,
+        concurrent: usize,
+    ) -> usize {
+        // Create scheduling context
+        let _ctx = IoSchedulingContext::new(file_size, self.config.base_buffer_size)
+            .with_sequential(matches!(pattern, AccessPattern::Sequential))
+            .with_media(media);
+
+        // Get base buffer size from core scheduler
+        let permit_wait = Duration::from_millis(10); // Default wait time
+        let is_sequential = matches!(pattern, AccessPattern::Sequential);
+        let core_strategy = self.scheduler.calculate_strategy(file_size, permit_wait, is_sequential);
+        let base_size = core_strategy.buffer_size;
+
+        // Apply multi-factor adjustments
+        let adjusted_size = self.apply_adjustments(base_size, media, pattern, load, concurrent);
+
+        // Record metrics
+        io_metrics::record_io_scheduler_decision(adjusted_size, load.as_str(), pattern.as_str());
+
+        adjusted_size.min(self.config.max_buffer_size)
+    }
+
+    fn apply_adjustments(
+        &self,
+        base_size: usize,
+        media: StorageMedia,
+        pattern: AccessPattern,
+        load: IoLoadLevel,
+        concurrent: usize,
+    ) -> usize {
+        let mut size = base_size;
+
+        // Media adjustment
+        size = match media {
+            StorageMedia::Nvme => (size as f64 * 1.5) as usize,
+            StorageMedia::Ssd => (size as f64 * 1.2) as usize,
+            StorageMedia::Hdd => size,
+            _ => size,
+        };
+
+        // Pattern adjustment
+        size = match pattern {
+            AccessPattern::Sequential => (size as f64 * 1.5) as usize,
+            AccessPattern::Random => (size as f64 * 0.5) as usize,
+            _ => size,
+        };
+
+        // Load adjustment
+        size = match load {
+            IoLoadLevel::Low => (size as f64 * 1.2) as usize,
+            IoLoadLevel::Medium => size,
+            IoLoadLevel::High => (size as f64 * 0.7) as usize,
+            IoLoadLevel::Critical => (size as f64 * 0.5) as usize,
+        };
+
+        // Concurrency adjustment
+        if concurrent > 10 {
+            size = (size as f64 * 0.8) as usize;
+        }
+
+        size
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &SchedulerConfig {
+        &self.config
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_scheduler_config() {
+        let config = SchedulerConfig::default();
+        assert!(config.base_buffer_size < config.max_buffer_size);
+    }
+
+    #[test]
+    fn test_scheduler_manager() {
+        let manager = SchedulerManager::new(1024, 4096, 512, 2048);
+        let priority = manager.get_priority(100);
+        assert!(priority.is_high());
+    }
+
+    #[test]
+    fn test_io_strategy() {
+        let manager = SchedulerManager::new(1024, 4096, 512, 2048);
+        let strategy = manager.create_strategy();
+
+        let size = strategy.calculate_buffer_size(1024 * 1024, StorageMedia::Ssd, AccessPattern::Sequential, IoLoadLevel::Low, 1);
+
+        assert!(size > 0);
+    }
+}

crates/concurrency/src/timeout.rs

@@ -0,0 +1,150 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Timeout management for operations
+
+use rustfs_io_core::{TimeoutError, calculate_adaptive_timeout};
+use std::time::{Duration, Instant};
+use tokio_util::sync::CancellationToken;
+
+/// Timeout configuration
+#[derive(Debug, Clone)]
+pub struct TimeoutConfig {
+    /// Default timeout duration
+    pub default_timeout: Duration,
+    /// Maximum timeout duration
+    pub max_timeout: Duration,
+    /// Enable dynamic timeout calculation
+    pub enable_dynamic: bool,
+}
+
+impl Default for TimeoutConfig {
+    fn default() -> Self {
+        Self {
+            default_timeout: Duration::from_secs(30),
+            max_timeout: Duration::from_secs(300),
+            enable_dynamic: true,
+        }
+    }
+}
+
+/// Timeout manager
+pub struct TimeoutManager {
+    config: TimeoutConfig,
+}
+
+impl TimeoutManager {
+    /// Create a new timeout manager
+    pub fn new(default_timeout: Duration, max_timeout: Duration, enable_dynamic: bool) -> Self {
+        Self {
+            config: TimeoutConfig {
+                default_timeout,
+                max_timeout,
+                enable_dynamic,
+            },
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &TimeoutConfig {
+        &self.config
+    }
+
+    /// Calculate timeout for a given size
+    pub fn calculate_timeout(&self, size: u64, _history: &[Duration]) -> Duration {
+        if !self.config.enable_dynamic {
+            return self.config.default_timeout;
+        }
+
+        calculate_adaptive_timeout(self.config.default_timeout, None, 0, size).min(self.config.max_timeout)
+    }
+
+    /// Wrap an operation with timeout control
+    pub async fn wrap_operation<F, T, E>(&self, operation: F, timeout: Option<Duration>) -> Result<T, TimeoutError>
+    where
+        F: std::future::Future<Output = Result<T, E>>,
+        E: Into<TimeoutError>,
+    {
+        let timeout = timeout.unwrap_or(self.config.default_timeout);
+
+        match tokio::time::timeout(timeout, operation).await {
+            Ok(Ok(result)) => Ok(result),
+            Ok(Err(e)) => Err(e.into()),
+            Err(_) => Err(TimeoutError::TimedOut(timeout)),
+        }
+    }
+
+    /// Create a timeout guard for manual timeout control
+    pub fn create_guard(&self, timeout: Option<Duration>) -> TimeoutGuard {
+        TimeoutGuard::new(timeout.unwrap_or(self.config.default_timeout))
+    }
+}
+
+/// Timeout guard for manual timeout control
+pub struct TimeoutGuard {
+    timeout: Duration,
+    start: Instant,
+    cancel_token: CancellationToken,
+}
+
+impl TimeoutGuard {
+    fn new(timeout: Duration) -> Self {
+        Self {
+            timeout,
+            start: Instant::now(),
+            cancel_token: CancellationToken::new(),
+        }
+    }
+
+    /// Check if timeout has elapsed
+    pub fn is_timed_out(&self) -> bool {
+        self.start.elapsed() > self.timeout
+    }
+
+    /// Get remaining time
+    pub fn remaining(&self) -> Duration {
+        self.timeout.saturating_sub(self.start.elapsed())
+    }
+
+    /// Get the cancellation token
+    pub fn cancel_token(&self) -> CancellationToken {
+        self.cancel_token.clone()
+    }
+
+    /// Cancel the operation
+    pub fn cancel(&self) {
+        self.cancel_token.cancel();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_timeout_config() {
+        let config = TimeoutConfig::default();
+        assert!(config.default_timeout < config.max_timeout);
+    }
+
+    #[tokio::test]
+    async fn test_wrap_operation_success() {
+        let manager = TimeoutManager::new(Duration::from_secs(5), Duration::from_secs(10), true);
+
+        let result = manager.wrap_operation(async { Ok::<_, TimeoutError>(42) }, None).await;
+
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), 42);
+    }
+}

crates/config/Cargo.toml

@@ -37,3 +37,6 @@ constants = ["dep:const-str"]
 notify = ["dep:const-str", "constants"]
 observability = ["constants"]
 opa = ["constants"]
+
+[lib]
+doctest = false

crates/config/src/audit/mqtt.rs

@@ -23,9 +23,15 @@ pub const ENV_AUDIT_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_AUDIT_MQTT_RECONNECT
 pub const ENV_AUDIT_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_AUDIT_MQTT_KEEP_ALIVE_INTERVAL";
 pub const ENV_AUDIT_MQTT_QUEUE_DIR: &str = "RUSTFS_AUDIT_MQTT_QUEUE_DIR";
 pub const ENV_AUDIT_MQTT_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_MQTT_QUEUE_LIMIT";
+pub const ENV_AUDIT_MQTT_TLS_POLICY: &str = "RUSTFS_AUDIT_MQTT_TLS_POLICY";
+pub const ENV_AUDIT_MQTT_TLS_CA: &str = "RUSTFS_AUDIT_MQTT_TLS_CA";
+pub const ENV_AUDIT_MQTT_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_MQTT_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_MQTT_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_MQTT_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_MQTT_TLS_TRUST_LEAF_AS_CA: &str = "RUSTFS_AUDIT_MQTT_TLS_TRUST_LEAF_AS_CA";
+pub const ENV_AUDIT_MQTT_WS_PATH_ALLOWLIST: &str = "RUSTFS_AUDIT_MQTT_WS_PATH_ALLOWLIST";
 
 /// A list of all valid configuration keys for an MQTT target.
-pub const ENV_AUDIT_MQTT_KEYS: &[&str; 10] = &[
+pub const ENV_AUDIT_MQTT_KEYS: &[&str; 16] = &[
     ENV_AUDIT_MQTT_ENABLE,
     ENV_AUDIT_MQTT_BROKER,
     ENV_AUDIT_MQTT_TOPIC,
@@ -36,6 +42,12 @@ pub const ENV_AUDIT_MQTT_KEYS: &[&str; 10] = &[
     ENV_AUDIT_MQTT_KEEP_ALIVE_INTERVAL,
     ENV_AUDIT_MQTT_QUEUE_DIR,
     ENV_AUDIT_MQTT_QUEUE_LIMIT,
+    ENV_AUDIT_MQTT_TLS_POLICY,
+    ENV_AUDIT_MQTT_TLS_CA,
+    ENV_AUDIT_MQTT_TLS_CLIENT_CERT,
+    ENV_AUDIT_MQTT_TLS_CLIENT_KEY,
+    ENV_AUDIT_MQTT_TLS_TRUST_LEAF_AS_CA,
+    ENV_AUDIT_MQTT_WS_PATH_ALLOWLIST,
 ];
 
 /// A list of all valid configuration keys for an MQTT target.
@@ -50,5 +62,11 @@ pub const AUDIT_MQTT_KEYS: &[&str] = &[
     crate::MQTT_KEEP_ALIVE_INTERVAL,
     crate::MQTT_QUEUE_DIR,
     crate::MQTT_QUEUE_LIMIT,
+    crate::MQTT_TLS_POLICY,
+    crate::MQTT_TLS_CA,
+    crate::MQTT_TLS_CLIENT_CERT,
+    crate::MQTT_TLS_CLIENT_KEY,
+    crate::MQTT_TLS_TRUST_LEAF_AS_CA,
+    crate::MQTT_WS_PATH_ALLOWLIST,
     crate::COMMENT_KEY,
 ];

crates/config/src/constants/app.rs

@@ -131,9 +131,64 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable for server access key.
+pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
+
+/// Environment variable for server access key file.
+pub const ENV_RUSTFS_ACCESS_KEY_FILE: &str = "RUSTFS_ACCESS_KEY_FILE";
+
+/// Environment variable for server root user.
+pub const ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
+
+/// Environment variable for server secret key.
+pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
+
+/// Environment variable for server secret key file.
+pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
+
+/// Environment variable for server root password.
+pub const ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
+
+/// Environment variable for server OBS endpoint.
+pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
+
+/// Environment variable for console server enable.
+pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
+
+/// Environment variable for console server address.
+pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
+
 /// Environment variable for server tls path.
 pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
 
+/// Environment variable for server KMS enable.
+pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
+
+/// Default KMS enable for server-side encryption
+/// This is the default value for enabling KMS encryption for server-side encryption.
+/// Default value: false
+pub const DEFAULT_KMS_ENABLE: bool = false;
+
+/// Environment variable for server KMS backend.
+pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
+
+/// Environment variable for Vault Transit mount path.
+pub const ENV_RUSTFS_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
+
+/// Default KMS backend for server-side encryption
+/// This is the default KMS backend for server-side encryption.
+/// Default value: local
+pub const DEFAULT_KMS_BACKEND: &str = "local";
+
+/// Environment variable for selecting the buffer profile used for adaptive buffer sizing.
+pub const ENV_RUSTFS_BUFFER_PROFILE: &str = "RUSTFS_BUFFER_PROFILE";
+
+/// Default buffer profile for adaptive buffer sizing
+/// This is the default buffer profile for adaptive buffer sizing.
+/// It is used to identify the workload profile for adaptive buffer sizing.
+/// Default value: GeneralPurpose
+pub const DEFAULT_BUFFER_PROFILE: &str = "GeneralPurpose";
+
 /// Default value for the server TLS path if `ENV_RUSTFS_TLS_PATH` is not set.
 pub const DEFAULT_RUSTFS_TLS_PATH: &str = "";
 

crates/config/src/constants/capacity.rs

@@ -0,0 +1,155 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Capacity calculation configuration constants
+
+// ============================================================================
+// Environment Variable Names
+// ============================================================================
+
+/// Environment variable for scheduled update interval
+pub const ENV_CAPACITY_SCHEDULED_INTERVAL: &str = "RUSTFS_CAPACITY_SCHEDULED_INTERVAL";
+
+/// Environment variable for write trigger delay
+pub const ENV_CAPACITY_WRITE_TRIGGER_DELAY: &str = "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY";
+
+/// Environment variable for write frequency threshold
+pub const ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD: &str = "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD";
+
+/// Environment variable for fast update threshold
+pub const ENV_CAPACITY_FAST_UPDATE_THRESHOLD: &str = "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD";
+
+/// Environment variable for max files threshold
+pub const ENV_CAPACITY_MAX_FILES_THRESHOLD: &str = "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD";
+
+/// Environment variable for statistics timeout
+pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
+
+/// Environment variable for sample rate
+pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
+
+/// Environment variable for following symbolic links during capacity calculation
+pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
+
+/// Environment variable for maximum symlink follow depth
+pub const ENV_CAPACITY_MAX_SYMLINK_DEPTH: &str = "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH";
+
+/// Environment variable for enabling dynamic timeout calculation
+pub const ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: &str = "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT";
+
+/// Environment variable for minimum capacity calculation timeout
+pub const ENV_CAPACITY_MIN_TIMEOUT: &str = "RUSTFS_CAPACITY_MIN_TIMEOUT";
+
+/// Environment variable for maximum capacity calculation timeout
+pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
+
+/// Environment variable for progress stall detection timeout
+pub const ENV_CAPACITY_STALL_TIMEOUT: &str = "RUSTFS_CAPACITY_STALL_TIMEOUT";
+
+// ============================================================================
+// Default Values
+// ============================================================================
+
+/// Scheduled update interval in seconds
+/// Default: 120 seconds (2 minutes)
+pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 120;
+
+/// Write trigger delay in seconds
+/// Default: 5 seconds
+pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 5;
+
+/// Write frequency threshold (writes per minute)
+/// Default: 5 writes/minute
+pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 5;
+
+/// Fast update threshold in seconds
+/// Default: 30 seconds
+pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 30;
+
+/// Maximum files threshold for sampling
+/// Default: 200,000 files
+pub const DEFAULT_MAX_FILES_THRESHOLD: usize = 200_000;
+
+/// Statistics timeout in seconds
+/// Default: 3 seconds
+pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
+
+/// Sampling rate (1 in every N files)
+/// Default: 200
+pub const DEFAULT_SAMPLE_RATE: usize = 200;
+
+/// Follow symbolic links during capacity calculation
+/// Default: false (disabled for safety)
+pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
+
+/// Maximum symlink follow depth
+/// Default: 3 levels
+pub const DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH: u8 = 3;
+
+/// Enable dynamic timeout calculation based on directory characteristics
+/// Default: true (enabled)
+pub const DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: bool = true;
+
+/// Minimum capacity calculation timeout in seconds
+/// Default: 2 seconds
+pub const DEFAULT_CAPACITY_MIN_TIMEOUT_SECS: u64 = 2;
+
+/// Maximum capacity calculation timeout in seconds
+/// Default: 15 seconds
+pub const DEFAULT_CAPACITY_MAX_TIMEOUT_SECS: u64 = 15;
+
+/// Progress stall detection timeout in seconds
+/// Default: 20 seconds
+pub const DEFAULT_CAPACITY_STALL_TIMEOUT_SECS: u64 = 20;
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_env_var_names() {
+        assert_eq!(ENV_CAPACITY_SCHEDULED_INTERVAL, "RUSTFS_CAPACITY_SCHEDULED_INTERVAL");
+        assert_eq!(ENV_CAPACITY_WRITE_TRIGGER_DELAY, "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY");
+        assert_eq!(ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_FAST_UPDATE_THRESHOLD, "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
+        assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
+        assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
+        assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_STALL_TIMEOUT, "RUSTFS_CAPACITY_STALL_TIMEOUT");
+    }
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, 120);
+        assert_eq!(DEFAULT_WRITE_TRIGGER_DELAY_SECS, 5);
+        assert_eq!(DEFAULT_WRITE_FREQUENCY_THRESHOLD, 5);
+        assert_eq!(DEFAULT_FAST_UPDATE_THRESHOLD_SECS, 30);
+        assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
+        assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_SAMPLE_RATE, 200);
+        assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
+        assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2);
+        assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15);
+        assert_eq!(DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, 20);
+    }
+}

crates/config/src/constants/mod.rs

@@ -14,6 +14,7 @@
 
 pub(crate) mod app;
 pub(crate) mod body_limits;
+pub(crate) mod capacity;
 pub(crate) mod compress;
 pub(crate) mod console;
 pub(crate) mod env;
@@ -28,3 +29,5 @@ pub(crate) mod runtime;
 pub(crate) mod scanner;
 pub(crate) mod targets;
 pub(crate) mod tls;
+pub(crate) mod workload;
+pub(crate) mod zero_copy;

crates/config/src/constants/object.rs

@@ -12,62 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/// Environment variable name to toggle object-level in-memory caching.
-///
-/// - Purpose: Enable or disable the object-level in-memory cache (moka).
-/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config.
-/// - Semantics: When enabled, the system keeps fully-read objects in memory to reduce backend requests; when disabled, reads bypass the object cache.
-/// - Example: `export RUSTFS_OBJECT_CACHE_ENABLE=true`
-/// - Note: Evaluate together with `RUSTFS_OBJECT_CACHE_CAPACITY_MB`, TTL/TTI and concurrency thresholds to balance memory usage and throughput.
-pub const ENV_OBJECT_CACHE_ENABLE: &str = "RUSTFS_OBJECT_CACHE_ENABLE";
-
-/// Environment variable name that specifies the object cache capacity in megabytes.
-///
-/// - Purpose: Set the maximum total capacity of the object cache (in MB).
-/// - Unit: MB (1 MB = 1_048_576 bytes).
-/// - Valid values: any positive integer (0 may indicate disabled or alternative handling).
-/// - Semantics: When the moka cache reaches this capacity, eviction policies will remove entries; tune according to available memory and object size distribution.
-/// - Example: `export RUSTFS_OBJECT_CACHE_CAPACITY_MB=512`
-/// - Note: Actual memory usage will be slightly higher due to object headers and indexing overhead.
-pub const ENV_OBJECT_CACHE_CAPACITY_MB: &str = "RUSTFS_OBJECT_CACHE_CAPACITY_MB";
-
-/// Environment variable name for maximum object size eligible for caching in megabytes.
-///
-/// - Purpose: Define the upper size limit for individual objects to be considered for caching.
-/// - Unit: MB (1 MB = 1_048_576 bytes).
-/// - Valid values: any positive integer; objects larger than this size will not be cached.
-/// - Semantics: Prevents caching of excessively large objects that could monopolize cache capacity; tune based on typical object size distribution.
-/// - Example: `export RUSTFS_OBJECT_CACHE_MAX_OBJECT_SIZE_MB=50`
-/// - Note: Setting this too low may reduce cache effectiveness; setting it too high may lead to inefficient memory usage.
-pub const ENV_OBJECT_CACHE_MAX_OBJECT_SIZE_MB: &str = "RUSTFS_OBJECT_CACHE_MAX_OBJECT_SIZE_MB";
-
-/// Environment variable name for object cache TTL (time-to-live) in seconds.
-///
-/// - Purpose: Specify the maximum lifetime of a cached entry from the moment it is written.
-/// - Unit: seconds (u64).
-/// - Semantics: TTL acts as a hard upper bound; entries older than TTL are considered expired and removed by periodic cleanup.
-/// - Example: `export RUSTFS_OBJECT_CACHE_TTL_SECS=300`
-/// - Note: TTL and TTI both apply; either policy can cause eviction.
-pub const ENV_OBJECT_CACHE_TTL_SECS: &str = "RUSTFS_OBJECT_CACHE_TTL_SECS";
-
-/// Environment variable name for object cache TTI (time-to-idle) in seconds.
-///
-/// - Purpose: Specify how long an entry may remain in cache without being accessed before it is evicted.
-/// - Unit: seconds (u64).
-/// - Semantics: TTI helps remove one-time or infrequently used entries; frequent accesses reset idle timers but do not extend beyond TTL unless additional logic exists.
-/// - Example: `export RUSTFS_OBJECT_CACHE_TTI_SECS=120`
-/// - Note: Works together with TTL to keep the cache populated with actively used objects.
-pub const ENV_OBJECT_CACHE_TTI_SECS: &str = "RUSTFS_OBJECT_CACHE_TTI_SECS";
-
-/// Environment variable name for threshold of "hot" object hit count used to extend life.
-///
-/// - Purpose: Define a hit-count threshold to mark objects as "hot" so they may be treated preferentially near expiration.
-/// - Valid values: positive integer (usize).
-/// - Semantics: Objects reaching this hit count can be considered for relaxed eviction to avoid thrashing hot items.
-/// - Example: `export RUSTFS_OBJECT_HOT_MIN_HITS_TO_EXTEND=5`
-/// - Note: This is an optional enhancement and requires cache-layer statistics and extension logic to take effect.
-pub const ENV_OBJECT_HOT_MIN_HITS_TO_EXTEND: &str = "RUSTFS_OBJECT_HOT_MIN_HITS_TO_EXTEND";
-
 /// Environment variable name for high concurrency threshold used in adaptive buffering.
 ///
 /// - Purpose: When concurrent request count exceeds this threshold, the system enters a "high concurrency" optimization mode to reduce per-request buffer sizes.
@@ -94,24 +38,6 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
 /// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
 pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";
 
-/// Default: object caching is disabled.
-///
-/// - Semantics: Safe default to avoid unexpected memory usage or cache consistency concerns when not explicitly enabled.
-/// - Default is set to false (disabled).
-pub const DEFAULT_OBJECT_CACHE_ENABLE: bool = false;
-
-/// Default object cache capacity in MB.
-///
-/// - Default: 100 MB (can be overridden by `RUSTFS_OBJECT_CACHE_CAPACITY_MB`).
-/// - Note: Choose a conservative default to reduce memory pressure in development/testing.
-pub const DEFAULT_OBJECT_CACHE_CAPACITY_MB: u64 = 100;
-
-/// Default maximum object size eligible for caching in MB.
-///
-/// - Default: 10 MB (can be overridden by `RUSTFS_OBJECT_CACHE_MAX_OBJECT_SIZE_MB`).
-/// - Note: Balances caching effectiveness with memory usage.
-pub const DEFAULT_OBJECT_CACHE_MAX_OBJECT_SIZE_MB: usize = 10;
-
 /// Maximum concurrent requests before applying aggressive optimization.
 ///
 /// When concurrent requests exceed this threshold (>8), the system switches to
@@ -141,33 +67,6 @@ pub const DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: usize = 4;
 /// Default is set to 64 concurrent reads.
 pub const DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS: usize = 64;
 
-/// Time-to-live for cached objects (5 minutes = 300 seconds).
-///
-/// After this duration, cached objects are automatically expired by Moka's
-/// background cleanup process, even if they haven't been accessed. This prevents
-/// stale data from consuming cache capacity indefinitely.
-///
-/// Default is set to 300 seconds.
-pub const DEFAULT_OBJECT_CACHE_TTL_SECS: u64 = 300;
-
-/// Time-to-idle for cached objects (2 minutes = 120 seconds).
-///
-/// Objects that haven't been accessed for this duration are automatically evicted,
-/// even if their TTL hasn't expired. This ensures cache is populated with actively
-/// used objects and clears out one-time reads efficiently.
-///
-/// Default is set to 120 seconds.
-pub const DEFAULT_OBJECT_CACHE_TTI_SECS: u64 = 120;
-
-/// Minimum hit count to extend object lifetime beyond TTL.
-///
-/// "Hot" objects that have been accessed at least this many times are treated
-/// specially - they can survive longer in cache even as they approach TTL expiration.
-/// This prevents frequently accessed objects from being evicted prematurely.
-///
-/// Default is set to 5 hits.
-pub const DEFAULT_OBJECT_HOT_MIN_HITS_TO_EXTEND: usize = 5;
-
 /// Skip bitrot hash verification on GetObject reads.
 ///
 /// When enabled, GetObject reads skip the per-shard hash
@@ -180,3 +79,425 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
+
+// =============================================================================
+// Concurrent Request Fix - Timeout and Backpressure Configuration
+// =============================================================================
+
+/// Environment variable for GetObject request timeout in seconds.
+///
+/// When a GetObject request exceeds this duration, it will be cancelled
+/// and return a 504 Gateway Timeout error. This prevents requests from
+/// hanging indefinitely due to deadlocks or resource exhaustion.
+///
+/// Default: 30 seconds (can be overridden by `RUSTFS_OBJECT_GET_TIMEOUT`).
+/// Set to 0 to disable timeout (not recommended for production).
+pub const ENV_OBJECT_GET_TIMEOUT: &str = "RUSTFS_OBJECT_GET_TIMEOUT";
+
+/// Default GetObject request timeout in seconds.
+///
+/// This value balances between allowing large object transfers to complete
+/// and preventing indefinite hangs. For 20-26MB objects with concurrent
+/// range reads, 30 seconds should be sufficient under normal conditions.
+pub const DEFAULT_OBJECT_GET_TIMEOUT: u64 = 30;
+
+/// Environment variable for disk read operation timeout in seconds.
+///
+/// Individual disk read operations that exceed this duration will be
+/// cancelled and treated as failures. This helps detect slow or hung
+/// disks without waiting indefinitely.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DISK_READ_TIMEOUT`).
+pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT";
+
+/// Default disk read timeout in seconds.
+pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
+
+/// Environment variable for minimum GetObject timeout in seconds.
+///
+/// When dynamic timeout calculation is enabled, this is the minimum timeout
+/// that will be used regardless of object size. This prevents excessively
+/// short timeouts for very small objects.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_MIN_TIMEOUT`).
+pub const ENV_OBJECT_MIN_TIMEOUT: &str = "RUSTFS_OBJECT_MIN_TIMEOUT";
+
+/// Default minimum GetObject timeout: 5 seconds.
+pub const DEFAULT_OBJECT_MIN_TIMEOUT: u64 = 5;
+
+/// Environment variable for maximum GetObject timeout in seconds.
+///
+/// When dynamic timeout calculation is enabled, this is the maximum timeout
+/// that will be used regardless of object size. This prevents excessively
+/// long timeouts for very large objects.
+///
+/// Default: 300 seconds (5 minutes, can be overridden by `RUSTFS_OBJECT_MAX_TIMEOUT`).
+pub const ENV_OBJECT_MAX_TIMEOUT: &str = "RUSTFS_OBJECT_MAX_TIMEOUT";
+
+/// Default maximum GetObject timeout: 300 seconds (5 minutes).
+pub const DEFAULT_OBJECT_MAX_TIMEOUT: u64 = 300;
+
+/// Environment variable for default bytes per second for timeout estimation.
+///
+/// This value is used to estimate timeout duration based on object size when
+/// dynamic timeout calculation is enabled. The timeout is calculated as:
+/// (object_size / bytes_per_second) * buffer_factor
+///
+/// Default: 1048576 (1 MB/s, can be overridden by `RUSTFS_OBJECT_BYTES_PER_SECOND`).
+pub const ENV_OBJECT_BYTES_PER_SECOND: &str = "RUSTFS_OBJECT_BYTES_PER_SECOND";
+
+/// Default bytes per second for timeout estimation: 1 MB/s.
+pub const DEFAULT_OBJECT_BYTES_PER_SECOND: u64 = 1024 * 1024;
+
+/// Environment variable to enable dynamic timeout calculation.
+///
+/// When enabled, timeout is calculated based on object size and transfer speed
+/// rather than using a fixed timeout value. This provides better timeout
+/// handling for objects of varying sizes.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_DYNAMIC_TIMEOUT_ENABLE`).
+pub const ENV_OBJECT_DYNAMIC_TIMEOUT_ENABLE: &str = "RUSTFS_OBJECT_DYNAMIC_TIMEOUT_ENABLE";
+
+/// Default: dynamic timeout calculation is enabled.
+pub const DEFAULT_OBJECT_DYNAMIC_TIMEOUT_ENABLE: bool = true;
+
+/// Environment variable for duplex pipe buffer size in bytes.
+///
+/// The duplex pipe connects the disk read task to the HTTP response stream.
+/// A larger buffer reduces backpressure but increases memory usage.
+/// For large objects (20-26MB), a 4MB buffer provides good throughput.
+///
+/// Default: 4194304 (4 MB, can be overridden by `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE`).
+/// Minimum recommended: 1048576 (1 MB).
+pub const ENV_OBJECT_DUPLEX_BUFFER_SIZE: &str = "RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE";
+
+/// Default duplex buffer size: 4 MB.
+///
+/// This is 4x larger than the original 1 MB buffer, providing better
+/// handling of large objects and reducing backpressure-related hangs.
+pub const DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+
+/// Environment variable for I/O buffer size in bytes.
+///
+/// This controls the buffer size used for individual I/O operations.
+/// A larger buffer improves throughput for sequential reads but may
+/// increase latency for small random reads.
+///
+/// Default: 131072 (128 KB, can be overridden by `RUSTFS_OBJECT_IO_BUFFER_SIZE`).
+pub const ENV_OBJECT_IO_BUFFER_SIZE: &str = "RUSTFS_OBJECT_IO_BUFFER_SIZE";
+
+/// Default I/O buffer size: 128 KB.
+pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024;
+
+/// Environment variable to enable/disable lock optimization.
+///
+/// When enabled, read locks are released immediately after metadata
+/// is read, rather than being held for the entire data transfer.
+/// This significantly reduces lock contention under high concurrency.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`).
+pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE";
+
+/// Default: lock optimization is enabled.
+pub const DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE: bool = true;
+
+/// Environment variable to enable/disable priority-based I/O scheduling.
+///
+/// When enabled, smaller requests (< 1MB) are given higher priority
+/// than larger requests (> 10MB), preventing "starvation" of small
+/// requests by large ones.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE`).
+pub const ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE: &str = "RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE";
+
+/// Default: priority scheduling is enabled.
+pub const DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE: bool = true;
+
+/// Environment variable to enable/disable deadlock detection.
+///
+/// When enabled, the system monitors active requests and detects
+/// potential deadlock situations (circular lock wait chains).
+/// This has some performance overhead and is intended for debugging.
+///
+/// Default: false (disabled, can be overridden by `RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE`).
+pub const ENV_OBJECT_DEADLOCK_DETECTION_ENABLE: &str = "RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE";
+
+/// Default: deadlock detection is disabled for performance.
+pub const DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE: bool = false;
+
+/// Environment variable for deadlock detection check interval in seconds.
+///
+/// How often the deadlock detector analyzes the lock wait graph.
+/// More frequent checks detect deadlocks faster but use more CPU.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL`).
+pub const ENV_OBJECT_DEADLOCK_CHECK_INTERVAL: &str = "RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL";
+
+/// Default deadlock check interval: 5 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_CHECK_INTERVAL: u64 = 5;
+
+/// Environment variable for deadlock detection hang threshold in seconds.
+///
+/// Requests that have been running longer than this threshold are
+/// considered "potentially hung" and included in deadlock analysis.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD`).
+pub const ENV_OBJECT_DEADLOCK_HANG_THRESHOLD: &str = "RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD";
+
+/// Default hang threshold: 10 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_HANG_THRESHOLD: u64 = 10;
+
+/// Environment variable for backpressure high watermark percentage.
+///
+/// When buffer usage exceeds this percentage, the system enters
+/// "high watermark" state and may apply backpressure to producers.
+///
+/// Default: 80 (80%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK";
+
+/// Default high watermark: 80%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK: u32 = 80;
+
+/// Environment variable for backpressure low watermark percentage.
+///
+/// When buffer usage drops below this percentage after being in
+/// high watermark state, backpressure is released.
+///
+/// Default: 50 (50%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK";
+
+/// Default low watermark: 50%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK: u32 = 50;
+
+/// Environment variable for lock acquisition timeout in seconds.
+///
+/// When a lock cannot be acquired within this duration, the operation
+/// will fail with a timeout error. This prevents indefinite waiting
+/// for locks that may never be released due to deadlocks.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`).
+pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT";
+
+/// Default lock acquisition timeout: 5 seconds.
+pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
+
+// ============================================================================
+// I/O priority scheduling configuration
+// ============================================================================
+
+/// Environment variable for I/O high priority size threshold in bytes.
+///
+/// Requests smaller than this threshold are classified as high priority.
+/// High priority requests are processed first to prevent starvation of small requests.
+///
+/// Default: 1048576 (1 MB, can be overridden by `RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD";
+
+/// Default high priority size threshold: 1 MB.
+pub const DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: usize = 1024 * 1024;
+
+/// Environment variable for I/O low priority size threshold in bytes.
+///
+/// Requests larger than this threshold are classified as low priority.
+/// Low priority requests are processed last to avoid blocking small requests.
+///
+/// Default: 10485760 (10 MB, can be overridden by `RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD";
+
+/// Default low priority size threshold: 10 MB.
+pub const DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: usize = 10 * 1024 * 1024;
+
+/// Environment variable for high priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the high priority queue.
+///
+/// Default: 32 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_HIGH_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY";
+
+/// Default high priority queue capacity: 32.
+pub const DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY: usize = 32;
+
+/// Environment variable for normal priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the normal priority queue.
+///
+/// Default: 64 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_NORMAL_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY";
+
+/// Default normal priority queue capacity: 64.
+pub const DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY: usize = 64;
+
+/// Environment variable for low priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the low priority queue.
+///
+/// Default: 16 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_LOW_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY";
+
+/// Default low priority queue capacity: 16.
+pub const DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY: usize = 16;
+
+/// Environment variable for starvation prevention check interval in milliseconds.
+///
+/// How often the system checks for starving low-priority requests.
+/// When a low-priority request has been waiting longer than the starvation threshold,
+/// it is promoted to normal priority.
+///
+/// Default: 100 ms (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL`).
+pub const ENV_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: &str = "RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL";
+
+/// Default starvation prevention interval: 100 ms.
+pub const DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: u64 = 100;
+
+/// Environment variable for starvation threshold in seconds.
+///
+/// Maximum time a low-priority request can wait before being promoted to normal priority.
+/// This prevents indefinite starvation of low-priority requests.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS`).
+pub const ENV_OBJECT_IO_STARVATION_THRESHOLD_SECS: &str = "RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS";
+
+/// Default starvation threshold: 5 seconds.
+pub const DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS: u64 = 5;
+
+/// Environment variable for load sampling window size.
+///
+/// Number of recent samples used to calculate I/O load metrics.
+///
+/// Default: 100 samples (can be overridden by `RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW`).
+pub const ENV_OBJECT_IO_LOAD_SAMPLE_WINDOW: &str = "RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW";
+
+/// Default load sampling window: 100 samples.
+pub const DEFAULT_OBJECT_IO_LOAD_SAMPLE_WINDOW: usize = 100;
+
+/// Environment variable for high load wait time threshold in milliseconds.
+///
+/// When average wait time exceeds this threshold, the system is considered to be under high load.
+///
+/// Default: 50 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS";
+
+/// Default high load threshold: 50 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: u64 = 50;
+
+/// Environment variable for low load wait time threshold in milliseconds.
+///
+/// When average wait time is below this threshold, the system is considered to be under low load.
+///
+/// Default: 10 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS";
+
+/// Default low load threshold: 10 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: u64 = 10;
+
+/// Environment variable for enabling storage media detection for adaptive I/O scheduling.
+///
+/// When disabled, the scheduler falls back to `Unknown` storage media unless an explicit
+/// override is provided.
+///
+/// Default: true (can be overridden by `RUSTFS_OBJECT_IO_STORAGE_DETECTION_ENABLE`).
+pub const ENV_OBJECT_IO_STORAGE_DETECTION_ENABLE: &str = "RUSTFS_OBJECT_IO_STORAGE_DETECTION_ENABLE";
+
+/// Default storage media detection setting: enabled.
+pub const DEFAULT_OBJECT_IO_STORAGE_DETECTION_ENABLE: bool = true;
+
+/// Environment variable for overriding detected storage media.
+///
+/// Supported values: `nvme`, `ssd`, `hdd`, `unknown`.
+/// Empty value means auto-detect or fallback to `Unknown`.
+///
+/// Default: empty string (can be overridden by `RUSTFS_OBJECT_IO_STORAGE_MEDIA_OVERRIDE`).
+pub const ENV_OBJECT_IO_STORAGE_MEDIA_OVERRIDE: &str = "RUSTFS_OBJECT_IO_STORAGE_MEDIA_OVERRIDE";
+
+/// Default storage media override: no override.
+pub const DEFAULT_OBJECT_IO_STORAGE_MEDIA_OVERRIDE: &str = "";
+
+/// Environment variable for access-pattern history size.
+///
+/// Controls how many recent offset/length observations are used to classify
+/// sequential, random, or mixed reads.
+///
+/// Default: 8 (can be overridden by `RUSTFS_OBJECT_IO_PATTERN_HISTORY_SIZE`).
+pub const ENV_OBJECT_IO_PATTERN_HISTORY_SIZE: &str = "RUSTFS_OBJECT_IO_PATTERN_HISTORY_SIZE";
+
+/// Default access-pattern history size: 8 samples.
+pub const DEFAULT_OBJECT_IO_PATTERN_HISTORY_SIZE: usize = 8;
+
+/// Environment variable for sequential access step tolerance in bytes.
+///
+/// Small gaps between adjacent reads within this tolerance are still treated as sequential.
+///
+/// Default: 131072 bytes (128 KiB, can be overridden by `RUSTFS_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES`).
+pub const ENV_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES: &str = "RUSTFS_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES";
+
+/// Default sequential step tolerance: 128 KiB.
+pub const DEFAULT_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES: u64 = 128 * 1024;
+
+/// Environment variable for bandwidth EMA beta.
+///
+/// Lower values react faster to recent throughput changes while higher values smooth
+/// short-term fluctuations more aggressively.
+///
+/// Default: 0.1 (can be overridden by `RUSTFS_OBJECT_IO_BANDWIDTH_EMA_BETA`).
+pub const ENV_OBJECT_IO_BANDWIDTH_EMA_BETA: &str = "RUSTFS_OBJECT_IO_BANDWIDTH_EMA_BETA";
+
+/// Default bandwidth EMA beta: 0.1.
+pub const DEFAULT_OBJECT_IO_BANDWIDTH_EMA_BETA: f64 = 0.1;
+
+/// Environment variable for the low bandwidth threshold in bytes per second.
+///
+/// Observed throughput below this value causes the scheduler to be more conservative
+/// with buffer growth and read-ahead.
+///
+/// Default: 67108864 bytes/sec (64 MiB/s, can be overridden by `RUSTFS_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS`).
+pub const ENV_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS: &str = "RUSTFS_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS";
+
+/// Default low bandwidth threshold: 64 MiB/s.
+pub const DEFAULT_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS: u64 = 64 * 1024 * 1024;
+
+/// Environment variable for the high bandwidth threshold in bytes per second.
+///
+/// Observed throughput above this value allows the scheduler to be more aggressive
+/// for sequential workloads.
+///
+/// Default: 536870912 bytes/sec (512 MiB/s, can be overridden by `RUSTFS_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS`).
+pub const ENV_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS: &str = "RUSTFS_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS";
+
+/// Default high bandwidth threshold: 512 MiB/s.
+pub const DEFAULT_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS: u64 = 512 * 1024 * 1024;
+
+/// Environment variable for NVMe buffer cap in bytes.
+///
+/// Sequential reads on NVMe can scale up to this buffer cap.
+///
+/// Default: 2097152 bytes (2 MiB, can be overridden by `RUSTFS_OBJECT_IO_NVME_BUFFER_CAP`).
+pub const ENV_OBJECT_IO_NVME_BUFFER_CAP: &str = "RUSTFS_OBJECT_IO_NVME_BUFFER_CAP";
+
+/// Default NVMe buffer cap: 2 MiB.
+pub const DEFAULT_OBJECT_IO_NVME_BUFFER_CAP: usize = 2 * 1024 * 1024;
+
+/// Environment variable for SSD buffer cap in bytes.
+///
+/// Default: 1048576 bytes (1 MiB, can be overridden by `RUSTFS_OBJECT_IO_SSD_BUFFER_CAP`).
+pub const ENV_OBJECT_IO_SSD_BUFFER_CAP: &str = "RUSTFS_OBJECT_IO_SSD_BUFFER_CAP";
+
+/// Default SSD buffer cap: 1 MiB.
+pub const DEFAULT_OBJECT_IO_SSD_BUFFER_CAP: usize = 1024 * 1024;
+
+/// Environment variable for HDD buffer cap in bytes.
+///
+/// Default: 524288 bytes (512 KiB, can be overridden by `RUSTFS_OBJECT_IO_HDD_BUFFER_CAP`).
+pub const ENV_OBJECT_IO_HDD_BUFFER_CAP: &str = "RUSTFS_OBJECT_IO_HDD_BUFFER_CAP";
+
+/// Default HDD buffer cap: 512 KiB.
+pub const DEFAULT_OBJECT_IO_HDD_BUFFER_CAP: usize = 512 * 1024;
+
+/// Environment variable for disabling read-ahead under random or mixed access with concurrency.
+///
+/// When concurrent requests reach this threshold, random-heavy workloads stop using read-ahead.
+///
+/// Default: 4 (can be overridden by `RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY`).
+pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY";
+
+/// Default read-ahead disable concurrency threshold: 4.
+pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;

crates/config/src/constants/runtime.rs

@@ -27,6 +27,16 @@ pub const ENV_RNG_SEED: &str = "RUSTFS_RUNTIME_RNG_SEED";
 /// Event polling interval
 pub const ENV_EVENT_INTERVAL: &str = "RUSTFS_RUNTIME_EVENT_INTERVAL";
 
+// Dial9 Tokio Telemetry Configuration
+pub const ENV_RUNTIME_DIAL9_ENABLED: &str = "RUSTFS_RUNTIME_DIAL9_ENABLED";
+pub const ENV_RUNTIME_DIAL9_OUTPUT_DIR: &str = "RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR";
+pub const ENV_RUNTIME_DIAL9_FILE_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_FILE_PREFIX";
+pub const ENV_RUNTIME_DIAL9_MAX_FILE_SIZE: &str = "RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE";
+pub const ENV_RUNTIME_DIAL9_ROTATION_COUNT: &str = "RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT";
+pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
+pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
+pub const ENV_RUNTIME_DIAL9_SAMPLING_RATE: &str = "RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE";
+
 // Default values for Tokio runtime
 pub const DEFAULT_WORKER_THREADS: usize = 16;
 pub const DEFAULT_MAX_BLOCKING_THREADS: usize = 1024;
@@ -40,6 +50,15 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
 pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
 pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 
+// Dial9 Tokio Telemetry Default values
+pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
+pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";
+pub const DEFAULT_RUNTIME_DIAL9_FILE_PREFIX: &str = "rustfs-tokio";
+pub const DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE: u64 = 100 * 1024 * 1024; // 100MB
+pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10;
+pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling
+// Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option<String>)
+
 /// Threshold for small object seek support in megabytes.
 ///
 /// When an object is smaller than this size, rustfs will provide seek support.

crates/config/src/constants/targets.rs

@@ -34,3 +34,16 @@ pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval";
 pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
 pub const MQTT_QUEUE_DIR: &str = "queue_dir";
 pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";
+pub const MQTT_TLS_POLICY: &str = "tls_policy";
+pub const MQTT_TLS_CA: &str = "tls_ca";
+pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
+pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
+pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
+
+/// Environment variable controlling whether target queue files are Snappy-compressed.
+/// Applies to both notify and audit target queue stores.
+pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
+
+/// Queue-store compression is enabled by default to reduce disk footprint.
+pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;

crates/config/src/constants/tls.rs

@@ -84,3 +84,69 @@ pub const ENV_SERVER_MTLS_ENABLE: &str = "RUSTFS_SERVER_MTLS_ENABLE";
 /// By default, RustFS server mTLS is disabled.
 /// To change this behavior, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1
 pub const DEFAULT_SERVER_MTLS_ENABLE: bool = false;
+
+// ── HTTP Transport Tuning Parameters ──
+
+/// Environment variable for HTTP/2 initial stream window size (bytes)
+/// Default: 4194304 (4 MB)
+pub const ENV_H2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_H2_INITIAL_STREAM_WINDOW_SIZE";
+pub const DEFAULT_H2_INITIAL_STREAM_WINDOW_SIZE: u32 = 4 * 1024 * 1024; // 4 MB
+
+/// Environment variable for HTTP/2 initial connection window size (bytes)
+/// Default: 8388608 (8 MB)
+pub const ENV_H2_INITIAL_CONN_WINDOW_SIZE: &str = "RUSTFS_H2_INITIAL_CONN_WINDOW_SIZE";
+pub const DEFAULT_H2_INITIAL_CONN_WINDOW_SIZE: u32 = 8 * 1024 * 1024; // 8 MB
+
+/// Environment variable for HTTP/2 max frame size (bytes)
+/// Range: 16384 (16 KB) to 16777216 (16 MB) per RFC 7540
+/// Default: 524288 (512 KB)
+pub const ENV_H2_MAX_FRAME_SIZE: &str = "RUSTFS_H2_MAX_FRAME_SIZE";
+pub const DEFAULT_H2_MAX_FRAME_SIZE: u32 = 512 * 1024; // 512 KB
+
+/// Environment variable for HTTP/2 max header list size (bytes)
+/// Default: 65536 (64 KB)
+pub const ENV_H2_MAX_HEADER_LIST_SIZE: &str = "RUSTFS_H2_MAX_HEADER_LIST_SIZE";
+pub const DEFAULT_H2_MAX_HEADER_LIST_SIZE: u32 = 64 * 1024; // 64 KB
+
+/// Environment variable for HTTP/2 max concurrent streams
+/// Default: 2048
+pub const ENV_H2_MAX_CONCURRENT_STREAMS: &str = "RUSTFS_H2_MAX_CONCURRENT_STREAMS";
+pub const DEFAULT_H2_MAX_CONCURRENT_STREAMS: u32 = 2048;
+
+/// Environment variable for HTTP/2 keep-alive interval (seconds)
+/// Default: 20
+pub const ENV_H2_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_H2_KEEP_ALIVE_INTERVAL";
+pub const DEFAULT_H2_KEEP_ALIVE_INTERVAL: u64 = 20;
+
+/// Environment variable for HTTP/2 keep-alive timeout (seconds)
+/// Default: 10
+pub const ENV_H2_KEEP_ALIVE_TIMEOUT: &str = "RUSTFS_H2_KEEP_ALIVE_TIMEOUT";
+pub const DEFAULT_H2_KEEP_ALIVE_TIMEOUT: u64 = 10;
+
+/// Environment variable for HTTP/1.1 header read timeout (seconds)
+/// Default: 5
+pub const ENV_HTTP1_HEADER_READ_TIMEOUT: &str = "RUSTFS_HTTP1_HEADER_READ_TIMEOUT";
+pub const DEFAULT_HTTP1_HEADER_READ_TIMEOUT: u64 = 5;
+
+/// Environment variable for HTTP/1.1 max buffer size (bytes)
+/// Default: 65536 (64 KB)
+pub const ENV_HTTP1_MAX_BUF_SIZE: &str = "RUSTFS_HTTP1_MAX_BUF_SIZE";
+pub const DEFAULT_HTTP1_MAX_BUF_SIZE: usize = 64 * 1024; // 64 KB
+
+// ── TLS Hot Reload Parameters ──
+
+/// Environment variable to enable TLS certificate hot reload
+/// Default: false
+/// To enable, set the environment variable RUSTFS_TLS_RELOAD_ENABLE=1
+pub const ENV_TLS_RELOAD_ENABLE: &str = "RUSTFS_TLS_RELOAD_ENABLE";
+
+/// Default value for TLS certificate hot reload
+/// By default, RustFS does not reload TLS certificates automatically.
+pub const DEFAULT_TLS_RELOAD_ENABLE: bool = false;
+
+/// Environment variable for TLS certificate reload interval (seconds)
+/// Default: 30 seconds. Minimum: 5 seconds.
+pub const ENV_TLS_RELOAD_INTERVAL: &str = "RUSTFS_TLS_RELOAD_INTERVAL";
+
+/// Default interval for TLS certificate reload check
+pub const DEFAULT_TLS_RELOAD_INTERVAL: u64 = 30;

crates/config/src/constants/workload.rs

@@ -0,0 +1,70 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Workload profile buffer configuration constants.
+//!
+//! This module defines environment variable keys and default values for
+//! custom buffer profile configuration.
+
+use crate::{KI_B, MI_B};
+
+/// Environment variable for minimum buffer size
+/// Default: 64KB (65536 bytes)
+pub const ENV_RUSTFS_BUFFER_MIN_SIZE: &str = "RUSTFS_BUFFER_MIN_SIZE";
+
+/// Environment variable for maximum buffer size
+/// Default: 1MB (1048576 bytes)
+pub const ENV_RUSTFS_BUFFER_MAX_SIZE: &str = "RUSTFS_BUFFER_MAX_SIZE";
+
+/// Environment variable for default buffer size (used when file size is unknown)
+/// Default: 256KB (262144 bytes)
+pub const ENV_RUSTFS_BUFFER_DEFAULT_SIZE: &str = "RUSTFS_BUFFER_DEFAULT_SIZE";
+
+/// Default minimum buffer size: 64KB
+pub const DEFAULT_BUFFER_MIN_SIZE: usize = 64 * KI_B;
+
+/// Default maximum buffer size: 1MB
+pub const DEFAULT_BUFFER_MAX_SIZE: usize = MI_B;
+
+/// Default buffer size for unknown file size: 256KB
+pub const DEFAULT_BUFFER_UNKNOWN_SIZE: usize = 256 * KI_B;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::MI_B;
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_BUFFER_MIN_SIZE, 65536); // 64KB
+        assert_eq!(DEFAULT_BUFFER_MAX_SIZE, 1048576); // 1MB
+        assert_eq!(DEFAULT_BUFFER_UNKNOWN_SIZE, 262144); // 256KB
+    }
+
+    #[test]
+    fn test_constants() {
+        assert_eq!(KI_B, 1024);
+        assert_eq!(MI_B, 1024 * 1024);
+        assert_eq!(64 * KI_B, DEFAULT_BUFFER_MIN_SIZE);
+        assert_eq!(MI_B, DEFAULT_BUFFER_MAX_SIZE);
+        assert_eq!(256 * KI_B, DEFAULT_BUFFER_UNKNOWN_SIZE);
+    }
+
+    #[test]
+    fn test_env_var_names() {
+        assert_eq!(ENV_RUSTFS_BUFFER_MIN_SIZE, "RUSTFS_BUFFER_MIN_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_MAX_SIZE, "RUSTFS_BUFFER_MAX_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, "RUSTFS_BUFFER_DEFAULT_SIZE");
+    }
+}

crates/config/src/constants/zero_copy.rs

@@ -0,0 +1,133 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Zero-copy I/O configuration constants.
+//!
+//! This module defines environment variables and default values for zero-copy
+//! read operations, which use memory mapping (mmap) to avoid data copying.
+
+// =============================================================================
+// Zero-Copy Configuration
+// =============================================================================
+
+/// Environment variable for zero-copy read enable.
+///
+/// When enabled, uses mmap (Unix) or optimized reads for zero-copy data access.
+/// This reduces memory copies from 3-4 to 1, lowering CPU usage by 20-30%
+/// and improving P95 latency by 15-25%.
+///
+/// - Purpose: Enable or disable zero-copy read operations
+/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config
+/// - Semantics: When enabled, uses mmap on Unix systems for memory-mapped file reads;
+///   falls back to regular I/O on non-Unix platforms or when mmap fails
+/// - Example: `export RUSTFS_OBJECT_ZERO_COPY_ENABLE=true`
+/// - Note: Zero-copy is safe for all workloads and provides significant performance
+///   benefits with minimal risk. Disable only if mmap-related issues are encountered.
+pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
+
+/// Default: zero-copy reads are enabled.
+///
+/// Zero-copy uses memory mapping (mmap) on Unix systems to avoid data copying
+/// between kernel and user space. This provides:
+/// - Reduced memory copies: from 3-4 copies to 1 copy
+/// - Lower CPU usage: 20-30% reduction expected
+/// - Improved latency P95: 15-25% reduction expected
+/// - Increased throughput: 10-20% improvement expected
+///
+/// On non-Unix platforms or when mmap fails, the system automatically falls back
+/// to regular I/O without errors.
+pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true;
+
+/// Environment variable for zero-copy read operating mode.
+///
+/// Supported values:
+/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path
+/// - `conservative`: allow a single mmap window per request
+/// - `balanced`: allow multiple mmap windows with the default size guardrails
+/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff
+pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE";
+
+/// Default zero-copy read mode.
+pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced";
+
+/// Environment variable for the maximum mmap window size used by the chunk fast path.
+///
+/// This controls the visible bytes per mapped chunk before the implementation emits a new window.
+pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES";
+
+/// Default mmap window size for chunk fast path reads: 8 MiB.
+pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024;
+
+/// Environment variable for the maximum total active mmap bytes.
+///
+/// Requests that would exceed this active window budget fall back to the compatibility path.
+pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES";
+
+/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB.
+pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024;
+
+// =============================================================================
+// Direct I/O Configuration
+// =============================================================================
+
+/// Environment variable for Direct I/O enable (Linux only).
+///
+/// When enabled, uses O_DIRECT flag to bypass OS page cache for large files.
+/// This is only beneficial for specific workloads (databases, large sequential reads).
+///
+/// - Purpose: Enable or disable Direct I/O for large file operations
+/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config
+/// - Semantics: When enabled, files larger than the threshold will use O_DIRECT flag;
+///   this bypasses the OS page cache and transfers data directly between disk and application
+/// - Example: `export RUSTFS_OBJECT_DIRECT_IO_ENABLE=true`
+/// - Note: Direct I/O is disabled by default because it's only beneficial for specific
+///   use cases. For most workloads, the OS page cache provides better performance.
+pub const ENV_OBJECT_DIRECT_IO_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_ENABLE";
+
+/// Default: Direct I/O is disabled.
+///
+/// Direct I/O is disabled by default because it's only beneficial for specific use cases:
+/// - Large file transfers (>128MB)
+/// - Databases with their own cache
+/// - Applications requiring predictable I/O latency
+///
+/// For most workloads, the OS page cache provides better performance through:
+/// - Read-ahead caching
+/// - Write buffering
+/// - Multi-use caching (same data cached for multiple operations)
+pub const DEFAULT_OBJECT_DIRECT_IO_ENABLE: bool = false;
+
+/// Environment variable for Direct I/O minimum file size threshold.
+///
+/// Files smaller than this size will use regular I/O even if Direct I/O is enabled.
+/// This avoids the overhead of Direct I/O for small files where the OS page cache
+/// is more effective.
+///
+/// - Purpose: Set the minimum file size for Direct I/O operations
+/// - Unit: Bytes
+/// - Valid values: any positive integer (default: 134,217,728 bytes = 128 MB)
+/// - Semantics: Only files larger than this threshold will use Direct I/O when enabled;
+///   smaller files use regular buffered I/O
+/// - Example: `export RUSTFS_OBJECT_DIRECT_IO_THRESHOLD=268435456`
+/// - Note: The default threshold of 128MB balances the overhead of Direct I/O setup
+///   against the benefits of bypassing the page cache for large files.
+pub const ENV_OBJECT_DIRECT_IO_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_THRESHOLD";
+
+/// Default Direct I/O threshold: 128 MB.
+///
+/// Only files larger than 128MB will use Direct I/O when enabled.
+/// Smaller files benefit from OS page cache.
+///
+/// Formula: 128 * 1024 * 1024 = 134,217,728 bytes
+pub const DEFAULT_OBJECT_DIRECT_IO_THRESHOLD: usize = 128 * 1024 * 1024;

crates/config/src/lib.rs

@@ -19,6 +19,8 @@ pub use constants::app::*;
 #[cfg(feature = "constants")]
 pub use constants::body_limits::*;
 #[cfg(feature = "constants")]
+pub use constants::capacity::*;
+#[cfg(feature = "constants")]
 pub use constants::compress::*;
 #[cfg(feature = "constants")]
 pub use constants::console::*;
@@ -45,6 +47,10 @@ pub use constants::targets::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;
 #[cfg(feature = "constants")]
+pub use constants::workload::*;
+#[cfg(feature = "constants")]
+pub use constants::zero_copy::*;
+#[cfg(feature = "constants")]
 pub mod oidc {
     pub use super::constants::oidc::*;
 }

crates/config/src/notify/mqtt.rs

@@ -24,6 +24,12 @@ pub const NOTIFY_MQTT_KEYS: &[&str] = &[
     crate::MQTT_KEEP_ALIVE_INTERVAL,
     crate::MQTT_QUEUE_DIR,
     crate::MQTT_QUEUE_LIMIT,
+    crate::MQTT_TLS_POLICY,
+    crate::MQTT_TLS_CA,
+    crate::MQTT_TLS_CLIENT_CERT,
+    crate::MQTT_TLS_CLIENT_KEY,
+    crate::MQTT_TLS_TRUST_LEAF_AS_CA,
+    crate::MQTT_WS_PATH_ALLOWLIST,
     crate::COMMENT_KEY,
 ];
 
@@ -38,8 +44,14 @@ pub const ENV_NOTIFY_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_RECONNE
 pub const ENV_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL";
 pub const ENV_NOTIFY_MQTT_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_DIR";
 pub const ENV_NOTIFY_MQTT_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_LIMIT";
+pub const ENV_NOTIFY_MQTT_TLS_POLICY: &str = "RUSTFS_NOTIFY_MQTT_TLS_POLICY";
+pub const ENV_NOTIFY_MQTT_TLS_CA: &str = "RUSTFS_NOTIFY_MQTT_TLS_CA";
+pub const ENV_NOTIFY_MQTT_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_MQTT_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_MQTT_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_MQTT_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_MQTT_TLS_TRUST_LEAF_AS_CA: &str = "RUSTFS_NOTIFY_MQTT_TLS_TRUST_LEAF_AS_CA";
+pub const ENV_NOTIFY_MQTT_WS_PATH_ALLOWLIST: &str = "RUSTFS_NOTIFY_MQTT_WS_PATH_ALLOWLIST";
 
-pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 10] = &[
+pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 16] = &[
     ENV_NOTIFY_MQTT_ENABLE,
     ENV_NOTIFY_MQTT_BROKER,
     ENV_NOTIFY_MQTT_TOPIC,
@@ -50,4 +62,10 @@ pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 10] = &[
     ENV_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL,
     ENV_NOTIFY_MQTT_QUEUE_DIR,
     ENV_NOTIFY_MQTT_QUEUE_LIMIT,
+    ENV_NOTIFY_MQTT_TLS_POLICY,
+    ENV_NOTIFY_MQTT_TLS_CA,
+    ENV_NOTIFY_MQTT_TLS_CLIENT_CERT,
+    ENV_NOTIFY_MQTT_TLS_CLIENT_KEY,
+    ENV_NOTIFY_MQTT_TLS_TRUST_LEAF_AS_CA,
+    ENV_NOTIFY_MQTT_WS_PATH_ALLOWLIST,
 ];

crates/crypto/Cargo.toml

@@ -56,3 +56,6 @@ crypto = [
     "dep:rand",
     "dep:sha2",
 ]
+
+[lib]
+doctest = false

crates/e2e_test/Cargo.toml

@@ -20,6 +20,11 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 
+[[bin]]
+name = "small_put_bench"
+path = "src/bin/small_put_bench.rs"
+test = false
+
 [lints]
 workspace = true
 
@@ -30,6 +35,7 @@ ftps = []
 [dependencies]
 rustfs-ecstore.workspace = true
 rustfs-common.workspace = true
+rustfs-rio.workspace = true
 flatbuffers.workspace = true
 futures.workspace = true
 rustfs-lock.workspace = true
@@ -40,23 +46,38 @@ serde_json.workspace = true
 tonic = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
+clap = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
 bytes.workspace = true
 serial_test = { workspace = true }
 aws-sdk-s3.workspace = true
 aws-config = { workspace = true }
+aws-smithy-http-client.workspace = true
+async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
+flate2.workspace = true
+http.workspace = true
+http-body.workspace = true
+http-body-util.workspace = true
 reqwest = { workspace = true }
+rustfs-signer.workspace = true
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 uuid = { workspace = true }
+urlencoding.workspace = true
+walkdir.workspace = true
 base64 = { workspace = true }
 rand = { workspace = true }
 chrono = { workspace = true }
 md5 = { workspace = true }
 sha2 = { workspace = true }
+astral-tokio-tar = { workspace = true }
+s3s.workspace = true
+zstd.workspace = true
+time.workspace = true
 suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
 rcgen.workspace = true
 anyhow.workspace = true
-rustls.workspace = true
+rustls.workspace = true
+zip.workspace = true

crates/e2e_test/src/anonymous_access_test.rs

@@ -16,7 +16,7 @@
 //! Verifies that anonymous access works correctly with bucket policies
 //! when PublicAccessBlock configuration is missing or explicitly set.
 
-use crate::common::{RustFSTestEnvironment, init_logging};
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::types::PublicAccessBlockConfiguration;
 use serial_test::serial;
 use tracing::info;
@@ -67,7 +67,7 @@ async fn anonymous_get_object(
     key: &str,
 ) -> Result<reqwest::Response, reqwest::Error> {
     let url = format!("{}/{}/{}", env.url, bucket_name, key);
-    reqwest::Client::new().get(&url).send().await
+    local_http_client().get(&url).send().await
 }
 
 /// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it

crates/e2e_test/src/archive_download_integrity_test.rs

@@ -0,0 +1,329 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+    use http::header::{CONTENT_TYPE, HOST};
+    use reqwest::StatusCode;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::{pre_sign_v4, sign_v4};
+    use s3s::Body;
+    use serial_test::serial;
+    use sha2::{Digest, Sha256};
+    use std::error::Error;
+    use std::io::{Cursor, Write};
+    use std::process::Command;
+    use time::OffsetDateTime;
+    use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
+
+    const ARCHIVE_TEST_BUCKET: &str = "archive-download-integrity";
+    const MULTIPART_ARCHIVE_TEST_BUCKET: &str = "archive-multipart-integrity";
+    const MULTIPART_PART_SIZE: usize = 5 * 1024 * 1024;
+
+    fn build_zip_bytes(files: &[(&str, &[u8])]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let cursor = Cursor::new(Vec::new());
+        let mut zip = ZipWriter::new(cursor);
+        let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
+
+        for (name, content) in files {
+            zip.start_file(*name, options)?;
+            zip.write_all(content)?;
+        }
+
+        Ok(zip.finish()?.into_inner())
+    }
+
+    fn random_bytes(size: usize) -> Vec<u8> {
+        (0..size).map(|idx| (idx % 251) as u8).collect()
+    }
+
+    async fn start_rustfs_server_with_env(
+        env: &mut RustFSTestEnvironment,
+        extra_env: &[(&str, &str)],
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let binary_path = rustfs_binary_path();
+        let mut command = Command::new(&binary_path);
+        command.env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
+        for (key, value) in extra_env {
+            command.env(key, value);
+        }
+
+        let process = command
+            .args([
+                "--address",
+                &env.address,
+                "--access-key",
+                &env.access_key,
+                "--secret-key",
+                &env.secret_key,
+                &env.temp_dir,
+            ])
+            .spawn()?;
+
+        env.process = Some(process);
+        env.wait_for_server_ready().await?;
+        Ok(())
+    }
+
+    async fn presigned_get_request_with_accept_encoding(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        accept_encoding: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let signed = pre_sign_v4(
+            http::Request::builder()
+                .method(http::Method::GET)
+                .uri(uri)
+                .header(HOST, authority)
+                .body(Body::empty())?,
+            access_key,
+            secret_key,
+            "",
+            "us-east-1",
+            600,
+            OffsetDateTime::now_utc(),
+        );
+
+        let client = reqwest::Client::builder()
+            .no_proxy()
+            .no_gzip()
+            .no_brotli()
+            .no_zstd()
+            .no_deflate()
+            .build()?;
+
+        Ok(client
+            .get(signed.uri().to_string())
+            .header("Accept-Encoding", accept_encoding)
+            .send()
+            .await?)
+    }
+
+    async fn signed_put_request_with_headers(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        body: Vec<u8>,
+        content_type: &str,
+        content_encoding: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::PUT)
+            .uri(uri)
+            .header(HOST, authority)
+            .header(CONTENT_TYPE, content_type)
+            .header("content-encoding", content_encoding)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+        let signed = sign_v4(request, body.len() as i64, access_key, secret_key, "", "us-east-1");
+
+        let client = reqwest::Client::builder().no_proxy().build()?;
+        let mut builder = client.put(url).body(body);
+        for (name, value) in signed.headers() {
+            builder = builder.header(name, value);
+        }
+
+        Ok(builder.send().await?)
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_rejects_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server_without_cleanup(vec![]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
+        let response =
+            signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip")
+                .await?;
+
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+        let body = response.text().await?;
+        assert!(
+            body.contains("InvalidArgument") || body.contains("Content-Encoding"),
+            "unexpected error body: {body}"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_download_roundtrip_with_http_compression_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(
+            &mut env,
+            &[
+                ("RUSTFS_COMPRESS_ENABLE", "on"),
+                ("RUSTFS_COMPRESS_MIME_TYPES", "text/*,application/json,application/zip"),
+                ("RUSTFS_COMPRESS_MIN_SIZE", "1"),
+            ],
+        )
+        .await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = build_zip_bytes(&[
+            ("docs/readme.txt", b"archive-download-integrity"),
+            ("docs/notes.txt", b"response-compression-must-not-alter-zip-bytes"),
+        ])?;
+        let expected_sha256 = Sha256::digest(&zip_bytes);
+
+        client
+            .put_object()
+            .bucket(ARCHIVE_TEST_BUCKET)
+            .key("bundle.zip")
+            .content_type("application/zip")
+            .body(ByteStream::from(zip_bytes.clone()))
+            .send()
+            .await?;
+
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
+        let response =
+            presigned_get_request_with_accept_encoding(&object_url, &env.access_key, &env.secret_key, "gzip, br, zstd").await?;
+
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(
+            response
+                .headers()
+                .get("content-encoding")
+                .and_then(|value| value.to_str().ok()),
+            None,
+            "archive download must not be HTTP-compressed"
+        );
+
+        let downloaded = response.bytes().await?;
+        assert_eq!(
+            downloaded.as_ref(),
+            zip_bytes.as_slice(),
+            "downloaded archive bytes must match uploaded bytes"
+        );
+        assert_eq!(
+            Sha256::digest(downloaded.as_ref()).as_slice(),
+            expected_sha256.as_slice(),
+            "archive SHA256 mismatch"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_roundtrip_preserves_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server_without_cleanup(vec![]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let payload = random_bytes(MULTIPART_PART_SIZE + 512 * 1024);
+        let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
+        assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
+        let expected_sha256 = Sha256::digest(&zip_bytes);
+
+        let create_output = client
+            .create_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .content_type("application/zip")
+            .send()
+            .await?;
+        let upload_id = create_output.upload_id().expect("multipart upload id");
+
+        let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
+        let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
+
+        let upload_part_1 = client
+            .upload_part()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(first_part))
+            .send()
+            .await?;
+
+        let upload_part_2 = client
+            .upload_part()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .upload_id(upload_id)
+            .part_number(2)
+            .body(ByteStream::from(second_part))
+            .send()
+            .await?;
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload_part_1.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload_part_2.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await?;
+
+        let downloaded = client
+            .get_object()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .send()
+            .await?
+            .body
+            .collect()
+            .await?
+            .into_bytes();
+
+        assert_eq!(
+            downloaded.as_ref(),
+            zip_bytes.as_slice(),
+            "multipart archive bytes must match uploaded bytes"
+        );
+        assert_eq!(
+            Sha256::digest(downloaded.as_ref()).as_slice(),
+            expected_sha256.as_slice(),
+            "multipart archive SHA256 mismatch"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+}

crates/e2e_test/src/bin/small_put_bench.rs

@@ -0,0 +1,441 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::{Context, Result, anyhow, bail};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Delete, ObjectIdentifier};
+use aws_sdk_s3::{Client, Config};
+use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use bytes::Bytes;
+use clap::Parser;
+use serde::Serialize;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+#[derive(Parser, Debug)]
+#[command(name = "small_put_bench")]
+#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")]
+struct Args {
+    #[arg(long, env = "RUSTFS_BENCH_ENDPOINT")]
+    endpoint: String,
+
+    #[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")]
+    access_key: String,
+
+    #[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")]
+    secret_key: String,
+
+    #[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")]
+    region: String,
+
+    #[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")]
+    bucket: String,
+
+    #[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")]
+    sizes: String,
+
+    #[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)]
+    concurrency: usize,
+
+    #[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)]
+    duration_secs: u64,
+
+    #[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)]
+    timeout_secs: u64,
+
+    #[arg(long, env = "RUSTFS_BENCH_PREFIX")]
+    prefix: Option<String>,
+
+    #[arg(long)]
+    output_json: Option<PathBuf>,
+
+    #[arg(long, default_value_t = false)]
+    cleanup: bool,
+}
+
+#[derive(Clone, Debug)]
+struct SizeSpec {
+    label: String,
+    slug: String,
+    bytes: usize,
+}
+
+#[derive(Debug)]
+struct Sample {
+    ok: bool,
+    duration_ms: f64,
+}
+
+#[derive(Debug, Serialize)]
+struct SizeSummary {
+    label: String,
+    bytes: usize,
+    total: usize,
+    succeeded: usize,
+    failed: usize,
+    wall_secs: f64,
+    object_rate: f64,
+    throughput_mib_per_sec: f64,
+    avg_ms: Option<f64>,
+    p50_ms: Option<f64>,
+    p90_ms: Option<f64>,
+    p99_ms: Option<f64>,
+}
+
+#[derive(Debug, Serialize)]
+struct RunSummary {
+    run_id: String,
+    endpoint: String,
+    bucket: String,
+    concurrency: usize,
+    duration_secs: u64,
+    timeout_secs: u64,
+    sizes: Vec<SizeSummary>,
+}
+
+fn main() -> Result<()> {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .context("failed to build tokio runtime")?;
+    runtime.block_on(async_main())
+}
+
+async fn async_main() -> Result<()> {
+    let args = Args::parse();
+    validate_args(&args)?;
+
+    let sizes = parse_size_list(&args.sizes)?;
+    let run_id = args.prefix.clone().unwrap_or_else(default_run_id);
+    let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region);
+
+    ensure_bucket(&client, &args.bucket).await?;
+
+    let mut size_summaries = Vec::with_capacity(sizes.len());
+    for size in &sizes {
+        let summary = run_size_benchmark(
+            client.clone(),
+            args.bucket.clone(),
+            run_id.clone(),
+            size.clone(),
+            args.concurrency,
+            Duration::from_secs(args.duration_secs),
+            Duration::from_secs(args.timeout_secs),
+        )
+        .await?;
+        print_size_summary(&summary);
+        size_summaries.push(summary);
+    }
+
+    if args.cleanup {
+        cleanup_prefix(&client, &args.bucket, &run_id).await?;
+    }
+
+    let summary = RunSummary {
+        run_id,
+        endpoint: args.endpoint,
+        bucket: args.bucket,
+        concurrency: args.concurrency,
+        duration_secs: args.duration_secs,
+        timeout_secs: args.timeout_secs,
+        sizes: size_summaries,
+    };
+
+    if let Some(path) = args.output_json {
+        let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?;
+        std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?;
+        println!("Wrote summary to {}", path.display());
+    }
+
+    Ok(())
+}
+
+fn validate_args(args: &Args) -> Result<()> {
+    if args.concurrency == 0 {
+        bail!("--concurrency must be greater than zero");
+    }
+    if args.duration_secs == 0 {
+        bail!("--duration-secs must be greater than zero");
+    }
+    if args.timeout_secs == 0 {
+        bail!("--timeout-secs must be greater than zero");
+    }
+    Ok(())
+}
+
+fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench");
+    let mut config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new(region.to_string()))
+        .endpoint_url(endpoint)
+        .force_path_style(true)
+        .behavior_version_latest();
+
+    if endpoint.starts_with("http://") {
+        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
+    }
+
+    Client::from_conf(config.build())
+}
+
+async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> {
+    if client.head_bucket().bucket(bucket).send().await.is_ok() {
+        return Ok(());
+    }
+
+    match client.create_bucket().bucket(bucket).send().await {
+        Ok(_) => Ok(()),
+        Err(err) => {
+            let rendered = err.to_string();
+            if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") {
+                Ok(())
+            } else {
+                Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}"))
+            }
+        }
+    }
+}
+
+async fn run_size_benchmark(
+    client: Client,
+    bucket: String,
+    run_id: String,
+    size: SizeSpec,
+    concurrency: usize,
+    duration: Duration,
+    timeout: Duration,
+) -> Result<SizeSummary> {
+    let payload = Bytes::from(vec![0_u8; size.bytes]);
+    let deadline = Instant::now() + duration;
+    let wall_start = Instant::now();
+
+    let mut handles = Vec::with_capacity(concurrency);
+    for worker in 0..concurrency {
+        let client = client.clone();
+        let bucket = bucket.clone();
+        let payload = payload.clone();
+        let prefix = format!("{run_id}/{}/worker-{worker}", size.slug);
+        handles.push(tokio::spawn(async move {
+            let mut samples = Vec::new();
+            let mut idx = 0usize;
+
+            while Instant::now() < deadline {
+                let key = format!("{prefix}/obj-{idx}.bin");
+                let started_at = Instant::now();
+                let request = client
+                    .put_object()
+                    .bucket(&bucket)
+                    .key(key)
+                    .body(ByteStream::from(payload.clone()))
+                    .content_type("application/octet-stream");
+
+                let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_)));
+                samples.push(Sample {
+                    ok,
+                    duration_ms: started_at.elapsed().as_secs_f64() * 1000.0,
+                });
+                idx += 1;
+            }
+
+            samples
+        }));
+    }
+
+    let mut samples = Vec::new();
+    for handle in handles {
+        samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?);
+    }
+
+    Ok(build_size_summary(&size, samples, wall_start.elapsed()))
+}
+
+fn build_size_summary(size: &SizeSpec, mut samples: Vec<Sample>, wall_elapsed: Duration) -> SizeSummary {
+    let total = samples.len();
+    let succeeded = samples.iter().filter(|sample| sample.ok).count();
+    let failed = total.saturating_sub(succeeded);
+    let wall_secs = wall_elapsed.as_secs_f64();
+    let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 };
+    let throughput_mib_per_sec = if wall_secs > 0.0 {
+        ((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs
+    } else {
+        0.0
+    };
+
+    let avg_ms = if total > 0 {
+        Some(samples.iter().map(|sample| sample.duration_ms).sum::<f64>() / total as f64)
+    } else {
+        None
+    };
+
+    samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms));
+    let durations: Vec<f64> = samples.into_iter().map(|sample| sample.duration_ms).collect();
+
+    SizeSummary {
+        label: size.label.clone(),
+        bytes: size.bytes,
+        total,
+        succeeded,
+        failed,
+        wall_secs,
+        object_rate,
+        throughput_mib_per_sec,
+        avg_ms,
+        p50_ms: percentile(&durations, 0.50),
+        p90_ms: percentile(&durations, 0.90),
+        p99_ms: percentile(&durations, 0.99),
+    }
+}
+
+async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
+    let mut continuation_token = None;
+    loop {
+        let response = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix(prefix)
+            .set_continuation_token(continuation_token.clone())
+            .send()
+            .await
+            .with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?;
+
+        let objects: Vec<ObjectIdentifier> = response
+            .contents
+            .unwrap_or_default()
+            .into_iter()
+            .filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok()))
+            .flatten()
+            .collect();
+
+        for chunk in objects.chunks(1_000) {
+            if chunk.is_empty() {
+                continue;
+            }
+
+            client
+                .delete_objects()
+                .bucket(bucket)
+                .delete(
+                    Delete::builder()
+                        .set_objects(Some(chunk.to_vec()))
+                        .quiet(true)
+                        .build()
+                        .context("failed to build delete request")?,
+                )
+                .send()
+                .await
+                .with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?;
+        }
+
+        if response.is_truncated.unwrap_or(false) {
+            continuation_token = response.next_continuation_token;
+        } else {
+            break;
+        }
+    }
+
+    Ok(())
+}
+
+fn parse_size_list(input: &str) -> Result<Vec<SizeSpec>> {
+    input
+        .split(',')
+        .map(str::trim)
+        .filter(|item| !item.is_empty())
+        .map(parse_size_spec)
+        .collect()
+}
+
+fn parse_size_spec(input: &str) -> Result<SizeSpec> {
+    let normalized = input.trim();
+    let lower = normalized.to_ascii_lowercase();
+
+    let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") {
+        (value, 1024usize)
+    } else if let Some(value) = lower.strip_suffix("mib") {
+        (value, 1024usize * 1024usize)
+    } else if let Some(value) = lower.strip_suffix('b') {
+        (value, 1usize)
+    } else {
+        (lower.as_str(), 1usize)
+    };
+
+    let value = number_part
+        .trim()
+        .parse::<usize>()
+        .with_context(|| format!("invalid size component: {input}"))?;
+    let bytes = value
+        .checked_mul(multiplier)
+        .ok_or_else(|| anyhow!("size overflow for {input}"))?;
+
+    Ok(SizeSpec {
+        label: normalized.to_string(),
+        slug: normalized
+            .chars()
+            .filter(|ch| ch.is_ascii_alphanumeric())
+            .collect::<String>()
+            .to_ascii_lowercase(),
+        bytes,
+    })
+}
+
+fn percentile(values: &[f64], percentile: f64) -> Option<f64> {
+    if values.is_empty() {
+        return None;
+    }
+
+    let index = ((values.len() - 1) as f64 * percentile).floor() as usize;
+    values.get(index).copied()
+}
+
+fn default_run_id() -> String {
+    format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))
+}
+
+fn print_size_summary(summary: &SizeSummary) {
+    println!(
+        "{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}",
+        summary.label,
+        summary.succeeded,
+        summary.failed,
+        summary.object_rate,
+        summary.throughput_mib_per_sec,
+        summary.avg_ms,
+        summary.p50_ms,
+        summary.p90_ms,
+        summary.p99_ms,
+    );
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_size_spec_supports_binary_units() {
+        let four_kib = parse_size_spec("4KiB").expect("4KiB should parse");
+        assert_eq!(four_kib.bytes, 4 * 1024);
+
+        let one_mib = parse_size_spec("1MiB").expect("1MiB should parse");
+        assert_eq!(one_mib.bytes, 1024 * 1024);
+    }
+
+    #[test]
+    fn percentile_returns_expected_bucket() {
+        let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
+        assert_eq!(percentile(&values, 0.50), Some(30.0));
+        assert_eq!(percentile(&values, 0.90), Some(40.0));
+    }
+}

crates/e2e_test/src/bucket_logging_test.rs

@@ -0,0 +1,544 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end tests for S3 dummy-compat bucket APIs.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::error::ProvideErrorMetadata;
+    use aws_sdk_s3::types::{
+        AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
+        RequestPaymentConfiguration, WebsiteConfiguration,
+    };
+    use serial_test::serial;
+    use std::path::PathBuf;
+    use std::process::Command;
+    use tracing::info;
+
+    fn awscurl_binary_path() -> PathBuf {
+        std::env::var_os("AWSCURL_PATH")
+            .map(PathBuf::from)
+            .unwrap_or_else(|| PathBuf::from("awscurl"))
+    }
+
+    fn awscurl_available() -> bool {
+        Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
+    }
+
+    fn execute_s3_awscurl(
+        method: &str,
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+        let output = Command::new(awscurl_binary_path())
+            .args([
+                "--service",
+                "s3",
+                "--region",
+                "us-east-1",
+                "--access_key",
+                access_key,
+                "--secret_key",
+                secret_key,
+                "-i",
+                "-X",
+                method,
+                url,
+            ])
+            .output()?;
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
+        }
+        Ok(String::from_utf8_lossy(&output.stdout).to_string())
+    }
+
+    fn parse_status(raw: &str) -> Option<u16> {
+        raw.lines()
+            .filter_map(|line| {
+                if line.starts_with("HTTP/") {
+                    line.split_whitespace().nth(1)?.parse::<u16>().ok()
+                } else {
+                    None
+                }
+            })
+            .next_back()
+    }
+
+    fn parse_body(raw: &str) -> String {
+        if let Some(pos) = raw.rfind("\r\n\r\n") {
+            return raw[pos + 4..].to_string();
+        }
+        if let Some(pos) = raw.rfind("\n\n") {
+            return raw[pos + 2..].to_string();
+        }
+        String::new()
+    }
+
+    fn parse_headers(raw: &str) -> String {
+        let start = raw.rfind("HTTP/").unwrap_or(0);
+        let tail = &raw[start..];
+        if let Some(pos) = tail.find("\r\n\r\n") {
+            return tail[..pos].to_string();
+        }
+        if let Some(pos) = tail.find("\n\n") {
+            return tail[..pos].to_string();
+        }
+        tail.to_string()
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_compatibility_endpoints() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "test-get-bucket-logging";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        let result = client.get_bucket_logging().bucket(bucket).send().await;
+        assert!(
+            result.is_ok(),
+            "GetBucketLogging should return success for existing bucket, got: {:?}",
+            result.err()
+        );
+
+        let output = result.unwrap();
+        assert!(
+            output.logging_enabled().is_none(),
+            "Default GetBucketLogging should return empty logging configuration"
+        );
+
+        let put_logging = client
+            .put_bucket_logging()
+            .bucket(bucket)
+            .bucket_logging_status(
+                BucketLoggingStatus::builder()
+                    .logging_enabled(
+                        LoggingEnabled::builder()
+                            .target_bucket(bucket)
+                            .target_prefix("logs/")
+                            .build()
+                            .expect("failed to build LoggingEnabled"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_logging.is_ok(),
+            "PutBucketLogging should return success for existing bucket, got: {:?}",
+            put_logging.err()
+        );
+
+        let output_after_put = client
+            .get_bucket_logging()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketLogging should succeed after PutBucketLogging");
+        let logging_after_put = output_after_put
+            .logging_enabled()
+            .expect("GetBucketLogging should return persisted logging_enabled");
+        assert_eq!(
+            logging_after_put.target_bucket(),
+            bucket,
+            "GetBucketLogging should preserve target bucket"
+        );
+        assert_eq!(
+            logging_after_put.target_prefix(),
+            "logs/",
+            "GetBucketLogging should preserve target prefix"
+        );
+
+        let accelerate = client
+            .get_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketAccelerateConfiguration should succeed");
+        assert!(
+            accelerate.status().is_none(),
+            "Default GetBucketAccelerateConfiguration should return empty status"
+        );
+
+        let payment = client
+            .get_bucket_request_payment()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketRequestPayment should succeed");
+        assert_eq!(
+            payment.payer().map(|p| p.as_str()),
+            Some("BucketOwner"),
+            "GetBucketRequestPayment should return BucketOwner by default"
+        );
+
+        let put_accelerate = client
+            .put_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .accelerate_configuration(
+                AccelerateConfiguration::builder()
+                    .status(BucketAccelerateStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_accelerate.is_ok(),
+            "PutBucketAccelerateConfiguration should return success for existing bucket, got: {:?}",
+            put_accelerate.err()
+        );
+
+        let put_request_payment = client
+            .put_bucket_request_payment()
+            .bucket(bucket)
+            .request_payment_configuration(
+                RequestPaymentConfiguration::builder()
+                    .payer(Payer::Requester)
+                    .build()
+                    .expect("failed to build RequestPaymentConfiguration"),
+            )
+            .send()
+            .await;
+        assert!(
+            put_request_payment.is_ok(),
+            "PutBucketRequestPayment should return success for existing bucket, got: {:?}",
+            put_request_payment.err()
+        );
+
+        let accelerate_after_put = client
+            .get_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketAccelerateConfiguration should succeed after put");
+        assert_eq!(
+            accelerate_after_put.status().map(|s| s.as_str()),
+            Some("Suspended"),
+            "GetBucketAccelerateConfiguration should preserve put status"
+        );
+
+        let payment_after_put = client
+            .get_bucket_request_payment()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketRequestPayment should succeed after put");
+        assert_eq!(
+            payment_after_put.payer().map(|p| p.as_str()),
+            Some("Requester"),
+            "GetBucketRequestPayment should preserve put payer"
+        );
+
+        let put_website = client
+            .put_bucket_website()
+            .bucket(bucket)
+            .website_configuration(
+                WebsiteConfiguration::builder()
+                    .index_document(
+                        IndexDocument::builder()
+                            .suffix("index.html")
+                            .build()
+                            .expect("failed to build IndexDocument"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_website.is_ok(),
+            "PutBucketWebsite should return success for existing bucket, got: {:?}",
+            put_website.err()
+        );
+
+        let website = client.get_bucket_website().bucket(bucket).send().await;
+        assert!(website.is_ok(), "GetBucketWebsite should return persisted website configuration");
+        let website_output = website.unwrap();
+        assert_eq!(
+            website_output.index_document().map(|doc| doc.suffix()),
+            Some("index.html"),
+            "GetBucketWebsite should preserve index document suffix"
+        );
+
+        client
+            .delete_bucket_website()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("DeleteBucketWebsite should return success");
+
+        let website_after_delete = client.get_bucket_website().bucket(bucket).send().await;
+        assert!(
+            website_after_delete.is_err(),
+            "GetBucketWebsite should return NoSuchWebsiteConfiguration after deletion"
+        );
+        let website_err = website_after_delete.err().unwrap();
+        let website_code = website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(website_code, Some("NoSuchWebsiteConfiguration")),
+            "Unexpected GetBucketWebsite error code: {:?}, err: {:?}",
+            website_code,
+            website_err
+        );
+
+        env.stop_server();
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let missing_bucket = "test-dummy-bucket-missing";
+
+        let get_logging = client.get_bucket_logging().bucket(missing_bucket).send().await;
+        assert!(get_logging.is_err(), "GetBucketLogging should fail for missing bucket");
+        let get_logging_err = get_logging.err().unwrap();
+        let get_logging_code = get_logging_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_logging_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketLogging error code: {:?}, err: {:?}",
+            get_logging_code,
+            get_logging_err
+        );
+
+        let put_logging = client
+            .put_bucket_logging()
+            .bucket(missing_bucket)
+            .bucket_logging_status(BucketLoggingStatus::builder().build())
+            .send()
+            .await;
+        assert!(put_logging.is_err(), "PutBucketLogging should fail for missing bucket");
+        let put_logging_err = put_logging.err().unwrap();
+        let put_logging_code = put_logging_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_logging_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketLogging error code: {:?}, err: {:?}",
+            put_logging_code,
+            put_logging_err
+        );
+
+        let get_accelerate = client
+            .get_bucket_accelerate_configuration()
+            .bucket(missing_bucket)
+            .send()
+            .await;
+        assert!(get_accelerate.is_err(), "GetBucketAccelerateConfiguration should fail for missing bucket");
+        let get_accelerate_err = get_accelerate.err().unwrap();
+        let get_accelerate_code = get_accelerate_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_accelerate_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketAccelerateConfiguration error code: {:?}, err: {:?}",
+            get_accelerate_code,
+            get_accelerate_err
+        );
+
+        let get_request_payment = client.get_bucket_request_payment().bucket(missing_bucket).send().await;
+        assert!(get_request_payment.is_err(), "GetBucketRequestPayment should fail for missing bucket");
+        let get_request_payment_err = get_request_payment.err().unwrap();
+        let get_request_payment_code = get_request_payment_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_request_payment_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketRequestPayment error code: {:?}, err: {:?}",
+            get_request_payment_code,
+            get_request_payment_err
+        );
+
+        let put_accelerate = client
+            .put_bucket_accelerate_configuration()
+            .bucket(missing_bucket)
+            .accelerate_configuration(
+                AccelerateConfiguration::builder()
+                    .status(BucketAccelerateStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(put_accelerate.is_err(), "PutBucketAccelerateConfiguration should fail for missing bucket");
+        let put_accelerate_err = put_accelerate.err().unwrap();
+        let put_accelerate_code = put_accelerate_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_accelerate_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketAccelerateConfiguration error code: {:?}, err: {:?}",
+            put_accelerate_code,
+            put_accelerate_err
+        );
+
+        let put_request_payment = client
+            .put_bucket_request_payment()
+            .bucket(missing_bucket)
+            .request_payment_configuration(
+                RequestPaymentConfiguration::builder()
+                    .payer(Payer::BucketOwner)
+                    .build()
+                    .expect("failed to build RequestPaymentConfiguration"),
+            )
+            .send()
+            .await;
+        assert!(put_request_payment.is_err(), "PutBucketRequestPayment should fail for missing bucket");
+        let put_request_payment_err = put_request_payment.err().unwrap();
+        let put_request_payment_code = put_request_payment_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_request_payment_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketRequestPayment error code: {:?}, err: {:?}",
+            put_request_payment_code,
+            put_request_payment_err
+        );
+
+        let put_website = client
+            .put_bucket_website()
+            .bucket(missing_bucket)
+            .website_configuration(
+                WebsiteConfiguration::builder()
+                    .index_document(
+                        IndexDocument::builder()
+                            .suffix("index.html")
+                            .build()
+                            .expect("failed to build IndexDocument"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(put_website.is_err(), "PutBucketWebsite should fail for missing bucket");
+        let put_website_err = put_website.err().unwrap();
+        let put_website_code = put_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_website_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketWebsite error code: {:?}, err: {:?}",
+            put_website_code,
+            put_website_err
+        );
+
+        let get_website = client.get_bucket_website().bucket(missing_bucket).send().await;
+        assert!(get_website.is_err(), "GetBucketWebsite should fail for missing bucket");
+        let get_website_err = get_website.err().unwrap();
+        let get_website_code = get_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_website_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketWebsite error code: {:?}, err: {:?}",
+            get_website_code,
+            get_website_err
+        );
+
+        let delete_website = client.delete_bucket_website().bucket(missing_bucket).send().await;
+        assert!(delete_website.is_err(), "DeleteBucketWebsite should fail for missing bucket");
+        let delete_website_err = delete_website.err().unwrap();
+        let delete_website_code = delete_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(delete_website_code, Some("NoSuchBucket")),
+            "Unexpected DeleteBucketWebsite error code: {:?}, err: {:?}",
+            delete_website_code,
+            delete_website_err
+        );
+
+        env.stop_server();
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_endpoints_http_contracts() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket API HTTP contracts");
+        if !awscurl_available() {
+            info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
+            return;
+        }
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "test-dummy-bucket-http-contracts";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketLogging HTTP request failed");
+        assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
+        let logging_body = parse_body(&logging_raw);
+        assert!(
+            logging_body.contains("<BucketLoggingStatus"),
+            "GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
+        );
+
+        let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketAccelerateConfiguration HTTP request failed");
+        assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
+        let accel_body = parse_body(&accel_raw);
+        assert!(
+            accel_body.contains("<AccelerateConfiguration"),
+            "GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
+        );
+
+        let payment_raw =
+            execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
+                .expect("GetBucketRequestPayment HTTP request failed");
+        assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
+        let payment_body = parse_body(&payment_raw);
+        assert!(
+            payment_body.contains("<Payer>BucketOwner</Payer>"),
+            "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
+        );
+
+        let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketWebsite HTTP request failed");
+        assert_eq!(
+            parse_status(&website_raw),
+            Some(404),
+            "GetBucketWebsite should return 404 when website config is absent"
+        );
+        let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
+        assert!(
+            website_content_type.contains("content-type:") && website_content_type.contains("xml"),
+            "GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
+        );
+        let website_body = parse_body(&website_raw);
+        assert!(
+            website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
+            "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
+        );
+
+        let delete_raw =
+            execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+                .expect("DeleteBucketWebsite HTTP request failed");
+        assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
+
+        env.stop_server();
+    }
+}

crates/e2e_test/src/bucket_policy_check_test.rs

@@ -54,6 +54,10 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
 #[serial]
 async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if !crate::common::awscurl_available() {
+        info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
+        return Ok(());
+    }
     info!("Starting test_bucket_policy_authenticated_user...");
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/checksum_upload_test.rs

@@ -19,9 +19,14 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
-    use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+    use aws_sdk_s3::primitives::{ByteStream, SdkBody};
+    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
     use base64::Engine;
+    use bytes::Bytes;
+    use futures::StreamExt;
+    use http_body::Frame;
+    use http_body_util::StreamBody;
+    use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
     use tracing::info;
@@ -57,6 +62,59 @@ mod tests {
         base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
     }
 
+    fn checksum_crc64nvme_base64(body: &[u8]) -> String {
+        Checksum::new_from_data(RioChecksumType::CRC64_NVME, body)
+            .expect("crc64nvme checksum")
+            .encoded
+    }
+
+    fn streamed_body_70kib_of_a() -> ByteStream {
+        let bytes = Bytes::from_static(&[b'a'; 1024]);
+        let stream = futures::stream::repeat_with(move || {
+            let frame = Frame::data(bytes.clone());
+            Ok::<_, std::io::Error>(frame)
+        });
+        let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
+        ByteStream::new(SdkBody::from_body_1_x(body))
+    }
+
+    struct WithSizeHint<T> {
+        inner: T,
+        size_hint: usize,
+    }
+
+    impl<T> WithSizeHint<T> {
+        fn new(inner: T, size_hint: usize) -> Self {
+            Self { inner, size_hint }
+        }
+    }
+
+    impl<T> http_body::Body for WithSizeHint<T>
+    where
+        T: http_body::Body + Unpin,
+    {
+        type Data = T::Data;
+        type Error = T::Error;
+
+        fn poll_frame(
+            self: std::pin::Pin<&mut Self>,
+            cx: &mut std::task::Context<'_>,
+        ) -> std::task::Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
+            let this = self.get_mut();
+            std::pin::Pin::new(&mut this.inner).poll_frame(cx)
+        }
+
+        fn is_end_stream(&self) -> bool {
+            self.inner.is_end_stream()
+        }
+
+        fn size_hint(&self) -> http_body::SizeHint {
+            let mut hint = self.inner.size_hint();
+            hint.set_exact(self.size_hint as u64);
+            hint
+        }
+    }
+
     /// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
     #[tokio::test]
     #[serial]
@@ -129,6 +187,121 @@ mod tests {
         info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
     }
 
+    /// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect
+    /// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum.
+    #[tokio::test]
+    #[serial]
+    async fn test_put_object_with_checksum_algorithm_only() {
+        init_logging();
+        info!("TEST: PutObject with checksum_algorithm only");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-checksum-algorithm-only";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let key = "obj-with-checksum-algorithm-only.txt";
+        let content = vec![b'a'; 70 * 1024];
+
+        let put_resp = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .body(ByteStream::from(content.clone()))
+            .send()
+            .await
+            .expect("PutObject with checksum_algorithm should succeed");
+
+        let put_checksum = put_resp
+            .checksum_crc32()
+            .expect("PutObject should return checksum_crc32 when checksum_algorithm is used")
+            .to_string();
+
+        let mut get_resp = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("GetObject should succeed");
+
+        let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty()))
+            .collect()
+            .await
+            .expect("collect body")
+            .into_bytes();
+
+        assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content");
+        assert_eq!(
+            get_resp.checksum_crc32().map(str::to_string),
+            Some(put_checksum),
+            "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum"
+        );
+    }
+
+    /// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec<u8>)`.
+    #[tokio::test]
+    #[serial]
+    async fn test_put_object_with_checksum_algorithm_only_streaming_body() {
+        init_logging();
+        info!("TEST: PutObject with checksum_algorithm only using streaming body");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-checksum-algorithm-streaming";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let key = "obj-with-checksum-algorithm-streaming.txt";
+        let expected_content = vec![b'a'; 70 * 1024];
+
+        let put_resp = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .body(streamed_body_70kib_of_a())
+            .send()
+            .await
+            .expect("PutObject with streaming checksum_algorithm should succeed");
+
+        let put_checksum = put_resp
+            .checksum_crc32()
+            .expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads")
+            .to_string();
+
+        let mut get_resp = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("GetObject should succeed");
+
+        let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty()))
+            .collect()
+            .await
+            .expect("collect body")
+            .into_bytes();
+
+        assert_eq!(
+            body_bytes.as_ref(),
+            expected_content.as_slice(),
+            "GetObject body must match uploaded content"
+        );
+        assert_eq!(
+            get_resp.checksum_crc32().map(str::to_string),
+            Some(put_checksum),
+            "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads"
+        );
+    }
+
     /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
     /// Uses part size >= 5MB (server minimum) for two parts.
     #[tokio::test]
@@ -226,4 +399,243 @@ mod tests {
         );
         info!("PASSED: MultipartUpload with checksum and GetObject content match");
     }
+
+    /// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation,
+    /// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed.
+    #[tokio::test]
+    #[serial]
+    async fn test_multipart_upload_with_crc32_algorithm_only() {
+        init_logging();
+        info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-multipart-checksum-crc32-auto";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        let key = "multipart-with-crc32-auto.bin";
+        let part1_content = "a".repeat(5 * 1024 * 1024 + 1);
+        let part2_content = "b".repeat(1024);
+
+        let create_resp = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .checksum_algorithm(ChecksumAlgorithm::Crc32)
+            .send()
+            .await
+            .expect("CreateMultipartUpload should succeed");
+
+        let upload_id = create_resp.upload_id().expect("upload_id should be present");
+
+        let part1_resp = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(part1_content.clone().into_bytes()))
+            .send()
+            .await
+            .expect("UploadPart 1 should succeed");
+        let part1_checksum = part1_resp
+            .checksum_crc32()
+            .expect("UploadPart 1 should return checksum_crc32")
+            .to_string();
+
+        let part2_resp = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(2)
+            .body(ByteStream::from(part2_content.clone().into_bytes()))
+            .send()
+            .await
+            .expect("UploadPart 2 should succeed");
+        let part2_checksum = part2_resp
+            .checksum_crc32()
+            .expect("UploadPart 2 should return checksum_crc32")
+            .to_string();
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(part1_resp.e_tag().expect("etag part 1"))
+                    .checksum_crc32(part1_checksum)
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(part2_resp.e_tag().expect("etag part 2"))
+                    .checksum_crc32(part2_checksum)
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await
+            .expect("CompleteMultipartUpload should succeed");
+
+        let body_bytes = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("GetObject should succeed")
+            .body
+            .collect()
+            .await
+            .expect("collect body")
+            .into_bytes();
+
+        let expected_content = format!("{part1_content}{part2_content}");
+        assert_eq!(
+            body_bytes.as_ref(),
+            expected_content.as_bytes(),
+            "completed multipart object must match concatenated parts"
+        );
+    }
+
+    /// Regression test for issue #2282:
+    /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
+    #[tokio::test]
+    #[serial]
+    async fn test_crc64nvme_matches_between_put_object_and_multipart_upload() {
+        init_logging();
+        info!("TEST: CRC64NVME matches between direct PutObject and multipart upload");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-crc64nvme-multipart-match";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        const PART_SIZE: usize = 6 * 1024 * 1024;
+        let part1: Vec<u8> = (0..PART_SIZE).map(|i| (i % 251) as u8).collect();
+        let part2: Vec<u8> = (0..PART_SIZE).map(|i| ((i + 17) % 251) as u8).collect();
+        let content: Vec<u8> = part1.iter().chain(part2.iter()).copied().collect();
+
+        let direct_key = "crc64nvme-direct.bin";
+        let multipart_key = "crc64nvme-multipart.bin";
+        let full_checksum = checksum_crc64nvme_base64(&content);
+        let part1_checksum = checksum_crc64nvme_base64(&part1);
+        let part2_checksum = checksum_crc64nvme_base64(&part2);
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(direct_key)
+            .body(ByteStream::from(content.clone()))
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .checksum_crc64_nvme(full_checksum.clone())
+            .send()
+            .await
+            .expect("Failed to put direct object with CRC64NVME");
+
+        let create_result = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .send()
+            .await
+            .expect("Failed to create multipart upload");
+
+        let upload_id = create_result.upload_id().expect("No upload_id").to_string();
+
+        let upload1 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(&upload_id)
+            .part_number(1)
+            .body(ByteStream::from(part1.clone()))
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .checksum_crc64_nvme(part1_checksum)
+            .send()
+            .await
+            .expect("Failed to upload multipart part 1");
+
+        let upload2 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(&upload_id)
+            .part_number(2)
+            .body(ByteStream::from(part2.clone()))
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .checksum_crc64_nvme(part2_checksum)
+            .send()
+            .await
+            .expect("Failed to upload multipart part 2");
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload1.e_tag().expect("No etag for part 1"))
+                    .checksum_crc64_nvme(upload1.checksum_crc64_nvme().expect("No CRC64NVME for part 1"))
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload2.e_tag().expect("No etag for part 2"))
+                    .checksum_crc64_nvme(upload2.checksum_crc64_nvme().expect("No CRC64NVME for part 2"))
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(&upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await
+            .expect("Failed to complete multipart upload");
+
+        let direct_head = client
+            .head_object()
+            .bucket(bucket)
+            .key(direct_key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("Failed to head direct object");
+
+        let multipart_head = client
+            .head_object()
+            .bucket(bucket)
+            .key(multipart_key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("Failed to head multipart object");
+
+        assert_eq!(
+            direct_head.checksum_crc64_nvme(),
+            Some(full_checksum.as_str()),
+            "Direct object should report the uploaded full-object CRC64NVME"
+        );
+        assert_eq!(
+            multipart_head.checksum_crc64_nvme(),
+            Some(full_checksum.as_str()),
+            "Multipart object should report the same full-object CRC64NVME as direct upload"
+        );
+    }
 }

crates/e2e_test/src/common.rs

@@ -23,7 +23,11 @@
 
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
-use std::path::PathBuf;
+use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use reqwest::Client as HttpClient;
+use std::ffi::OsStr;
+use std::fs as stdfs;
+use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use std::sync::Once;
 use std::time::Duration;
@@ -32,11 +36,29 @@ use tokio::net::TcpStream;
 use tokio::time::sleep;
 use tracing::{error, info, warn};
 use uuid::Uuid;
+use walkdir::WalkDir;
 
 // Common constants for all E2E tests
 pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
 pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
 pub const TEST_BUCKET: &str = "e2e-test-bucket";
+
+fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
+    let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
+    let mut config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(endpoint_url)
+        .force_path_style(true)
+        .behavior_version_latest();
+
+    if endpoint_url.starts_with("http://") {
+        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
+    }
+
+    config.build()
+}
+
 pub fn workspace_root() -> PathBuf {
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     path.pop(); // e2e_test
@@ -44,29 +66,125 @@ pub fn workspace_root() -> PathBuf {
     path
 }
 
+pub fn local_http_client() -> HttpClient {
+    HttpClient::builder()
+        .no_proxy()
+        .build()
+        .expect("failed to build local reqwest client")
+}
+
 /// Resolve the RustFS binary relative to the workspace.
-/// Always builds the binary to ensure it's up to date.
 pub fn rustfs_binary_path() -> PathBuf {
+    rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())
+}
+
+/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
+pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
     if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
         return PathBuf::from(path);
     }
 
-    // Always build the binary to ensure it's up to date
-    info!("Building RustFS binary to ensure it's up to date...");
-    build_rustfs_binary();
-
     let mut binary_path = workspace_root();
     binary_path.push("target");
     let profile_dir = if cfg!(debug_assertions) { "debug" } else { "release" };
     binary_path.push(profile_dir);
     binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
 
+    let features_match = binary_features_match(&binary_path, requested_features);
+    let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
+    let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match;
+    if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) {
+        if source_is_newer {
+            warn!(
+                "RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds",
+                binary_path
+            );
+        }
+        info!("Using existing RustFS binary at {:?}", binary_path);
+        return binary_path;
+    }
+
+    info!("Building RustFS binary to ensure it's up to date...");
+    build_rustfs_binary(requested_features);
+
     info!("Using RustFS binary at {:?}", binary_path);
     binary_path
 }
 
+fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool {
+    let Ok(binary_meta) = std::fs::metadata(binary_path) else {
+        return true;
+    };
+    let Ok(binary_modified) = binary_meta.modified() else {
+        return true;
+    };
+
+    let workspace = workspace_root();
+    let watch_roots = [
+        workspace.join("Cargo.toml"),
+        workspace.join("Cargo.lock"),
+        workspace.join("rustfs"),
+        workspace.join("crates"),
+    ];
+
+    watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path))
+}
+
+fn running_inside_e2e_test_binary() -> bool {
+    std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test")
+}
+
+fn requested_rustfs_build_features() -> Option<String> {
+    std::env::var("RUSTFS_BUILD_FEATURES")
+        .ok()
+        .map(|value| value.trim().to_string())
+        .filter(|value| !value.is_empty())
+}
+
+fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf {
+    binary_path.with_extension("features")
+}
+
+fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool {
+    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
+    let recorded = stdfs::read_to_string(stamp_path).ok().map(|value| value.trim().to_string());
+
+    match requested_features {
+        Some(features) => recorded.as_deref() == Some(features),
+        None => recorded.as_deref().is_none_or(str::is_empty),
+    }
+}
+
+fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool {
+    if path.is_file() {
+        return std::fs::metadata(path)
+            .and_then(|meta| meta.modified())
+            .map(|modified| modified > binary_modified)
+            .unwrap_or(false);
+    }
+
+    if !path.is_dir() {
+        return false;
+    }
+
+    WalkDir::new(path)
+        .into_iter()
+        .filter_entry(|entry| {
+            let name = entry.file_name();
+            name != OsStr::new("target") && name != OsStr::new(".git")
+        })
+        .filter_map(Result::ok)
+        .filter(|entry| entry.file_type().is_file())
+        .any(|entry| {
+            std::fs::metadata(entry.path())
+                .and_then(|meta| meta.modified())
+                .map(|modified| modified > binary_modified)
+                .unwrap_or(false)
+        })
+}
+
 /// Build the RustFS binary using cargo
-fn build_rustfs_binary() {
+fn build_rustfs_binary(requested_features: Option<&str>) {
     let workspace = workspace_root();
     info!("Building RustFS binary from workspace: {:?}", workspace);
 
@@ -81,11 +199,8 @@ fn build_rustfs_binary() {
     let mut cmd = Command::new("cargo");
     cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);
 
-    // Read features from environment variable for e2e tests
-    if let Ok(features) = std::env::var("RUSTFS_BUILD_FEATURES")
-        && !features.is_empty()
-    {
-        cmd.arg("--features").arg(&features);
+    if let Some(features) = requested_features {
+        cmd.arg("--features").arg(features);
         info!("Building with features: {}", features);
     }
 
@@ -105,6 +220,15 @@ fn build_rustfs_binary() {
         panic!("Failed to build RustFS binary. Error: {stderr}");
     }
 
+    let mut binary_path = workspace;
+    binary_path.push("target");
+    binary_path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
+    binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
+    let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
+    if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
+        warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
+    }
+
     info!("✅ RustFS binary built successfully");
 }
 
@@ -114,6 +238,17 @@ fn awscurl_binary_path() -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("awscurl"))
 }
 
+pub fn awscurl_available() -> bool {
+    let path = awscurl_binary_path();
+    if path.components().count() > 1 || path.is_absolute() {
+        return path.is_file();
+    }
+
+    std::env::var_os("PATH")
+        .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
+        .unwrap_or(false)
+}
+
 // Global initialization
 static INIT: Once = Once::new();
 
@@ -183,24 +318,22 @@ impl RustFSTestEnvironment {
 
     /// Kill any existing RustFS processes
     pub async fn cleanup_existing_processes(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        info!("Cleaning up any existing RustFS processes");
-        let binary_path = rustfs_binary_path();
-        let binary_name = binary_path.to_string_lossy();
-        let output = Command::new("pkill").args(["-f", &binary_name]).output();
+        info!("Cleaning up any existing RustFS processes for {}", self.address);
 
-        if let Ok(output) = output
-            && output.status.success()
-        {
-            info!("Killed existing RustFS processes: {}", binary_name);
-            sleep(Duration::from_millis(1000)).await;
+        for pattern in [&self.address, &self.temp_dir] {
+            let output = Command::new("pkill").args(["-f", pattern]).output();
+
+            if let Ok(output) = output
+                && output.status.success()
+            {
+                info!("Killed existing RustFS processes matching: {}", pattern);
+                sleep(Duration::from_millis(250)).await;
+            }
         }
         Ok(())
     }
 
-    /// Start RustFS server with basic configuration
-    pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        self.cleanup_existing_processes().await?;
-
+    fn build_start_args<'a>(&'a self, extra_args: Vec<&'a str>) -> Vec<&'a str> {
         let mut args = vec![
             "--address",
             &self.address,
@@ -210,16 +343,29 @@ impl RustFSTestEnvironment {
             &self.secret_key,
         ];
 
-        // Add extra arguments
         args.extend(extra_args);
-
-        // Add temp directory as the last argument
         args.push(&self.temp_dir);
+        args
+    }
+
+    async fn start_rustfs_server_inner(
+        &mut self,
+        extra_args: Vec<&str>,
+        cleanup_existing: bool,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        if cleanup_existing {
+            self.cleanup_existing_processes().await?;
+        }
+
+        let args = self.build_start_args(extra_args);
 
         info!("Starting RustFS server with args: {:?}", args);
 
         let binary_path = rustfs_binary_path();
-        let process = Command::new(&binary_path).args(&args).spawn()?;
+        let process = Command::new(&binary_path)
+            .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+            .args(&args)
+            .spawn()?;
 
         self.process = Some(process);
 
@@ -229,18 +375,40 @@ impl RustFSTestEnvironment {
         Ok(())
     }
 
-    /// Wait for RustFS server to be ready by checking TCP connectivity
+    /// Start RustFS server with basic configuration
+    pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner(extra_args, true).await
+    }
+
+    /// Start RustFS server without cleaning up other running RustFS processes.
+    ///
+    /// This is useful for tests that need multiple independent RustFS instances
+    /// alive at the same time.
+    pub async fn start_rustfs_server_without_cleanup(
+        &mut self,
+        extra_args: Vec<&str>,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner(extra_args, false).await
+    }
+
+    /// Wait for RustFS server to be ready.
+    ///
+    /// A listening TCP port is not sufficient here: the process may accept
+    /// connections before the S3 stack is fully initialized, which causes
+    /// early requests to fail intermittently. Treat readiness as "S3 API
+    /// responds successfully" instead.
     pub async fn wait_for_server_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         info!("Waiting for RustFS server to be ready on {}", self.address);
+        let client = self.create_s3_client();
 
-        for i in 0..30 {
-            if TcpStream::connect(&self.address).await.is_ok() {
+        for i in 0..60 {
+            if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
                 info!("✅ RustFS server is ready after {} attempts", i + 1);
                 return Ok(());
             }
 
-            if i == 29 {
-                return Err("RustFS server failed to become ready within 30 seconds".into());
+            if i == 59 {
+                return Err("RustFS server failed to become ready within 60 seconds".into());
             }
 
             sleep(Duration::from_secs(1)).await;
@@ -251,16 +419,7 @@ impl RustFSTestEnvironment {
 
     /// Create an AWS S3 client configured for this RustFS instance
     pub fn create_s3_client(&self) -> Client {
-        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "e2e-test");
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&self.url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-
-        Client::from_conf(config)
+        Client::from_conf(build_test_s3_config(&self.url, &self.access_key, &self.secret_key, "e2e-test"))
     }
 
     /// Create test bucket
@@ -304,18 +463,18 @@ impl Drop for RustFSTestEnvironment {
     }
 }
 
-/// Utility function to execute awscurl commands
-pub async fn execute_awscurl(
+async fn execute_awscurl_with_service(
     url: &str,
     method: &str,
     body: Option<&str>,
     access_key: &str,
     secret_key: &str,
+    service: &str,
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
     let mut args = vec![
         "--fail-with-body",
         "--service",
-        "s3",
+        service,
         "--region",
         "us-east-1",
         "--access_key",
@@ -331,7 +490,64 @@ pub async fn execute_awscurl(
         args.extend(&["-d", body_content]);
     }
 
-    info!("Executing awscurl: {} {}", method, url);
+    info!("Executing awscurl: {} {} (service={})", method, url, service);
+    let awscurl_path = awscurl_binary_path();
+    let output = Command::new(&awscurl_path).args(&args).output()?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
+    }
+
+    let response = String::from_utf8_lossy(&output.stdout).to_string();
+    Ok(response)
+}
+
+/// Utility function to execute awscurl commands (SigV4 service `s3` for admin APIs).
+pub async fn execute_awscurl(
+    url: &str,
+    method: &str,
+    body: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    execute_awscurl_with_service(url, method, body, access_key, secret_key, "s3").await
+}
+
+/// `POST` with SigV4 `--service sts` and explicit `Content-Type: application/x-www-form-urlencoded`.
+///
+/// RustFS `AssumeRole` is handled on `POST /` by the admin router; `is_match` requires this
+/// content type so s3s routes to the custom handler instead of `Unknown operation`.
+pub async fn awscurl_post_sts_form_urlencoded(
+    url: &str,
+    body: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    let args = vec![
+        "--fail-with-body",
+        "--service",
+        "sts",
+        "--region",
+        "us-east-1",
+        "--access_key",
+        access_key,
+        "--secret_key",
+        secret_key,
+        "-H",
+        "Content-Type: application/x-www-form-urlencoded",
+        "-X",
+        "POST",
+        url,
+        "-d",
+        body,
+    ];
+
+    info!(
+        "Executing awscurl: POST {} (service=sts, Content-Type=application/x-www-form-urlencoded)",
+        url
+    );
     let awscurl_path = awscurl_binary_path();
     let output = Command::new(&awscurl_path).args(&args).output()?;
 
@@ -406,6 +622,7 @@ pub struct RustFSTestClusterEnvironment {
     pub temp_dir: String,
     pub access_key: String,
     pub secret_key: String,
+    pub extra_env: Vec<(String, String)>,
 }
 
 impl RustFSTestClusterEnvironment {
@@ -454,9 +671,19 @@ impl RustFSTestClusterEnvironment {
             temp_dir,
             access_key: DEFAULT_ACCESS_KEY.to_string(),
             secret_key: DEFAULT_SECRET_KEY.to_string(),
+            extra_env: Vec::new(),
         })
     }
 
+    /// Add an extra environment variable applied to every cluster node process.
+    pub fn set_env<K, V>(&mut self, key: K, value: V)
+    where
+        K: Into<String>,
+        V: Into<String>,
+    {
+        self.extra_env.push((key.into(), value.into()));
+    }
+
     /// Build the volumes argument string for RustFS binary (internal helper method).
     ///
     /// Concatenates the address and data directory of all cluster nodes into a single string
@@ -487,14 +714,20 @@ impl RustFSTestClusterEnvironment {
         for (i, node) in self.nodes.iter_mut().enumerate() {
             info!("Starting cluster node {} on {}", i, node.address);
 
-            let process = Command::new(&binary_path)
+            let mut command = Command::new(&binary_path);
+            command
                 .env("RUSTFS_VOLUMES", &volumes_arg)
                 .env("RUSTFS_ADDRESS", &node.address)
                 .env("RUSTFS_ACCESS_KEY", &self.access_key)
                 .env("RUSTFS_SECRET_KEY", &self.secret_key)
                 .env("RUSTFS_CONSOLE_ENABLE", "false")
-                .current_dir(&node.data_dir)
-                .spawn()?;
+                .env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
+
+            for (key, value) in &self.extra_env {
+                command.env(key, value);
+            }
+
+            let process = command.current_dir(&node.data_dir).spawn()?;
 
             node.process = Some(process);
         }
@@ -503,7 +736,9 @@ impl RustFSTestClusterEnvironment {
             self.wait_for_node_ready(&node.address, i).await?;
         }
 
-        self.wait_for_service_ready().await?;
+        for node_idx in 0..self.nodes.len() {
+            self.wait_for_node_service_ready(node_idx).await?;
+        }
 
         Ok(())
     }
@@ -523,17 +758,17 @@ impl RustFSTestClusterEnvironment {
         Err(format!("Node {} failed to become ready", idx).into())
     }
 
-    /// Wait for the entire cluster's S3-compatible service to be ready (internal helper method).
+    /// Wait for a specific node's S3-compatible service to be ready (internal helper method).
     ///
-    /// Verifies service availability by calling the S3 `list_buckets` API, retries up to 120 times
-    /// with a 1-second interval between attempts. Fails if the API call remains unsuccessful after all retries.
-    async fn wait_for_service_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let client = self.create_s3_client(0)?;
+    /// Verifies service availability by calling the S3 `list_buckets` API against the requested node,
+    /// retries up to 120 times with a 1-second interval between attempts.
+    async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let client = self.create_s3_client(node_idx)?;
 
         for attempt in 0..120 {
             match client.list_buckets().send().await {
                 Ok(_) => {
-                    info!("Cluster service ready after {} attempts", attempt + 1);
+                    info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);
                     return Ok(());
                 }
                 Err(_) => {
@@ -541,7 +776,8 @@ impl RustFSTestClusterEnvironment {
                 }
             }
         }
-        Err("Cluster service failed to become ready".into())
+
+        Err(format!("Cluster node {} service failed to become ready", node_idx).into())
     }
 
     /// Create an S3 client configured to communicate with a specific cluster node.
@@ -562,15 +798,12 @@ impl RustFSTestClusterEnvironment {
         if node_idx >= self.nodes.len() {
             return Err("node_idx is invalid".into());
         }
-        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "cluster-test");
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&self.nodes[node_idx].url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-        Ok(Client::from_conf(config))
+        Ok(Client::from_conf(build_test_s3_config(
+            &self.nodes[node_idx].url,
+            &self.access_key,
+            &self.secret_key,
+            "cluster-test",
+        )))
     }
 
     /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector.

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -16,15 +16,13 @@
 //! "In a versioned Bucket, DeleteMarkers are not appearing straight after
 //! a delete_objects is called."
 //!
-//! Root cause: `delete_versions_internal` wrote new xl.meta to disk via
-//! `write_all_private` without invalidating the `GlobalFileCache`. Subsequent
-//! calls to `read_metadata` returned the stale cached xl.meta (without the
-//! delete marker), making `list_object_versions` show the old version as
-//! `IsLatest=true` rather than the new delete marker.
+//! Root cause: metadata updates could become temporarily invisible to
+//! `list_object_versions`, so the old version was still reported as
+//! `IsLatest=true` instead of the newly-created delete marker.
 //!
-//! Fix: `write_all_private` now calls `get_global_file_cache().invalidate()`
-//! after every successful write, and `rename_data` also invalidates the cache
-//! for the destination path after the atomic rename.
+//! Fix: metadata write, delete, and rename paths now make the updated
+//! `xl.meta` immediately visible, and the old file-cache shortcut has been
+//! removed from the read path.
 
 #[cfg(test)]
 mod tests {

crates/e2e_test/src/existing_object_tag_policy_test.rs

@@ -0,0 +1,364 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! E2E: `s3:ExistingObjectTag` with **IAM identity policy**, **bucket policy**, and **STS AssumeRole
+//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
+//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
+
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
+};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Tag, Tagging};
+use aws_sdk_s3::{Client, Config};
+use serial_test::serial;
+use tracing::info;
+use uuid::Uuid;
+
+fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
+    let open = format!("<{tag}>");
+    let close = format!("</{tag}>");
+    let start = xml.find(&open)? + open.len();
+    let end = xml[start..].find(&close)? + start;
+    Some(xml[start..end].to_string())
+}
+
+fn parse_assume_role_credentials(xml: &str) -> Result<(String, String, String), Box<dyn std::error::Error + Send + Sync>> {
+    let ak = extract_xml_tag(xml, "AccessKeyId").ok_or("missing AccessKeyId in AssumeRole response")?;
+    let sk = extract_xml_tag(xml, "SecretAccessKey").ok_or("missing SecretAccessKey in AssumeRole response")?;
+    let token = extract_xml_tag(xml, "SessionToken").ok_or("missing SessionToken in AssumeRole response")?;
+    Ok((ak, sk, token))
+}
+
+async fn assume_role_with_session_policy(
+    env: &RustFSTestEnvironment,
+    parent_ak: &str,
+    parent_sk: &str,
+    session_policy_json: &str,
+) -> Result<(String, String, String), Box<dyn std::error::Error + Send + Sync>> {
+    let policy_enc = urlencoding::encode(session_policy_json);
+    let body = format!("Action=AssumeRole&Version=2011-06-15&DurationSeconds=3600&Policy={}", policy_enc);
+    let url = format!("{}/", env.url.trim_end_matches('/'));
+    let xml = awscurl_post_sts_form_urlencoded(&url, &body, parent_ak, parent_sk).await?;
+    parse_assume_role_credentials(&xml)
+}
+
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    password: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string();
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_add_canned_policy(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    policy_json: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_attach_policy_to_user(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    username: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+        env.url, policy_name, username
+    );
+    awscurl_put(&url, "", &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) {
+    let url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, username);
+    let _ = awscurl_delete(&url, &env.access_key, &env.secret_key).await;
+}
+
+async fn admin_remove_policy(env: &RustFSTestEnvironment, policy_name: &str) {
+    let url = format!("{}/rustfs/admin/v3/remove-canned-policy?name={}", env.url, policy_name);
+    let _ = awscurl_delete(&url, &env.access_key, &env.secret_key).await;
+}
+
+async fn put_object_with_tagging_str(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    data: &[u8],
+    tagging: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(data.to_vec()))
+        .tagging(tagging)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn put_object_tag_kv(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    tag_key: &str,
+    tag_value: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let tag = Tag::builder()
+        .key(tag_key)
+        .value(tag_value)
+        .build()
+        .map_err(|e| format!("Tag build: {e}"))?;
+    let tagging = Tagging::builder()
+        .tag_set(tag)
+        .build()
+        .map_err(|e| format!("Tagging build: {e}"))?;
+    client
+        .put_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .tagging(tagging)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
+    let _ = admin.delete_object().bucket(bucket).key(key).send().await;
+    let _ = admin.delete_bucket().bucket(bucket).send().await;
+}
+
+/// IAM identity policy: GetObject allowed only when `s3:ExistingObjectTag/security` == `public`.
+#[tokio::test]
+#[serial]
+async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let user = format!("e2eiamtag-{suffix}");
+    let user_secret = "longSecretKeyForTest123!";
+    let policy_name = format!("e2e-iam-tag-pol-{suffix}");
+    let bucket = format!("e2e-iam-tag-bkt-{suffix}");
+    let key = "tagged-object.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &user, user_secret).await?;
+
+    let policy_doc = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    admin_add_canned_policy(&env, &policy_name, &policy_doc).await?;
+    admin_attach_policy_to_user(&env, &policy_name, &user).await?;
+
+    admin.create_bucket().bucket(&bucket).send().await?;
+    put_object_with_tagging_str(&admin, &bucket, key, b"hello-iam-tag", "security=public").await?;
+
+    let uclient = user_client(&env, &user, user_secret);
+    let out = uclient.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = out.body.collect().await?;
+
+    put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
+    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
+    assert!(
+        denied.is_err(),
+        "GetObject must be denied when ExistingObjectTag no longer matches IAM policy"
+    );
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &user).await;
+    admin_remove_policy(&env, &policy_name).await;
+
+    info!("test_e2e_iam_policy_existing_object_tag_get_object passed");
+    Ok(())
+}
+
+/// Bucket policy: same `ExistingObjectTag` condition; user has no canned IAM policy attached.
+#[tokio::test]
+#[serial]
+async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let user = format!("e2ebptag-{suffix}");
+    let user_secret = "longSecretKeyForTest456!";
+    let bucket = format!("e2e-bp-tag-bkt-{suffix}");
+    let key = "obj.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &user, user_secret).await?;
+
+    admin.create_bucket().bucket(&bucket).send().await?;
+
+    let deny_before = user_client(&env, &user, user_secret)
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .send()
+        .await;
+    assert!(deny_before.is_err(), "without bucket policy, user must be denied");
+
+    let bp = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Principal": { "AWS": [user.clone()] },
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    admin.put_bucket_policy().bucket(&bucket).policy(&bp).send().await?;
+    put_object_with_tagging_str(&admin, &bucket, key, b"data", "security=public").await?;
+
+    let uclient = user_client(&env, &user, user_secret);
+    let ok = uclient.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = ok.body.collect().await?;
+
+    put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
+    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
+    assert!(denied.is_err(), "GetObject must fail when tag no longer satisfies bucket policy");
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &user).await;
+
+    info!("test_e2e_bucket_policy_existing_object_tag_get_object passed");
+    Ok(())
+}
+
+/// STS `AssumeRole` with inline `Policy` (session policy): GetObject only when `ExistingObjectTag/security` is `public`.
+#[tokio::test]
+#[serial]
+async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let parent = format!("e2e-sts-par-{suffix}");
+    let parent_secret = "longSecretKeyForParentSts99!";
+    let policy_readwrite = format!("e2e-sts-rw-{suffix}");
+    let bucket = format!("e2e-sts-tag-bkt-{suffix}");
+    let key = "sts-obj.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &parent, parent_secret).await?;
+
+    let rw = serde_json::to_string(&serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": ["arn:aws:s3:::*"]
+        }]
+    }))?;
+    admin_add_canned_policy(&env, &policy_readwrite, &rw).await?;
+    admin_attach_policy_to_user(&env, &policy_readwrite, &parent).await?;
+
+    let parent_client = user_client(&env, &parent, parent_secret);
+    parent_client.create_bucket().bucket(&bucket).send().await?;
+    put_object_with_tagging_str(&parent_client, &bucket, key, b"sts-e2e-data", "security=public").await?;
+
+    let session_policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    let (ak, sk, token) = assume_role_with_session_policy(&env, &parent, parent_secret, &session_policy).await?;
+
+    let session_client = sts_session_client(&env, &ak, &sk, &token);
+    let ok = session_client.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = ok.body.collect().await?;
+
+    put_object_tag_kv(&parent_client, &bucket, key, "security", "private").await?;
+    let denied = session_client.get_object().bucket(&bucket).key(key).send().await;
+    assert!(
+        denied.is_err(),
+        "session policy must deny GetObject when ExistingObjectTag no longer matches"
+    );
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &parent).await;
+    admin_remove_policy(&env, &policy_readwrite).await;
+
+    info!("test_e2e_sts_assume_role_session_policy_existing_object_tag passed");
+    Ok(())
+}

crates/e2e_test/src/kms/common.rs

@@ -22,11 +22,13 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities
 
-use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging};
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
+};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
-use base64::Engine;
+use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
 use serde_json;
 use std::process::{Child, Command};
 use std::time::Duration;
@@ -44,6 +46,7 @@ pub const VAULT_ADDRESS: &str = "127.0.0.1:8200";
 pub const VAULT_TOKEN: &str = "dev-root-token";
 pub const VAULT_TRANSIT_PATH: &str = "transit";
 pub const VAULT_KEY_NAME: &str = "rustfs-master-key";
+pub const ENV_TEST_VAULT_BIN: &str = "RUSTFS_TEST_VAULT_BIN";
 
 /// Initialize tracing for KMS tests with KMS-specific log levels
 pub fn init_logging() {
@@ -51,6 +54,19 @@ pub fn init_logging() {
     // Additional KMS-specific logging configuration can be added here if needed
 }
 
+pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
+    if awscurl_available() {
+        return false;
+    }
+
+    info!("Skipping {} because awscurl is not available in PATH", test_name);
+    true
+}
+
+pub fn sse_customer_key_md5_base64(key: &str) -> String {
+    BASE64.encode(md5::compute(key).0)
+}
+
 // KMS-specific helper functions
 /// Configure KMS backend via admin API
 pub async fn configure_kms(
@@ -133,10 +149,10 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
         "usage": "EncryptDecrypt",
         "status": "Active",
         "metadata": HashMap::<String, String>::new(),
-        "created_at": chrono::Utc::now().to_rfc3339(),
+        "created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
         "rotated_at": serde_json::Value::Null,
         "created_by": "e2e-test",
-        "encrypted_key_material": key_data.to_vec(),
+        "encrypted_key_material": BASE64.encode(key_data),
         "nonce": Vec::<u8>::new()
     });
 
@@ -155,7 +171,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
 
     let test_key = "01234567890123456789012345678901"; // 32-byte key
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let test_data = b"Hello, KMS SSE-C World!";
     let object_key = "test-sse-c-object";
 
@@ -272,6 +288,10 @@ pub async fn test_kms_key_management(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
+        return Ok(());
+    }
+
     info!("Testing KMS key management APIs");
 
     // Test CreateKey
@@ -324,8 +344,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
     let wrong_key = "98765432109876543210987654321098";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
     let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
     let test_data = b"Test data for error scenarios";
     let object_key = "test-error-object";
 
@@ -366,6 +386,10 @@ pub struct VaultTestEnvironment {
 }
 
 impl VaultTestEnvironment {
+    fn resolve_vault_binary() -> String {
+        std::env::var(ENV_TEST_VAULT_BIN).unwrap_or_else(|_| "vault".to_string())
+    }
+
     /// Create a new Vault test environment
     pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
         let base_env = RustFSTestEnvironment::new().await?;
@@ -380,7 +404,8 @@ impl VaultTestEnvironment {
     pub async fn start_vault(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         info!("Starting Vault server in development mode");
 
-        let vault_process = Command::new("vault")
+        let vault_binary = Self::resolve_vault_binary();
+        let vault_process = Command::new(&vault_binary)
             .args([
                 "server",
                 "-dev",
@@ -406,7 +431,7 @@ impl VaultTestEnvironment {
             let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
             if port_check {
                 // Additional check by making a health request
-                if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await
+                if let Ok(response) = local_http_client().get(format!("{VAULT_URL}/v1/sys/health")).send().await
                     && response.status().is_success()
                 {
                     info!("Vault server is ready after {} seconds", i);
@@ -426,7 +451,7 @@ impl VaultTestEnvironment {
 
     /// Setup Vault transit secrets engine
     pub async fn setup_vault_transit(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let client = reqwest::Client::new();
+        let client = local_http_client();
 
         info!("Enabling Vault transit secrets engine");
 
@@ -471,10 +496,10 @@ impl VaultTestEnvironment {
         self.base_env.start_rustfs_server(Vec::new()).await
     }
 
-    /// Configure Vault KMS backend
-    pub async fn configure_vault_kms(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    /// Configure Vault Transit KMS backend
+    pub async fn configure_vault_transit_kms(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let kms_config = serde_json::json!({
-            "backend_type": "vault",
+            "backend_type": "VaultTransit",
             "address": VAULT_URL,
             "auth_method": {
                 "Token": {
@@ -482,8 +507,6 @@ impl VaultTestEnvironment {
                 }
             },
             "mount_path": VAULT_TRANSIT_PATH,
-            "kv_mount": "secret",
-            "key_path_prefix": "rustfs/kms/keys",
             "default_key_id": VAULT_KEY_NAME,
             "skip_tls_verify": true
         })
@@ -687,7 +710,7 @@ pub async fn test_multipart_upload_with_config(
 /// Create a standard SSE-C encryption configuration for testing
 pub fn create_sse_c_config() -> EncryptionType {
     let key = "01234567890123456789012345678901"; // 32-byte key
-    let key_md5 = format!("{:x}", md5::compute(key));
+    let key_md5 = sse_customer_key_md5_base64(key);
     EncryptionType::SSEC {
         key: key.to_string(),
         key_md5,
@@ -768,7 +791,7 @@ impl LocalKMSTestEnvironment {
 
         // Configure KMS with the default key in one step
         let kms_config = serde_json::json!({
-            "backend_type": "local",
+            "backend_type": "Local",
             "key_dir": self.kms_keys_dir,
             "file_permissions": 0o600,
             "default_key_id": default_key_id

crates/e2e_test/src/kms/encryption_metadata_test.rs

@@ -26,27 +26,20 @@ use serial_test::serial;
 use std::collections::{HashMap, VecDeque};
 use tracing::info;
 
-fn assert_encryption_metadata(metadata: &HashMap<String, String>, expected_size: usize) {
+fn assert_managed_encryption_metadata_hidden(metadata: Option<&HashMap<String, String>>) {
+    let Some(metadata) = metadata else { return };
+
     for key in [
         "x-rustfs-encryption-key",
         "x-rustfs-encryption-iv",
         "x-rustfs-encryption-context",
         "x-rustfs-encryption-original-size",
     ] {
-        assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present");
         assert!(
-            !metadata.get(key).unwrap().is_empty(),
-            "managed encryption metadata '{key}' should not be empty"
+            !metadata.contains_key(key),
+            "managed encryption metadata '{key}' should not be exposed to clients"
         );
     }
-
-    let size_value = metadata
-        .get("x-rustfs-encryption-original-size")
-        .expect("managed encryption metadata should include original size");
-    let parsed_size: usize = size_value
-        .parse()
-        .expect("x-rustfs-encryption-original-size should be numeric");
-    assert_eq!(parsed_size, expected_size, "recorded original size should match uploaded payload length");
 }
 
 fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &str, plaintext: &[u8]) {
@@ -142,10 +135,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
         "head_object should advertise SSE-S3"
     );
 
-    let metadata = head
-        .metadata()
-        .expect("head_object should return managed encryption metadata");
-    assert_encryption_metadata(metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     assert_storage_encrypted(std::path::Path::new(&kms_env.base_env.temp_dir), TEST_BUCKET, key, payload);
 
@@ -210,10 +200,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "source object should maintain the configured KMS key id"
     );
-    let source_metadata = head_source
-        .metadata()
-        .expect("source object should include managed encryption metadata");
-    assert_encryption_metadata(source_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_source.metadata());
 
     let dest_key = "metadata-sse-kms-object-copy";
     let copy_source = format!("{TEST_BUCKET}/{source_key}");
@@ -238,10 +225,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "copied object should keep the default KMS key id"
     );
-    let dest_metadata = head_dest
-        .metadata()
-        .expect("copied object should include managed encryption metadata");
-    assert_encryption_metadata(dest_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_dest.metadata());
 
     let copied_body = s3_client
         .get_object()
@@ -358,10 +342,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
         "multipart object should retain bucket default KMS key"
     );
 
-    assert_encryption_metadata(
-        head.metadata().expect("multipart head_object should expose managed metadata"),
-        combined.len(),
-    );
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     // Data returned to clients should decrypt back to original payload
     let fetched = s3_client

crates/e2e_test/src/kms/kms_comprehensive_test.rs

@@ -19,9 +19,9 @@
 //! complex workflows.
 
 use super::common::{
-    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, test_all_multipart_encryption_types,
-    test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption, test_sse_kms_encryption,
-    test_sse_s3_encryption,
+    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, sse_customer_key_md5_base64,
+    test_all_multipart_encryption_types, test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
@@ -149,8 +149,8 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     // Test different SSE-C keys to ensure isolation
     let key1 = "01234567890123456789012345678901";
     let key2 = "98765432109876543210987654321098";
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let config1 = MultipartTestConfig::new(
         "isolation-test-key1",
@@ -183,7 +183,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     info!("🔒 Verify key isolation");
     let wrong_key = "11111111111111111111111111111111";
     let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
 
     // Try to read file encrypted with key1 using wrong key
     let wrong_read_result = s3_client

crates/e2e_test/src/kms/kms_edge_cases_test.rs

@@ -21,7 +21,7 @@
 //! - Concurrent encryption operations
 //! - Security validation tests
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use aws_sdk_s3::types::ServerSideEncryption;
 use base64::Engine;
@@ -72,7 +72,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
     info!("📤 Testing SSE-C with zero-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "zero-byte-sse-c";
 
     let _put_response_c = s3_client
@@ -166,7 +166,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
     info!("📤 Testing SSE-C with single-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "single-byte-sse-c";
 
     s3_client
@@ -334,7 +334,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
     info!("🔍 Testing access to SSE-C object without key");
 
     // First upload a valid SSE-C object
-    let valid_key_md5 = format!("{:x}", compute(valid_key));
+    let valid_key_md5 = sse_customer_key_md5_base64(valid_key);
     s3_client
         .put_object()
         .bucket(TEST_BUCKET)
@@ -421,7 +421,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
                     // SSE-C
                     let key = format!("testkey{i:026}"); // 32-byte key
                     let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
-                    let key_md5 = format!("{:x}", compute(&key));
+                    let key_md5 = sse_customer_key_md5_base64(&key);
 
                     client
                         .put_object()
@@ -493,8 +493,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
 
     let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
     let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     // Upload same data with different keys
     s3_client

crates/e2e_test/src/kms/kms_local_test.rs

@@ -19,7 +19,10 @@
 //! - S3 object upload/download with SSE-S3, SSE-KMS, SSE-C encryption
 //! - Complete encryption/decryption lifecycle
 
-use super::common::{LocalKMSTestEnvironment, get_kms_status, test_kms_key_management, test_sse_c_encryption};
+use super::common::{
+    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    test_kms_key_management, test_sse_c_encryption,
+};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{error, info};
@@ -28,6 +31,9 @@ use tracing::{error, info};
 #[serial]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Local KMS End-to-End Test");
 
     // Create LocalKMS test environment
@@ -140,8 +146,8 @@ async fn test_local_kms_key_isolation() {
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Data encrypted with key 1";
     let data2 = b"Data encrypted with key 2";
@@ -338,6 +344,9 @@ async fn test_local_kms_multipart_upload() {
     test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET)
         .await
         .expect("SSE-C multipart upload test failed");
+    test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET)
+        .await
+        .expect("SSE-C multipart wrong-key download test failed");
 
     // Test 4: Large multipart upload (test streaming encryption with multiple blocks)
     // TODO: Re-enable after fixing streaming encryption issues with large files
@@ -562,7 +571,7 @@ async fn test_multipart_upload_with_sse_c(
     // SSE-C encryption key
     let encryption_key = "01234567890123456789012345678901";
     let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
-    let key_md5 = format!("{:x}", md5::compute(encryption_key));
+    let key_md5 = sse_customer_key_md5_base64(encryption_key);
 
     // Generate test data
     let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 3) % 256) as u8).collect();
@@ -642,6 +651,99 @@ async fn test_multipart_upload_with_sse_c(
     Ok(())
 }
 
+async fn test_multipart_download_with_wrong_sse_c_key_fails(
+    s3_client: &aws_sdk_s3::Client,
+    bucket: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let object_key = "multipart-sse-c-bad-download-test";
+    let part_size = 5 * 1024 * 1024;
+    let total_parts = 2;
+    let total_size = part_size * total_parts;
+
+    let encryption_key = "01234567890123456789012345678901";
+    let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
+    let key_md5 = sse_customer_key_md5_base64(encryption_key);
+
+    let wrong_key = "abcdefghijklmnopqrstuvwxyz012345";
+    let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
+
+    let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect();
+
+    let create_multipart_output = s3_client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(object_key)
+        .sse_customer_algorithm("AES256")
+        .sse_customer_key(&key_b64)
+        .sse_customer_key_md5(&key_md5)
+        .send()
+        .await?;
+
+    let upload_id = create_multipart_output.upload_id().unwrap();
+    let mut completed_parts = Vec::new();
+
+    for part_number in 1..=total_parts {
+        let start = (part_number - 1) * part_size;
+        let end = std::cmp::min(start + part_size, total_size);
+        let part_data = &test_data[start..end];
+
+        let upload_part_output = s3_client
+            .upload_part()
+            .bucket(bucket)
+            .key(object_key)
+            .upload_id(upload_id)
+            .part_number(part_number as i32)
+            .body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()))
+            .sse_customer_algorithm("AES256")
+            .sse_customer_key(&key_b64)
+            .sse_customer_key_md5(&key_md5)
+            .send()
+            .await?;
+
+        completed_parts.push(
+            aws_sdk_s3::types::CompletedPart::builder()
+                .part_number(part_number as i32)
+                .e_tag(upload_part_output.e_tag().unwrap())
+                .build(),
+        );
+    }
+
+    let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
+        .set_parts(Some(completed_parts))
+        .build();
+
+    s3_client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(object_key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_multipart_upload)
+        .send()
+        .await?;
+
+    let err = s3_client
+        .get_object()
+        .bucket(bucket)
+        .key(object_key)
+        .sse_customer_algorithm("AES256")
+        .sse_customer_key(&wrong_key_b64)
+        .sse_customer_key_md5(&wrong_key_md5)
+        .send()
+        .await
+        .expect_err("multipart SSE-C download with the wrong key should fail");
+
+    let service_err = err.into_service_error();
+    assert_eq!(
+        service_err.meta().code(),
+        Some("InvalidRequest"),
+        "wrong-key multipart SSE-C download should return InvalidRequest, got {:?}",
+        service_err.meta().code()
+    );
+
+    Ok(())
+}
+
 /// Test large multipart upload to verify streaming encryption works correctly
 #[allow(dead_code)]
 async fn test_large_multipart_upload(

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -19,14 +19,14 @@
 //! multipart upload behaviour.
 
 use crate::common::{TEST_BUCKET, init_logging};
-use md5::compute;
 use serial_test::serial;
 use tokio::time::{Duration, sleep};
 use tracing::{error, info};
 
 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, start_kms, test_all_multipart_encryption_types, test_error_scenarios,
-    test_kms_key_management, test_sse_c_encryption, test_sse_kms_encryption, test_sse_s3_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 
 /// Helper that brings up Vault, configures RustFS, and starts the KMS service.
@@ -42,7 +42,7 @@ impl VaultKmsTestContext {
         env.setup_vault_transit().await?;
 
         env.start_rustfs_for_vault().await?;
-        env.configure_vault_kms().await?;
+        env.configure_vault_transit_kms().await?;
 
         start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
 
@@ -65,6 +65,9 @@ impl VaultKmsTestContext {
 #[serial]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
 
     let context = VaultKmsTestContext::new().await?;
@@ -118,6 +121,9 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
+        return Ok(());
+    }
     info!("Starting Vault KMS SSE-C key isolation test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -133,8 +139,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Vault data encrypted with key 1";
     let data2 = b"Vault data encrypted with key 2";
@@ -202,6 +208,9 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[serial]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
+        return Ok(());
+    }
     info!("Starting Vault KMS large file SSE-S3 test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -264,6 +273,9 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
+        return Ok(());
+    }
     info!("Starting Vault KMS multipart upload encryption suite");
 
     let context = VaultKmsTestContext::new().await?;
@@ -292,6 +304,9 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[serial]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
+        return Ok(());
+    }
     info!("Starting Vault KMS key operations test (CRUD)");
 
     let context = VaultKmsTestContext::new().await?;

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -21,7 +21,7 @@
 //! 3. Test the saving and reading of encrypted metadata
 //! 4. Test the complete sharded upload encryption process
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{debug, info};
@@ -504,7 +504,7 @@ async fn test_multipart_encryption_type(
     let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
         let key = "01234567890123456789012345678901";
         let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
-        let key_md5 = format!("{:x}", md5::compute(key));
+        let key_md5 = sse_customer_key_md5_base64(key);
         (Some(key_b64), Some(key_md5))
     } else {
         (None, None)

crates/e2e_test/src/lib.rs

@@ -40,6 +40,10 @@ mod quota_test;
 #[cfg(test)]
 mod bucket_policy_check_test;
 
+/// IAM / bucket / STS session policy with `s3:ExistingObjectTag` conditions (E2E).
+#[cfg(test)]
+mod existing_object_tag_policy_test;
+
 // Regression tests for Issue #2036: anonymous access with PublicAccessBlock
 #[cfg(test)]
 mod anonymous_access_test;
@@ -52,6 +56,9 @@ mod special_chars_test;
 #[cfg(test)]
 mod content_encoding_test;
 
+#[cfg(test)]
+mod archive_download_integrity_test;
+
 // ListObjectsV2 pagination test (Issue #1596)
 #[cfg(test)]
 mod list_objects_v2_pagination_test;
@@ -66,6 +73,19 @@ mod compression_test;
 // Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
 #[cfg(test)]
 mod delete_objects_versioning_test;
+
+// Regression test for Issue #2252: ListObjectVersions misses newest version after put -> delete -> put
+#[cfg(test)]
+mod list_object_versions_regression_test;
+
+// versions&metadata=true extension regression test
+#[cfg(test)]
+mod list_object_versions_metadata_extension_test;
+
+// list-type=2&metadata=true extension regression test
+#[cfg(test)]
+mod list_objects_v2_metadata_extension_test;
+
 #[cfg(test)]
 mod protocols;
 
@@ -80,6 +100,32 @@ mod cluster_concurrency_test;
 #[cfg(test)]
 mod checksum_upload_test;
 
+// Range request regression tests
+#[cfg(test)]
+mod range_request_test;
+
 // Group deletion tests
 #[cfg(test)]
 mod group_delete_test;
+
+// S3 dummy-compat bucket API tests
+#[cfg(test)]
+mod bucket_logging_test;
+
+// Multipart control API auth regression tests
+#[cfg(test)]
+mod multipart_auth_test;
+
+#[cfg(test)]
+mod stale_multipart_cleanup_cluster_test;
+
+// Object lambda end-to-end regression tests
+#[cfg(test)]
+mod object_lambda_test;
+
+// Replication extension end-to-end regression tests
+#[cfg(test)]
+mod replication_extension_test;
+
+#[cfg(test)]
+mod snowball_auto_extract_test;

crates/e2e_test/src/list_object_versions_metadata_extension_test.rs

@@ -0,0 +1,122 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end regression test for the versions metadata extension:
+//! `GET /{bucket}?versions&metadata=true`
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use http::header::HOST;
+    use reqwest::StatusCode;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
+    use serial_test::serial;
+    use std::error::Error;
+    use tracing::info;
+
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_metadata_extension_returns_metadata_tags_and_internal()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        info!("🧪 TEST: versions&metadata=true returns versions metadata extensions");
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "test-list-versions-metadata-ext";
+        let key = "versions/metadata-object.txt";
+
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .metadata("project", "alpha")
+            .metadata("owner", "ops")
+            .tagging("env=test&project=alpha")
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"metadata extension body"))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}?versions&metadata=true&prefix={}", env.url, bucket, urlencoding::encode(key));
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(response.status(), StatusCode::OK, "versions&metadata=true should succeed");
+
+        let body = response.text().await?;
+        info!("versions&metadata=true response body: {}", body);
+
+        assert!(body.contains("<ListVersionsResult"), "expected ListVersionsResult XML, got: {body}");
+        assert!(body.contains("<Version>"), "expected at least one Version entry, got: {body}");
+        assert!(body.contains("<UserMetadata>"), "expected UserMetadata extension, got: {body}");
+        assert!(
+            body.contains("<project>alpha</project>"),
+            "expected stripped user metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<owner>ops</owner>"),
+            "expected second metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<UserTags>env=test&amp;project=alpha</UserTags>"),
+            "expected UserTags extension in response, got: {body}"
+        );
+        assert!(body.contains("<Internal>"), "expected Internal extension, got: {body}");
+        assert!(body.contains("<K>1</K>"), "expected Internal/K in response, got: {body}");
+        assert!(body.contains("<M>0</M>"), "expected Internal/M in response, got: {body}");
+
+        Ok(())
+    }
+}

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -0,0 +1,182 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for Issue #2252:
+//! "ListObjectVersions misses the newest version after create -> delete -> create."
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_immediately_returns_latest_put_after_delete_marker() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns the newest version immediately after put -> delete -> put");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-object-versions-2252";
+        let key = "test-prefix/test-object.txt";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+
+        let first_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"first version"))
+            .send()
+            .await
+            .expect("Failed to put first object version");
+        let first_version_id = first_put
+            .version_id()
+            .map(str::to_string)
+            .expect("First put should return a version_id");
+
+        let delete_resp = client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("Failed to create delete marker");
+        let delete_marker_version_id = delete_resp
+            .version_id()
+            .map(str::to_string)
+            .expect("DeleteObject should return a delete marker version_id");
+
+        let second_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"second version"))
+            .send()
+            .await
+            .expect("Failed to put second object version");
+        let second_version_id = second_put
+            .version_id()
+            .map(str::to_string)
+            .expect("Second put should return a version_id");
+
+        let first_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions immediately after second put");
+        let second_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions a second time");
+
+        let first_versions = first_listing.versions().to_vec();
+        let first_delete_markers = first_listing.delete_markers().to_vec();
+        let second_versions = second_listing.versions().to_vec();
+        let second_delete_markers = second_listing.delete_markers().to_vec();
+
+        info!(
+            "First listing: {} versions, {} delete markers",
+            first_versions.len(),
+            first_delete_markers.len()
+        );
+        info!(
+            "Second listing: {} versions, {} delete markers",
+            second_versions.len(),
+            second_delete_markers.len()
+        );
+
+        assert_eq!(
+            first_versions.len(),
+            2,
+            "First ListObjectVersions call should return both object versions immediately (regression #2252)"
+        );
+        assert_eq!(
+            first_delete_markers.len(),
+            1,
+            "First ListObjectVersions call should return the delete marker immediately (regression #2252)"
+        );
+        assert_eq!(
+            second_versions.len(),
+            2,
+            "Second ListObjectVersions call should still return both object versions"
+        );
+        assert_eq!(
+            second_delete_markers.len(),
+            1,
+            "Second ListObjectVersions call should still return the delete marker"
+        );
+
+        let first_latest_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(second_version_id.as_str()))
+            .expect("First listing should include the newest object version");
+        assert_eq!(
+            first_latest_version.is_latest(),
+            Some(true),
+            "Newest object version should be latest on the first listing"
+        );
+
+        let first_original_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(first_version_id.as_str()))
+            .expect("First listing should include the original object version");
+        assert_eq!(
+            first_original_version.is_latest(),
+            Some(false),
+            "Original object version should no longer be latest"
+        );
+
+        let first_delete_marker = first_delete_markers
+            .iter()
+            .find(|marker| marker.version_id() == Some(delete_marker_version_id.as_str()))
+            .expect("First listing should include the delete marker");
+        assert_eq!(
+            first_delete_marker.is_latest(),
+            Some(false),
+            "Delete marker should no longer be latest after the second put"
+        );
+    }
+}

crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs

@@ -0,0 +1,110 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end regression test for the objects metadata extension:
+//! `GET /{bucket}?list-type=2&metadata=true`
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+    use http::header::HOST;
+    use reqwest::StatusCode;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
+    use serial_test::serial;
+    use std::error::Error;
+    use tracing::info;
+
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_metadata_extension_returns_metadata_tags_and_internal()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        info!("🧪 TEST: list-type=2&metadata=true returns object metadata extensions");
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "test-list-objects-v2-metadata-ext";
+        let key = "objects/metadata-object.txt";
+
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .metadata("project", "alpha")
+            .metadata("owner", "ops")
+            .tagging("env=test&project=alpha")
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"metadata extension body"))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}?list-type=2&metadata=true&prefix={}", env.url, bucket, urlencoding::encode(key));
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(response.status(), StatusCode::OK, "list-type=2&metadata=true should succeed");
+
+        let body = response.text().await?;
+        info!("list-type=2&metadata=true response body: {}", body);
+
+        assert!(body.contains("<ListBucketResult"), "expected ListBucketResult XML, got: {body}");
+        assert!(body.contains("<Contents>"), "expected at least one Contents entry, got: {body}");
+        assert!(body.contains("<UserMetadata>"), "expected UserMetadata extension, got: {body}");
+        assert!(
+            body.contains("<project>alpha</project>"),
+            "expected stripped user metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<owner>ops</owner>"),
+            "expected second metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<UserTags>env=test&amp;project=alpha</UserTags>"),
+            "expected UserTags extension in response, got: {body}"
+        );
+        assert!(body.contains("<Internal>"), "expected Internal extension, got: {body}");
+        assert!(body.contains("<K>1</K>"), "expected Internal/K in response, got: {body}");
+        assert!(body.contains("<M>0</M>"), "expected Internal/M in response, got: {body}");
+
+        Ok(())
+    }
+}

crates/e2e_test/src/object_lock/object_lock_test.rs

@@ -25,8 +25,11 @@
 //! - Default bucket retention is applied to new objects
 
 use super::common::*;
+use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{Delete, ObjectIdentifier, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
+use aws_sdk_s3::types::{
+    CompletedMultipartUpload, CompletedPart, Delete, ObjectIdentifier, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
+};
 use serial_test::serial;
 use tracing::info;
 
@@ -37,6 +40,45 @@ fn init_logging() {
         .try_init();
 }
 
+async fn put_bucket_deny_policy(
+    client: &Client,
+    bucket: &str,
+    sid: &str,
+    action: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Sid": sid,
+            "Effect": "Deny",
+            "Principal": "*",
+            "Action": action,
+            "Resource": format!("arn:aws:s3:::{}/*", bucket)
+        }]
+    })
+    .to_string();
+
+    client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
+    Ok(())
+}
+
+fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
+    let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
+    aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
+        .expect("retention timestamp should parse")
+}
+
+fn assert_access_denied<T, E: std::fmt::Debug>(result: Result<T, E>, context: &str) {
+    let err = match result {
+        Ok(_) => panic!("{context}"),
+        Err(err) => format!("{err:?}"),
+    };
+    assert!(
+        err.contains("AccessDenied") || err.to_lowercase().contains("access denied"),
+        "{context}: expected AccessDenied, got: {err}"
+    );
+}
+
 // ============================================================================
 // DeleteObject Tests
 // ============================================================================
@@ -149,6 +191,57 @@ async fn test_delete_object_allowed_by_governance_with_bypass() {
     info!("✅ Test passed: GOVERNANCE retention allows deletion with bypass");
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_object_creates_delete_marker_for_retained_current_version() {
+    init_logging();
+    info!("🧪 Test: DeleteObject creates delete marker for retained current version");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-retention-delete-marker";
+    let key = "retained-object";
+    let data = b"test data for retained current version";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+
+    let retain_until = future_retain_until(30);
+    let retained_version_id =
+        put_object_with_retention(&client, bucket, key, data, ObjectLockRetentionMode::Governance, retain_until)
+            .await
+            .unwrap();
+
+    let delete_marker_output = client.delete_object().bucket(bucket).key(key).send().await.unwrap();
+    assert_eq!(delete_marker_output.delete_marker(), Some(true));
+
+    let delete_marker_version_id = delete_marker_output
+        .version_id()
+        .expect("delete marker should have a version id")
+        .to_string();
+
+    let protected_delete = delete_object_with_bypass(&client, bucket, key, Some(&retained_version_id), false).await;
+    assert!(protected_delete.is_err(), "Retained version should still reject direct deletion");
+
+    delete_object_with_bypass(&client, bucket, key, Some(&delete_marker_version_id), false)
+        .await
+        .unwrap();
+
+    let still_protected = delete_object_with_bypass(&client, bucket, key, Some(&retained_version_id), false).await;
+    assert!(
+        still_protected.is_err(),
+        "Retained version should remain protected after delete marker removal"
+    );
+
+    delete_object_with_bypass(&client, bucket, key, Some(&retained_version_id), true)
+        .await
+        .unwrap();
+
+    info!("✅ Test passed: Delete marker is allowed while retained version stays protected");
+}
+
 #[tokio::test]
 #[serial]
 async fn test_delete_object_blocked_by_legal_hold() {
@@ -182,6 +275,42 @@ async fn test_delete_object_blocked_by_legal_hold() {
     info!("✅ Test passed: Legal Hold blocks deletion");
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_object_allowed_with_legal_hold_off() {
+    init_logging();
+    info!("🧪 Test: DeleteObject allowed with Legal Hold OFF");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-legal-hold-off-delete";
+    let key = "legal-hold-off-object";
+    let data = b"test data for legal hold off";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+
+    let version_id = put_object_with_legal_hold(&client, bucket, key, data, ObjectLockLegalHoldStatus::Off)
+        .await
+        .unwrap();
+
+    let delete_result = delete_object_with_bypass(&client, bucket, key, Some(&version_id), false).await;
+    assert!(delete_result.is_ok(), "Delete should succeed when legal hold is OFF");
+
+    let head_result = client
+        .head_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await;
+    assert!(head_result.is_err(), "Object should be deleted when legal hold is OFF");
+
+    info!("✅ Test passed: Legal Hold OFF allows deletion");
+}
+
 #[tokio::test]
 #[serial]
 async fn test_delete_object_after_legal_hold_removed() {
@@ -216,6 +345,737 @@ async fn test_delete_object_after_legal_hold_removed() {
     info!("✅ Test passed: Deletion succeeds after Legal Hold removal");
 }
 
+#[tokio::test]
+#[serial]
+async fn test_get_object_legal_hold_returns_updated_status() {
+    init_logging();
+    info!("🧪 Test: GetObjectLegalHold returns updated status");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-get-legal-hold";
+    let key = "legal-hold-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let version_id = put_object_with_legal_hold(&client, bucket, key, b"test data", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let on_hold = client
+        .get_object_legal_hold()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(
+        on_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("ON")
+    );
+
+    put_object_legal_hold(&client, bucket, key, Some(&version_id), ObjectLockLegalHoldStatus::Off)
+        .await
+        .unwrap();
+
+    let off_hold = client
+        .get_object_legal_hold()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(
+        off_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("OFF")
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_retention_returns_configured_values() {
+    init_logging();
+    info!("🧪 Test: GetObjectRetention returns configured values");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-get-retention";
+    let key = "retained-object";
+    let retain_until = future_retain_until(30);
+    let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let version_id =
+        put_object_with_retention(&client, bucket, key, b"test data", ObjectLockRetentionMode::Governance, retain_until)
+            .await
+            .unwrap();
+
+    let retention = client
+        .get_object_retention()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await
+        .unwrap();
+    let retention = retention.retention().expect("retention should be present");
+
+    assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
+    assert_eq!(
+        retention
+            .retain_until_date()
+            .expect("retain_until_date should be present")
+            .fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)
+            .unwrap(),
+        retain_until_expected
+    );
+}
+
+// ============================================================================
+// Put/Copy/Multipart Legal Hold Tests
+// ============================================================================
+
+#[tokio::test]
+#[serial]
+async fn test_put_object_overwrite_blocked_by_legal_hold() {
+    init_logging();
+    info!("🧪 Test: PutObject overwrite blocked by Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-put-overwrite-legal-hold";
+    let key = "overwrite-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+
+    put_object_with_legal_hold(&client, bucket, key, b"locked-body", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let overwrite_result = client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(b"replacement-body".to_vec()))
+        .send()
+        .await;
+
+    assert!(overwrite_result.is_err(), "PutObject overwrite should fail while legal hold is ON");
+
+    let error_str = format!("{:?}", overwrite_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("legal") || error_str.to_lowercase().contains("hold"),
+        "overwrite error should mention legal hold, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_copy_object_applies_requested_legal_hold() {
+    init_logging();
+    info!("🧪 Test: CopyObject applies requested Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-copy-object-legal-hold";
+    let src_key = "src-object";
+    let dst_key = "dst-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    client
+        .copy_object()
+        .copy_source(format!("{bucket}/{src_key}"))
+        .bucket(bucket)
+        .key(dst_key)
+        .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+        .send()
+        .await
+        .unwrap();
+
+    let legal_hold = client
+        .get_object_legal_hold()
+        .bucket(bucket)
+        .key(dst_key)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(
+        legal_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("ON")
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_copy_object_overwrite_blocked_by_legal_hold() {
+    init_logging();
+    info!("🧪 Test: CopyObject overwrite blocked by Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-copy-overwrite-legal-hold";
+    let src_key = "src-object";
+    let dst_key = "dst-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_object_with_legal_hold(&client, bucket, dst_key, b"locked-destination", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let copy_result = client
+        .copy_object()
+        .copy_source(format!("{bucket}/{src_key}"))
+        .bucket(bucket)
+        .key(dst_key)
+        .send()
+        .await;
+
+    assert!(
+        copy_result.is_err(),
+        "CopyObject overwrite should fail while destination legal hold is ON"
+    );
+
+    let error_str = format!("{:?}", copy_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("legal") || error_str.to_lowercase().contains("hold"),
+        "copy overwrite error should mention legal hold, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_create_multipart_upload_applies_requested_legal_hold() {
+    init_logging();
+    info!("🧪 Test: CreateMultipartUpload applies requested Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-legal-hold";
+    let key = "multipart-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+        .send()
+        .await
+        .unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await
+        .unwrap();
+
+    let legal_hold = client.get_object_legal_hold().bucket(bucket).key(key).send().await.unwrap();
+
+    assert_eq!(
+        legal_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("ON")
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_create_multipart_upload_blocked_by_compliance_retention() {
+    init_logging();
+    info!("🧪 Test: CreateMultipartUpload blocked by COMPLIANCE retention");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-create-compliance";
+    let key = "protected-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    put_object_with_retention(
+        &client,
+        bucket,
+        key,
+        b"locked-destination",
+        ObjectLockRetentionMode::Compliance,
+        future_retain_until(30),
+    )
+    .await
+    .unwrap();
+
+    let create_result = client.create_multipart_upload().bucket(bucket).key(key).send().await;
+
+    assert!(
+        create_result.is_err(),
+        "CreateMultipartUpload should fail while destination is under active COMPLIANCE retention"
+    );
+
+    let error_str = format!("{:?}", create_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("retention") || error_str.to_lowercase().contains("compliance"),
+        "multipart create error should mention retention, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_delete_completed_multipart_object_blocked_by_legal_hold() {
+    init_logging();
+    info!("🧪 Test: Delete completed multipart object blocked by Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-delete-legal-hold";
+    let key = "multipart-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+        .send()
+        .await
+        .unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_output = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await
+        .unwrap();
+
+    let version_id = complete_output.version_id().expect("multipart object should be versioned");
+    let delete_result = delete_object_with_bypass(&client, bucket, key, Some(version_id), false).await;
+    assert!(delete_result.is_err(), "Delete should fail for multipart object protected by legal hold");
+}
+
+#[tokio::test]
+#[serial]
+async fn test_delete_completed_multipart_object_blocked_by_retention() {
+    init_logging();
+    info!("🧪 Test: Delete completed multipart object blocked by retention");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-delete-retention";
+    let key = "multipart-object";
+    let retain_until = retention_timestamp(30);
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Compliance)
+        .object_lock_retain_until_date(retain_until)
+        .send()
+        .await
+        .unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_output = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await
+        .unwrap();
+
+    let version_id = complete_output.version_id().expect("multipart object should be versioned");
+    let delete_result = delete_object_with_bypass(&client, bucket, key, Some(version_id), false).await;
+    assert!(delete_result.is_err(), "Delete should fail for multipart object protected by retention");
+}
+
+#[tokio::test]
+#[serial]
+async fn test_complete_multipart_upload_blocked_when_legal_hold_added_after_create() {
+    init_logging();
+    info!("🧪 Test: CompleteMultipartUpload blocked when Legal Hold appears after MPU creation");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-complete-multipart-legal-hold";
+    let key = "multipart-race-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_object_with_legal_hold(&client, bucket, key, b"locked-current-version", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_result = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await;
+
+    assert!(complete_result.is_err(), "CompleteMultipartUpload should fail once legal hold is enabled");
+
+    let error_str = format!("{:?}", complete_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("legal") || error_str.to_lowercase().contains("hold"),
+        "complete error should mention legal hold, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_complete_multipart_upload_blocked_when_compliance_retention_added_after_create() {
+    init_logging();
+    info!("🧪 Test: CompleteMultipartUpload blocked when COMPLIANCE retention appears after MPU creation");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-complete-multipart-compliance";
+    let key = "multipart-race-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_object_with_retention(
+        &client,
+        bucket,
+        key,
+        b"locked-current-version",
+        ObjectLockRetentionMode::Compliance,
+        future_retain_until(30),
+    )
+    .await
+    .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_result = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await;
+
+    assert!(
+        complete_result.is_err(),
+        "CompleteMultipartUpload should fail once COMPLIANCE retention is enabled"
+    );
+
+    let error_str = format!("{:?}", complete_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("retention") || error_str.to_lowercase().contains("compliance"),
+        "complete error should mention retention, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_write_paths_require_put_object_legal_hold_permission() {
+    init_logging();
+    info!("🧪 Test: write paths require PutObjectLegalHold permission");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-legal-hold-permissions";
+    let src_key = "src-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_bucket_deny_policy(&client, bucket, "DenyPutObjectLegalHold", "s3:PutObjectLegalHold")
+        .await
+        .unwrap();
+
+    assert_access_denied(
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("put-target")
+            .body(ByteStream::from(b"put-body".to_vec()))
+            .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+            .send()
+            .await,
+        "PutObject with legal hold should require s3:PutObjectLegalHold",
+    );
+
+    assert_access_denied(
+        client
+            .copy_object()
+            .copy_source(format!("{bucket}/{src_key}"))
+            .bucket(bucket)
+            .key("copy-target")
+            .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+            .send()
+            .await,
+        "CopyObject with legal hold should require s3:PutObjectLegalHold",
+    );
+
+    assert_access_denied(
+        client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key("multipart-target")
+            .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+            .send()
+            .await,
+        "CreateMultipartUpload with legal hold should require s3:PutObjectLegalHold",
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_write_paths_require_put_object_retention_permission() {
+    init_logging();
+    info!("🧪 Test: write paths require PutObjectRetention permission");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-retention-permissions";
+    let src_key = "src-object";
+    let retain_until = retention_timestamp(30);
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_bucket_deny_policy(&client, bucket, "DenyPutObjectRetention", "s3:PutObjectRetention")
+        .await
+        .unwrap();
+
+    assert_access_denied(
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("put-target")
+            .body(ByteStream::from(b"put-body".to_vec()))
+            .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
+            .object_lock_retain_until_date(retain_until)
+            .send()
+            .await,
+        "PutObject with retention should require s3:PutObjectRetention",
+    );
+
+    assert_access_denied(
+        client
+            .copy_object()
+            .copy_source(format!("{bucket}/{src_key}"))
+            .bucket(bucket)
+            .key("copy-target")
+            .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
+            .object_lock_retain_until_date(retain_until)
+            .send()
+            .await,
+        "CopyObject with retention should require s3:PutObjectRetention",
+    );
+
+    assert_access_denied(
+        client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key("multipart-target")
+            .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
+            .object_lock_retain_until_date(retain_until)
+            .send()
+            .await,
+        "CreateMultipartUpload with retention should require s3:PutObjectRetention",
+    );
+}
+
 // ============================================================================
 // DeleteObjects (Batch Delete) Tests
 // ============================================================================

crates/e2e_test/src/policy/policy_variables_test.rs

@@ -14,7 +14,7 @@
 
 //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
 
-use crate::common::{awscurl_put, init_logging};
+use crate::common::{awscurl_delete, awscurl_put, init_logging};
 use crate::policy::test_env::PolicyTestEnvironment;
 use aws_sdk_s3::primitives::ByteStream;
 use serial_test::serial;
@@ -113,11 +113,11 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
 
     // Remove user
     let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, username);
-    let _ = awscurl_put(&remove_user_url, "", &env.access_key, &env.secret_key).await;
+    let _ = awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await;
 
     // Remove policy
     let remove_policy_url = format!("{}/rustfs/admin/v3/remove-canned-policy?name={}", env.url, policy_name);
-    let _ = awscurl_put(&remove_policy_url, "", &env.access_key, &env.secret_key).await;
+    let _ = awscurl_delete(&remove_policy_url, &env.access_key, &env.secret_key).await;
 }
 
 /// Test AWS policy variables with single-value scenarios

crates/e2e_test/src/protocols/ftps_core.rs

@@ -14,11 +14,13 @@
 
 //! Core FTPS tests
 
-use crate::common::rustfs_binary_path;
+use crate::common::rustfs_binary_path_with_features;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use rcgen::generate_simple_self_signed;
-use rustls::{ClientConfig, RootCertStore, pki_types::CertificateDer, pki_types::pem::PemObject};
+use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
+use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
+use rustls::{ClientConfig, DigitallySignedStruct, Error as RustlsError, SignatureScheme};
 use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -31,6 +33,46 @@ use tracing::info;
 const FTPS_PORT: u16 = 9021;
 const FTPS_ADDRESS: &str = "127.0.0.1:9021";
 
+#[derive(Debug)]
+struct AcceptAnyServerCertVerifier;
+
+impl ServerCertVerifier for AcceptAnyServerCertVerifier {
+    fn verify_server_cert(
+        &self,
+        _end_entity: &CertificateDer<'_>,
+        _intermediates: &[CertificateDer<'_>],
+        _server_name: &ServerName<'_>,
+        _ocsp_response: &[u8],
+        _now: UnixTime,
+    ) -> Result<ServerCertVerified, RustlsError> {
+        Ok(ServerCertVerified::assertion())
+    }
+
+    fn verify_tls12_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn verify_tls13_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+        rustls::crypto::aws_lc_rs::default_provider()
+            .signature_verification_algorithms
+            .supported_schemes()
+    }
+}
+
 /// Test FTPS: put, ls, mkdir, rmdir, delete operations
 pub async fn test_ftps_core_operations() -> Result<()> {
     let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
@@ -58,7 +100,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
 
     // Start server manually
     info!("Starting FTPS server on {}", FTPS_ADDRESS);
-    let binary_path = rustfs_binary_path();
+    let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav"));
     let mut server_process = Command::new(&binary_path)
         .env("RUSTFS_FTPS_ENABLE", "true")
         .env("RUSTFS_FTPS_ADDRESS", FTPS_ADDRESS)
@@ -78,19 +120,9 @@ pub async fn test_ftps_core_operations() -> Result<()> {
             .install_default()
             .map_err(|e| anyhow::anyhow!("Failed to install crypto provider: {:?}", e))?;
 
-        // Create a simple rustls config that accepts any certificate for testing
-        let mut root_store = RootCertStore::empty();
-        // Add the self-signed certificate to the trust store for e2e
-        // Note: In a real environment, you'd use proper root certificates
-        let cert_pem = default_cert.cert.pem();
-        let cert_der = CertificateDer::pem_reader_iter(&mut Cursor::new(cert_pem))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?;
-
-        root_store.add_parsable_certificates(cert_der);
-
         let config = ClientConfig::builder()
-            .with_root_certificates(root_store)
+            .dangerous()
+            .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
             .with_no_client_auth();
 
         // Wrap in suppaftp's RustlsConnector

crates/e2e_test/src/protocols/webdav_core.rs

@@ -14,7 +14,7 @@
 
 //! Core WebDAV tests
 
-use crate::common::rustfs_binary_path;
+use crate::common::rustfs_binary_path_with_features;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use base64::Engine;
@@ -47,7 +47,7 @@ pub async fn test_webdav_core_operations() -> Result<()> {
 
     // Start server manually
     info!("Starting WebDAV server on {}", WEBDAV_ADDRESS);
-    let binary_path = rustfs_binary_path();
+    let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav"));
     let mut server_process = Command::new(&binary_path)
         .env("RUSTFS_WEBDAV_ENABLE", "true")
         .env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)

crates/e2e_test/src/quota_test.rs

@@ -17,6 +17,15 @@ use aws_sdk_s3::Client;
 use serial_test::serial;
 use tracing::{debug, info};
 
+fn skip_without_awscurl() -> bool {
+    if crate::common::awscurl_available() {
+        return false;
+    }
+
+    info!("Skipping quota test because awscurl is not available");
+    true
+}
+
 /// Test environment setup for quota tests
 pub struct QuotaTestEnv {
     pub env: RustFSTestEnvironment,
@@ -233,6 +242,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         // Create test bucket
@@ -269,6 +281,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -302,6 +317,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -336,6 +354,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -372,6 +393,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -403,6 +427,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -441,6 +468,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         // Create two buckets in the same environment
@@ -479,6 +509,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -512,6 +545,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -575,6 +611,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -628,6 +667,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -671,6 +713,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -727,6 +772,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;

crates/e2e_test/src/range_request_test.rs

@@ -0,0 +1,80 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end regression test for invalid GET object ranges.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::error::SdkError;
+    use aws_sdk_s3::primitives::ByteStream;
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        match client.create_bucket().bucket(bucket).send().await {
+            Ok(_) => Ok(()),
+            Err(err) => {
+                if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") {
+                    Ok(())
+                } else {
+                    Err(Box::new(err))
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() {
+        init_logging();
+        info!("TEST: GetObject invalid range should return InvalidRange/416");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-invalid-range";
+        let key = "range.txt";
+        let content = b"testcontent";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PutObject should succeed");
+
+        let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await;
+
+        let err = result.expect_err("GetObject with an unsatisfiable range should fail");
+        match err {
+            SdkError::ServiceError(service_err) => {
+                assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416");
+
+                let s3_err = service_err.into_err();
+                assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange");
+            }
+            other_err => panic!("Expected S3 service error, got: {other_err:?}"),
+        }
+    }
+}

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -21,7 +21,7 @@ use rustfs_lock::{
     LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockMetadata, LockPriority},
 };
-use rustfs_protos::proto_gen::node_service::{GenerallyLockRequest, PingRequest};
+use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
 use tonic::Request;
 use tracing::{info, warn};
 
@@ -61,6 +61,45 @@ impl GrpcLockClient {
             metadata: LockMetadata::default(),
             priority: LockPriority::Normal,
             deadlock_detection: false,
+            suppress_contention_logs: false,
+        }
+    }
+
+    fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
+        if let Some(lock_info_json) = lock_info_json {
+            match serde_json::from_str::<LockInfo>(&lock_info_json) {
+                Ok(info) => info,
+                Err(e) => {
+                    warn!("Failed to deserialize lock_info from response: {}, using request data", e);
+                    LockInfo {
+                        id: request.lock_id.clone(),
+                        resource: request.resource.clone(),
+                        lock_type: request.lock_type,
+                        status: LockStatus::Acquired,
+                        owner: request.owner.clone(),
+                        acquired_at: std::time::SystemTime::now(),
+                        expires_at: std::time::SystemTime::now() + request.ttl,
+                        last_refreshed: std::time::SystemTime::now(),
+                        metadata: request.metadata.clone(),
+                        priority: request.priority,
+                        wait_start_time: None,
+                    }
+                }
+            }
+        } else {
+            LockInfo {
+                id: request.lock_id.clone(),
+                resource: request.resource.clone(),
+                lock_type: request.lock_type,
+                status: LockStatus::Acquired,
+                owner: request.owner.clone(),
+                acquired_at: std::time::SystemTime::now(),
+                expires_at: std::time::SystemTime::now() + request.ttl,
+                last_refreshed: std::time::SystemTime::now(),
+                metadata: request.metadata.clone(),
+                priority: request.priority,
+                wait_start_time: None,
+            }
         }
     }
 }
@@ -88,46 +127,10 @@ impl LockClient for GrpcLockClient {
 
         // Check if the lock acquisition was successful
         if resp.success {
-            // Try to deserialize lock_info from response
-            let lock_info = if let Some(lock_info_json) = resp.lock_info {
-                match serde_json::from_str::<LockInfo>(&lock_info_json) {
-                    Ok(info) => info,
-                    Err(e) => {
-                        // If deserialization fails, fall back to constructing from request
-                        warn!("Failed to deserialize lock_info from response: {}, using request data", e);
-                        LockInfo {
-                            id: request.lock_id.clone(),
-                            resource: request.resource.clone(),
-                            lock_type: request.lock_type,
-                            status: LockStatus::Acquired,
-                            owner: request.owner.clone(),
-                            acquired_at: std::time::SystemTime::now(),
-                            expires_at: std::time::SystemTime::now() + request.ttl,
-                            last_refreshed: std::time::SystemTime::now(),
-                            metadata: request.metadata.clone(),
-                            priority: request.priority,
-                            wait_start_time: None,
-                        }
-                    }
-                }
-            } else {
-                // If lock_info is not provided, construct from request
-                LockInfo {
-                    id: request.lock_id.clone(),
-                    resource: request.resource.clone(),
-                    lock_type: request.lock_type,
-                    status: LockStatus::Acquired,
-                    owner: request.owner.clone(),
-                    acquired_at: std::time::SystemTime::now(),
-                    expires_at: std::time::SystemTime::now() + request.ttl,
-                    last_refreshed: std::time::SystemTime::now(),
-                    metadata: request.metadata.clone(),
-                    priority: request.priority,
-                    wait_start_time: None,
-                }
-            };
-
-            Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
+            Ok(LockResponse::success(
+                Self::build_lock_info(request, resp.lock_info),
+                std::time::Duration::ZERO,
+            ))
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
@@ -137,6 +140,45 @@ impl LockClient for GrpcLockClient {
         }
     }
 
+    async fn acquire_locks_batch(&self, requests: &[LockRequest]) -> Result<Vec<LockResponse>> {
+        let mut client = self.get_client().await?;
+        let req = Request::new(BatchGenerallyLockRequest {
+            args: requests
+                .iter()
+                .map(|request| {
+                    serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
+                })
+                .collect::<Result<Vec<_>>>()?,
+        });
+
+        let resp = client
+            .lock_batch(req)
+            .await
+            .map_err(|e| LockError::internal(e.to_string()))?
+            .into_inner();
+
+        Ok(requests
+            .iter()
+            .enumerate()
+            .map(|(idx, request)| match resp.results.get(idx) {
+                Some(result) if result.success => {
+                    LockResponse::success(Self::build_lock_info(request, result.lock_info.clone()), std::time::Duration::ZERO)
+                }
+                Some(result) => LockResponse::failure(
+                    result
+                        .error_info
+                        .clone()
+                        .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
+                    std::time::Duration::ZERO,
+                ),
+                None => LockResponse::failure(
+                    format!("Lock batch response missing entry for request index {idx}"),
+                    std::time::Duration::ZERO,
+                ),
+            })
+            .collect())
+    }
+
     async fn release(&self, lock_id: &LockId) -> Result<bool> {
         info!("grpc release for {}", lock_id);
 
@@ -160,6 +202,31 @@ impl LockClient for GrpcLockClient {
         Ok(resp.success)
     }
 
+    async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
+        let mut client = self.get_client().await?;
+        let req = Request::new(BatchGenerallyLockRequest {
+            args: lock_ids
+                .iter()
+                .map(|lock_id| {
+                    serde_json::to_string(&Self::create_unlock_request(lock_id))
+                        .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
+                })
+                .collect::<Result<Vec<_>>>()?,
+        });
+
+        let resp = client
+            .un_lock_batch(req)
+            .await
+            .map_err(|e| LockError::internal(e.to_string()))?
+            .into_inner();
+
+        Ok(lock_ids
+            .iter()
+            .enumerate()
+            .map(|(idx, _)| resp.results.get(idx).map(|result| result.success).unwrap_or(false))
+            .collect())
+    }
+
     async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
         info!("grpc refresh for {}", lock_id);
         let refresh_request = Self::create_unlock_request(lock_id);

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -21,7 +21,8 @@ use rustfs_lock::{LockClient, LockRequest};
 use rustfs_protos::{
     models::PingBodyBuilder,
     proto_gen::node_service::{
-        GenerallyLockRequest, GenerallyLockResponse, PingRequest, PingResponse, node_service_server::NodeService,
+        BatchGenerallyLockRequest, BatchGenerallyLockResponse, GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult,
+        PingRequest, PingResponse, node_service_server::NodeService,
     },
 };
 use std::pin::Pin;
@@ -33,6 +34,22 @@ use tracing::debug;
 
 type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send>>;
 
+fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLockResult {
+    GenerallyLockResult {
+        success: response.success,
+        error_info: response.error,
+        lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
+    }
+}
+
+fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
+    GenerallyLockResult {
+        success: false,
+        error_info: Some(error.into()),
+        lock_info: None,
+    }
+}
+
 /// Minimal NodeService implementation that only supports Lock RPCs
 /// Used for testing distributed lock scenarios with real gRPC
 #[derive(Debug)]
@@ -187,6 +204,92 @@ impl NodeService for MinimalLockNodeService {
         }
     }
 
+    async fn lock_batch(
+        &self,
+        request: Request<BatchGenerallyLockRequest>,
+    ) -> Result<Response<BatchGenerallyLockResponse>, Status> {
+        let request = request.into_inner();
+        let mut results = vec![lock_result_from_error("request was not processed"); request.args.len()];
+        let mut valid_requests = Vec::with_capacity(request.args.len());
+        let mut valid_indices = Vec::with_capacity(request.args.len());
+
+        for (idx, arg) in request.args.iter().enumerate() {
+            match serde_json::from_str::<LockRequest>(arg) {
+                Ok(args) => {
+                    valid_requests.push(args);
+                    valid_indices.push(idx);
+                }
+                Err(err) => {
+                    results[idx] = lock_result_from_error(format!("can not decode args, err: {err}"));
+                }
+            }
+        }
+
+        if !valid_requests.is_empty() {
+            match self.lock_client.acquire_locks_batch(&valid_requests).await {
+                Ok(batch_results) => {
+                    for (result_idx, response) in batch_results.into_iter().enumerate() {
+                        if let Some(request_idx) = valid_indices.get(result_idx) {
+                            results[*request_idx] = lock_result_from_response(response);
+                        }
+                    }
+                }
+                Err(err) => {
+                    for request_idx in valid_indices {
+                        results[request_idx] = lock_result_from_error(format!("can not batch lock, err: {err}"));
+                    }
+                }
+            }
+        }
+
+        Ok(Response::new(BatchGenerallyLockResponse { results }))
+    }
+
+    async fn un_lock_batch(
+        &self,
+        request: Request<BatchGenerallyLockRequest>,
+    ) -> Result<Response<BatchGenerallyLockResponse>, Status> {
+        let request = request.into_inner();
+        let mut results = vec![lock_result_from_error("request was not processed"); request.args.len()];
+        let mut lock_ids = Vec::with_capacity(request.args.len());
+        let mut valid_indices = Vec::with_capacity(request.args.len());
+
+        for (idx, arg) in request.args.iter().enumerate() {
+            match serde_json::from_str::<LockRequest>(arg) {
+                Ok(args) => {
+                    lock_ids.push(args.lock_id);
+                    valid_indices.push(idx);
+                }
+                Err(err) => {
+                    results[idx] = lock_result_from_error(format!("can not decode args, err: {err}"));
+                }
+            }
+        }
+
+        if !lock_ids.is_empty() {
+            match self.lock_client.release_locks_batch(&lock_ids).await {
+                Ok(batch_results) => {
+                    for (result_idx, success) in batch_results.into_iter().enumerate() {
+                        if let Some(request_idx) = valid_indices.get(result_idx) {
+                            results[*request_idx] = GenerallyLockResult {
+                                success,
+                                error_info: None,
+                                lock_info: None,
+                            };
+                        }
+                    }
+                }
+                Err(err) => {
+                    for request_idx in valid_indices {
+                        results[request_idx] = lock_result_from_error(format!("can not batch unlock, err: {err}"));
+                    }
+                }
+            }
+        }
+
+        Ok(Response::new(BatchGenerallyLockResponse { results }))
+    }
+
     // All other methods return unimplemented
     async fn heal_bucket(
         &self,
@@ -551,6 +654,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn get_live_events(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::GetLiveEventsRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::GetLiveEventsResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn start_profiling(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::StartProfilingRequest>,

crates/e2e_test/src/reliant/lock.rs

@@ -14,10 +14,71 @@
 // limitations under the License.
 
 use super::{grpc_lock_client::GrpcLockClient, grpc_lock_server::spawn_lock_server};
-use rustfs_lock::{GlobalLockManager, NamespaceLock, ObjectKey, client::local::LocalClient};
+use rustfs_lock::client::{LockClient, local::LocalClient};
+use rustfs_lock::{
+    GlobalLockManager, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey,
+};
 use std::sync::Arc;
 use std::time::Duration;
 
+fn test_resource() -> ObjectKey {
+    ObjectKey {
+        bucket: Arc::from("test-bucket"),
+        object: Arc::from("test-object"),
+        version: None,
+    }
+}
+
+#[derive(Debug, Default)]
+struct FailingClient;
+
+#[async_trait::async_trait]
+impl rustfs_lock::LockClient for FailingClient {
+    async fn acquire_lock(&self, _request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<LockResponse> {
+        Err(LockError::internal("simulated gRPC node failure"))
+    }
+
+    async fn release(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+        Ok(false)
+    }
+
+    async fn refresh(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+        Ok(false)
+    }
+
+    async fn force_release(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+        Ok(false)
+    }
+
+    async fn check_status(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<Option<LockInfo>> {
+        Ok(None)
+    }
+
+    async fn get_stats(&self) -> rustfs_lock::Result<LockStats> {
+        Ok(LockStats::default())
+    }
+
+    async fn close(&self) -> rustfs_lock::Result<()> {
+        Ok(())
+    }
+
+    async fn is_online(&self) -> bool {
+        false
+    }
+
+    async fn is_local(&self) -> bool {
+        false
+    }
+}
+
+async fn failing_grpc_client() -> (Arc<dyn rustfs_lock::LockClient>, tokio::task::JoinHandle<()>) {
+    let failing_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(FailingClient);
+    let (addr, handle) = spawn_lock_server(failing_client)
+        .await
+        .expect("Failed to spawn failing gRPC lock server");
+    (Arc::new(GrpcLockClient::new(addr)), handle)
+}
+
 #[tokio::test]
 async fn test_distributed_lock_4_nodes_grpc() {
     // Spawn 4 gRPC lock servers, each with its own GlobalLockManager
@@ -52,11 +113,7 @@ async fn test_distributed_lock_4_nodes_grpc() {
     let lock = NamespaceLock::with_clients_and_quorum("grpc-4-node".to_string(), clients, 3);
     assert_eq!(lock.namespace(), "grpc-4-node");
 
-    let resource = ObjectKey {
-        bucket: Arc::from("test-bucket"),
-        object: Arc::from("test-object"),
-        version: None,
-    };
+    let resource = test_resource();
 
     // Test 1: Owner A acquires write lock successfully
     let mut guard_a = lock
@@ -128,3 +185,138 @@ async fn test_distributed_lock_4_nodes_grpc() {
     handle3.abort();
     handle4.abort();
 }
+
+#[tokio::test]
+async fn test_distributed_lock_2_nodes_grpc_read_survives_failed_node() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+    let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client_ok: Arc<dyn rustfs_lock::LockClient> = Arc::new(GrpcLockClient::new(addr));
+    let (grpc_client_bad, failing_handle) = failing_grpc_client().await;
+    let lock = NamespaceLock::with_clients_and_quorum("grpc-2-node".to_string(), vec![grpc_client_ok, grpc_client_bad], 2);
+    let resource = test_resource();
+
+    let guard = lock
+        .get_read_lock(resource.clone(), "owner-a", Duration::from_secs(2))
+        .await
+        .expect("Read lock should succeed with one healthy node in a two-node gRPC cluster");
+
+    match guard {
+        rustfs_lock::NamespaceLockGuard::Standard(_) => {}
+        rustfs_lock::NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
+    }
+
+    let err = lock
+        .get_write_lock(resource, "owner-a", Duration::from_secs(2))
+        .await
+        .expect_err("Write lock should fail with one healthy node in a two-node gRPC cluster");
+
+    let err_str = err.to_string().to_lowercase();
+    assert!(
+        err_str.contains("quorum") || err_str.contains("not reached"),
+        "Error should be quorum related, got: {}",
+        err
+    );
+
+    handle.abort();
+    failing_handle.abort();
+}
+
+#[tokio::test]
+async fn test_grpc_lock_client_batch_acquire_and_release() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+    let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client = GrpcLockClient::new(addr);
+    let requests = vec![
+        LockRequest::new(test_resource(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(2)),
+        LockRequest::new(
+            ObjectKey {
+                bucket: Arc::from("test-bucket"),
+                object: Arc::from("test-object-2"),
+                version: None,
+            },
+            LockType::Exclusive,
+            "owner-a",
+        )
+        .with_acquire_timeout(Duration::from_secs(2)),
+    ];
+
+    let responses = grpc_client
+        .acquire_locks_batch(&requests)
+        .await
+        .expect("batch acquire should succeed");
+    assert_eq!(responses.len(), requests.len());
+    assert!(responses.iter().all(|response| response.success));
+
+    let lock_ids = responses
+        .iter()
+        .map(|response| {
+            response
+                .lock_info
+                .as_ref()
+                .expect("batch response should include lock info")
+                .id
+                .clone()
+        })
+        .collect::<Vec<_>>();
+    let released = grpc_client
+        .release_locks_batch(&lock_ids)
+        .await
+        .expect("batch release should succeed");
+    assert_eq!(released, vec![true, true]);
+
+    handle.abort();
+}
+
+#[tokio::test]
+async fn test_distributed_lock_4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes() {
+    let manager1 = Arc::new(GlobalLockManager::new());
+    let manager2 = Arc::new(GlobalLockManager::new());
+    let client1: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager1));
+    let client2: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager2));
+
+    let (addr1, handle1) = spawn_lock_server(client1).await.expect("Failed to spawn server 1");
+    let (addr2, handle2) = spawn_lock_server(client2).await.expect("Failed to spawn server 2");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client1: Arc<dyn rustfs_lock::LockClient> = Arc::new(GrpcLockClient::new(addr1));
+    let grpc_client2: Arc<dyn rustfs_lock::LockClient> = Arc::new(GrpcLockClient::new(addr2));
+    let (grpc_client3, handle3) = failing_grpc_client().await;
+    let (grpc_client4, handle4) = failing_grpc_client().await;
+
+    let lock = NamespaceLock::with_clients(
+        "grpc-4-node-partial".to_string(),
+        vec![grpc_client1, grpc_client2, grpc_client3, grpc_client4],
+    );
+    let resource = test_resource();
+
+    let mut read_guard = lock
+        .get_read_lock(resource.clone(), "owner-a", Duration::from_secs(2))
+        .await
+        .expect("Read lock should succeed with two healthy nodes in a four-node gRPC cluster");
+    assert!(read_guard.release(), "Read guard should release cleanly");
+
+    let err = lock
+        .get_write_lock(resource, "owner-b", Duration::from_secs(2))
+        .await
+        .expect_err("Write lock should fail when only two of four gRPC nodes are healthy");
+
+    let err_str = err.to_string().to_lowercase();
+    assert!(
+        err_str.contains("quorum") || err_str.contains("not reached"),
+        "Error should be quorum related, got: {}",
+        err
+    );
+
+    handle1.abort();
+    handle2.abort();
+    handle3.abort();
+    handle4.abort();
+}

crates/e2e_test/src/snowball_auto_extract_test.rs

@@ -0,0 +1,321 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::error::ProvideErrorMetadata;
+    use aws_sdk_s3::primitives::ByteStream;
+    use serial_test::serial;
+    use std::error::Error;
+    use std::io::Cursor;
+
+    async fn build_test_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+
+        for dir in ["dir/", "empty-dir/"] {
+            let mut header = tokio_tar::Header::new_gnu();
+            header.set_entry_type(tokio_tar::EntryType::Directory);
+            header.set_size(0);
+            header.set_mode(0o755);
+            header.set_cksum();
+            builder.append_data(&mut header, dir, Cursor::new(Vec::new())).await?;
+        }
+
+        for (path, data) in [
+            ("dir/file.txt", b"nested payload\n".as_slice()),
+            ("root.txt", b"root payload\n".as_slice()),
+        ] {
+            let mut header = tokio_tar::Header::new_gnu();
+            header.set_size(data.len() as u64);
+            header.set_mode(0o644);
+            header.set_cksum();
+            builder.append_data(&mut header, path, Cursor::new(data)).await?;
+        }
+
+        Ok(builder.into_inner().await?.into_inner())
+    }
+
+    async fn build_archive_with_invalid_entry() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+
+        let mut valid_header = tokio_tar::Header::new_gnu();
+        valid_header.set_size(b"valid-body".len() as u64);
+        valid_header.set_mode(0o644);
+        valid_header.set_cksum();
+        builder
+            .append_data(&mut valid_header, "valid.txt", Cursor::new(b"valid-body".as_slice()))
+            .await?;
+
+        let long_name = format!("{}.txt", "a".repeat(1100));
+        let mut invalid_header = tokio_tar::Header::new_gnu();
+        invalid_header.set_size(b"ignored-body".len() as u64);
+        invalid_header.set_mode(0o644);
+        invalid_header.set_cksum();
+        builder
+            .append_data(&mut invalid_header, long_name, Cursor::new(b"ignored-body".as_slice()))
+            .await?;
+
+        Ok(builder.into_inner().await?.into_inner())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-prefix-test";
+        let archive = build_test_archive().await?;
+
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Prefix", "/tenant-a/")
+            .body(ByteStream::from(archive))
+            .send()
+            .await?;
+
+        let root = client.get_object().bucket(bucket).key("tenant-a/root.txt").send().await?;
+        assert_eq!(root.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
+
+        let nested = client.get_object().bucket(bucket).key("tenant-a/dir/file.txt").send().await?;
+        assert_eq!(nested.body.collect().await?.into_bytes().as_ref(), b"nested payload\n");
+
+        let dir_marker = client.head_object().bucket(bucket).key("tenant-a/empty-dir/").send().await?;
+        assert_eq!(dir_marker.content_length(), Some(0));
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-standard-options";
+        let extracted_prefix = "/tenant-standard/";
+
+        client.create_bucket().bucket(bucket).send().await?;
+
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+
+        let mut dir_header = tokio_tar::Header::new_gnu();
+        dir_header.set_entry_type(tokio_tar::EntryType::Directory);
+        dir_header.set_size(0);
+        dir_header.set_mode(0o755);
+        dir_header.set_cksum();
+        builder
+            .append_data(&mut dir_header, "ignored-dir/", Cursor::new(Vec::new()))
+            .await?;
+
+        let mut valid_header = tokio_tar::Header::new_gnu();
+        valid_header.set_size(b"standard-body".len() as u64);
+        valid_header.set_mode(0o644);
+        valid_header.set_cksum();
+        builder
+            .append_data(&mut valid_header, "valid.txt", Cursor::new(b"standard-body".as_slice()))
+            .await?;
+
+        let long_name = format!("{}.txt", "a".repeat(1100));
+        let mut invalid_header = tokio_tar::Header::new_gnu();
+        invalid_header.set_size(b"ignored-body".len() as u64);
+        invalid_header.set_mode(0o644);
+        invalid_header.set_cksum();
+        builder
+            .append_data(&mut invalid_header, long_name, Cursor::new(b"ignored-body".as_slice()))
+            .await?;
+
+        let archive = builder.into_inner().await?.into_inner();
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .body(ByteStream::from(archive))
+            .customize()
+            .mutate_request(move |req| {
+                req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+                req.headers_mut().insert("x-amz-meta-snowball-prefix", extracted_prefix);
+                req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true");
+                req.headers_mut().insert("x-amz-meta-snowball-ignore-errors", "true");
+            })
+            .send()
+            .await?;
+
+        let valid = client
+            .get_object()
+            .bucket(bucket)
+            .key("tenant-standard/valid.txt")
+            .send()
+            .await?;
+        assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"standard-body");
+
+        let dir_err = client
+            .head_object()
+            .bucket(bucket)
+            .key("tenant-standard/ignored-dir/")
+            .send()
+            .await
+            .expect_err("directory marker should be skipped when standard ignore-dirs=true");
+        let dir_service_err = dir_err.into_service_error();
+        assert_eq!(dir_service_err.code(), Some("NotFound"));
+
+        let listed = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix("tenant-standard/")
+            .send()
+            .await?;
+        let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
+        assert_eq!(keys, vec!["tenant-standard/valid.txt"]);
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_ignores_directories_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-ignore-dirs-default";
+        let archive = build_test_archive().await?;
+
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Prefix", "tenant-b")
+            .metadata("Minio-Snowball-Ignore-Dirs", "true")
+            .body(ByteStream::from(archive))
+            .send()
+            .await?;
+
+        let err = client
+            .head_object()
+            .bucket(bucket)
+            .key("tenant-b/empty-dir/")
+            .send()
+            .await
+            .expect_err("directory marker should be skipped when ignore-dirs=true");
+        let service_err = err.into_service_error();
+        assert_eq!(service_err.code(), Some("NotFound"));
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-ignore-errors";
+        let archive = build_archive_with_invalid_entry().await?;
+
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Prefix", "tenant-c")
+            .metadata("Minio-Snowball-Ignore-Errors", "true")
+            .body(ByteStream::from(archive))
+            .send()
+            .await?;
+
+        let valid = client.get_object().bucket(bucket).key("tenant-c/valid.txt").send().await?;
+        assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
+
+        let listed = client.list_objects_v2().bucket(bucket).prefix("tenant-c/").send().await?;
+        let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
+        assert_eq!(keys, vec!["tenant-c/valid.txt"]);
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn snowball_auto_extract_prefers_exact_minio_prefix_over_suffix_fallback() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "snowball-prefix-precedence";
+        let archive = build_test_archive().await?;
+
+        client.create_bucket().bucket(bucket).send().await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .body(ByteStream::from(archive))
+            .customize()
+            .mutate_request(|req| {
+                req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+                req.headers_mut()
+                    .insert("x-amz-meta-acme-snowball-prefix", "/tenant-fallback/");
+                req.headers_mut().insert("x-amz-meta-minio-snowball-prefix", "/tenant-exact/");
+            })
+            .send()
+            .await?;
+
+        let exact = client.get_object().bucket(bucket).key("tenant-exact/root.txt").send().await?;
+        assert_eq!(exact.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
+
+        let fallback_err = client
+            .head_object()
+            .bucket(bucket)
+            .key("tenant-fallback/root.txt")
+            .send()
+            .await
+            .expect_err("fallback suffix header should not override exact MinIO prefix");
+        let fallback_service_err = fallback_err.into_service_error();
+        assert_eq!(fallback_service_err.code(), Some("NotFound"));
+
+        env.stop_server();
+        Ok(())
+    }
+}

crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs

@@ -0,0 +1,155 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{RustFSTestClusterEnvironment, init_logging};
+use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::CompletedMultipartUpload;
+use serial_test::serial;
+use tokio::time::{Duration, sleep};
+use tracing::info;
+use uuid::Uuid;
+
+const CLEANUP_BUCKET: &str = "stale-multipart-cleanup-cluster";
+
+async fn list_parts_reports_missing_upload(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    upload_id: &str,
+) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+    let result = client.list_parts().bucket(bucket).key(key).upload_id(upload_id).send().await;
+    match result {
+        Ok(_) => Ok(false),
+        Err(SdkError::ServiceError(err)) => {
+            let code = err.err().meta().code().unwrap_or("");
+            if code == "NoSuchUpload" {
+                Ok(true)
+            } else {
+                Err(format!("unexpected list_parts service error: code={code}, err={err:?}").into())
+            }
+        }
+        Err(err) => Err(format!("unexpected list_parts error: {err:?}").into()),
+    }
+}
+
+async fn complete_reports_missing_upload(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    upload_id: &str,
+) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+    let result = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(CompletedMultipartUpload::builder().build())
+        .send()
+        .await;
+    match result {
+        Ok(_) => Ok(false),
+        Err(SdkError::ServiceError(err)) => {
+            let code = err.err().meta().code().unwrap_or("");
+            if code == "NoSuchUpload" {
+                Ok(true)
+            } else {
+                Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
+            }
+        }
+        Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
+    }
+}
+
+async fn wait_for_cleanup_on_all_nodes(
+    clients: &[aws_sdk_s3::Client],
+    bucket: &str,
+    key: &str,
+    upload_id: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    for attempt in 0..30 {
+        let mut all_cleaned = true;
+        for (idx, client) in clients.iter().enumerate() {
+            let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
+            let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
+            if !(list_parts_missing && complete_missing) {
+                info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
+                all_cleaned = false;
+                break;
+            }
+        }
+
+        if all_cleaned {
+            return Ok(());
+        }
+
+        sleep(Duration::from_secs(1)).await;
+    }
+
+    Err("stale multipart upload was not cleaned up on all nodes within timeout".into())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.set_env("RUSTFS_API_STALE_UPLOADS_EXPIRY", "5s");
+    cluster.set_env("RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL", "1s");
+    cluster.start().await?;
+    cluster.create_test_bucket(CLEANUP_BUCKET).await?;
+
+    let clients = cluster.create_all_clients()?;
+    let key = format!("multipart/stale-{}.txt", Uuid::new_v4().simple());
+
+    let create_output = clients[0]
+        .create_multipart_upload()
+        .bucket(CLEANUP_BUCKET)
+        .key(&key)
+        .send()
+        .await?;
+    let upload_id = create_output
+        .upload_id()
+        .ok_or("create_multipart_upload response missing upload_id")?
+        .to_string();
+
+    clients[1]
+        .upload_part()
+        .bucket(CLEANUP_BUCKET)
+        .key(&key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from_static(b"stale multipart part"))
+        .send()
+        .await?;
+
+    let parts_before_cleanup = clients[2]
+        .list_parts()
+        .bucket(CLEANUP_BUCKET)
+        .key(&key)
+        .upload_id(&upload_id)
+        .send()
+        .await?;
+    assert_eq!(
+        parts_before_cleanup.parts().len(),
+        1,
+        "multipart upload should be visible before background cleanup"
+    );
+
+    wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
+
+    Ok(())
+}

crates/e2e_test/src/version_id_regression_test.rs

@@ -454,12 +454,12 @@ mod tests {
         Ok(())
     }
 
-    /// Test 7: PutObject should return "null" version_id when versioning is Suspended
+    /// Test 7: PutObject should omit version_id when versioning is Suspended
     #[tokio::test]
     #[serial]
-    async fn test_put_object_returns_null_version_id_with_suspended_versioning() {
+    async fn test_put_object_omits_version_id_with_suspended_versioning() {
         init_logging();
-        info!("🧪 TEST: PutObject returns null version_id with versioning suspended");
+        info!("🧪 TEST: PutObject omits version_id with versioning suspended");
 
         let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
         env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
@@ -489,13 +489,130 @@ mod tests {
 
         info!("📥 PutObject response - version_id: {:?}", output.version_id);
 
-        // When suspended, version_id must be "null"
+        // When suspended, version_id must be omitted
         assert_eq!(
-            output.version_id.as_deref(),
-            Some("null"),
-            "❌ FAILED: version_id should be 'null' when versioning is suspended"
+            output.version_id, None,
+            "❌ FAILED: version_id should be omitted when versioning is suspended"
         );
 
-        info!("✅ PASSED: PutObject correctly returns 'null' version_id");
+        info!("✅ PASSED: PutObject correctly omits version_id");
+    }
+
+    /// Test 8: CopyObject should omit version_id when versioning is Suspended
+    #[tokio::test]
+    #[serial]
+    async fn test_copy_object_omits_version_id_with_suspended_versioning() {
+        init_logging();
+        info!("🧪 TEST: CopyObject omits version_id with versioning suspended");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-copy-suspended-version-id";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+        suspend_versioning(&client, bucket)
+            .await
+            .expect("Failed to suspend versioning");
+
+        let source_key = "source-file.txt";
+        let dest_key = "dest-file.txt";
+        let content = b"Content to copy into suspended bucket";
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(source_key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("Failed to create source object");
+
+        let result = client
+            .copy_object()
+            .bucket(bucket)
+            .key(dest_key)
+            .copy_source(format!("{}/{}", bucket, source_key))
+            .send()
+            .await;
+
+        assert!(result.is_ok(), "CopyObject failed: {:?}", result.err());
+        let output = result.unwrap();
+
+        info!("📥 CopyObject response - version_id: {:?}", output.version_id);
+        assert_eq!(
+            output.version_id, None,
+            "❌ FAILED: version_id should be omitted when versioning is suspended"
+        );
+
+        info!("✅ PASSED: CopyObject correctly omits version_id");
+    }
+
+    /// Test 9: CompleteMultipartUpload should omit version_id when versioning is Suspended
+    #[tokio::test]
+    #[serial]
+    async fn test_multipart_upload_omits_version_id_with_suspended_versioning() {
+        init_logging();
+        info!("🧪 TEST: CompleteMultipartUpload omits version_id with versioning suspended");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-multipart-suspended-version-id";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+        suspend_versioning(&client, bucket)
+            .await
+            .expect("Failed to suspend versioning");
+
+        let key = "multipart-file.txt";
+        let content = b"Part 1 content for suspended multipart upload test";
+
+        let create_result = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("Failed to create multipart upload");
+
+        let upload_id = create_result.upload_id().expect("No upload_id returned");
+
+        let upload_part_result = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("Failed to upload part");
+
+        let etag = upload_part_result.e_tag().expect("No etag returned").to_string();
+        let completed_part = CompletedPart::builder().part_number(1).e_tag(etag).build();
+        let completed_upload = CompletedMultipartUpload::builder().parts(completed_part).build();
+
+        let result = client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await;
+
+        assert!(result.is_ok(), "CompleteMultipartUpload failed: {:?}", result.err());
+        let output = result.unwrap();
+
+        info!("📥 CompleteMultipartUpload response - version_id: {:?}", output.version_id);
+        assert_eq!(
+            output.version_id, None,
+            "❌ FAILED: version_id should be omitted when versioning is suspended"
+        );
+
+        info!("✅ PASSED: CompleteMultipartUpload correctly omits version_id");
     }
 }

crates/ecstore/Cargo.toml

@@ -70,11 +70,12 @@ reed-solomon-erasure = { workspace = true }
 reed-solomon-simd = { workspace = true }
 lazy_static.workspace = true
 rustfs-lock.workspace = true
+rustfs-io-core.workspace = true
+rustfs-io-metrics.workspace = true
 regex = { workspace = true }
 path-absolutize = { workspace = true }
 rmp.workspace = true
 rmp-serde.workspace = true
-serde_bytes.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat"] }
 base64 = { workspace = true }
 hmac = { workspace = true }
@@ -96,6 +97,8 @@ num_cpus = { workspace = true }
 rand.workspace = true
 pin-project-lite.workspace = true
 md-5.workspace = true
+memmap2 = { workspace = true }
+libc.workspace = true
 rustfs-madmin.workspace = true
 rustfs-workers.workspace = true
 reqwest = { workspace = true }
@@ -107,7 +110,6 @@ async-recursion.workspace = true
 aws-credential-types = { workspace = true }
 aws-smithy-types = { workspace = true }
 parking_lot = { workspace = true }
-moka = { workspace = true }
 base64-simd.workspace = true
 serde_urlencoded.workspace = true
 google-cloud-storage = { workspace = true }
@@ -117,6 +119,9 @@ faster-hex = { workspace = true }
 ratelimit = { workspace = true }
 aws-smithy-http-client.workspace = true
 
+# Observability and Metrics
+metrics = { workspace = true }
+
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
@@ -134,3 +139,18 @@ harness = false
 [[bench]]
 name = "comparison_benchmark"
 harness = false
+
+[[bench]]
+name = "direct_chunk_benchmark"
+harness = false
+
+[[bench]]
+name = "reconstructed_chunk_benchmark"
+harness = false
+
+[[bench]]
+name = "bitrot_chunk_benchmark"
+harness = false
+
+[lib]
+doctest = false

crates/ecstore/README.md

@@ -32,6 +32,43 @@
 
 For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
+## 📈 Benchmarks
+
+ECStore ships several Criterion benchmarks under [`crates/ecstore/benches/`](./benches/).
+
+### Direct Chunk Path
+
+Use the direct chunk benchmark to compare the current slice-forwarding path against the previous assembled-copy path:
+
+```bash
+cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark
+```
+
+To run only the end-to-end ECStore range-read benchmark:
+
+```bash
+cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark ecstore_get_object_chunks
+```
+
+To run the reconstructed multi-disk range-read benchmark:
+
+```bash
+cargo bench -p rustfs-ecstore --bench reconstructed_chunk_benchmark
+```
+
+### Saved Comparison Points
+
+Latest local measurements on this branch:
+
+- `direct_chunk_path/slice_forwarding/single_block_aligned`: about `477 ns`
+- `direct_chunk_path/assembled_copy/single_block_aligned`: about `3.25 us`
+- `direct_chunk_path/slice_forwarding/multi_block_unaligned`: about `963 ns`
+- `direct_chunk_path/assembled_copy/multi_block_unaligned`: about `7.25 us`
+- `ecstore_get_object_chunks/drain/multi_disk_range`: about `644-654 us`, throughput about `2.86-2.90 GiB/s`
+- `reconstructed_chunk_path/drain/multi_disk_missing_shard`: about `1.292-1.304 ms`, throughput about `1.43-1.45 GiB/s`
+
+These numbers are intended as branch-local reference points. Re-run the benchmark on your target machine before treating them as a regression baseline.
+
 ## 📄 License
 
 This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.