test(filemeta): flush xl meta test fixture before reopen (#2557)

Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>

.agents/skills/pr-creation-checker/SKILL.md

@@ -26,6 +26,8 @@ Use this skill before `gh pr create`, before `gh pr edit`, or when reviewing whe
 - Review the diff and summarize what changed.
 - Call out unrelated edits, generated artifacts, logs, or secrets as blockers.
 - Mark risky areas explicitly: auth, storage, config, network, migrations, breaking changes.
+- Scan the diff for newly added string literals and confirm whether they duplicate values already defined as constants/enums/typed wrappers in the same module or shared modules.
+- Treat introducing a new hardcoded literal where a project constant already exists as a likely regression risk; require either a refactor to reuse the constant or an explicit exception explanation in the PR body.
 
 3. Verify readiness requirements
 - Require `make pre-commit` before marking the PR ready.
@@ -82,6 +84,7 @@ Use this skill before `gh pr create`, before `gh pr edit`, or when reviewing whe
 - Return `BLOCKED` if required template sections are missing.
 - Return `BLOCKED` if the title/body is not in English.
 - Return `BLOCKED` if the title does not follow the repository's Conventional Commit rule.
+- Return `BLOCKED` if the diff introduces string literals that should use existing constants but did not.
 
 ## Reference
 

.agents/skills/pr-creation-checker/references/pr-readiness-checklist.md

@@ -12,3 +12,4 @@
 - Confirm non-applicable sections are filled with `N/A`.
 - Confirm the PR body does not include local absolute paths unless explicitly required.
 - Confirm multiline GitHub CLI commands use `--body-file`.
+- Confirm new hardcoded string literals were not introduced for values already represented by existing constants/enums (including protocol labels, error identifiers, headers, and metric names), or record a justified exception.

.cargo/config.toml

@@ -0,0 +1,26 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# RustFS Cargo configuration
+
+# Enable tokio_unstable cfg for dial9-tokio-telemetry support
+# This allows dial9 to hook into Tokio's internal runtime events
+[build]
+# Enable Tokio unstable features required by dial9-tokio-telemetry for runtime tracing.
+# See: https://docs.rs/tokio/latest/tokio/#unstable-features
+rustflags = ["--cfg", "tokio_unstable"]
+
+# Enable frame pointers for CPU profiling (Linux only, optional but recommended)
+# Uncomment the following line for better CPU profiling data
+# rustflags = ["--cfg", "tokio_unstable", "-C", "force-frame-pointers=yes"]

.docker/observability/docker-compose.yml

@@ -15,10 +15,11 @@
 services:
 
   # --- Tracing ---
-
   tempo:
-    image: grafana/tempo:latest
+    image: grafana/tempo:2.10.3
     container_name: tempo
+    depends_on:
+      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -37,6 +38,38 @@ services:
       timeout: 5s
       retries: 3
       start_period: 15s
+  redpanda:
+    image: redpandadata/redpanda:latest
+    ports:
+      - "9092:9092" # Kafka API for clients
+    command: >
+      redpanda start --overprovisioned
+      --mode=dev-container
+      --kafka-addr=PLAINTEXT://0.0.0.0:9092
+      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
+
+  redpanda-console:
+    image: docker.redpanda.com/redpandadata/console:latest
+    environment:
+      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
+    volumes:
+      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
+    ports:
+      - "8080:8080"
+    depends_on:
+      - redpanda
+
+  vulture:
+    image: grafana/tempo-vulture:latest
+    restart: always
+    command:
+      [
+        "-prometheus-listen-address=:8080",
+        "-tempo-query-url=http://tempo:3200",
+        "-tempo-push-url=http://tempo:4317",
+      ]
+    depends_on:
+      - tempo
 
   jaeger:
     image: jaegertracing/jaeger:latest

.docker/observability/grafana/dashboards/rustfs.json

@@ -91,7 +91,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "rustfs_process_uptime_seconds{job=~\"$job\"}",
+          "expr": "rustfs_system_process_uptime_seconds{job=~\"$job\"}",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -2473,7 +2473,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (process_cpu_usage_percent{job=~\"$job\"})",
+          "expr": "sum by (job) (rustfs_system_process_cpu_usage{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "A"
@@ -2572,7 +2572,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rustfs_process_memory_bytes{job=~\"$job\"})",
+          "expr": "sum by (job) (rustfs_system_process_resident_memory_bytes{job=~\"$job\"})",
           "legendFormat": "{{job}}",
           "range": true,
           "refId": "B"
@@ -2670,7 +2670,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"received\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
           "legendFormat": "RX - {{job}}",
           "range": true,
           "refId": "C"
@@ -2681,7 +2681,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
           "legendFormat": "TX - {{job}}",
           "range": true,
           "refId": "D"
@@ -3339,7 +3339,7 @@
                   "index": 0,
                   "text": "INACTIVE"
                 },
-                "to": 1e-9
+                "to": 1e-09
               },
               "type": "range"
             }
@@ -3905,6 +3905,879 @@
       ],
       "title": "Active File Size",
       "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 60
+      },
+      "id": 100,
+      "panels": [],
+      "title": "Performance Monitoring (S3 & Zero-Copy)",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 61
+      },
+      "id": 101,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_get_object_total{job=~\"$job\"}[5m])",
+          "legendFormat": "GetObject - {{tier}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_put_object_total{job=~\"$job\"}[5m])",
+          "legendFormat": "PutObject - {{zero_copy_enabled}}",
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Operations Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 100
+              },
+              {
+                "color": "red",
+                "value": 500
+              }
+            ]
+          },
+          "unit": "ms"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 61
+      },
+      "id": 102,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "histogram_quantile(0.95, rate(rustfs_s3_get_object_duration_ms_bucket{job=~\"$job\"}[5m]))",
+          "legendFormat": "GetObject P95",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "histogram_quantile(0.95, rate(rustfs_s3_put_object_duration_ms_bucket{job=~\"$job\"}[5m]))",
+          "legendFormat": "PutObject P95",
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Operation Latency (P95)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 69
+      },
+      "id": 103,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_get_object_size_bytes_sum{job=~\"$job\"}[5m]) / rate(rustfs_s3_get_object_size_bytes_count{job=~\"$job\"}[5m])",
+          "legendFormat": "GetObject Avg Size",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_s3_put_object_size_bytes_sum{job=~\"$job\"}[5m]) / rate(rustfs_s3_put_object_size_bytes_count{job=~\"$job\"}[5m])",
+          "legendFormat": "PutObject Avg Size",
+          "refId": "B"
+        }
+      ],
+      "title": "S3 Operation Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 69
+      },
+      "id": 104,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_zero_copy_memory_saved_bytes{job=~\"$job\"}",
+          "legendFormat": "Memory Saved ({{operation}})",
+          "refId": "A"
+        }
+      ],
+      "title": "Zero-Copy Memory Savings",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 77
+      },
+      "id": 105,
+      "options": {
+        "legend": {
+          "calcs": ["mean"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_hit_rate{job=~\"$job\",tier=\"small\"}",
+          "legendFormat": "Hit Rate (Small)",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_hit_rate{job=~\"$job\",tier=\"medium\"}",
+          "legendFormat": "Hit Rate (Medium)",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_hit_rate{job=~\"$job\",tier=\"large\"}",
+          "legendFormat": "Hit Rate (Large)",
+          "refId": "C"
+        }
+      ],
+      "title": "BytesPool Hit Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 77
+      },
+      "id": 106,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_allocated_bytes{job=~\"$job\",tier=\"small\"}",
+          "legendFormat": "Allocated (Small)",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_allocated_bytes{job=~\"$job\",tier=\"medium\"}",
+          "legendFormat": "Allocated (Medium)",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_bytes_pool_allocated_bytes{job=~\"$job\",tier=\"large\"}",
+          "legendFormat": "Allocated (Large)",
+          "refId": "C"
+        }
+      ],
+      "title": "BytesPool Allocated Memory",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 85
+      },
+      "id": 107,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_io_buffer_size_bytes{job=~\"$job\"}",
+          "legendFormat": "Buffer Size ({{storage_media}})",
+          "refId": "A"
+        }
+      ],
+      "title": "I/O Buffer Size (Adaptive)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "MB/s"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 85
+      },
+      "id": 108,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_io_bandwidth_bytes_sum{job=~\"$job\"}[5m]) / 1024 / 1024",
+          "legendFormat": "Bandwidth",
+          "refId": "A"
+        }
+      ],
+      "title": "I/O Bandwidth",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 93
+      },
+      "id": 109,
+      "options": {
+        "legend": {
+          "calcs": ["mean"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(rustfs_cache_hits_total{job=~\"$job\"}[5m]) / (rate(rustfs_cache_hits_total{job=~\"$job\"}[5m]) + rate(rustfs_cache_misses_total{job=~\"$job\"}[5m])) * 100",
+          "legendFormat": "Cache Hit Rate ({{cache}})",
+          "refId": "A"
+        }
+      ],
+      "title": "Cache Hit Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 93
+      },
+      "id": 110,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_cache_size_bytes{job=~\"$job\",cache=\"l1\"}",
+          "legendFormat": "L1 Cache Size",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rustfs_cache_size_bytes{job=~\"$job\",cache=\"l2\"}",
+          "legendFormat": "L2 Cache Size",
+          "refId": "B"
+        }
+      ],
+      "title": "Cache Size",
+      "type": "timeseries"
     }
   ],
   "preload": false,
@@ -4048,5 +4921,5 @@
   "timezone": "browser",
   "title": "RustFS",
   "uid": "rustfs-s3",
-  "version": 11
+  "version": 12
 }

.docker/observability/prometheus.yml

@@ -50,6 +50,11 @@ scrape_configs:
     static_configs:
       - targets: [ 'localhost:9090' ]
 
+  - job_name: 'vulture'
+    static_configs:
+      - targets:
+          - 'vulture:8080'
+
 otlp:
   promote_resource_attributes:
     - service.instance.id

.docker/observability/redpanda-console.yaml

@@ -0,0 +1,3 @@
+kafka:
+  brokers:
+    - redpanda:9092

.docker/observability/tempo.yaml

@@ -25,32 +25,18 @@ memberlist:
   join_members:
     - tempo:7946
 
-# Distributor configuration - receives traces and writes directly to ingesters
 distributor:
-  ingester_write_path_enabled: true
-  kafka_write_path_enabled: false
   receivers:
     otlp:
       protocols:
         grpc:
-          endpoint: "tempo:4317"
+          endpoint: "0.0.0.0:4317"
         http:
-          endpoint: "tempo:4318"
-  ring:
-    kvstore:
-      store: memberlist
-
-# Ingester configuration - consumes from Kafka and stores traces
-ingester:
-  lifecycler:
-    ring:
-      kvstore:
-        store: memberlist
-      replication_factor: 1
-    tokens_file_path: /var/tempo/tokens.json
-  trace_idle_period: 10s
-  max_block_bytes: 1_000_000
-  max_block_duration: 5m
+          endpoint: "0.0.0.0:4318"
+  #log_received_spans:
+  #    enabled: true
+  #  log_discarded_spans:
+  #    enabled: true
 
 backend_scheduler:
   provider:
@@ -67,8 +53,7 @@ backend_worker:
       store: memberlist
 
 querier:
-  frontend_worker:
-    frontend_address: tempo:3200
+  query_live_store: true
 
 metrics_generator:
   registry:
@@ -90,9 +75,9 @@ storage:
   trace:
     backend: local
     wal:
-      path: /var/tempo/wal
+      path: /var/tempo/wal # where to store the wal locally
     local:
-      path: /var/tempo/blocks
+      path: /var/tempo/blocks # where to store the traces locally
 
 overrides:
   defaults:
@@ -101,24 +86,14 @@ overrides:
       generate_native_histograms: both
 
 ingest:
-  enabled: false
-  # Disabled because using direct ingester write path
-  # If you want Kafka path, enable this and set:
-  # kafka:
-  #   brokers: [redpanda:9092]
-  #   topic: tempo-ingest
-  #   encoding: protobuf
-  #   consumer_group: tempo-ingest-consumer
+  enabled: true
+  kafka:
+    address: redpanda:9092
+    topic: tempo-ingest
 
 block_builder:
   consume_cycle_duration: 30s
 
-compactor:
-  compaction:
-    block_retention: 168h  # 7 days
-  ring:
-    kvstore:
-      store: memberlist
-
 usage_report:
   reporting_enabled: false
+

.github/AGENTS.md

@@ -25,6 +25,12 @@ Current `test-and-lint` gate includes:
 
 - `cargo nextest run --all --exclude e2e_test`
 - `cargo test --all --doc`
+- `cargo test -p rustfs get_object_chunk_fast_path`
+- `cargo test -p rustfs materialize_chunk_stream_before_commit`
+- `touch rustfs/build.rs`
+- `cargo build -p rustfs --bins --jobs 2`
+- `cargo test -p e2e_test archive_multipart_roundtrip_preserves_bytes`
+- `cargo test -p e2e_test presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path`
 - `cargo fmt --all --check`
 - `cargo clippy --all-targets --all-features -- -D warnings`
 - `./scripts/check_layer_dependencies.sh`

.github/actions/setup/action.yml

@@ -62,7 +62,7 @@ runs:
         repo-token: ${{ inputs.github-token }}
 
     - name: Install flatc
-      uses: Nugine/setup-flatc@v1
+      uses: Nugine/setup-flatc@v1.2.4
       with:
         version: "25.9.23"
 

.github/cla.yml

@@ -0,0 +1,17 @@
+enabled: true
+
+document:
+  version: v1
+  url: https://github.com/rustfs/cla/blob/main/cla/v1.md
+
+signing:
+  mode: comment
+  comment_pattern: I have read and agree to the CLA.
+
+registry:
+  type: json-repo
+  repository: rustfs/cla
+  path_prefix: signatures
+
+status:
+  check_name: CLA Check

.github/dependabot.yml

@@ -26,11 +26,19 @@ updates:
       day: "monday"
       timezone: "Asia/Shanghai"
       time: "08:00"
+    assignees:
+      - "heihutu"
+    reviewers:
+      - "houseme"
+      - "overtrue"
+      - "majinghe"
     ignore:
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"
         versions: [ "0.23.x" ]
+      - dependency-name: "ratelimit"
+        versions: [ "1.x" ]
     groups:
       s3s:
         update-types:

.github/pull_request_template.md

@@ -34,4 +34,4 @@ Pull Request Template for RustFS
 
 ---
 
-Thank you for your contribution! Please ensure your PR follows the community standards ([CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)) and sign the CLA if this is your first contribution.
+Thank you for your contribution! Please ensure your PR follows the community standards ([CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)). If this is your first contribution, review the [CLA document](https://github.com/rustfs/cla/blob/main/cla/v1.md) and sign it by commenting `I have read and agree to the CLA.` on the PR.

.github/workflows/build.yml

@@ -204,7 +204,12 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     env:
-      RUSTFLAGS: ${{ matrix.rustflags }}
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      # Always enable Tokio unstable features (required by dial9-tokio-telemetry).
+      # The RUSTFLAGS env var takes precedence over .cargo/config.toml [build] rustflags,
+      # so we must include --cfg tokio_unstable here explicitly; otherwise an empty
+      # RUSTFLAGS value would shadow the config-file flag and silently break tracing.
+      RUSTFLAGS: "--cfg tokio_unstable ${{ matrix.rustflags }}"
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.prepare-platform-matrix.outputs.matrix) }}
@@ -316,19 +321,26 @@ jobs:
               ;;
           esac
 
+          # Normalize version used for package filenames
+          PACKAGE_VERSION="${VERSION}"
+          if [[ "$PACKAGE_VERSION" == v* ]]; then
+            PACKAGE_VERSION="${PACKAGE_VERSION#v}"
+          fi
+
           # Generate package name based on build type
           if [[ -n "$VARIANT" ]]; then
             ARCH_WITH_VARIANT="${ARCH}-${VARIANT}"
           else
             ARCH_WITH_VARIANT="${ARCH}"
           fi
+          PACKAGE_BASENAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}"
 
           if [[ "$BUILD_TYPE" == "development" ]]; then
             # Development build: rustfs-${platform}-${arch}-${variant}-dev-${short_sha}.zip
             PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}-dev-${SHORT_SHA}"
           else
             # Release/Prerelease build: rustfs-${platform}-${arch}-${variant}-v${version}.zip
-            PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}-v${VERSION}"
+            PACKAGE_NAME="${PACKAGE_BASENAME}-v${PACKAGE_VERSION}"
           fi
 
           # Create zip packages for all platforms
@@ -398,7 +410,7 @@ jobs:
           if [[ "$BUILD_TYPE" == "release" ]] || [[ "$BUILD_TYPE" == "prerelease" ]]; then
             # Create latest version filename
             # Convert from rustfs-linux-x86_64-musl-v1.0.0 to rustfs-linux-x86_64-musl-latest
-            LATEST_FILE="${PACKAGE_NAME%-v*}-latest.zip"
+            LATEST_FILE="${PACKAGE_BASENAME}-latest.zip"
 
             echo "🔄 Creating latest version: ${PACKAGE_NAME}.zip -> $LATEST_FILE"
             cp "${PACKAGE_NAME}.zip" "$LATEST_FILE"
@@ -411,7 +423,7 @@ jobs:
             # Development builds (only main branch triggers development builds)
             # Create main-latest version filename
             # Convert from rustfs-linux-x86_64-dev-abc123 to rustfs-linux-x86_64-main-latest
-            MAIN_LATEST_FILE="${PACKAGE_NAME%-dev-*}-main-latest.zip"
+            MAIN_LATEST_FILE="${PACKAGE_BASENAME}-main-latest.zip"
 
             echo "🔄 Creating main-latest version: ${PACKAGE_NAME}.zip -> $MAIN_LATEST_FILE"
             cp "${PACKAGE_NAME}.zip" "$MAIN_LATEST_FILE"

.github/workflows/ci.yml

@@ -53,6 +53,8 @@ on:
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
       - ".github/workflows/performance.yml"
+  merge_group:
+    types: [checks_requested]
   schedule:
     - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
   workflow_dispatch:
@@ -106,6 +108,8 @@ jobs:
     if: needs.skip-check.outputs.should_skip != 'true'
     runs-on: ubicloud-standard-4
     timeout-minutes: 60
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -138,6 +142,8 @@ jobs:
     if: needs.skip-check.outputs.should_skip != 'true'
     runs-on: ubicloud-standard-4
     timeout-minutes: 30
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6

.github/workflows/cla.yml

@@ -0,0 +1,70 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CLA Check
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+  merge_group:
+    types: [checks_requested]
+  issue_comment:
+    types: [created, edited]
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+  checks: write
+
+jobs:
+  cla:
+    if: ${{ github.event_name != 'issue_comment' || github.event.issue.pull_request }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report CLA result for merge queue
+        if: github.event_name == 'merge_group'
+        uses: actions/github-script@v8
+        with:
+          script: |
+            await github.rest.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'CLA Check',
+              head_sha: context.sha,
+              status: 'completed',
+              conclusion: 'success',
+              output: {
+                title: 'CLA requirements satisfied for merge queue',
+                summary: 'Queued pull requests must satisfy the required CLA check before they enter the merge queue. This reports the existing CLA result on the merge-group SHA.'
+              }
+            });
+
+      - name: Create token for rustfs/cla
+        if: github.event_name != 'merge_group'
+        id: registry-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ vars.CLA_BOT_APP_ID }}
+          private-key: ${{ secrets.CLA_BOT_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: cla
+          permission-contents: write
+
+      - name: Run CLA Bot
+        if: github.event_name != 'merge_group'
+        uses: overtrue/cla-bot@v0.0.9
+        with:
+          github-token: ${{ github.token }}
+          registry-token: ${{ steps.registry-token.outputs.token }}

.github/workflows/nix-flake-update.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
-        
+
       - name: Install Nix
         uses: DeterminateSystems/determinate-nix-action@v3
 
@@ -46,10 +46,10 @@ jobs:
         id: update
         uses: DeterminateSystems/update-flake-lock@main
         with:
-          git-author-name: heihutu
-          git-author-email: heihutu@gmail.com
-          git-committer-name: heihutu
-          git-committer-email: heihutu@gmail.com
+          git-author-name: houseme
+          git-author-email: housemecn@gmail.com
+          git-committer-name: houseme
+          git-committer-email: housemecn@gmail.com
           pr-title: "chore(deps): update flake.lock"
           pr-labels: |
             dependencies

.github/workflows/performance.yml

@@ -42,6 +42,8 @@ jobs:
     name: Performance Profiling
     runs-on: ubicloud-standard-2
     timeout-minutes: 30
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -117,6 +119,8 @@ jobs:
     name: Benchmark Tests
     runs-on: ubicloud-standard-2
     timeout-minutes: 45
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6

.gitignore

@@ -20,7 +20,8 @@ deploy/data/*
 *jsonl
 .env
 .rustfs.sys
-.cargo
+.cargo/
+!.cargo/config.toml
 profile.json
 .docker/openobserve-otel/data
 *.zst
@@ -39,7 +40,6 @@ artifacts/
 *.audit
 *.snappy
 PR_DESCRIPTION.md
-IMPLEMENTATION_PLAN.md
 scripts/s3-tests/selected_tests.txt
 docs
 
@@ -48,4 +48,8 @@ result*
 *.gz
 rustfs-webdav.code-workspace
 
-.aiexclude
+.aiexclude
+*.bak
+# Local test/benchmark artifacts
+benchmarks.logs
+tmp/

.vscode/launch.json

@@ -197,19 +197,27 @@
                 // "__RUSTFS_SSE_SIMPLE_CMK": "2dfNXGHlsEflGVCxb+5DIdGEl1sIvtwX+QfmYasi5QM=",
 
                 // 2. kms local backend test key
-                "RUSTFS_KMS_ENABLE": "true",
-                "RUSTFS_KMS_BACKEND": "local",
-                "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
-                "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
-                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+                // "RUSTFS_KMS_ENABLE": "true",
+                // "RUSTFS_KMS_BACKEND": "local",
+                // "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
+                // "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
+                // "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
 
                 // 3. kms vault backend test key
                 // "RUSTFS_KMS_ENABLE": "true",
-                // "RUSTFS_KMS_BACKEND": "vault",
+                // "RUSTFS_KMS_BACKEND": "vault-kv2",
                 // "RUSTFS_KMS_VAULT_ADDRESS": "http://127.0.0.1:8200",
                 // "RUSTFS_KMS_VAULT_TOKEN": "Dev Token",
                 // "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
 
+                // 4. kms vault transit backend test key
+                "RUSTFS_KMS_ENABLE": "true",
+                "RUSTFS_KMS_BACKEND": "vault-transit",
+                "RUSTFS_KMS_VAULT_ADDRESS": "http://127.0.0.1:8200",
+                "RUSTFS_KMS_VAULT_TOKEN": "Dev Token",
+                "RUSTFS_KMS_VAULT_MOUNT_PATH": "transit",
+                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+
             },
             "sourceLanguages": [
                 "rust"

AGENTS.md

@@ -16,6 +16,22 @@ If repo-level instructions conflict, follow the nearest file and keep behavior a
 - Respond in the same language used by the requester.
 - Keep source code, comments, commit messages, and PR title/body in English.
 
+## Change Style for Existing Logic
+
+- Prefer direct, local code over extracting one-off helpers.
+- Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow.
+- Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths.
+- Do not refactor existing code only to make it easier to unit test.
+- Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths.
+- Keep code elegant, concise, and direct. Prefer minimal, readable implementations over over-engineering and excessive abstraction. Use comments to clarify non-obvious intent and invariants, not to compensate for unclear code.
+
+## Constant and String Usage
+
+- Before introducing new string literals, search for existing constants/enums that already represent the same semantic value.
+- Reuse existing constants for protocol labels, error identifiers, header keys, event names, metric names, command tags, and similar fixed tokens.
+- If a new string is truly unique, define a local constant near related logic and avoid scattering the literal across multiple sites.
+- When changing existing behavior, keep naming and format consistency by aligning with established project constants.
+
 ## Sources of Truth
 
 - Workspace layout and crate membership: `Cargo.toml` (`[workspace].members`)
@@ -26,16 +42,23 @@ If repo-level instructions conflict, follow the nearest file and keep behavior a
 Avoid duplicating long crate lists or command matrices in instruction files.
 Reference the source files above instead.
 
-## Mandatory Before Commit
+## Verification Before PR
 
-Run and pass:
+For code changes, run and pass the following before opening a PR:
 
 ```bash
 make pre-commit
 ```
 
+Before pushing code changes, make sure formatting is clean:
+
+- Run `cargo fmt --all`.
+- Run `cargo fmt --all --check` and ensure no files are modified unexpectedly.
+
 If `make` is unavailable, run the equivalent checks defined under `.config/make/`.
-Do not commit when required checks fail.
+Documentation-only or instruction-only changes are exempt from the verification commands above (including the `.config/make/` equivalents), though any installed git pre-commit hooks (for example, from `make setup-hooks`) may still run on commit unless explicitly skipped.
+After build-based verification completes, clean generated build artifacts before wrapping up to avoid unnecessary disk usage.
+Do not open a PR with code changes when the required checks fail.
 
 ## Git and PR Baseline
 
@@ -46,6 +69,11 @@ Do not commit when required checks fail.
 - Use `N/A` for non-applicable template sections.
 - Include verification commands in the PR description.
 - When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
+- After fixing code review comments or CI findings, always mark corresponding review
+  comments/threads as resolved before returning to the user.
+- In handling review comments, confirm the underlying issue before changing code.
+  If a suggested change is not appropriate for behavior or risk, reply with a
+  concise rationale instead of blindly applying it.
 
 ## Security Baseline
 

ARCHITECTURE.md

@@ -0,0 +1,377 @@
+# ARCHITECTURE.md
+
+> Last updated: 2026-04-13 · Revision: 1 (draft)
+>
+> This document describes the high-level architecture of RustFS.
+> If you want to familiarize yourself with the code base, you are in the right place!
+>
+> See also [CONTRIBUTING.md](CONTRIBUTING.md) for development workflow.
+
+## Bird's Eye View
+
+RustFS is a high-performance, S3-compatible distributed object storage system written
+in Rust. It uses erasure coding for data durability, supports multi-tenancy through
+IAM/STS, and provides a web-based admin console.
+
+A running RustFS node exposes:
+
+- **S3 API** (port 9000) — the primary data path for object CRUD
+- **Admin API** (port 9000, `/minio/` prefix) — cluster management, IAM, metrics
+- **Console** (port 9001) — web UI backed by the Admin API
+- **Inter-node RPC** (gRPC/tonic) — cluster communication for distributed mode
+
+The core data flow for a PUT request looks like:
+
+```
+HTTP request
+  → server (TLS, auth, routing, compression)
+    → app/object_usecase (validation, policy, lifecycle)
+      → storage/ecfs (erasure coding, encryption, checksums)
+        → ecstore (disk pool selection, data distribution)
+          → rio (reader pipeline: encrypt → compress → hash → write)
+            → io-core (zero-copy I/O, buffer pool, direct I/O)
+              → local disk / remote disk via RPC
+```
+
+## Code Map
+
+The repository is a Cargo workspace with a flat `crates/` layout:
+
+```
+rustfs/                      # Workspace root (virtual manifest)
+├── rustfs/                  # Main binary + library crate (75K lines)
+│   └── src/
+│       ├── main.rs          # Entry point, startup sequence
+│       ├── lib.rs           # Module tree root
+│       ├── server/          # HTTP server, TLS, routing, middleware
+│       ├── admin/           # Admin API handlers and console
+│       ├── app/             # Use-case layer (object, bucket, multipart)
+│       ├── storage/         # Storage engine interface and implementation
+│       ├── auth.rs          # S3 request authentication
+│       ├── config/          # CLI args, config parsing, workload profiles
+│       └── ...
+├── crates/                  # 39 library crates
+│   ├── ecstore/             # Erasure-coded storage engine (⚠️ 87K lines)
+│   ├── rio/                 # Reader I/O pipeline (encrypt, compress, hash)
+│   ├── io-core/             # Zero-copy I/O, scheduling, buffer pool
+│   ├── io-metrics/          # I/O metrics collection
+│   ├── common/              # Shared runtime state, globals, data usage types
+│   ├── config/              # Configuration types and parsing
+│   ├── utils/               # Pure utility functions
+│   ├── ...                  # (see "Crate Reference" below)
+│   └── e2e_test/            # End-to-end integration tests
+└── docs/                    # Design documents and analysis
+```
+
+### Main Crate Layers (`rustfs/src/`)
+
+The main crate is organized in layers, top to bottom:
+
+| Layer | Directory | Responsibility |
+|-------|-----------|----------------|
+| **Server** | `server/` | HTTP listener, TLS, CORS, compression, middleware, graceful shutdown |
+| **Admin** | `admin/` | Admin API routing, 30+ handler modules, web console |
+| **App** | `app/` | Use-case orchestration: object_usecase, bucket_usecase, multipart_usecase |
+| **Storage** | `storage/` | S3 API translation, erasure-coded FS, SSE encryption, RPC, concurrency |
+| **Auth** | `auth.rs` | S3 signature verification, credential validation |
+| **Config** | `config/` | CLI parsing, config struct, workload profiles |
+
+A request flows **downward** through the layers. No layer should reach upward
+(e.g., storage must not import from admin).
+
+### Crate Reference
+
+Crates are organized in a dependency DAG with 9 depth levels (0 = leaf, 8 = top):
+
+```
+Depth 0 — LEAF (no internal deps):
+  appauth, checksums, config, credentials, crypto, io-metrics,
+  madmin, mcp, s3-common, workers, zip
+
+Depth 1:
+  io-core (→ io-metrics)
+  policy (→ config, credentials, crypto)
+  utils (→ config)                        ⚠️ inverted: utils should be leaf
+
+Depth 2:
+  concurrency, filemeta, keystone, kms, lock, obs,
+  signer, targets, trusted-proxies
+
+Depth 3:
+  common (→ filemeta, madmin)             ⚠️ inverted: common should be leaf
+
+Depth 4:
+  object-capacity, protos, rio
+
+Depth 5 — CORE:
+  ecstore (16 internal deps, 11 dependents — the architectural heart)
+
+Depth 6:
+  audit, heal, iam, metrics, notify, s3select-api, scanner
+
+Depth 7:
+  object-io, protocols, s3select-query
+
+Depth 8 — TOP:
+  rustfs (35 internal deps — the binary, depends on almost everything)
+```
+
+#### By Domain
+
+**Core Infrastructure:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `config` | 3.3K | Configuration types and environment parsing |
+| `utils` | 8.7K | Pure utilities (paths, compression, network, retry) |
+| `common` | 4.4K | Shared runtime state, globals, data usage types, metrics |
+| `madmin` | 5.5K | Admin API request/response types |
+
+**I/O Pipeline:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `io-core` | 6.5K | Zero-copy I/O, buffer pool, direct I/O, scheduling, backpressure |
+| `io-metrics` | 4.5K | I/O operation metrics and counters |
+| `rio` | 6.9K | Composable reader chain (encrypt → compress → hash → limit) |
+| `object-io` | 2.4K | High-level object read/write using rio + ecstore |
+| `concurrency` | 1.8K | Concurrency control wrappers over io-core |
+
+**Storage Engine:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `ecstore` | 87K | ⚠️ Erasure-coded storage: disks, pools, buckets, replication, lifecycle |
+| `filemeta` | 10K | File/object metadata types and versioning |
+| `checksums` | 732 | Checksum computation |
+| `lock` | 7.1K | Distributed lock manager |
+| `heal` | 5.9K | Data healing / bitrot repair |
+| `scanner` | 5.4K | Background data usage scanner |
+| `object-capacity` | 2.5K | Capacity tracking and management |
+
+**Security & Auth:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `crypto` | 1.6K | Encryption primitives |
+| `credentials` | 713 | Credential types (access key / secret key) |
+| `signer` | 1.4K | S3 v4 request signing |
+| `iam` | 9.0K | Identity and access management |
+| `policy` | 8.8K | Policy engine (S3 bucket/IAM policies) |
+| `kms` | 8.1K | Key management service integration |
+| `keystone` | 1.9K | OpenStack Keystone auth |
+| `appauth` | 143 | Application-level auth tokens |
+
+**Protocol & API:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `protos` | 5.7K | Protobuf/gRPC definitions for inter-node RPC |
+| `protocols` | 18K | FTP/FTPS, WebDAV, Swift API support |
+| `s3-common` | 738 | Shared S3 types |
+| `s3select-api` | 1.9K | S3 Select interface |
+| `s3select-query` | 3.6K | S3 Select query engine |
+
+**Observability:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `metrics` | 8.4K | Prometheus metric collectors |
+| `io-metrics` | 4.5K | I/O-specific metrics |
+| `obs` | 5.6K | OpenTelemetry tracing and telemetry |
+| `audit` | 2.4K | Audit logging |
+
+**Events:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `notify` | 5.5K | Event notification system |
+| `targets` | 3.2K | Notification targets (Kafka, AMQP, webhook, etc.) |
+
+**Other:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `trusted-proxies` | 4.0K | Trusted proxy / IP forwarding |
+| `zip` | 986 | ZIP archive support for bulk downloads |
+| `workers` | 136 | Simple worker abstraction |
+| `mcp` | 2.0K | Model Context Protocol server (AI tooling) |
+
+## Architecture Invariants
+
+> These are rules that the codebase should follow. Some are currently violated
+> (marked with ⚠️). Documenting them here makes the violations explicit and
+> trackable.
+
+1. **Layers flow downward.** Server → Admin/App → Storage → ecstore → rio/io-core.
+   No upward imports.
+
+2. **Leaf crates have zero internal dependencies.** `config`, `credentials`, `crypto`,
+   `io-metrics`, `madmin`, `s3-common` should depend only on external crates.
+   - ⚠️ VIOLATED: `utils` depends on `config`, `common` depends on `filemeta` and `madmin`.
+
+3. **Each type has exactly one definition.** Types shared across crates must be defined
+   in one crate and re-exported or imported by others.
+   - ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
+     `BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
+
+4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
+   storage-level abstractions (objects, buckets, disks, pools).
+
+5. **The `rustfs` binary crate is the only place that wires everything together.**
+   Individual crates should be testable in isolation.
+
+6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
+   not bare `Error`).
+   - ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
+     `mcp` and `heal` use `anyhow` in library code.
+
+## Known Structural Issues
+
+> This section documents known problems in the current architecture.
+> It exists so the team can track and address them deliberately.
+
+### Critical
+
+- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
+  but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
+  types instead of importing them.
+
+- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
+  bucket management, erasure coding, replication, lifecycle, RPC, and configuration
+  — all in one crate. It should be decomposed along its existing subdirectories.
+
+### High
+
+- **Dependency inversions.** `utils → config` and `common → filemeta/madmin` break
+  the layering model. These need to be untangled.
+
+- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
+  concurrency, and rustfs/storage. Should be defined once with builder/composition.
+
+### Medium
+
+- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
+  mixed naming (bare `Error` vs descriptive names).
+
+- **Ambiguous common vs utils boundary.** Both described as "utilities and data
+  structures." Need clear ownership rules.
+
+## Cross-Cutting Concerns
+
+### Error Handling
+
+The project convention is `thiserror` for typed errors with descriptive names.
+See `AGENTS.md`: "Prefer thiserror for library-facing error types."
+
+```rust
+// GOOD
+#[derive(Debug, thiserror::Error)]
+pub enum StorageError {
+    #[error("disk not found: {0}")]
+    DiskNotFound(String),
+}
+
+// AVOID
+pub enum Error { ... }        // too generic
+anyhow::Result<T>             // in library code (OK in tests/CLI)
+```
+
+### Logging & Tracing
+
+- Use `tracing` crate (`info!`, `warn!`, `error!`, `debug!`, `trace!`)
+- Structured fields: `tracing::info!(bucket = %name, "created bucket")`
+- Spans for request-scoped context
+
+### Metrics
+
+- Prometheus-style metrics via `rustfs-metrics` crate
+- I/O-specific counters via `rustfs-io-metrics`
+- Registration happens at crate level, collection in `metrics` crate
+
+### Testing
+
+- Unit tests: `#[cfg(test)] mod tests` in the same file
+- Integration tests: inside respective crates (not top-level `tests/`)
+- E2E tests: `crates/e2e_test/` — tests against a running server
+- Run all: `make test` or `cargo nextest run`
+
+## Startup Sequence
+
+The binary (`main.rs`) boots in this order:
+
+1. Environment variable compatibility (`MINIO_*` → `RUSTFS_*`)
+2. Tokio runtime construction
+3. CLI argument parsing
+4. License, observability, TLS, trusted proxies initialization
+5. Config parsing, server address resolution
+6. Credentials, endpoints, local disks, lock client initialization
+7. Capacity management initialization
+8. HTTP server start (S3 API + optional console)
+9. ECStore initialization (erasure coding storage engine)
+10. Global config, background replication, KMS
+11. Optional: FTP/FTPS/WebDAV servers
+12. Event notifier, audit system, deadlock detector
+13. Bucket metadata, IAM, Keystone, OIDC
+14. Scanner and heal manager
+15. Metrics system, mark `FullReady`
+16. Wait for shutdown signal → graceful shutdown
+
+## Dependency Diagram (Simplified)
+
+```
+                            ┌─────────┐
+                            │  rustfs │  (binary + lib, 75K lines)
+                            │  main   │
+                            └────┬────┘
+                                 │
+                 ┌───────────────┼───────────────┐
+                 │               │               │
+            ┌────▼────┐    ┌────▼────┐    ┌─────▼─────┐
+            │ server  │    │  admin  │    │    app     │
+            │ (HTTP)  │    │(console)│    │(use-cases) │
+            └────┬────┘    └────┬────┘    └─────┬─────┘
+                 │               │               │
+                 └───────────────┼───────────────┘
+                                 │
+                          ┌──────▼──────┐
+                          │   storage   │
+                          │ (ecfs, SSE, │
+                          │  RPC, ACL)  │
+                          └──────┬──────┘
+                                 │
+              ┌──────────────────┼──────────────────┐
+              │                  │                  │
+        ┌─────▼─────┐    ┌──────▼──────┐    ┌──────▼──────┐
+        │  ecstore   │    │     rio     │    │   io-core   │
+        │ (87K,core) │    │  (readers)  │    │ (zero-copy) │
+        └─────┬──────┘    └─────────────┘    └─────────────┘
+              │
+    ┌─────┬──┼──┬─────┬──────┐
+    │     │  │  │     │      │
+ common utils config policy filemeta ...
+```
+
+## How to Navigate
+
+- **"Where does S3 PutObject go?"**
+  `server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
+  `ecstore` distributes → `rio` encrypts/compresses → `io-core` writes
+
+- **"Where are bucket policies enforced?"**
+  `app/bucket_usecase` calls into `crates/policy/`
+
+- **"Where is replication configured?"**
+  `admin/handlers/replication.rs` and `admin/handlers/site_replication.rs` for API,
+  `ecstore/src/bucket/replication/` for engine
+
+- **"Where do I add a new admin endpoint?"**
+  Add handler in `admin/handlers/`, register in `admin/router.rs`
+
+- **"Where do I add a new metric?"**
+  Define in `crates/metrics/`, register collector, expose via `/minio/v2/metrics`
+
+---
+
+*Inspired by [matklad's ARCHITECTURE.md](https://matklad.github.io/2021/02/06/ARCHITECTURE.md.html)
+and [rust-analyzer's architecture.md](https://github.com/rust-analyzer/rust-analyzer/blob/master/docs/book/src/contributing/architecture.md).*

Cargo.toml

@@ -35,10 +35,12 @@ members = [
     "crates/metrics", # Metrics collection and reporting
     "crates/notify", # Notification system for events
     "crates/obs", # Observability utilities
+    "crates/object-capacity", # Capacity scan and refresh core
     "crates/policy", # Policy management
     "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
     "crates/protos", # Protocol buffer definitions
     "crates/rio", # Rust I/O utilities and abstractions
+    "crates/concurrency", # Rust I/O utilities and abstractions
     "crates/s3-common", # Common utilities and data structures for S3 compatibility
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -48,6 +50,8 @@ members = [
     "crates/trusted-proxies", # Trusted proxies management
     "crates/utils", # Utility functions and helpers
     "crates/workers", # Worker thread pools and task scheduling
+    "crates/io-metrics", # Zero-copy metrics collection for performance analysis
+    "crates/io-core", # Zero-copy core reader and writer implementations
     "crates/zip", # ZIP file handling and compression
 ]
 resolver = "3"
@@ -69,6 +73,8 @@ unsafe_code = "deny"
 
 [workspace.lints.clippy]
 all = "warn"
+needless_collect = "warn"
+redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
@@ -79,6 +85,7 @@ rustfs-audit = { path = "crates/audit", version = "0.0.5" }
 rustfs-checksums = { path = "crates/checksums", version = "0.0.5" }
 rustfs-common = { path = "crates/common", version = "0.0.5" }
 rustfs-config = { path = "./crates/config", version = "0.0.5" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "0.0.5" }
 rustfs-credentials = { path = "crates/credentials", version = "0.0.5" }
 rustfs-crypto = { path = "crates/crypto", version = "0.0.5" }
 rustfs-ecstore = { path = "crates/ecstore", version = "0.0.5" }
@@ -91,6 +98,9 @@ rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
 rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
 rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
 rustfs-notify = { path = "crates/notify", version = "0.0.5" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
+rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
 rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
 rustfs-protos = { path = "crates/protos", version = "0.0.5" }
@@ -112,20 +122,20 @@ async-channel = "2.5.0"
 async-compression = { version = "0.4.41" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
-axum = "0.8.8"
+axum = "0.8.9"
 futures = "0.3.32"
 futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
-hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
-hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
+hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
+hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 socket2 = { version = "0.6.3", features = ["all"] }
-tokio = { version = "1.50.0", features = ["fs", "rt-multi-thread"] }
+tokio = { version = "1.52.0", features = ["fs", "rt-multi-thread", "io-uring"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
@@ -144,31 +154,29 @@ flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
 quick-xml = "0.39.2"
-rmcp = { version = "1.2.0" }
+rmcp = { version = "1.4.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_bytes = "0.11"
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
-argon2 = { version = "0.6.0-rc.7" }
+argon2 = { version = "0.6.0-rc.8" }
 blake2 = "0.11.0-rc.5"
-blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
-hmac = { version = "0.13.0-rc.5" }
+hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
-pbkdf2 = "0.13.0-rc.9"
+pbkdf2 = "0.13.0-rc.10"
 rsa = { version = "0.10.0-rc.17" }
-rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls = { version = "0.23.38", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = "1.14.0"
-sha1 = "0.11.0-rc.5"
-sha2 = "0.11.0-rc.5"
+sha1 = "0.11.0"
+sha2 = "0.11.0"
 subtle = "2.6"
 zeroize = { version = "1.8.2", features = ["derive"] }
 
@@ -180,13 +188,13 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros
 
 # Utilities and Tools
 anyhow = "1.0.102"
-arc-swap = "1.8.2"
+arc-swap = "1.9.1"
 astral-tokio-tar = "0.6.0"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.8.15" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.127.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
@@ -202,50 +210,51 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "52.3.0"
+datafusion = "53.0.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
 glob = "0.3.3"
-google-cloud-storage = "1.9.0"
-google-cloud-auth = "1.7.0"
-hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
+google-cloud-storage = "1.11.0"
+google-cloud-auth = "1.9.0"
+hashbrown = { version = "0.17.0", features = ["serde", "rayon"] }
 hex = "0.4.3"
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.183"
+libc = "0.2.185"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.10"
+local-ip-address = "0.6.11"
+memmap2 = "0.9.10"
 lz4 = "1.28.1"
-matchit = "0.9.1"
-md-5 = "0.11.0-rc.5"
+matchit = "0.9.2"
+md-5 = "0.11.0"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
-moka = { version = "0.12.14", features = ["future"] }
+moka = { version = "0.12.15", features = ["future"] }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.12.0"
-object_store = "0.12.5"
+nvml-wrapper = "0.12.1"
+object_store = "0.13.2"
 parking_lot = "0.12.5"
 path-absolutize = "3.1.1"
 path-clean = "1.0.1"
 percent-encoding = "2.3.2"
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
-rand = { version = "0.10.0", features = ["serde"] }
-ratelimit = "0.10.0"
-rayon = "1.11.0"
+rand = { version = "0.10.1", features = ["serde"] }
+ratelimit = "0.10.1"
+rayon = "1.12.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
-rumqttc = { version = "0.25.1" }
+rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
-rustc-hash = { version = "2.1.1" }
-s3s = { git = "https://github.com/rustfs/s3s", rev = "d9556e3c0036bd3f2b330966009cbaa5aebf19a3", features = ["minio"] }
+rustc-hash = { version = "2.1.2" }
+s3s = { git = "https://github.com/rustfs/s3s", rev = "0dd8fcaaa72eda68fafd49c38daea43bb8697558", features = ["minio"] }
 serial_test = "3.4.0"
 shadow-rs = { version = "1.7.1", default-features = false }
 siphasher = "1.0.2"
@@ -268,20 +277,21 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.22.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
-vaultrs = { version = "0.7.4" }
+uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "8.2.0"
+zip = "8.5.1"
 zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
+dial9-tokio-telemetry = "0.2"
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
-opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rustls"] }
+opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }

Dockerfile

@@ -72,7 +72,8 @@ LABEL name="RustFS" \
       url="https://rustfs.com" \
       license="Apache-2.0"
 
-RUN apk add --no-cache ca-certificates coreutils curl
+RUN apk update && \
+    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

README.md

@@ -100,28 +100,28 @@ To get started with RustFS, follow these steps:
 curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
 ```
 
-### 2\. Docker Quick Start (Option 2)
+### 2. Docker Quick Start (Option 2)
 
 The RustFS container runs as a non-root user `rustfs` (UID `10001`). If you run Docker with `-v` to mount a host directory, please ensure the host directory owner is set to `10001`, otherwise you will encounter permission denied errors.
 
 ```bash
- # Create data and logs directories
- mkdir -p data logs
+# Create data and logs directories
+mkdir -p data logs
 
- # Change the owner of these directories
- chown -R 10001:10001 data logs
+# Change the owner of these directories
+chown -R 10001:10001 data logs
 
- # Using latest version
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
+# Using latest version
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
- # Using specific version
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
+# Using specific version
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
 
 ```bash
- podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
+podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 ```
 
 You can also use Docker Compose. Using the `docker-compose.yml` file in the root directory:
@@ -136,7 +136,7 @@ Similarly, you can run the command with podman
 podman compose --profile observability up -d
 ```
 
-**NOTE**: We recommend reviewing the `docker-compose.yaml` file before running. It defines several services including Grafana, Prometheus, and Jaeger, which are helpful for RustFS observability. If you wish to start Redis or Nginx containers, you can specify the corresponding profiles.
+**NOTE**: We recommend reviewing the `docker-compose.yml` file before running. It defines several services including Grafana, Prometheus, and Jaeger, which are helpful for RustFS observability. If you wish to start Redis or Nginx containers, you can specify the corresponding profiles.
 
 ### 3\. Build from Source (Option 3) - Advanced Users
 
@@ -195,6 +195,19 @@ nix build
 nix run
 ```
 
+### 6\. X-CMD (Option 6)
+
+If you are an [x-cmd](https://www.x-cmd.com/install/rustfs) user:
+
+```bash
+# Run directly without installing
+x rustfs
+
+# Download the binary and install it to the global environment
+x env use rustfs
+rustfs --help
+```
+
 ---
 
 ### Accessing RustFS
@@ -206,6 +219,48 @@ nix run
 
 **NOTE**: To access the RustFS instance via `https`, please refer to the [TLS Configuration Docs](https://docs.rustfs.com/integration/tls-configured.html).
 
+### OIDC Roles Claim (Microsoft Entra ID)
+
+RustFS supports mapping an OIDC claim containing role values into the existing
+authorization pipeline. The `roles_claim` setting is **optional**: when unset or
+empty, only the `groups` claim contributes to authorization (same as older
+RustFS releases). For Microsoft Entra ID app roles, set `roles_claim=roles` so
+both console admin checks and bucket IAM policies can evaluate those roles.
+
+Example environment configuration (opt-in roles claim):
+
+```bash
+RUSTFS_IDENTITY_OPENID_ENABLE=on
+RUSTFS_IDENTITY_OPENID_CONFIG_URL="https://login.microsoftonline.com/<tenant-id>/v2.0/.well-known/openid-configuration"
+RUSTFS_IDENTITY_OPENID_CLIENT_ID="<client-id>"
+RUSTFS_IDENTITY_OPENID_CLIENT_SECRET="<client-secret>"
+RUSTFS_IDENTITY_OPENID_SCOPES="openid,profile,email"
+RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM="groups"
+RUSTFS_IDENTITY_OPENID_ROLES_CLAIM="roles"
+```
+
+Policy condition example (evaluate app roles directly with `jwt:roles`; when
+`roles_claim` is configured, RustFS also merges those values into `jwt:groups`
+for backward compatibility with older policies):
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["admin:*"],
+      "Resource": ["arn:aws:s3:::*"],
+      "Condition": {
+        "ForAnyValue:StringEquals": {
+          "jwt:roles": ["RustFS.ConsoleAdmin"]
+        }
+      }
+    }
+  ]
+}
+```
+
 ## Documentation
 
 For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).

README_ZH.md

@@ -122,7 +122,7 @@ RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用
 docker compose --profile observability up -d
 ```
 
-**注意**: 我们建议您在运行前查看 `docker-compose.yaml` 文件。该文件定义了包括 Grafana、Prometheus 和 Jaeger 在内的多个服务，有助于 RustFS 的可观测性监控。如果您还想启动 Redis 或 Nginx 容器，可以指定相应的 profile。
+**注意**: 我们建议您在运行前查看 `docker-compose.yml` 文件。该文件定义了包括 Grafana、Prometheus 和 Jaeger 在内的多个服务，有助于 RustFS 的可观测性监控。如果您还想启动 Redis 或 Nginx 容器，可以指定相应的 profile。
 
 ### 3\. 源码编译 (选项 3) - 进阶用户
 
@@ -163,11 +163,41 @@ make help-docker                      # 显示所有 Docker 相关命令
 
 请按照 [Helm Chart README](https://charts.rustfs.com) 上的说明在 Kubernetes 集群上安装 RustFS。
 
+### 5\. Nix Flake (Option 5)
+
+如果你已经 启用了 [Nix Flakes 功能](https://nixos.wiki/wiki/Flakes#Enable_flakes):
+
+```bash
+# 直接运行，无需安装
+nix run github:rustfs/rustfs
+
+# 编译二进制文件
+nix build github:rustfs/rustfs
+./result/bin/rustfs --help
+
+# 或者从本地检出的代码库运行/编译
+nix build
+nix run
+```
+
+### 6\. X-CMD (Option 6)
+
+如果你是 [x-cmd](https://www.x-cmd.com/install/rustfs) 用户：
+
+```bash
+# 直接运行，无需安装
+x rustfs
+
+# 下载二进制文件并安装到全局环境中
+x env use rustfs
+rustfs --help
+```
+
 ---
 
 ### 访问 RustFS
 
-1. **访问控制台**: 打开浏览器并访问 `http://localhost:9000` 进入 RustFS 控制台。
+1. **访问控制台**: 打开浏览器并访问 `http://localhost:9001` 进入 RustFS 控制台。
    - 默认账号/密码: `rustfsadmin` / `rustfsadmin`
 2. **创建存储桶**: 使用控制台为您​​的对象创建一个新的存储桶 (Bucket)。
 3. **上传对象**: 您可以直接通过控制台上传文件，或使用 S3 兼容的 API/客户端与您的 RustFS 实例进行交互。
@@ -198,7 +228,7 @@ make help-docker                      # 显示所有 Docker 相关命令
 - **商务合作**: [hello@rustfs.com](mailto:hello@rustfs.com)
 - **工作机会**: [jobs@rustfs.com](mailto:jobs@rustfs.com)
 - **一般讨论**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
-- **贡献指南**: [CONTRIBUTING.md](https://www.google.com/search?q=CONTRIBUTING.md)
+- **贡献指南**: [CONTRIBUTING.md](CONTRIBUTING.md)
 
 ## 贡献者
 

build-rustfs.sh

@@ -213,7 +213,7 @@ setup_rust_environment() {
     # Set up environment variables for musl targets
     if [[ "$PLATFORM" == *"musl"* ]]; then
         print_message $YELLOW "Setting up environment for musl target..."
-        export RUSTFLAGS="-C target-feature=-crt-static"
+        export RUSTFLAGS="'--cfg tokio_unstable -C target-feature=-crt-static'"
 
         # For cargo-zigbuild, set up additional environment variables
         if command -v cargo-zigbuild &> /dev/null; then
@@ -430,7 +430,7 @@ build_binary() {
         fi
     else
         # Native compilation
-        build_cmd="RUSTFLAGS=-Clink-arg=-lm cargo build"
+        build_cmd="RUSTFLAGS='--cfg tokio_unstable -Clink-arg=-lm' cargo build"
     fi
 
     if [ "$BUILD_TYPE" = "release" ]; then

crates/appauth/Cargo.toml

@@ -33,3 +33,6 @@ rand.workspace = true
 
 [lints]
 workspace = true
+
+[lib]
+doctest = false

crates/audit/Cargo.toml

@@ -44,6 +44,12 @@ tracing = { workspace = true, features = ["std", "attributes"] }
 url = { workspace = true }
 rumqttc = { workspace = true }
 
+[dev-dependencies]
+temp-env = { workspace = true }
 
 [lints]
 workspace = true
+
+[lib]
+test = false
+doctest = false

crates/audit/src/factory.rs

@@ -19,7 +19,8 @@ use rumqttc::QoS;
 use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
 use rustfs_config::{
     AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
+    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
+    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
     WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
     WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
@@ -27,7 +28,10 @@ use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
     error::TargetError,
-    target::{mqtt::MQTTArgs, webhook::WebhookArgs},
+    target::{
+        mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
+        webhook::WebhookArgs,
+    },
 };
 use std::time::Duration;
 use tracing::{debug, warn};
@@ -166,6 +170,14 @@ impl TargetFactory for MQTTTargetFactory {
                 .and_then(|v| v.parse::<u64>().ok())
                 .map(Duration::from_secs)
                 .unwrap_or_else(|| Duration::from_secs(30)),
+            tls: MQTTTlsConfig::from_values(
+                config.lookup(MQTT_TLS_POLICY).as_deref(),
+                config.lookup(MQTT_TLS_CA).as_deref(),
+                config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
+                config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
+                config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
+                config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
+            )?,
             queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
             queue_limit: config
                 .lookup(MQTT_QUEUE_LIMIT)
@@ -185,12 +197,15 @@ impl TargetFactory for MQTTTargetFactory {
         let url = Url::parse(&broker)
             .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
 
-        match url.scheme() {
-            "tcp" | "ssl" | "ws" | "wss" | "mqtt" | "mqtts" => {}
-            _ => {
-                return Err(TargetError::Configuration("Unsupported broker URL scheme".to_string()));
-            }
-        }
+        let tls = MQTTTlsConfig::from_values(
+            config.lookup(MQTT_TLS_POLICY).as_deref(),
+            config.lookup(MQTT_TLS_CA).as_deref(),
+            config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
+            config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
+            config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
+            config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
+        )?;
+        validate_mqtt_broker_url(&url, &tls)?;
 
         if config.lookup(MQTT_TOPIC).is_none() {
             return Err(TargetError::Configuration("Missing MQTT topic".to_string()));

crates/audit/src/registry.rs

@@ -109,9 +109,6 @@ impl AuditRegistry {
         let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
         // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // let final_config = config.clone(); // Clone a configuration for aggregating the final result
-        // Record the defaults for each segment so that the segment can eventually be rebuilt
-        let mut section_defaults: HashMap<String, KVS> = HashMap::new();
         // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
@@ -125,9 +122,6 @@ impl AuditRegistry {
             let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
             debug!(?default_cfg, "Get the default configuration");
 
-            // Save defaults for eventual write back
-            section_defaults.insert(section_name.clone(), default_cfg.clone());
-
             // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
             debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -235,97 +229,28 @@ impl AuditRegistry {
                 if enabled {
                     info!(instance_id = %id, "Target is enabled, ready to create a task");
                     // 5.3. Create asynchronous tasks for enabled instances
-                    let target_type_clone = target_type.clone();
                     let tid = id.clone();
                     let merged_config_arc = Arc::new(merged_config);
                     tasks.push(async move {
                         let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (target_type_clone, tid, result, Arc::clone(&merged_config_arc))
+                        (tid, result)
                     });
                 } else {
-                    info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
-                    // Remove disabled target from final configuration
-                    // final_config.0.entry(section_name.clone()).or_default().remove(&id);
+                    info!(instance_id = %id, "Skip disabled target");
                 }
             }
         }
 
         // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
-        let mut successful_configs = Vec::new();
-        while let Some((target_type, id, result, final_config)) = tasks.next().await {
+        while let Some((id, result)) = tasks.next().await {
             match result {
                 Ok(target) => {
-                    info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
+                    info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully");
                     successful_targets.push(target);
-                    successful_configs.push((target_type, id, final_config));
                 }
                 Err(e) => {
-                    error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
-                }
-            }
-        }
-
-        // 7. Aggregate new configuration and write back to system configuration
-        if !successful_configs.is_empty() || !section_defaults.is_empty() {
-            info!(
-                "Prepare to update {} successfully created target configurations to the system configuration...",
-                successful_configs.len()
-            );
-
-            let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
-
-            for (target_type, id, kvs) in successful_configs {
-                let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
-                successes_by_section
-                    .entry(section_name)
-                    .or_default()
-                    .insert(id.to_lowercase(), (*kvs).clone());
-            }
-
-            let mut new_config = config.clone();
-            // Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
-            let mut sections: HashSet<String> = HashSet::new();
-            sections.extend(section_defaults.keys().cloned());
-            sections.extend(successes_by_section.keys().cloned());
-
-            for section in sections {
-                let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
-                // Add default item
-                if let Some(default_kvs) = section_defaults.get(&section)
-                    && !default_kvs.is_empty()
-                {
-                    section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
-                }
-
-                // Add successful instance item
-                if let Some(instances) = successes_by_section.get(&section) {
-                    for (id, kvs) in instances {
-                        section_map.insert(id.clone(), kvs.clone());
-                    }
-                }
-
-                // Empty breaks are removed and non-empty breaks are replaced entirely.
-                if section_map.is_empty() {
-                    new_config.0.remove(&section);
-                } else {
-                    new_config.0.insert(section, section_map);
-                }
-            }
-
-            let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
-                return Err(AuditError::StorageNotAvailable(
-                    "Failed to save target configuration: server storage not initialized".to_string(),
-                ));
-            };
-
-            match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
-                Ok(_) => {
-                    info!("The new configuration was saved to the system successfully.")
-                }
-                Err(e) => {
-                    error!("Failed to save the new configuration: {}", e);
-                    return Err(AuditError::SaveConfig(Box::new(e)));
+                    error!(instance_id = %id, error = %e, "Failed to create a target");
                 }
             }
         }
@@ -365,6 +290,11 @@ impl AuditRegistry {
         self.targets.get(id).map(|t| t.as_ref())
     }
 
+    /// Lists cloned target values for runtime inspection without exposing mutable registry access.
+    pub fn list_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
+        self.targets.values().map(|target| target.clone_dyn()).collect()
+    }
+
     /// Lists all target IDs
     ///
     /// # Returns

crates/audit/src/system.rs

@@ -13,14 +13,15 @@
 //  limitations under the License.
 
 use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
+use hashbrown::HashMap;
 use rustfs_ecstore::config::Config;
 use rustfs_targets::{
     StoreError, Target, TargetError,
     store::{Key, Store},
-    target::EntityTarget,
+    target::{EntityTarget, QueuedPayload},
 };
 use std::sync::Arc;
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::{Mutex, RwLock, mpsc};
 use tracing::{error, info, warn};
 
 /// State of the audit system
@@ -39,6 +40,8 @@ pub struct AuditSystem {
     registry: Arc<Mutex<AuditRegistry>>,
     state: Arc<RwLock<AuditSystemState>>,
     config: Arc<RwLock<Option<Config>>>,
+    /// Cancellation senders for active audit stream tasks (target_id -> cancel tx)
+    stream_cancellers: Arc<RwLock<HashMap<String, mpsc::Sender<()>>>>,
 }
 
 impl Default for AuditSystem {
@@ -54,6 +57,7 @@ impl AuditSystem {
             registry: Arc::new(Mutex::new(AuditRegistry::new())),
             state: Arc::new(RwLock::new(AuditSystemState::Stopped)),
             config: Arc::new(RwLock::new(None)),
+            stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
         }
     }
 
@@ -110,27 +114,7 @@ impl AuditSystem {
 
                 // Initialize all targets
                 for target in targets {
-                    let target_id = target.id().to_string();
-                    if let Err(e) = target.init().await {
-                        error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
-                    } else {
-                        // After successful initialization, if enabled and there is a store, start the send from storage task
-                        if target.is_enabled() {
-                            if let Some(store) = target.store() {
-                                info!(target_id = %target_id, "Start audit stream processing for target");
-                                let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
-                                    store.boxed_clone();
-                                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
-                                self.start_audit_stream_with_batching(store_clone, target_arc);
-                                info!(target_id = %target_id, "Audit stream processing started");
-                            } else {
-                                info!(target_id = %target_id, "No store configured, skip audit stream processing");
-                            }
-                        } else {
-                            info!(target_id = %target_id, "Target disabled, skip audit stream processing");
-                        }
-                        registry.add_target(target_id, target);
-                    }
+                    self.init_and_register_target(target, &mut registry).await;
                 }
 
                 // Update state to running
@@ -214,6 +198,9 @@ impl AuditSystem {
 
         info!("Stopping audit system");
 
+        // Stop all stream tasks first
+        self.stop_all_streams().await;
+
         // Close all targets
         let mut registry = self.registry.lock().await;
         if let Err(e) = registry.close_all().await {
@@ -258,52 +245,49 @@ impl AuditSystem {
         let state = self.state.read().await;
 
         match *state {
-            AuditSystemState::Running => {
-                // Continue with dispatch
-                info!("Dispatching audit log entry");
-            }
+            AuditSystemState::Running => {}
             AuditSystemState::Paused => {
-                // Skip dispatch when paused
                 return Ok(());
             }
             _ => {
-                // Don't dispatch when not running
                 return Err(AuditError::NotInitialized("Audit system is not running".to_string()));
             }
         }
         drop(state);
 
-        let registry = self.registry.lock().await;
-        let target_keys = registry.list_targets();
+        // Collect cloned targets under lock, then dispatch without holding it
+        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
+            let registry = self.registry.lock().await;
+            let target_keys = registry.list_targets();
 
-        if target_keys.is_empty() {
-            warn!("No audit targets configured for dispatch");
-            return Ok(());
-        }
+            if target_keys.is_empty() {
+                warn!("No audit targets configured for dispatch");
+                return Ok(());
+            }
 
-        // Dispatch to all targets concurrently
+            target_keys
+                .into_iter()
+                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
+                .collect()
+        };
+
+        // Dispatch to all targets concurrently (no lock held)
         let mut tasks = Vec::new();
 
-        for target_key in target_keys {
-            if let Some(target) = registry.get_target(&target_key) {
-                let entry_clone = Arc::clone(&entry);
-                let target_key_clone = target_key.clone();
+        for (target_key, target) in targets {
+            let entity_target = EntityTarget {
+                object_name: entry.api.name.clone().unwrap_or_default(),
+                bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                event_name: entry.event,
+                data: (*entry).clone(),
+            };
 
-                // Create EntityTarget for the audit log entry
-                let entity_target = EntityTarget {
-                    object_name: entry.api.name.clone().unwrap_or_default(),
-                    bucket_name: entry.api.bucket.clone().unwrap_or_default(),
-                    event_name: entry.event, // Default, should be derived from entry
-                    data: (*entry_clone).clone(),
-                };
+            let task = async move {
+                let result = target.save(Arc::new(entity_target)).await;
+                (target_key, result)
+            };
 
-                let task = async move {
-                    let result = target.save(Arc::new(entity_target)).await;
-                    (target_key_clone, result)
-                };
-
-                tasks.push(task);
-            }
+            tasks.push(task);
         }
 
         // Execute all dispatch tasks
@@ -359,39 +343,45 @@ impl AuditSystem {
         }
         drop(state);
 
-        let registry = self.registry.lock().await;
-        let target_keys = registry.list_targets();
+        // Collect targets under lock, then dispatch without holding it
+        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
+            let registry = self.registry.lock().await;
+            let target_keys = registry.list_targets();
 
-        if target_keys.is_empty() {
-            warn!("No audit targets configured for batch dispatch");
-            return Ok(());
-        }
+            if target_keys.is_empty() {
+                warn!("No audit targets configured for batch dispatch");
+                return Ok(());
+            }
+
+            target_keys
+                .into_iter()
+                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
+                .collect()
+        };
 
         let mut tasks = Vec::new();
-        for target_key in target_keys {
-            if let Some(target) = registry.get_target(&target_key) {
-                let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
-                let target_key_clone = target_key.clone();
+        for (target_key, target) in targets {
+            let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
+            let target_key_clone = target_key.clone();
 
-                let task = async move {
-                    let mut success_count = 0;
-                    let mut errors = Vec::new();
-                    for entry in entries_clone {
-                        let entity_target = EntityTarget {
-                            object_name: entry.api.name.clone().unwrap_or_default(),
-                            bucket_name: entry.api.bucket.clone().unwrap_or_default(),
-                            event_name: entry.event,
-                            data: (*entry).clone(),
-                        };
-                        match target.save(Arc::new(entity_target)).await {
-                            Ok(_) => success_count += 1,
-                            Err(e) => errors.push(e),
-                        }
+            let task = async move {
+                let mut success_count = 0;
+                let mut errors = Vec::new();
+                for entry in entries_clone {
+                    let entity_target = EntityTarget {
+                        object_name: entry.api.name.clone().unwrap_or_default(),
+                        bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                        event_name: entry.event,
+                        data: (*entry).clone(),
+                    };
+                    match target.save(Arc::new(entity_target)).await {
+                        Ok(_) => success_count += 1,
+                        Err(e) => errors.push(e),
                     }
-                    (target_key_clone, success_count, errors)
-                };
-                tasks.push(task);
-            }
+                }
+                (target_key_clone, success_count, errors)
+            };
+            tasks.push(task);
         }
 
         let results = futures::future::join_all(tasks).await;
@@ -417,6 +407,60 @@ impl AuditSystem {
         Ok(())
     }
 
+    /// Stops all active audit stream tasks by sending cancellation signals.
+    async fn stop_all_streams(&self) {
+        let mut cancellers = self.stream_cancellers.write().await;
+        for (target_id, cancel_tx) in cancellers.drain() {
+            info!(target_id = %target_id, "Stopping audit stream");
+            let _ = cancel_tx.send(()).await;
+        }
+    }
+
+    /// Initializes a single target: runs init(), starts stream if store is present,
+    /// and adds it to the registry. For store-backed targets, registration and stream
+    /// startup proceed even if init() fails so queued entries can be drained later.
+    async fn init_and_register_target(
+        &self,
+        target: Box<dyn Target<AuditEntry> + Send + Sync>,
+        registry: &mut AuditRegistry,
+    ) -> Option<String> {
+        let target_id = target.id().to_string();
+        let has_store = target.store().is_some();
+
+        if let Err(e) = target.init().await {
+            error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
+            // Non-store targets: init failure is fatal.
+            if !has_store {
+                return None;
+            }
+            // Store-backed targets: still register and start the stream so queued
+            // entries can be drained when connectivity recovers.
+            warn!(
+                target_id = %target_id,
+                "Proceeding with store-backed audit target despite init failure"
+            );
+        }
+
+        if target.is_enabled() {
+            if let Some(store) = target.store() {
+                info!(target_id = %target_id, "Start audit stream processing for target");
+                let store_clone: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send> = store.boxed_clone();
+                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
+                let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc);
+
+                self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx);
+                info!(target_id = %target_id, "Audit stream processing started");
+            } else {
+                info!(target_id = %target_id, "No store configured, skip audit stream processing");
+            }
+        } else {
+            info!(target_id = %target_id, "Target disabled, skip audit stream processing");
+        }
+
+        registry.add_target(target_id.clone(), target);
+        Some(target_id)
+    }
+
     /// Starts the audit stream processing for a target with batching and retry logic
     ///
     /// # Arguments
@@ -427,9 +471,10 @@ impl AuditSystem {
     /// and attempts to send them to the specified target. It implements retry logic with exponential backoff
     fn start_audit_stream_with_batching(
         &self,
-        store: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send>,
+        store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
         target: Arc<dyn Target<AuditEntry> + Send + Sync>,
-    ) {
+    ) -> mpsc::Sender<()> {
+        let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
         let state = self.state.clone();
 
         tokio::spawn(async move {
@@ -442,6 +487,12 @@ impl AuditSystem {
             const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
 
             loop {
+                // Check for cancellation signal
+                if cancel_rx.try_recv().is_ok() {
+                    info!("Audit stream cancelled for target: {}", target.id());
+                    break;
+                }
+
                 match *state.read().await {
                     AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {}
                     _ => {
@@ -452,11 +503,22 @@ impl AuditSystem {
 
                 let keys: Vec<Key> = store.list();
                 if keys.is_empty() {
-                    sleep(Duration::from_millis(500)).await;
+                    tokio::select! {
+                        _ = sleep(Duration::from_millis(500)) => {},
+                        _ = cancel_rx.recv() => {
+                            info!("Audit stream cancelled during idle for target: {}", target.id());
+                            return;
+                        }
+                    }
                     continue;
                 }
 
                 for key in keys {
+                    if cancel_rx.try_recv().is_ok() {
+                        info!("Audit stream cancelled during processing for target: {}", target.id());
+                        return;
+                    }
+
                     let mut retries = 0usize;
                     let mut success = false;
 
@@ -497,6 +559,8 @@ impl AuditSystem {
                 sleep(Duration::from_millis(100)).await;
             }
         });
+
+        cancel_tx
     }
 
     /// Enables a specific target
@@ -594,6 +658,12 @@ impl AuditSystem {
         registry.list_targets()
     }
 
+    /// Returns cloned target values for read-only runtime inspection.
+    pub async fn get_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
+        let registry = self.registry.lock().await;
+        registry.list_target_values()
+    }
+
     /// Gets information about a specific target
     ///
     /// # Arguments
@@ -616,9 +686,11 @@ impl AuditSystem {
     pub async fn reload_config(&self, new_config: Config) -> AuditResult<()> {
         info!("Reloading audit system configuration");
 
-        // Record config reload
         observability::record_config_reload();
 
+        // Stop all existing stream tasks first
+        self.stop_all_streams().await;
+
         // Store new configuration
         {
             let mut config_guard = self.config.write().await;
@@ -636,28 +708,9 @@ impl AuditSystem {
             Ok(targets) => {
                 info!(target_count = targets.len(), "Reloaded audit targets successfully");
 
-                // Initialize all new targets
                 for target in targets {
-                    let target_id = target.id().to_string();
-                    if let Err(e) = target.init().await {
-                        error!(target_id = %target_id, error = %e, "Failed to initialize reloaded audit target");
-                    } else {
-                        // Same starts the storage stream after a heavy load
-                        if target.is_enabled() {
-                            if let Some(store) = target.store() {
-                                info!(target_id = %target_id, "Start audit stream processing for target (reload)");
-                                let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
-                                    store.boxed_clone();
-                                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
-                                self.start_audit_stream_with_batching(store_clone, target_arc);
-                                info!(target_id = %target_id, "Audit stream processing started (reload)");
-                            } else {
-                                info!(target_id = %target_id, "No store configured, skip audit stream processing (reload)");
-                            }
-                        } else {
-                            info!(target_id = %target_id, "Target disabled, skip audit stream processing (reload)");
-                        }
-                        registry.add_target(target.id().to_string(), target);
+                    if let Some(target_id) = self.init_and_register_target(target, &mut registry).await {
+                        info!(target_id = %target_id, "Target initialized (reload)");
                     }
                 }
 

crates/audit/tests/config_parsing_test.rs

@@ -53,6 +53,12 @@ fn test_mqtt_valid_fields() {
         "reconnect_interval",
         "queue_dir",
         "queue_limit",
+        "tls_policy",
+        "tls_ca",
+        "tls_client_cert",
+        "tls_client_key",
+        "tls_trust_leaf_as_ca",
+        "ws_path_allowlist",
     ];
 
     // This tests the MQTT configuration fields we support

crates/audit/tests/integration_test.rs

@@ -15,6 +15,7 @@
 use rustfs_audit::*;
 use rustfs_ecstore::config::{Config, KVS};
 use std::collections::HashMap;
+use temp_env::with_vars;
 
 #[tokio::test]
 async fn test_audit_system_creation() {
@@ -35,34 +36,42 @@ async fn test_config_parsing_webhook() {
     let mut config = Config(HashMap::new());
     let mut audit_webhook_section = HashMap::new();
 
-    // Create default configuration
     let mut default_kvs = KVS::new();
-    default_kvs.insert("enable".to_string(), "on".to_string());
-    default_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
-
+    default_kvs.insert("enable".to_string(), "off".to_string());
+    default_kvs.insert("endpoint".to_string(), "".to_string());
     audit_webhook_section.insert("_".to_string(), default_kvs);
+    let mut instance_kvs = KVS::new();
+    instance_kvs.insert("enable".to_string(), "on".to_string());
+    instance_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
+    audit_webhook_section.insert("primary".to_string(), instance_kvs);
     config.0.insert("audit_webhook".to_string(), audit_webhook_section);
 
     let registry = AuditRegistry::new();
 
-    // This should not fail even if server storage is not initialized
-    // as it's an integration test
     let result = registry.create_audit_targets_from_config(&config).await;
+    assert!(result.is_ok(), "audit target creation should not require server storage");
+}
 
-    // We expect this to fail due to server storage not being initialized
-    // but the parsing should work correctly
-    match result {
-        Err(AuditError::StorageNotAvailable(_)) => {
-            // This is expected in test environment
-        }
-        Err(e) => {
-            // Other errors might indicate parsing issues
-            println!("Unexpected error: {e}");
-        }
-        Ok(_) => {
-            // Unexpected success in test environment without server storage
-        }
-    }
+#[test]
+fn test_env_only_audit_target_does_not_require_server_storage() {
+    with_vars(
+        [
+            ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")),
+            ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("http://localhost:3020/webhook")),
+        ],
+        || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("failed to create tokio runtime");
+            runtime.block_on(async {
+                let config = Config(HashMap::new());
+                let registry = AuditRegistry::new();
+                let result = registry.create_audit_targets_from_config(&config).await;
+                assert!(result.is_ok(), "env-only audit target creation should not require server storage");
+            });
+        },
+    )
 }
 
 #[test]

crates/audit/tests/performance_test.rs

@@ -159,7 +159,8 @@ async fn test_audit_log_dispatch_performance() {
 
     let start = Instant::now();
 
-    // Dispatch audit log (should be fast since no targets are configured)
+    // Dispatch audit log against an unstarted system state. Empty config keeps
+    // the audit system stopped, so dispatch should fail fast without targets.
     let result = system.dispatch(Arc::new(audit_entry)).await;
     let elapsed = start.elapsed();
 
@@ -168,8 +169,10 @@ async fn test_audit_log_dispatch_performance() {
     // Should be very fast (sub-millisecond for no targets)
     assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
 
-    // Should succeed even with no targets
-    assert!(result.is_ok(), "Dispatch should succeed with no targets");
+    assert!(
+        matches!(result, Err(AuditError::NotInitialized(_))),
+        "Dispatch on a stopped system should return NotInitialized, got: {result:?}"
+    );
 
     // Clean up
     let _ = system.close().await;
@@ -186,11 +189,11 @@ async fn test_system_state_transitions() {
     let config = rustfs_ecstore::config::Config(std::collections::HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should be running (or failed due to server storage)
+    // Empty config keeps the audit system stopped even when start() succeeds.
     let state = system.get_state().await;
     match start_result {
         Ok(_) => {
-            assert_eq!(state, rustfs_audit::system::AuditSystemState::Running);
+            assert_eq!(state, rustfs_audit::system::AuditSystemState::Stopped);
         }
         Err(_) => {
             // Expected in test environment due to server storage not being initialized

crates/audit/tests/system_integration_test.rs

@@ -29,27 +29,21 @@ async fn test_complete_audit_system_lifecycle() {
     assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
     assert!(!system.is_running().await);
 
-    // 2. Start with empty config (will fail due to no server storage in test)
+    // 2. Start with empty config. The current implementation returns Ok(())
+    // but keeps the system stopped when no audit targets are enabled.
     let config = Config(HashMap::new());
     let start_result = system.start(config).await;
 
-    // Should fail in test environment but state handling should work
+    // State handling should remain consistent for both empty-config success and
+    // storage-unavailable failure paths.
     match start_result {
         Err(AuditError::StorageNotAvailable(_)) => {
             // Expected in test environment
             assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
         }
         Ok(_) => {
-            // If it somehow succeeds, verify running state
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
-            assert!(system.is_running().await);
-
-            // Test pause/resume
-            system.pause().await.expect("Should pause successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Paused);
-
-            system.resume().await.expect("Should resume successfully");
-            assert_eq!(system.get_state().await, system::AuditSystemState::Running);
+            assert_eq!(system.get_state().await, system::AuditSystemState::Stopped);
+            assert!(!system.is_running().await);
         }
         Err(e) => {
             panic!("Unexpected error: {e}");

crates/checksums/Cargo.toml

@@ -36,3 +36,6 @@ sha2 = { workspace = true }
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }
+
+[lib]
+doctest = false

crates/checksums/src/lib.rs

@@ -349,7 +349,7 @@ mod tests {
     fn base64_encoded_checksum_to_hex_string(header_value: &HeaderValue) -> String {
         let decoded_checksum = base64::decode(header_value.to_str().unwrap()).unwrap();
         let decoded_checksum = decoded_checksum.into_iter().fold(String::new(), |mut acc, byte| {
-            write!(acc, "{byte:02X?}").expect("string will always be writeable");
+            write!(acc, "{byte:02X?}").expect("string will always be writable");
             acc
         });
 

crates/common/Cargo.toml

@@ -41,3 +41,6 @@ rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 s3s = { workspace = true }
 tracing = { workspace = true }
+
+[lib]
+doctest = false

crates/common/src/capacity_scope.rs

@@ -0,0 +1,308 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::{Mutex, OnceLock};
+use std::time::{Duration, Instant};
+use uuid::Uuid;
+
+const CAPACITY_SCOPE_REGISTRY_SOFT_LIMIT: usize = 2_048;
+const CAPACITY_SCOPE_REGISTRY_HARD_LIMIT: usize = 4_096;
+const CAPACITY_SCOPE_TTL: Duration = Duration::from_secs(300);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CapacityScopeDisk {
+    pub endpoint: String,
+    pub drive_path: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct CapacityScope {
+    pub disks: Vec<CapacityScopeDisk>,
+}
+
+#[derive(Debug, Clone)]
+struct CapacityScopeEntry {
+    scope: CapacityScope,
+    recorded_at: Instant,
+}
+
+fn capacity_scope_registry() -> &'static Mutex<HashMap<Uuid, CapacityScopeEntry>> {
+    static REGISTRY: OnceLock<Mutex<HashMap<Uuid, CapacityScopeEntry>>> = OnceLock::new();
+    REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
+}
+
+fn global_dirty_scope_registry() -> &'static Mutex<HashSet<CapacityScopeDisk>> {
+    static REGISTRY: OnceLock<Mutex<HashSet<CapacityScopeDisk>>> = OnceLock::new();
+    REGISTRY.get_or_init(|| Mutex::new(HashSet::new()))
+}
+
+fn prune_expired_entries(entries: &mut HashMap<Uuid, CapacityScopeEntry>, now: Instant) {
+    entries.retain(|_, entry| now.duration_since(entry.recorded_at) <= CAPACITY_SCOPE_TTL);
+}
+
+fn enforce_hard_limit(entries: &mut HashMap<Uuid, CapacityScopeEntry>, max_len: usize) {
+    if entries.len() < max_len {
+        return;
+    }
+
+    let evict_count = entries.len() - max_len + 1;
+    let mut eviction_order: Vec<_> = entries.iter().map(|(token, entry)| (*token, entry.recorded_at)).collect();
+    eviction_order.sort_unstable_by_key(|(_, recorded_at)| *recorded_at);
+
+    for (token, _) in eviction_order.into_iter().take(evict_count) {
+        entries.remove(&token);
+    }
+}
+
+fn merge_capacity_scopes(existing: &mut CapacityScope, incoming: CapacityScope) {
+    let mut seen: HashSet<CapacityScopeDisk> = existing.disks.iter().cloned().collect();
+    for disk in incoming.disks {
+        if seen.insert(disk.clone()) {
+            existing.disks.push(disk);
+        }
+    }
+}
+
+pub fn record_capacity_scope(token: Uuid, scope: CapacityScope) {
+    let now = Instant::now();
+    let mut entries = capacity_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
+    if !entries.contains_key(&token) && entries.len() >= CAPACITY_SCOPE_REGISTRY_SOFT_LIMIT {
+        prune_expired_entries(&mut entries, now);
+        enforce_hard_limit(&mut entries, CAPACITY_SCOPE_REGISTRY_HARD_LIMIT);
+    }
+    if let Some(entry) = entries.get_mut(&token) {
+        merge_capacity_scopes(&mut entry.scope, scope);
+        entry.recorded_at = now;
+    } else {
+        entries.insert(token, CapacityScopeEntry { scope, recorded_at: now });
+    }
+}
+
+pub fn take_capacity_scope(token: Uuid) -> Option<CapacityScope> {
+    let now = Instant::now();
+    let mut entries = capacity_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
+    let entry = entries.remove(&token)?;
+    if now.duration_since(entry.recorded_at) > CAPACITY_SCOPE_TTL {
+        return None;
+    }
+    Some(entry.scope)
+}
+
+pub fn record_global_dirty_scope(scope: CapacityScope) {
+    if scope.disks.is_empty() {
+        return;
+    }
+
+    let mut dirty_scopes = global_dirty_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
+    dirty_scopes.extend(scope.disks);
+}
+
+pub fn drain_global_dirty_scopes() -> Vec<CapacityScopeDisk> {
+    let mut dirty_scopes = global_dirty_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
+    dirty_scopes.drain().collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Mutex;
+    use std::thread;
+
+    fn test_lock() -> &'static Mutex<()> {
+        static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
+        LOCK.get_or_init(|| Mutex::new(()))
+    }
+
+    fn clear_capacity_scope_registry_for_test() {
+        capacity_scope_registry()
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+            .clear();
+        global_dirty_scope_registry()
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+            .clear();
+    }
+
+    fn poison_capacity_scope_registry_for_test() {
+        let _ = thread::spawn(|| {
+            let _guard = capacity_scope_registry()
+                .lock()
+                .expect("capacity scope registry lock should succeed");
+            panic!("poison capacity scope registry");
+        })
+        .join();
+    }
+
+    fn poison_global_dirty_scope_registry_for_test() {
+        let _ = thread::spawn(|| {
+            let _guard = global_dirty_scope_registry()
+                .lock()
+                .expect("global dirty scope registry lock should succeed");
+            panic!("poison global dirty scope registry");
+        })
+        .join();
+    }
+
+    #[test]
+    fn record_and_take_capacity_scope_round_trips() {
+        let _guard = test_lock().lock().expect("test lock poisoned");
+        clear_capacity_scope_registry_for_test();
+        let token = Uuid::new_v4();
+        let scope = CapacityScope {
+            disks: vec![CapacityScopeDisk {
+                endpoint: "node-a".to_string(),
+                drive_path: "/tmp/disk-a".to_string(),
+            }],
+        };
+
+        record_capacity_scope(token, scope.clone());
+
+        assert_eq!(take_capacity_scope(token), Some(scope));
+        assert_eq!(take_capacity_scope(token), None);
+        clear_capacity_scope_registry_for_test();
+    }
+
+    #[test]
+    fn record_capacity_scope_merges_disks_for_same_token() {
+        let _guard = test_lock().lock().expect("test lock poisoned");
+        clear_capacity_scope_registry_for_test();
+        let token = Uuid::new_v4();
+        record_capacity_scope(
+            token,
+            CapacityScope {
+                disks: vec![CapacityScopeDisk {
+                    endpoint: "node-a".to_string(),
+                    drive_path: "/tmp/disk-a".to_string(),
+                }],
+            },
+        );
+        record_capacity_scope(
+            token,
+            CapacityScope {
+                disks: vec![
+                    CapacityScopeDisk {
+                        endpoint: "node-b".to_string(),
+                        drive_path: "/tmp/disk-b".to_string(),
+                    },
+                    CapacityScopeDisk {
+                        endpoint: "node-a".to_string(),
+                        drive_path: "/tmp/disk-a".to_string(),
+                    },
+                ],
+            },
+        );
+
+        let scope = take_capacity_scope(token).expect("scope should exist");
+        assert_eq!(scope.disks.len(), 2);
+        assert!(scope.disks.iter().any(|disk| disk.endpoint == "node-a"));
+        assert!(scope.disks.iter().any(|disk| disk.endpoint == "node-b"));
+        clear_capacity_scope_registry_for_test();
+    }
+
+    #[test]
+    fn record_capacity_scope_enforces_hard_limit() {
+        let _guard = test_lock().lock().expect("test lock poisoned");
+        clear_capacity_scope_registry_for_test();
+
+        for _ in 0..(CAPACITY_SCOPE_REGISTRY_HARD_LIMIT + 32) {
+            record_capacity_scope(
+                Uuid::new_v4(),
+                CapacityScope {
+                    disks: vec![CapacityScopeDisk {
+                        endpoint: "node-a".to_string(),
+                        drive_path: "/tmp/disk-a".to_string(),
+                    }],
+                },
+            );
+        }
+
+        let entries = capacity_scope_registry()
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        assert!(entries.len() <= CAPACITY_SCOPE_REGISTRY_HARD_LIMIT);
+        drop(entries);
+        clear_capacity_scope_registry_for_test();
+    }
+
+    #[test]
+    fn record_capacity_scope_recovers_from_poisoned_registry() {
+        let _guard = test_lock().lock().expect("test lock poisoned");
+        clear_capacity_scope_registry_for_test();
+        poison_capacity_scope_registry_for_test();
+        let token = Uuid::new_v4();
+        let scope = CapacityScope {
+            disks: vec![CapacityScopeDisk {
+                endpoint: "node-a".to_string(),
+                drive_path: "/tmp/disk-a".to_string(),
+            }],
+        };
+
+        record_capacity_scope(token, scope.clone());
+
+        assert_eq!(take_capacity_scope(token), Some(scope));
+        clear_capacity_scope_registry_for_test();
+    }
+
+    #[test]
+    fn record_and_drain_global_dirty_scope_round_trips() {
+        let _guard = test_lock().lock().expect("test lock poisoned");
+        clear_capacity_scope_registry_for_test();
+        record_global_dirty_scope(CapacityScope {
+            disks: vec![CapacityScopeDisk {
+                endpoint: "node-a".to_string(),
+                drive_path: "/tmp/disk-a".to_string(),
+            }],
+        });
+        record_global_dirty_scope(CapacityScope {
+            disks: vec![
+                CapacityScopeDisk {
+                    endpoint: "node-b".to_string(),
+                    drive_path: "/tmp/disk-b".to_string(),
+                },
+                CapacityScopeDisk {
+                    endpoint: "node-a".to_string(),
+                    drive_path: "/tmp/disk-a".to_string(),
+                },
+            ],
+        });
+
+        let drained = drain_global_dirty_scopes();
+        assert_eq!(drained.len(), 2);
+        assert!(drained.iter().any(|disk| disk.endpoint == "node-a"));
+        assert!(drained.iter().any(|disk| disk.endpoint == "node-b"));
+        assert!(drain_global_dirty_scopes().is_empty());
+        clear_capacity_scope_registry_for_test();
+    }
+
+    #[test]
+    fn record_global_dirty_scope_recovers_from_poisoned_registry() {
+        let _guard = test_lock().lock().expect("test lock poisoned");
+        clear_capacity_scope_registry_for_test();
+        poison_global_dirty_scope_registry_for_test();
+
+        record_global_dirty_scope(CapacityScope {
+            disks: vec![CapacityScopeDisk {
+                endpoint: "node-a".to_string(),
+                drive_path: "/tmp/disk-a".to_string(),
+            }],
+        });
+
+        let drained = drain_global_dirty_scopes();
+        assert_eq!(drained.len(), 1);
+        assert_eq!(drained[0].endpoint, "node-a");
+        clear_capacity_scope_registry_for_test();
+    }
+}

crates/common/src/data_usage.rs

@@ -1097,7 +1097,7 @@ impl DataUsageInfo {
 
     /// Add bucket usage info
     pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) {
-        self.buckets_usage.insert(bucket.clone(), usage);
+        self.buckets_usage.insert(bucket, usage);
         self.buckets_count = self.buckets_usage.len() as u64;
         self.last_update = Some(SystemTime::now());
     }

crates/common/src/internode_metrics.rs

@@ -0,0 +1,170 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::{counter, gauge};
+use std::sync::{
+    Arc, LazyLock,
+    atomic::{AtomicU64, Ordering},
+};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct InternodeMetricsSnapshot {
+    pub sent_bytes_total: u64,
+    pub recv_bytes_total: u64,
+    pub outgoing_requests_total: u64,
+    pub incoming_requests_total: u64,
+    pub errors_total: u64,
+    pub dial_errors_total: u64,
+    pub dial_avg_time_nanos: u64,
+    pub last_dial_unix_millis: u64,
+}
+
+#[derive(Debug, Default)]
+pub struct InternodeMetrics {
+    sent_bytes_total: AtomicU64,
+    recv_bytes_total: AtomicU64,
+    outgoing_requests_total: AtomicU64,
+    incoming_requests_total: AtomicU64,
+    errors_total: AtomicU64,
+    dial_errors_total: AtomicU64,
+    dial_total_time_nanos: AtomicU64,
+    dial_samples_total: AtomicU64,
+    last_dial_unix_millis: AtomicU64,
+}
+
+impl InternodeMetrics {
+    pub fn record_sent_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.sent.bytes.total").increment(bytes);
+    }
+
+    pub fn record_recv_bytes(&self, bytes: usize) {
+        let bytes = bytes as u64;
+        if bytes == 0 {
+            return;
+        }
+        self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
+        counter!("rustfs.internode.recv.bytes.total").increment(bytes);
+    }
+
+    pub fn record_outgoing_request(&self) {
+        self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.outgoing.total").increment(1);
+    }
+
+    pub fn record_incoming_request(&self) {
+        self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.requests.incoming.total").increment(1);
+    }
+
+    pub fn record_error(&self) {
+        self.errors_total.fetch_add(1, Ordering::Relaxed);
+        counter!("rustfs.internode.errors.total").increment(1);
+    }
+
+    pub fn record_dial_result(&self, duration: Duration, success: bool) {
+        let elapsed_nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
+        self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
+        let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
+        let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        gauge!("rustfs.internode.dial.avg_time.nanos").set(total as f64 / samples as f64);
+
+        if !success {
+            self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
+            counter!("rustfs.internode.dial.errors.total").increment(1);
+        }
+
+        let now_ms = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_millis()
+            .min(u128::from(u64::MAX)) as u64;
+        self.last_dial_unix_millis.store(now_ms, Ordering::Relaxed);
+    }
+
+    pub fn snapshot(&self) -> InternodeMetricsSnapshot {
+        let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
+        let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
+        let dial_avg_time_nanos = if dial_samples_total == 0 {
+            0
+        } else {
+            dial_total_time_nanos / dial_samples_total
+        };
+
+        InternodeMetricsSnapshot {
+            sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),
+            recv_bytes_total: self.recv_bytes_total.load(Ordering::Relaxed),
+            outgoing_requests_total: self.outgoing_requests_total.load(Ordering::Relaxed),
+            incoming_requests_total: self.incoming_requests_total.load(Ordering::Relaxed),
+            errors_total: self.errors_total.load(Ordering::Relaxed),
+            dial_errors_total: self.dial_errors_total.load(Ordering::Relaxed),
+            dial_avg_time_nanos,
+            last_dial_unix_millis: self.last_dial_unix_millis.load(Ordering::Relaxed),
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn reset_for_test(&self) {
+        self.sent_bytes_total.store(0, Ordering::Relaxed);
+        self.recv_bytes_total.store(0, Ordering::Relaxed);
+        self.outgoing_requests_total.store(0, Ordering::Relaxed);
+        self.incoming_requests_total.store(0, Ordering::Relaxed);
+        self.errors_total.store(0, Ordering::Relaxed);
+        self.dial_errors_total.store(0, Ordering::Relaxed);
+        self.dial_total_time_nanos.store(0, Ordering::Relaxed);
+        self.dial_samples_total.store(0, Ordering::Relaxed);
+        self.last_dial_unix_millis.store(0, Ordering::Relaxed);
+    }
+}
+
+pub fn global_internode_metrics() -> &'static Arc<InternodeMetrics> {
+    static GLOBAL_INTERNODE_METRICS: LazyLock<Arc<InternodeMetrics>> = LazyLock::new(|| Arc::new(InternodeMetrics::default()));
+    &GLOBAL_INTERNODE_METRICS
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn snapshot_reports_recorded_values() {
+        let metrics = global_internode_metrics();
+        metrics.reset_for_test();
+
+        metrics.record_sent_bytes(64);
+        metrics.record_recv_bytes(32);
+        metrics.record_outgoing_request();
+        metrics.record_incoming_request();
+        metrics.record_error();
+        metrics.record_dial_result(Duration::from_millis(9), true);
+        metrics.record_dial_result(Duration::from_millis(3), false);
+
+        let snapshot = metrics.snapshot();
+        assert_eq!(snapshot.sent_bytes_total, 64);
+        assert_eq!(snapshot.recv_bytes_total, 32);
+        assert_eq!(snapshot.outgoing_requests_total, 1);
+        assert_eq!(snapshot.incoming_requests_total, 1);
+        assert_eq!(snapshot.errors_total, 1);
+        assert_eq!(snapshot.dial_errors_total, 1);
+        assert_eq!(snapshot.dial_avg_time_nanos, 6_000_000);
+        assert!(snapshot.last_dial_unix_millis > 0);
+
+        metrics.reset_for_test();
+    }
+}

crates/common/src/lib.rs

@@ -13,10 +13,12 @@
 // limitations under the License.
 
 pub mod bucket_stats;
+pub mod capacity_scope;
 // pub mod error;
 pub mod data_usage;
 pub mod globals;
 pub mod heal_channel;
+pub mod internode_metrics;
 pub mod last_minute;
 pub mod metrics;
 mod readiness;

crates/common/src/metrics.rs

@@ -22,12 +22,12 @@ use std::{
     future::Future,
     pin::Pin,
     sync::{
-        Arc, OnceLock,
+        Arc, Mutex, OnceLock,
         atomic::{AtomicU64, Ordering},
     },
     time::{Duration, SystemTime},
 };
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::RwLock;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum IlmAction {
@@ -110,7 +110,7 @@ pub fn global_metrics() -> &'static Arc<Metrics> {
     GLOBAL_METRICS.get_or_init(|| Arc::new(Metrics::new()))
 }
 
-#[derive(Clone, Debug, PartialEq, PartialOrd)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Metric {
     // START Realtime metrics, that only records
     // last minute latencies and total operation count.
@@ -188,7 +188,6 @@ impl Metric {
         if index >= Self::Last as usize {
             return None;
         }
-        // Safe conversion using match instead of unsafe transmute
         match index {
             0 => Some(Self::ReadMetadata),
             1 => Some(Self::CheckMissing),
@@ -220,16 +219,48 @@ impl Metric {
     }
 }
 
-/// Thread-safe wrapper for LastMinuteLatency with atomic operations
-#[derive(Default)]
+// ---------------------------------------------------------------------------
+// LockedLastMinuteLatency
+// ---------------------------------------------------------------------------
+//
+// Uses std::sync::Mutex instead of tokio::sync::Mutex.
+//
+// Rationale: the critical section is a handful of integer additions inside
+// LastMinuteLatency::add_all / get_total — no I/O, no blocking syscalls, no
+// awaiting.  A std blocking mutex is cheaper (no task-wakeup overhead) and,
+// crucially, lets every caller be *synchronous*.  That eliminates the need to
+// spawn a background task just to record a duration, which was the only reason
+// tokio::spawn appeared in log/time/time_size/time_n/time_ilm.
+//
+// Note: vec![LockedLastMinuteLatency::default(); N] with the old Arc-based
+// Clone made every element share the *same* inner mutex — a latent bug where
+// all metrics slots wrote to one counter.  The new Clone creates a fresh
+// independent Mutex per element, matching the intent.
+
+/// Thread-safe wrapper for LastMinuteLatency backed by a std blocking mutex.
 pub struct LockedLastMinuteLatency {
+    // Arc so Clone is cheap *and* each cloned value stays independent (its own
+    // allocation).  We never hand out the Arc to two Metrics at once; the Arc
+    // is purely a convenience for the Clone impl below.
     latency: Arc<Mutex<LastMinuteLatency>>,
 }
 
+impl Default for LockedLastMinuteLatency {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// Produce a fresh, independent slot — *not* a shared alias.  This is what
+// vec![val; N] and #[derive(Clone)] on the parent struct both need.
 impl Clone for LockedLastMinuteLatency {
     fn clone(&self) -> Self {
+        let inner = match self.latency.lock() {
+            Ok(guard) => guard.clone(),
+            Err(poisoned) => poisoned.into_inner().clone(),
+        };
         Self {
-            latency: Arc::clone(&self.latency),
+            latency: Arc::new(Mutex::new(inner)),
         }
     }
 }
@@ -241,14 +272,13 @@ impl LockedLastMinuteLatency {
         }
     }
 
-    /// Add a duration measurement
-    pub async fn add(&self, duration: Duration) {
-        self.add_size(duration, 0).await;
+    /// Record a duration sample (no size).
+    pub fn add(&self, duration: Duration) {
+        self.add_size(duration, 0);
     }
 
-    /// Add a duration measurement with size
-    pub async fn add_size(&self, duration: Duration, size: u64) {
-        let mut latency = self.latency.lock().await;
+    /// Record a duration sample with an associated byte count.
+    pub fn add_size(&self, duration: Duration, size: u64) {
         let now = SystemTime::now()
             .duration_since(SystemTime::UNIX_EPOCH)
             .unwrap_or_default()
@@ -259,17 +289,27 @@ impl LockedLastMinuteLatency {
             total: duration.as_secs(),
             size,
         };
-        latency.add_all(now, &elem);
+
+        match self.latency.lock() {
+            Ok(mut guard) => guard.add_all(now, &elem),
+            Err(poisoned) => poisoned.into_inner().add_all(now, &elem),
+        }
     }
 
-    /// Get total accumulated metrics for the last minute
-    pub async fn total(&self) -> AccElem {
-        let mut latency = self.latency.lock().await;
-        latency.get_total()
+    /// Return accumulated totals for the last minute window.
+    pub fn total(&self) -> AccElem {
+        match self.latency.lock() {
+            Ok(mut latency) => latency.get_total(),
+            Err(poisoned) => poisoned.into_inner().get_total(),
+        }
     }
 }
 
-/// Current path tracker for monitoring active scan paths
+// ---------------------------------------------------------------------------
+// CurrentPathTracker — unchanged, still uses tokio::sync::RwLock because it
+// lives inside async path-update callbacks.
+// ---------------------------------------------------------------------------
+
 struct CurrentPathTracker {
     current_path: Arc<RwLock<String>>,
 }
@@ -290,17 +330,16 @@ impl CurrentPathTracker {
     }
 }
 
-/// Main scanner metrics structure
+// ---------------------------------------------------------------------------
+// Metrics
+// ---------------------------------------------------------------------------
+
 pub struct Metrics {
-    // All fields must be accessed atomically and aligned.
     operations: Vec<AtomicU64>,
     latency: Vec<LockedLastMinuteLatency>,
     actions: Vec<AtomicU64>,
     actions_latency: Vec<LockedLastMinuteLatency>,
-    // Current paths contains disk -> tracker mappings
     current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
-
-    // Cycle information
     cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
 }
 
@@ -331,8 +370,6 @@ const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
 
-/// Emit an OTEL counter increment for the given scanner metric.
-/// ScanCycle and ScanBucketDrive are handled by dedicated emit functions with labels.
 fn emit_otel_counter(metric: usize, count: u64) {
     match Metric::from_index(metric) {
         Some(Metric::ScanObject) => {
@@ -345,8 +382,6 @@ fn emit_otel_counter(metric: usize, count: u64) {
     }
 }
 
-/// Emit OTel metrics for a completed scan cycle.
-/// Counter with result label + gauge for last successful cycle duration.
 pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
     let result = if success { "success" } else { "error" };
     metrics::counter!(OTEL_SCANNER_CYCLES, "result" => result).increment(1);
@@ -355,8 +390,6 @@ pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
     }
 }
 
-/// Emit OTel metrics for a completed bucket-drive scan.
-/// Counter with result/bucket/disk labels + histogram for duration.
 pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
     let result = if success { "success" } else { "error" };
     metrics::counter!(
@@ -376,226 +409,202 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
 
 impl Metrics {
     pub fn new() -> Self {
-        let operations = (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect();
-
-        let latency = (0..Metric::LastRealtime as usize)
-            .map(|_| LockedLastMinuteLatency::new())
-            .collect();
-
         Self {
-            operations,
-            latency,
+            operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
+            // Each slot gets its own fresh LockedLastMinuteLatency so that
+            // different metrics never accidentally share state.
+            latency: (0..Metric::LastRealtime as usize)
+                .map(|_| LockedLastMinuteLatency::new())
+                .collect(),
             actions: (0..IlmAction::ActionCount as usize).map(|_| AtomicU64::new(0)).collect(),
-            actions_latency: vec![LockedLastMinuteLatency::default(); IlmAction::ActionCount as usize],
+            actions_latency: (0..IlmAction::ActionCount as usize)
+                .map(|_| LockedLastMinuteLatency::new())
+                .collect(),
             current_paths: Arc::new(RwLock::new(HashMap::new())),
             cycle_info: Arc::new(RwLock::new(None)),
         }
     }
 
-    /// Log scanner action with custom metadata - compatible with existing usage
+    // -----------------------------------------------------------------------
+    // Metric recording helpers
+    //
+    // All of these are now pure sync closures.  No tokio::spawn, no heap-
+    // allocated future, no scheduler overhead — just an atomic increment and a
+    // std-mutex lock for fewer than ~10 ns of work.
+    // -----------------------------------------------------------------------
+
+    /// Return a closure that records one observation of `metric` (with
+    /// optional caller-supplied metadata).  Call it once the operation ends.
     pub fn log(metric: Metric) -> impl Fn(&HashMap<String, String>) {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         move |_custom: &HashMap<String, String>| {
-            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-            // Update operation count
-            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-            emit_otel_counter(metric, 1);
-
-            // Update latency for realtime metrics (spawn async task for this)
-            if (metric) < Metric::LastRealtime as usize {
-                let metric_index = metric;
-                tokio::spawn(async move {
-                    global_metrics().latency[metric_index].add(duration).await;
-                });
-            }
-
-            // Log trace metrics
-            if metric as u8 > Metric::StartTrace as u8 {
-                //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric");
+            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric_idx, 1);
+            if metric_idx < Metric::LastRealtime as usize {
+                global_metrics().latency[metric_idx].add(duration);
             }
         }
     }
 
-    /// Time scanner action with size - returns function that takes size
+    /// Return a closure that records one observation of `metric` together with
+    /// a byte count.  Call `done(size_bytes)` when the operation ends.
     pub fn time_size(metric: Metric) -> impl Fn(u64) {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         move |size: u64| {
-            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-            // Update operation count
-            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-            emit_otel_counter(metric, 1);
-
-            // Update latency for realtime metrics with size (spawn async task)
-            if (metric) < Metric::LastRealtime as usize {
-                let metric_index = metric;
-                tokio::spawn(async move {
-                    global_metrics().latency[metric_index].add_size(duration, size).await;
-                });
+            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric_idx, 1);
+            if metric_idx < Metric::LastRealtime as usize {
+                global_metrics().latency[metric_idx].add_size(duration, size);
             }
         }
     }
 
-    /// Time a scanner action - returns a closure to call when done
+    /// Return a closure that records one observation of `metric`.
+    /// Call `done()` when the operation ends.
     pub fn time(metric: Metric) -> impl Fn() {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         move || {
-            let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-            // Update operation count
-            global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-            emit_otel_counter(metric, 1);
-
-            // Update latency for realtime metrics (spawn async task)
-            if (metric) < Metric::LastRealtime as usize {
-                let metric_index = metric;
-                tokio::spawn(async move {
-                    global_metrics().latency[metric_index].add(duration).await;
-                });
+            let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+            global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+            emit_otel_counter(metric_idx, 1);
+            if metric_idx < Metric::LastRealtime as usize {
+                global_metrics().latency[metric_idx].add(duration);
             }
         }
     }
 
-    /// Time N scanner actions - returns function that takes count, then returns completion function
+    /// Return a two-stage closure: first call takes an item count, second call
+    /// (returned closure) fires when the batch of `count` operations ends.
     pub fn time_n(metric: Metric) -> Box<dyn Fn(usize) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
-        let metric = metric as usize;
-        let start_time = SystemTime::now();
+        let metric_idx = metric as usize;
+        let start = SystemTime::now();
         Box::new(move |count: usize| {
             Box::new(move || {
-                let duration = SystemTime::now().duration_since(start_time).unwrap_or_default();
-
-                // Update operation count
-                global_metrics().operations[metric].fetch_add(count as u64, Ordering::Relaxed);
-                emit_otel_counter(metric, count as u64);
-
-                // Update latency for realtime metrics (spawn async task)
-                if (metric) < Metric::LastRealtime as usize {
-                    let metric_index = metric;
-                    tokio::spawn(async move {
-                        global_metrics().latency[metric_index].add(duration).await;
-                    });
+                let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+                global_metrics().operations[metric_idx].fetch_add(count as u64, Ordering::Relaxed);
+                emit_otel_counter(metric_idx, count as u64);
+                if metric_idx < Metric::LastRealtime as usize {
+                    global_metrics().latency[metric_idx].add(duration);
                 }
             })
         })
     }
 
-    /// Time ILM action with versions - returns function that takes versions, then returns completion function
+    /// Return a two-stage closure for ILM actions: first call takes a version
+    /// count, second call fires when the action completes.
     pub fn time_ilm(a: IlmAction) -> Box<dyn Fn(u64) -> Box<dyn Fn() + Send + Sync> + Send + Sync> {
-        let a_clone = a as usize;
-        if a_clone == IlmAction::NoneAction as usize || a_clone >= IlmAction::ActionCount as usize {
-            return Box::new(move |_: u64| Box::new(move || {}));
+        let a_idx = a as usize;
+        if a_idx == IlmAction::NoneAction as usize || a_idx >= IlmAction::ActionCount as usize {
+            return Box::new(|_| Box::new(|| {}));
         }
         let start = SystemTime::now();
         Box::new(move |versions: u64| {
             Box::new(move || {
-                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
-                tokio::spawn(async move {
-                    global_metrics().actions[a_clone].fetch_add(versions, Ordering::Relaxed);
-                    global_metrics().actions_latency[a_clone].add(duration).await;
-                });
+                let duration = SystemTime::now().duration_since(start).unwrap_or_default();
+                global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
+                global_metrics().actions_latency[a_idx].add(duration);
             })
         })
     }
 
-    /// Increment time with specific duration
-    pub async fn inc_time(metric: Metric, duration: Duration) {
-        let metric = metric as usize;
-        // Update operation count
-        global_metrics().operations[metric].fetch_add(1, Ordering::Relaxed);
-        emit_otel_counter(metric, 1);
-
-        // Update latency for realtime metrics
-        if (metric) < Metric::LastRealtime as usize {
-            global_metrics().latency[metric].add(duration).await;
+    /// Record a single observation of `metric` with a caller-supplied duration.
+    /// No longer async — nothing inside requires it.
+    pub fn inc_time(metric: Metric, duration: Duration) {
+        let metric_idx = metric as usize;
+        global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
+        emit_otel_counter(metric_idx, 1);
+        if metric_idx < Metric::LastRealtime as usize {
+            global_metrics().latency[metric_idx].add(duration);
         }
     }
 
-    /// Get lifetime operation count for a metric
+    // -----------------------------------------------------------------------
+    // Read-side helpers
+    // -----------------------------------------------------------------------
+
+    /// Lifetime operation count for `metric`.
     pub fn lifetime(&self, metric: Metric) -> u64 {
-        let metric = metric as usize;
-        if (metric) >= Metric::Last as usize {
+        let idx = metric as usize;
+        if idx >= Metric::Last as usize {
             return 0;
         }
-        self.operations[metric].load(Ordering::Relaxed)
+        self.operations[idx].load(Ordering::Relaxed)
     }
 
-    /// Get last minute statistics for a metric
-    pub async fn last_minute(&self, metric: Metric) -> AccElem {
-        let metric = metric as usize;
-        if (metric) >= Metric::LastRealtime as usize {
+    /// Last-minute accumulated stats for a realtime metric.
+    /// No longer async — LockedLastMinuteLatency::total() is now synchronous.
+    pub fn last_minute(&self, metric: Metric) -> AccElem {
+        let idx = metric as usize;
+        if idx >= Metric::LastRealtime as usize {
             return AccElem::default();
         }
-        self.latency[metric].total().await
+        self.latency[idx].total()
     }
 
-    /// Set current cycle information
+    /// Replace the current cycle record.
     pub async fn set_cycle(&self, cycle: Option<CurrentCycle>) {
         *self.cycle_info.write().await = cycle;
     }
 
-    /// Get current cycle information
+    /// Read the current cycle record.
     pub async fn get_cycle(&self) -> Option<CurrentCycle> {
         self.cycle_info.read().await.clone()
     }
 
-    /// Get current active paths
+    /// Snapshot of every path currently being scanned.
     pub async fn get_current_paths(&self) -> Vec<String> {
-        let mut result = Vec::new();
         let paths = self.current_paths.read().await;
-
+        let mut result = Vec::with_capacity(paths.len());
         for (disk, tracker) in paths.iter() {
-            let path = tracker.get_path().await;
-            result.push(format!("{disk}/{path}"));
+            result.push(format!("{disk}/{}", tracker.get_path().await));
         }
-
         result
     }
 
-    /// Get number of active drives
+    /// Number of drives with an active scan in progress.
     pub async fn active_drives(&self) -> usize {
         self.current_paths.read().await.len()
     }
 
-    /// Generate metrics report
+    /// Build a full metrics report snapshot.
     pub async fn report(&self) -> M_ScannerMetrics {
-        let mut metrics = M_ScannerMetrics::default();
+        let mut m = M_ScannerMetrics::default();
 
-        // Set cycle information
         if let Some(cycle) = self.get_cycle().await {
-            metrics.current_cycle = cycle.current;
-            metrics.cycles_completed_at = cycle.cycle_completed;
-            metrics.current_started = cycle.started;
+            m.current_cycle = cycle.current;
+            m.cycles_completed_at = cycle.cycle_completed;
+            m.current_started = cycle.started;
         }
 
-        // Replace default start time with global init time if it's the placeholder
         if let Some(init_time) = crate::get_global_init_time().await {
-            metrics.current_started = init_time;
+            m.current_started = init_time;
         }
 
-        metrics.collected_at = Utc::now();
-        metrics.active_paths = self.get_current_paths().await;
+        m.collected_at = Utc::now();
+        m.active_paths = self.get_current_paths().await;
 
-        // Lifetime operations
+        // Lifetime operation counts
         for i in 0..Metric::Last as usize {
             let count = self.operations[i].load(Ordering::Relaxed);
             if count > 0
                 && let Some(metric) = Metric::from_index(i)
             {
-                metrics.life_time_ops.insert(metric.as_str().to_string(), count);
+                m.life_time_ops.insert(metric.as_str().to_string(), count);
             }
         }
 
-        // Last minute statistics for realtime metrics
+        // Last-minute stats for realtime metrics — now plain sync calls
         for i in 0..Metric::LastRealtime as usize {
-            let last_min = self.latency[i].total().await;
+            let last_min = self.latency[i].total();
             if last_min.n > 0
                 && let Some(metric) = Metric::from_index(i)
             {
-                metrics.last_minute.actions.insert(
+                m.last_minute.actions.insert(
                     metric.as_str().to_string(),
                     TimedAction {
                         count: last_min.n,
@@ -606,23 +615,23 @@ impl Metrics {
             }
         }
 
-        // Lifetime ILM operations
+        // Lifetime ILM counts
         for i in 0..IlmAction::ActionCount as usize {
             let count = self.actions[i].load(Ordering::Relaxed);
             if count > 0
                 && let Some(action) = IlmAction::from_index(i)
             {
-                metrics.life_time_ilm.insert(action.as_str().to_string(), count);
+                m.life_time_ilm.insert(action.as_str().to_string(), count);
             }
         }
 
-        // Last minute ILM latency
+        // Last-minute ILM latency — plain sync calls
         for i in 0..IlmAction::ActionCount as usize {
-            let last_min = self.actions_latency[i].total().await;
+            let last_min = self.actions_latency[i].total();
             if last_min.n > 0
                 && let Some(action) = IlmAction::from_index(i)
             {
-                metrics.last_minute.ilm.insert(
+                m.last_minute.ilm.insert(
                     action.as_str().to_string(),
                     TimedAction {
                         count: last_min.n,
@@ -633,56 +642,65 @@ impl Metrics {
             }
         }
 
-        metrics
+        m
     }
 }
 
-// Type aliases for compatibility with existing code
-pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
-pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
-
-/// Create a current path updater for tracking scan progress
-pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
-    let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
-    let disk_name = disk.to_string();
-
-    // Store the tracker in global metrics
-    let tracker_clone = Arc::clone(&tracker);
-    let disk_clone = disk_name.clone();
-    tokio::spawn(async move {
-        global_metrics().current_paths.write().await.insert(disk_clone, tracker_clone);
-    });
-
-    let update_fn = {
-        let tracker = Arc::clone(&tracker);
-        Arc::new(move |path: &str| -> Pin<Box<dyn Future<Output = ()> + Send>> {
-            let tracker = Arc::clone(&tracker);
-            let path = path.to_string();
-            Box::pin(async move {
-                tracker.update_path(path).await;
-            })
-        })
-    };
-
-    let done_fn = {
-        let disk_name = disk_name.clone();
-        Arc::new(move || -> Pin<Box<dyn Future<Output = ()> + Send>> {
-            let disk_name = disk_name.clone();
-            Box::pin(async move {
-                global_metrics().current_paths.write().await.remove(&disk_name);
-            })
-        })
-    };
-
-    (update_fn, done_fn)
-}
-
 impl Default for Metrics {
     fn default() -> Self {
         Self::new()
     }
 }
 
+// ---------------------------------------------------------------------------
+// Path tracking helpers
+// ---------------------------------------------------------------------------
+
+pub type UpdateCurrentPathFn = Arc<dyn Fn(&str) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
+pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
+
+/// Register a new disk in the global path tracker and return two callbacks:
+/// one to update the current path and one to deregister the disk when done.
+pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
+    let tracker = Arc::new(CurrentPathTracker::new(initial.to_string()));
+    let disk_name = disk.to_string();
+
+    let tracker_clone = Arc::clone(&tracker);
+    let disk_insert = disk_name.clone();
+    tokio::spawn(async move {
+        global_metrics()
+            .current_paths
+            .write()
+            .await
+            .insert(disk_insert, tracker_clone);
+    });
+
+    let update_fn: UpdateCurrentPathFn = {
+        let tracker = Arc::clone(&tracker);
+        Arc::new(move |path: &str| {
+            let tracker = Arc::clone(&tracker);
+            let path = path.to_string();
+            Box::pin(async move { tracker.update_path(path).await })
+        })
+    };
+
+    let done_fn: CloseDiskFn = {
+        let disk = disk_name;
+        Arc::new(move || {
+            let disk = disk.clone();
+            Box::pin(async move {
+                global_metrics().current_paths.write().await.remove(&disk);
+            })
+        })
+    };
+
+    (update_fn, done_fn)
+}
+
+// ---------------------------------------------------------------------------
+// CloseDiskGuard
+// ---------------------------------------------------------------------------
+
 pub struct CloseDiskGuard(CloseDiskFn);
 
 impl CloseDiskGuard {
@@ -697,16 +715,10 @@ impl CloseDiskGuard {
 
 impl Drop for CloseDiskGuard {
     fn drop(&mut self) {
-        // Drop cannot be async, so we spawn the async cleanup task
-        // The task will run in the background and complete asynchronously
         if let Ok(handle) = tokio::runtime::Handle::try_current() {
             let close_fn = self.0.clone();
-            handle.spawn(async move {
-                close_fn().await;
-            });
-        } else {
-            // If we're not in a tokio runtime context, we can't spawn
-            // This is a best-effort cleanup, so we just skip it
+            handle.spawn(async move { close_fn().await });
         }
+        // If there is no runtime we are in a test or shutdown path; skip cleanup.
     }
 }

crates/common/src/readiness.rs

@@ -60,6 +60,23 @@ impl GlobalReadiness {
     pub fn is_ready(&self) -> bool {
         self.status.load(Ordering::SeqCst) == SystemStage::FullReady as u8
     }
+
+    /// Get the current system stage
+    /// # Returns
+    /// The current SystemStage of the service
+    pub fn current_stage(&self) -> SystemStage {
+        match self.status.load(Ordering::SeqCst) {
+            0 => SystemStage::Booting,
+            1 => SystemStage::StorageReady,
+            2 => SystemStage::IamReady,
+            3 => SystemStage::FullReady,
+            invalid => {
+                debug_assert!(false, "GlobalReadiness::current_stage: invalid status value {}", invalid);
+                // Fallback to the most conservative stage on invalid values
+                SystemStage::Booting
+            }
+        }
+    }
 }
 
 #[cfg(test)]

crates/concurrency/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "rustfs-concurrency"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling"
+keywords = ["rustfs", "concurrency", "timeout", "backpressure", "scheduling"]
+categories = ["concurrency", "filesystem"]
+
+[dependencies]
+# Internal crates
+rustfs-io-core = { workspace = true }
+rustfs-io-metrics = { workspace = true }
+
+# Async runtime
+tokio = { workspace = true, features = ["sync", "time", "rt"] }
+tokio-util = { workspace = true }
+
+# Error handling
+thiserror = { workspace = true }
+
+# Logging
+tracing = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["full"] }
+
+[features]
+default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"]
+
+# Feature modules
+timeout = []
+lock = []
+deadlock = []
+backpressure = []
+scheduler = []
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]

crates/concurrency/src/backpressure.rs

@@ -0,0 +1,224 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Backpressure management
+
+use rustfs_io_core::{BackpressureMonitor as CoreBackpressureMonitor, BackpressureState};
+use rustfs_io_metrics::backpressure_metrics;
+use std::sync::Arc;
+use std::time::Instant;
+use tokio::io::{DuplexStream, duplex};
+
+/// Backpressure configuration
+#[derive(Debug, Clone)]
+pub struct BackpressureConfig {
+    /// Buffer size in bytes
+    pub buffer_size: usize,
+    /// High watermark percentage
+    pub high_watermark: u32,
+    /// Low watermark percentage
+    pub low_watermark: u32,
+}
+
+impl Default for BackpressureConfig {
+    fn default() -> Self {
+        Self {
+            buffer_size: 4 * 1024 * 1024, // 4MB
+            high_watermark: 80,
+            low_watermark: 50,
+        }
+    }
+}
+
+impl BackpressureConfig {
+    /// Calculate high watermark threshold in bytes
+    pub fn high_watermark_bytes(&self) -> usize {
+        (self.buffer_size as u64 * self.high_watermark as u64 / 100) as usize
+    }
+
+    /// Calculate low watermark threshold in bytes
+    pub fn low_watermark_bytes(&self) -> usize {
+        (self.buffer_size as u64 * self.low_watermark as u64 / 100) as usize
+    }
+}
+
+/// Backpressure manager
+pub struct BackpressureManager {
+    config: BackpressureConfig,
+    monitor: Arc<CoreBackpressureMonitor>,
+}
+
+impl BackpressureManager {
+    /// Create a new backpressure manager
+    pub fn new(buffer_size: usize, high_watermark: u32, low_watermark: u32) -> Self {
+        let config = BackpressureConfig {
+            buffer_size,
+            high_watermark,
+            low_watermark,
+        };
+
+        let core_config = rustfs_io_core::BackpressureConfig {
+            max_concurrent: 32,
+            high_water_mark: high_watermark as f64 / 100.0,
+            low_water_mark: low_watermark as f64 / 100.0,
+            cooldown: std::time::Duration::from_millis(100),
+            enabled: true,
+        };
+
+        Self {
+            config,
+            monitor: Arc::new(CoreBackpressureMonitor::new(core_config)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &BackpressureConfig {
+        &self.config
+    }
+
+    /// Get the monitor
+    pub fn monitor(&self) -> Arc<CoreBackpressureMonitor> {
+        self.monitor.clone()
+    }
+
+    /// Create a backpressure pipe
+    pub fn create_pipe(&self) -> BackpressurePipe {
+        BackpressurePipe::new(self.config.clone(), self.monitor.clone())
+    }
+
+    /// Get current state
+    pub fn state(&self) -> BackpressureState {
+        self.monitor.state()
+    }
+
+    /// Check if backpressure is active
+    pub fn is_active(&self) -> bool {
+        self.monitor.is_active()
+    }
+}
+
+/// Backpressure pipe wrapping tokio's duplex
+pub struct BackpressurePipe {
+    reader: DuplexStream,
+    writer: DuplexStream,
+    config: BackpressureConfig,
+    monitor: Arc<CoreBackpressureMonitor>,
+    created_at: Instant,
+}
+
+impl BackpressurePipe {
+    fn new(config: BackpressureConfig, monitor: Arc<CoreBackpressureMonitor>) -> Self {
+        let (reader, writer) = duplex(config.buffer_size);
+
+        Self {
+            reader,
+            writer,
+            config,
+            monitor,
+            created_at: Instant::now(),
+        }
+    }
+
+    /// Get the reader end
+    pub fn reader(&mut self) -> &mut DuplexStream {
+        &mut self.reader
+    }
+
+    /// Get the writer end
+    pub fn writer(&mut self) -> &mut DuplexStream {
+        &mut self.writer
+    }
+
+    /// Split into reader and writer
+    pub fn into_split(self) -> (DuplexStream, DuplexStream) {
+        (self.reader, self.writer)
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &BackpressureConfig {
+        &self.config
+    }
+
+    /// Get current state
+    pub fn state(&self) -> BackpressureState {
+        self.monitor.state()
+    }
+
+    /// Get the age of this pipe
+    pub fn age(&self) -> std::time::Duration {
+        self.created_at.elapsed()
+    }
+
+    /// Check if should apply backpressure
+    pub fn should_apply_backpressure(&self) -> bool {
+        let should = self.monitor.should_apply_backpressure();
+        if should {
+            backpressure_metrics::record_backpressure_activation();
+        }
+        should
+    }
+}
+
+/// Backpressure event
+#[allow(dead_code)]
+#[derive(Debug, Clone)]
+pub struct BackpressureEvent {
+    /// Event timestamp
+    pub timestamp: Instant,
+    /// Event type
+    pub event_type: BackpressureEventType,
+    /// Buffer usage
+    pub buffer_usage: usize,
+    /// Buffer capacity
+    pub buffer_capacity: usize,
+}
+
+/// Backpressure event type
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy)]
+pub enum BackpressureEventType {
+    /// High watermark reached
+    HighWatermarkReached,
+    /// High watermark exited
+    HighWatermarkExited,
+    /// Backpressure applied
+    BackpressureApplied,
+    /// Backpressure released
+    BackpressureReleased,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_backpressure_config() {
+        let config = BackpressureConfig::default();
+        assert_eq!(config.buffer_size, 4 * 1024 * 1024);
+        assert!(config.high_watermark > config.low_watermark);
+    }
+
+    #[test]
+    fn test_backpressure_manager() {
+        let manager = BackpressureManager::new(1024, 80, 50);
+        assert_eq!(manager.state(), BackpressureState::Normal);
+    }
+
+    #[test]
+    fn test_backpressure_pipe() {
+        let manager = BackpressureManager::new(1024, 80, 50);
+        let pipe = manager.create_pipe();
+        assert_eq!(pipe.state(), BackpressureState::Normal);
+    }
+}

crates/concurrency/src/config.rs

@@ -0,0 +1,256 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Configuration for concurrency management
+
+use std::time::Duration;
+
+/// Feature flags for concurrency modules
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ConcurrencyFeatures {
+    /// Enable timeout control
+    pub timeout: bool,
+    /// Enable lock optimization
+    pub lock: bool,
+    /// Enable deadlock detection
+    pub deadlock: bool,
+    /// Enable backpressure management
+    pub backpressure: bool,
+    /// Enable I/O scheduling
+    pub scheduler: bool,
+}
+
+impl Default for ConcurrencyFeatures {
+    fn default() -> Self {
+        Self {
+            timeout: cfg!(feature = "timeout"),
+            lock: cfg!(feature = "lock"),
+            deadlock: cfg!(feature = "deadlock"),
+            backpressure: cfg!(feature = "backpressure"),
+            scheduler: cfg!(feature = "scheduler"),
+        }
+    }
+}
+
+impl ConcurrencyFeatures {
+    /// Create with all features enabled
+    pub fn all() -> Self {
+        Self {
+            timeout: true,
+            lock: true,
+            deadlock: true,
+            backpressure: true,
+            scheduler: true,
+        }
+    }
+
+    /// Create with no features enabled
+    pub fn none() -> Self {
+        Self {
+            timeout: false,
+            lock: false,
+            deadlock: false,
+            backpressure: false,
+            scheduler: false,
+        }
+    }
+
+    /// Check if any feature is enabled
+    pub fn any_enabled(&self) -> bool {
+        self.timeout || self.lock || self.deadlock || self.backpressure || self.scheduler
+    }
+}
+
+/// Main configuration for concurrency management
+#[derive(Debug, Clone)]
+pub struct ConcurrencyConfig {
+    /// Feature flags
+    pub features: ConcurrencyFeatures,
+
+    // Timeout configuration
+    /// Default timeout duration
+    pub default_timeout: Duration,
+    /// Maximum timeout duration
+    pub max_timeout: Duration,
+    /// Enable dynamic timeout
+    pub enable_dynamic_timeout: bool,
+
+    // Lock configuration
+    /// Enable lock optimization
+    pub enable_lock_optimization: bool,
+    /// Lock acquisition timeout
+    pub lock_acquire_timeout: Duration,
+
+    // Deadlock configuration
+    /// Enable deadlock detection
+    pub enable_deadlock_detection: bool,
+    /// Deadlock check interval
+    pub deadlock_check_interval: Duration,
+    /// Hang threshold
+    pub hang_threshold: Duration,
+
+    // Backpressure configuration
+    /// Buffer size for backpressure
+    pub backpressure_buffer_size: usize,
+    /// High watermark percentage
+    pub high_watermark: u32,
+    /// Low watermark percentage
+    pub low_watermark: u32,
+
+    // Scheduler configuration
+    /// Base buffer size for I/O
+    pub io_buffer_size: usize,
+    /// Maximum buffer size
+    pub max_buffer_size: usize,
+    /// High priority size threshold
+    pub high_priority_threshold: usize,
+    /// Low priority size threshold
+    pub low_priority_threshold: usize,
+}
+
+impl Default for ConcurrencyConfig {
+    fn default() -> Self {
+        Self {
+            features: ConcurrencyFeatures::default(),
+
+            // Timeout defaults
+            default_timeout: Duration::from_secs(30),
+            max_timeout: Duration::from_secs(300),
+            enable_dynamic_timeout: true,
+
+            // Lock defaults
+            enable_lock_optimization: true,
+            lock_acquire_timeout: Duration::from_secs(5),
+
+            // Deadlock defaults
+            enable_deadlock_detection: false,
+            deadlock_check_interval: Duration::from_secs(10),
+            hang_threshold: Duration::from_secs(60),
+
+            // Backpressure defaults
+            backpressure_buffer_size: 4 * 1024 * 1024, // 4MB
+            high_watermark: 80,
+            low_watermark: 50,
+
+            // Scheduler defaults
+            io_buffer_size: 64 * 1024,                // 64KB
+            max_buffer_size: 4 * 1024 * 1024,         // 4MB
+            high_priority_threshold: 1024 * 1024,     // 1MB
+            low_priority_threshold: 10 * 1024 * 1024, // 10MB
+        }
+    }
+}
+
+impl ConcurrencyConfig {
+    /// Create configuration from environment variables
+    pub fn from_env() -> Self {
+        let mut config = Self::default();
+
+        // Read from environment if available
+        if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_DEFAULT")
+            && let Ok(secs) = val.parse::<u64>()
+        {
+            config.default_timeout = Duration::from_secs(secs);
+        }
+
+        if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_MAX")
+            && let Ok(secs) = val.parse::<u64>()
+        {
+            config.max_timeout = Duration::from_secs(secs);
+        }
+
+        if let Ok(val) = std::env::var("RUSTFS_BACKPRESSURE_BUFFER_SIZE")
+            && let Ok(size) = val.parse::<usize>()
+        {
+            config.backpressure_buffer_size = size;
+        }
+
+        if let Ok(val) = std::env::var("RUSTFS_IO_BUFFER_SIZE")
+            && let Ok(size) = val.parse::<usize>()
+        {
+            config.io_buffer_size = size;
+        }
+
+        config
+    }
+
+    /// Validate configuration
+    pub fn validate(&self) -> Result<(), ConfigError> {
+        if self.default_timeout > self.max_timeout {
+            return Err(ConfigError::InvalidTimeout("default_timeout cannot exceed max_timeout".to_string()));
+        }
+
+        if self.high_watermark <= self.low_watermark || self.high_watermark > 100 {
+            return Err(ConfigError::InvalidBackpressure(
+                "high_watermark must be > low_watermark and <= 100".to_string(),
+            ));
+        }
+
+        if self.io_buffer_size > self.max_buffer_size {
+            return Err(ConfigError::InvalidScheduler("io_buffer_size cannot exceed max_buffer_size".to_string()));
+        }
+
+        Ok(())
+    }
+}
+
+/// Configuration error
+#[allow(clippy::enum_variant_names)]
+#[derive(Debug, Clone, thiserror::Error)]
+pub enum ConfigError {
+    /// Invalid timeout configuration
+    #[error("Invalid timeout config: {0}")]
+    InvalidTimeout(String),
+
+    /// Invalid backpressure configuration
+    #[error("Invalid backpressure config: {0}")]
+    InvalidBackpressure(String),
+
+    /// Invalid scheduler configuration
+    #[error("Invalid scheduler config: {0}")]
+    InvalidScheduler(String),
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_default_config() {
+        let config = ConcurrencyConfig::default();
+        assert!(config.validate().is_ok());
+    }
+
+    #[test]
+    fn test_invalid_timeout() {
+        let config = ConcurrencyConfig {
+            default_timeout: Duration::from_secs(100),
+            max_timeout: Duration::from_secs(50),
+            ..Default::default()
+        };
+        assert!(
+            config.validate().is_err(),
+            "validate() should return an error when default_timeout > max_timeout"
+        );
+    }
+
+    #[test]
+    fn test_features() {
+        let features = ConcurrencyFeatures::all();
+        assert!(features.any_enabled());
+
+        let features = ConcurrencyFeatures::none();
+        assert!(!features.any_enabled());
+    }
+}

crates/concurrency/src/deadlock.rs

@@ -0,0 +1,207 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Deadlock detection management
+
+use rustfs_io_core::{DeadlockDetector as CoreDeadlockDetector, LockType};
+use rustfs_io_metrics::deadlock_metrics;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+/// Deadlock configuration
+#[derive(Debug, Clone)]
+pub struct DeadlockConfig {
+    /// Enable deadlock detection
+    pub enabled: bool,
+    /// Check interval
+    pub check_interval: Duration,
+    /// Hang threshold
+    pub hang_threshold: Duration,
+}
+
+impl Default for DeadlockConfig {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            check_interval: Duration::from_secs(10),
+            hang_threshold: Duration::from_secs(60),
+        }
+    }
+}
+
+/// Deadlock manager
+pub struct DeadlockManager {
+    config: DeadlockConfig,
+    detector: Arc<CoreDeadlockDetector>,
+    running: Arc<tokio::sync::Mutex<bool>>,
+}
+
+impl DeadlockManager {
+    /// Create a new deadlock manager
+    pub fn new(enabled: bool, check_interval: Duration, hang_threshold: Duration) -> Self {
+        let config = DeadlockConfig {
+            enabled,
+            check_interval,
+            hang_threshold,
+        };
+
+        let core_config = rustfs_io_core::DeadlockDetectorConfig {
+            enabled,
+            detection_interval: check_interval,
+            max_hold_time: hang_threshold,
+        };
+
+        Self {
+            config,
+            detector: Arc::new(CoreDeadlockDetector::new(core_config)),
+            running: Arc::new(tokio::sync::Mutex::new(false)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &DeadlockConfig {
+        &self.config
+    }
+
+    /// Get the core detector
+    pub fn detector(&self) -> Arc<CoreDeadlockDetector> {
+        self.detector.clone()
+    }
+
+    /// Start the deadlock detection background task
+    pub async fn start(&self) {
+        if !self.config.enabled {
+            return;
+        }
+
+        let mut running = self.running.lock().await;
+        if *running {
+            return;
+        }
+        *running = true;
+        drop(running);
+
+        tracing::info!("Deadlock detection started");
+    }
+
+    /// Stop the deadlock detection
+    pub async fn stop(&self) {
+        let mut running = self.running.lock().await;
+        *running = false;
+
+        tracing::info!("Deadlock detection stopped");
+    }
+
+    /// Create a request tracker
+    pub fn track_request(&self, request_id: String, description: String) -> RequestTracker {
+        RequestTracker::new(request_id, description, self.detector.clone())
+    }
+
+    /// Register a lock
+    pub fn register_lock(&self, lock_type: LockType) -> u64 {
+        self.detector.register_lock(lock_type)
+    }
+
+    /// Unregister a lock
+    pub fn unregister_lock(&self, lock_id: u64) {
+        self.detector.unregister_lock(lock_id);
+    }
+
+    /// Detect deadlock
+    pub fn detect_deadlock(&self) -> Option<Vec<u64>> {
+        let result = self.detector.detect_deadlock();
+        if let Some(ref cycle) = result {
+            deadlock_metrics::record_deadlock_detected(cycle.len());
+        }
+        result
+    }
+}
+
+/// Request tracker for tracking resources
+pub struct RequestTracker {
+    request_id: String,
+    description: String,
+    start_time: Instant,
+    resources: HashMap<String, Vec<String>>,
+    detector: Arc<CoreDeadlockDetector>,
+}
+
+impl RequestTracker {
+    fn new(request_id: String, description: String, detector: Arc<CoreDeadlockDetector>) -> Self {
+        let start_time = Instant::now();
+        detector.register_request(&request_id, 1); // Use placeholder thread ID
+
+        Self {
+            request_id,
+            description,
+            start_time,
+            resources: HashMap::new(),
+            detector,
+        }
+    }
+
+    /// Get the request ID
+    pub fn request_id(&self) -> &str {
+        &self.request_id
+    }
+
+    /// Get the description
+    pub fn description(&self) -> &str {
+        &self.description
+    }
+
+    /// Get the elapsed time
+    pub fn elapsed(&self) -> Duration {
+        self.start_time.elapsed()
+    }
+
+    /// Record a lock acquisition
+    pub fn record_lock_acquire(&mut self, lock_id: u64, resource: String) {
+        self.resources.entry("locks".to_string()).or_default().push(resource);
+        self.detector.record_acquire(lock_id, 1); // Use placeholder thread ID
+        deadlock_metrics::record_lock_acquisition("read");
+    }
+
+    /// Record a lock release
+    pub fn record_lock_release(&mut self, lock_id: u64) {
+        self.detector.record_release(lock_id);
+    }
+}
+
+impl Drop for RequestTracker {
+    fn drop(&mut self) {
+        self.detector.unregister_request(&self.request_id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_deadlock_manager_creation() {
+        let manager = DeadlockManager::new(false, Duration::from_secs(10), Duration::from_secs(60));
+        assert!(!manager.config().enabled);
+    }
+
+    #[tokio::test]
+    async fn test_request_tracker() {
+        let manager = DeadlockManager::new(true, Duration::from_secs(10), Duration::from_secs(60));
+        let tracker = manager.track_request("req-1".to_string(), "test request".to_string());
+
+        assert_eq!(tracker.request_id(), "req-1");
+        assert_eq!(tracker.description(), "test request");
+    }
+}

crates/concurrency/src/lib.rs

@@ -0,0 +1,171 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! # RustFS Concurrency Management
+//!
+//! This crate provides comprehensive concurrency management for RustFS,
+//! including timeout control, lock optimization, deadlock detection,
+//! backpressure management, and I/O scheduling.
+//!
+//! ## Features
+//!
+//! All features are controlled by feature flags and can be enabled/disabled at compile time:
+//!
+//! - **timeout**: Dynamic timeout calculation based on data size and transfer rate
+//! - **lock**: Early lock release to reduce contention
+//! - **deadlock**: Request tracking and cycle detection
+//! - **backpressure**: Buffer-based flow control
+//! - **scheduler**: Adaptive buffer sizing and priority queuing
+//!
+//! ## Architecture
+//!
+//! ```text
+//! rustfs-concurrency (Business Layer)
+//!     ├── timeout (Timeout Control)
+//!     ├── lock (Lock Optimization)
+//!     ├── deadlock (Deadlock Detection)
+//!     ├── backpressure (Backpressure Management)
+//!     └── scheduler (I/O Scheduling)
+//!          │
+//!          ├── rustfs-io-core (Core Algorithms)
+//!          └── rustfs-io-metrics (Metrics Collection)
+//! ```
+//!
+//! ## Usage
+//!
+//! ```rust,no_run
+//! use rustfs_concurrency::{ConcurrencyConfig, ConcurrencyManager};
+//!
+//! # #[tokio::main]
+//! # async fn main() {
+//! // Create manager with all features enabled
+//! let config = ConcurrencyConfig::default();
+//! let manager = ConcurrencyManager::new(config);
+//!
+//! // Start services
+//! manager.start().await;
+//!
+//! // Use timeout control (if enabled)
+//! if manager.is_timeout_enabled() {
+//!     let timeout_manager = manager.timeout();
+//!     let _ = timeout_manager;
+//! }
+//!
+//! // Use lock optimization (if enabled)
+//! if manager.is_lock_enabled() {
+//!     let lock_manager = manager.lock();
+//!     let _ = lock_manager;
+//! }
+//!
+//! // Stop services
+//! manager.stop().await;
+//! # }
+//! ```
+
+#![deny(missing_docs)]
+#![deny(unsafe_code)]
+#![cfg_attr(docsrs, feature(doc_cfg))]
+
+// Re-export core types from io-core
+pub use rustfs_io_core::{
+    // Backpressure types
+    BackpressureConfig as CoreBackpressureConfig,
+    BackpressureMonitor as CoreBackpressureMonitor,
+    BackpressureState,
+
+    // Deadlock types
+    DeadlockDetector as CoreDeadlockDetector,
+    IoLoadLevel,
+    IoLoadMetrics,
+    IoPriority,
+    // Scheduler types
+    IoScheduler,
+    IoSchedulingContext,
+    LockInfo,
+    LockOptimizer as CoreLockOptimizer,
+
+    // Lock types
+    LockStats as CoreLockStats,
+    LockType,
+    // Timeout types
+    OperationProgress,
+    TimeoutError,
+    TimeoutStats,
+    WaitGraphEdge,
+
+    calculate_adaptive_timeout,
+    estimate_bytes_per_second,
+};
+
+// Module declarations with feature gates
+#[cfg(feature = "timeout")]
+mod timeout;
+
+#[cfg(feature = "lock")]
+mod lock;
+
+#[cfg(feature = "deadlock")]
+mod deadlock;
+
+#[cfg(feature = "backpressure")]
+mod backpressure;
+
+#[cfg(feature = "scheduler")]
+mod scheduler;
+
+// Public module exports with feature gates
+#[cfg(feature = "timeout")]
+pub use timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager};
+
+#[cfg(feature = "lock")]
+pub use lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard};
+
+#[cfg(feature = "deadlock")]
+pub use deadlock::{DeadlockConfig, DeadlockManager, RequestTracker};
+
+#[cfg(feature = "backpressure")]
+pub use backpressure::{BackpressureConfig, BackpressureManager, BackpressurePipe};
+
+#[cfg(feature = "scheduler")]
+pub use scheduler::{IoStrategy, SchedulerConfig, SchedulerManager};
+
+// Configuration
+mod config;
+pub use config::{ConcurrencyConfig, ConcurrencyFeatures};
+
+// Manager
+mod manager;
+pub use manager::{ConcurrencyManager, GetObjectQueueSnapshot};
+
+// Prelude for convenient imports
+pub mod prelude {
+    //! Prelude module for convenient imports
+
+    #[cfg(feature = "timeout")]
+    pub use crate::timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager};
+
+    #[cfg(feature = "lock")]
+    pub use crate::lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard};
+
+    #[cfg(feature = "deadlock")]
+    pub use crate::deadlock::{DeadlockConfig, DeadlockManager, RequestTracker};
+
+    #[cfg(feature = "backpressure")]
+    pub use crate::backpressure::{BackpressureConfig, BackpressureManager, BackpressurePipe};
+
+    #[cfg(feature = "scheduler")]
+    pub use crate::scheduler::{IoStrategy, SchedulerConfig, SchedulerManager};
+
+    pub use crate::{ConcurrencyConfig, ConcurrencyFeatures, ConcurrencyManager};
+}

crates/concurrency/src/lock.rs

@@ -0,0 +1,219 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Lock optimization management
+
+use rustfs_io_core::{LockOptimizer as CoreLockOptimizer, LockStats};
+use rustfs_io_metrics::lock_metrics;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+/// Lock configuration
+#[derive(Debug, Clone)]
+pub struct LockConfig {
+    /// Enable lock optimization
+    pub enabled: bool,
+    /// Lock acquisition timeout
+    pub acquire_timeout: Duration,
+}
+
+impl Default for LockConfig {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            acquire_timeout: Duration::from_secs(5),
+        }
+    }
+}
+
+/// Lock manager
+pub struct LockManager {
+    config: LockConfig,
+    optimizer: Arc<CoreLockOptimizer>,
+}
+
+impl LockManager {
+    /// Create a new lock manager
+    pub fn new(enabled: bool, acquire_timeout: Duration) -> Self {
+        let config = LockConfig {
+            enabled,
+            acquire_timeout,
+        };
+
+        let core_config = rustfs_io_core::LockOptimizeConfig {
+            enabled,
+            acquire_timeout,
+            max_hold_time_warning: Duration::from_millis(100),
+            adaptive_spin: true,
+            max_spin_iterations: 1000,
+        };
+
+        Self {
+            config,
+            optimizer: Arc::new(CoreLockOptimizer::new(core_config)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &LockConfig {
+        &self.config
+    }
+
+    /// Get the core optimizer
+    pub fn optimizer(&self) -> Arc<CoreLockOptimizer> {
+        self.optimizer.clone()
+    }
+
+    /// Get lock statistics
+    pub fn stats(&self) -> &LockStats {
+        self.optimizer.stats()
+    }
+
+    /// Optimize a lock guard
+    pub fn optimize<G>(&self, guard: G, resource: impl Into<String>) -> OptimizedLockGuard<G> {
+        OptimizedLockGuard::new(guard, resource, self.optimizer.clone())
+    }
+
+    /// Check if optimization is enabled
+    pub fn is_enabled(&self) -> bool {
+        self.config.enabled
+    }
+}
+
+/// Optimized lock guard with early release support
+pub struct OptimizedLockGuard<G> {
+    guard: Option<G>,
+    acquire_time: Instant,
+    released: bool,
+    resource: String,
+    optimizer: Arc<CoreLockOptimizer>,
+}
+
+impl<G> OptimizedLockGuard<G> {
+    fn new(guard: G, resource: impl Into<String>, optimizer: Arc<CoreLockOptimizer>) -> Self {
+        optimizer.on_acquire();
+        lock_metrics::record_lock_optimization_enabled(optimizer.config().enabled);
+
+        Self {
+            guard: Some(guard),
+            acquire_time: Instant::now(),
+            released: false,
+            resource: resource.into(),
+            optimizer,
+        }
+    }
+
+    /// Get the lock hold time
+    pub fn hold_time(&self) -> Duration {
+        self.acquire_time.elapsed()
+    }
+
+    /// Check if the lock has been released
+    pub fn is_released(&self) -> bool {
+        self.released
+    }
+
+    /// Release the lock early
+    pub fn early_release(&mut self) {
+        if self.released {
+            return;
+        }
+
+        let hold_time = self.hold_time();
+        self.guard.take();
+        self.released = true;
+
+        self.optimizer.on_release(hold_time);
+        lock_metrics::record_lock_hold_time(hold_time);
+
+        tracing::debug!(
+            resource = %self.resource,
+            hold_time_ms = hold_time.as_millis(),
+            "Lock released early (optimization active)"
+        );
+    }
+
+    /// Get a reference to the underlying guard
+    pub fn as_ref(&self) -> Option<&G> {
+        if self.released { None } else { self.guard.as_ref() }
+    }
+}
+
+impl<G> Drop for OptimizedLockGuard<G> {
+    fn drop(&mut self) {
+        if !self.released {
+            let hold_time = self.hold_time();
+            self.guard.take();
+            self.released = true;
+
+            self.optimizer.on_release(hold_time);
+            lock_metrics::record_lock_hold_time(hold_time);
+
+            tracing::debug!(
+                resource = %self.resource,
+                hold_time_ms = hold_time.as_millis(),
+                "Lock released on drop (normal release)"
+            );
+        }
+    }
+}
+
+/// Lock scope guard for RAII semantics
+pub struct LockScopeGuard<G> {
+    guard: Option<G>,
+}
+
+impl<G> LockScopeGuard<G> {
+    /// Create a new scope guard
+    pub fn new(guard: G) -> Self {
+        Self { guard: Some(guard) }
+    }
+
+    /// Get a reference to the guard
+    pub fn as_ref(&self) -> Option<&G> {
+        self.guard.as_ref()
+    }
+}
+
+impl<G> Drop for LockScopeGuard<G> {
+    fn drop(&mut self) {
+        self.guard.take();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Mutex;
+
+    #[test]
+    fn test_lock_manager_creation() {
+        let manager = LockManager::new(true, Duration::from_secs(5));
+        assert!(manager.is_enabled());
+    }
+
+    #[test]
+    fn test_optimized_lock_guard() {
+        let manager = LockManager::new(true, Duration::from_secs(5));
+        let mutex = Mutex::new(42);
+        let guard = mutex.lock().unwrap();
+
+        let optimized = manager.optimize(guard, "test_resource");
+        assert!(!optimized.is_released());
+
+        let mut optimized = optimized;
+        optimized.early_release();
+        assert!(optimized.is_released());
+    }
+}

crates/concurrency/src/manager.rs

@@ -0,0 +1,315 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Main concurrency manager
+
+use crate::config::ConcurrencyConfig;
+use std::sync::Arc;
+
+/// Snapshot of disk permit queue usage for GetObject orchestration.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct GetObjectQueueSnapshot {
+    /// Total permits configured for disk reads.
+    pub total_permits: usize,
+    /// Permits currently in use.
+    pub permits_in_use: usize,
+}
+
+impl GetObjectQueueSnapshot {
+    /// Create a queue snapshot from total and available permits.
+    pub fn from_available_permits(total_permits: usize, available_permits: usize) -> Self {
+        Self {
+            total_permits,
+            permits_in_use: total_permits.saturating_sub(available_permits),
+        }
+    }
+
+    /// Return currently available permits.
+    pub fn permits_available(&self) -> usize {
+        self.total_permits.saturating_sub(self.permits_in_use)
+    }
+
+    /// Return queue utilization percentage in the 0-100 range.
+    pub fn utilization_percent(&self) -> f64 {
+        if self.total_permits == 0 {
+            0.0
+        } else {
+            (self.permits_in_use as f64 / self.total_permits as f64) * 100.0
+        }
+    }
+
+    /// Return whether the queue is considered congested.
+    pub fn is_congested(&self, threshold_percent: f64) -> bool {
+        self.utilization_percent() > threshold_percent
+    }
+}
+
+/// Main concurrency manager that provides access to all concurrency features
+pub struct ConcurrencyManager {
+    config: ConcurrencyConfig,
+
+    #[cfg(feature = "timeout")]
+    timeout: Arc<crate::timeout::TimeoutManager>,
+
+    #[cfg(feature = "lock")]
+    lock: Arc<crate::lock::LockManager>,
+
+    #[cfg(feature = "deadlock")]
+    deadlock: Arc<crate::deadlock::DeadlockManager>,
+
+    #[cfg(feature = "backpressure")]
+    backpressure: Arc<crate::backpressure::BackpressureManager>,
+
+    #[cfg(feature = "scheduler")]
+    scheduler: Arc<crate::scheduler::SchedulerManager>,
+}
+
+impl ConcurrencyManager {
+    /// Create a new concurrency manager with the given configuration
+    pub fn new(config: ConcurrencyConfig) -> Self {
+        // Validate configuration
+        if let Err(e) = config.validate() {
+            panic!("Invalid concurrency configuration: {}", e);
+        }
+
+        Self {
+            #[cfg(feature = "timeout")]
+            timeout: Arc::new(crate::timeout::TimeoutManager::new(
+                config.default_timeout,
+                config.max_timeout,
+                config.enable_dynamic_timeout,
+            )),
+
+            #[cfg(feature = "lock")]
+            lock: Arc::new(crate::lock::LockManager::new(
+                config.enable_lock_optimization,
+                config.lock_acquire_timeout,
+            )),
+
+            #[cfg(feature = "deadlock")]
+            deadlock: Arc::new(crate::deadlock::DeadlockManager::new(
+                config.enable_deadlock_detection,
+                config.deadlock_check_interval,
+                config.hang_threshold,
+            )),
+
+            #[cfg(feature = "backpressure")]
+            backpressure: Arc::new(crate::backpressure::BackpressureManager::new(
+                config.backpressure_buffer_size,
+                config.high_watermark,
+                config.low_watermark,
+            )),
+
+            #[cfg(feature = "scheduler")]
+            scheduler: Arc::new(crate::scheduler::SchedulerManager::new(
+                config.io_buffer_size,
+                config.max_buffer_size,
+                config.high_priority_threshold,
+                config.low_priority_threshold,
+            )),
+
+            config,
+        }
+    }
+
+    /// Create with default configuration
+    pub fn with_defaults() -> Self {
+        Self::new(ConcurrencyConfig::default())
+    }
+
+    /// Create from environment variables
+    pub fn from_env() -> Self {
+        Self::new(ConcurrencyConfig::from_env())
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &ConcurrencyConfig {
+        &self.config
+    }
+
+    // ============================================
+    // Feature enable checks
+    // ============================================
+
+    /// Check if timeout feature is enabled
+    pub fn is_timeout_enabled(&self) -> bool {
+        #[cfg(feature = "timeout")]
+        {
+            self.config.features.timeout
+        }
+        #[cfg(not(feature = "timeout"))]
+        {
+            false
+        }
+    }
+
+    /// Check if lock feature is enabled
+    pub fn is_lock_enabled(&self) -> bool {
+        #[cfg(feature = "lock")]
+        {
+            self.config.features.lock
+        }
+        #[cfg(not(feature = "lock"))]
+        {
+            false
+        }
+    }
+
+    /// Check if deadlock feature is enabled
+    pub fn is_deadlock_enabled(&self) -> bool {
+        #[cfg(feature = "deadlock")]
+        {
+            self.config.features.deadlock
+        }
+        #[cfg(not(feature = "deadlock"))]
+        {
+            false
+        }
+    }
+
+    /// Check if backpressure feature is enabled
+    pub fn is_backpressure_enabled(&self) -> bool {
+        #[cfg(feature = "backpressure")]
+        {
+            self.config.features.backpressure
+        }
+        #[cfg(not(feature = "backpressure"))]
+        {
+            false
+        }
+    }
+
+    /// Check if scheduler feature is enabled
+    pub fn is_scheduler_enabled(&self) -> bool {
+        #[cfg(feature = "scheduler")]
+        {
+            self.config.features.scheduler
+        }
+        #[cfg(not(feature = "scheduler"))]
+        {
+            false
+        }
+    }
+
+    // ============================================
+    // Feature accessors
+    // ============================================
+
+    /// Get timeout manager
+    #[cfg(feature = "timeout")]
+    pub fn timeout(&self) -> Arc<crate::timeout::TimeoutManager> {
+        self.timeout.clone()
+    }
+
+    /// Get lock manager
+    #[cfg(feature = "lock")]
+    pub fn lock(&self) -> Arc<crate::lock::LockManager> {
+        self.lock.clone()
+    }
+
+    /// Get deadlock manager
+    #[cfg(feature = "deadlock")]
+    pub fn deadlock(&self) -> Arc<crate::deadlock::DeadlockManager> {
+        self.deadlock.clone()
+    }
+
+    /// Get backpressure manager
+    #[cfg(feature = "backpressure")]
+    pub fn backpressure(&self) -> Arc<crate::backpressure::BackpressureManager> {
+        self.backpressure.clone()
+    }
+
+    /// Get scheduler manager
+    #[cfg(feature = "scheduler")]
+    pub fn scheduler(&self) -> Arc<crate::scheduler::SchedulerManager> {
+        self.scheduler.clone()
+    }
+
+    // ============================================
+    // Lifecycle management
+    // ============================================
+
+    /// Start all enabled services (e.g., deadlock detection background task)
+    pub async fn start(&self) {
+        #[cfg(feature = "deadlock")]
+        {
+            if self.config.enable_deadlock_detection {
+                self.deadlock.start().await;
+            }
+        }
+
+        tracing::info!(
+            "Concurrency manager started (timeout={}, lock={}, deadlock={}, backpressure={}, scheduler={})",
+            self.is_timeout_enabled(),
+            self.is_lock_enabled(),
+            self.is_deadlock_enabled(),
+            self.is_backpressure_enabled(),
+            self.is_scheduler_enabled()
+        );
+    }
+
+    /// Stop all services
+    pub async fn stop(&self) {
+        #[cfg(feature = "deadlock")]
+        {
+            self.deadlock.stop().await;
+        }
+
+        tracing::info!("Concurrency manager stopped");
+    }
+}
+
+impl Default for ConcurrencyManager {
+    fn default() -> Self {
+        Self::with_defaults()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_queue_snapshot() {
+        let snapshot = GetObjectQueueSnapshot::from_available_permits(64, 16);
+        assert_eq!(snapshot.permits_in_use, 48);
+        assert_eq!(snapshot.permits_available(), 16);
+        assert!(snapshot.is_congested(70.0));
+    }
+
+    #[test]
+    fn test_manager_creation() {
+        let manager = ConcurrencyManager::with_defaults();
+        assert!(manager.config().validate().is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_manager_lifecycle() {
+        let manager = ConcurrencyManager::with_defaults();
+        manager.start().await;
+        manager.stop().await;
+    }
+
+    #[test]
+    fn test_feature_checks() {
+        let manager = ConcurrencyManager::with_defaults();
+
+        // These should return the feature flag status
+        let _ = manager.is_timeout_enabled();
+        let _ = manager.is_lock_enabled();
+        let _ = manager.is_deadlock_enabled();
+        let _ = manager.is_backpressure_enabled();
+        let _ = manager.is_scheduler_enabled();
+    }
+}

crates/concurrency/src/scheduler.rs

@@ -0,0 +1,225 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! I/O scheduler management
+
+use rustfs_io_core::{
+    IoLoadLevel, IoPriority, IoScheduler as CoreIoScheduler, IoSchedulingContext,
+    io_profile::{AccessPattern, StorageMedia},
+};
+use rustfs_io_metrics::io_metrics;
+use std::sync::Arc;
+use std::time::Duration;
+
+/// Scheduler configuration
+#[derive(Debug, Clone)]
+pub struct SchedulerConfig {
+    /// Base buffer size
+    pub base_buffer_size: usize,
+    /// Maximum buffer size
+    pub max_buffer_size: usize,
+    /// High priority threshold
+    pub high_priority_threshold: usize,
+    /// Low priority threshold
+    pub low_priority_threshold: usize,
+}
+
+impl Default for SchedulerConfig {
+    fn default() -> Self {
+        Self {
+            base_buffer_size: 64 * 1024,              // 64KB
+            max_buffer_size: 4 * 1024 * 1024,         // 4MB
+            high_priority_threshold: 1024 * 1024,     // 1MB
+            low_priority_threshold: 10 * 1024 * 1024, // 10MB
+        }
+    }
+}
+
+/// Scheduler manager
+pub struct SchedulerManager {
+    config: SchedulerConfig,
+    scheduler: Arc<CoreIoScheduler>,
+}
+
+impl SchedulerManager {
+    /// Create a new scheduler manager
+    pub fn new(
+        base_buffer_size: usize,
+        max_buffer_size: usize,
+        high_priority_threshold: usize,
+        low_priority_threshold: usize,
+    ) -> Self {
+        let config = SchedulerConfig {
+            base_buffer_size,
+            max_buffer_size,
+            high_priority_threshold,
+            low_priority_threshold,
+        };
+
+        let core_config = rustfs_io_core::IoSchedulerConfig::default();
+
+        Self {
+            config,
+            scheduler: Arc::new(CoreIoScheduler::new(core_config)),
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &SchedulerConfig {
+        &self.config
+    }
+
+    /// Get the scheduler
+    pub fn scheduler(&self) -> Arc<CoreIoScheduler> {
+        self.scheduler.clone()
+    }
+
+    /// Create an I/O strategy
+    pub fn create_strategy(&self) -> IoStrategy {
+        IoStrategy::new(self.config.clone(), self.scheduler.clone())
+    }
+
+    /// Calculate buffer size
+    pub fn calculate_buffer_size(
+        &self,
+        file_size: i64,
+        media: StorageMedia,
+        pattern: AccessPattern,
+        load: IoLoadLevel,
+        concurrent: usize,
+    ) -> usize {
+        let strategy = self.create_strategy();
+        strategy.calculate_buffer_size(file_size, media, pattern, load, concurrent)
+    }
+
+    /// Get I/O priority
+    pub fn get_priority(&self, size: i64) -> IoPriority {
+        IoPriority::from_size(size, self.config.high_priority_threshold, self.config.low_priority_threshold)
+    }
+}
+
+/// I/O strategy
+pub struct IoStrategy {
+    config: SchedulerConfig,
+    scheduler: Arc<CoreIoScheduler>,
+}
+
+impl IoStrategy {
+    fn new(config: SchedulerConfig, scheduler: Arc<CoreIoScheduler>) -> Self {
+        Self { config, scheduler }
+    }
+
+    /// Calculate buffer size with multi-factor strategy
+    pub fn calculate_buffer_size(
+        &self,
+        file_size: i64,
+        media: StorageMedia,
+        pattern: AccessPattern,
+        load: IoLoadLevel,
+        concurrent: usize,
+    ) -> usize {
+        // Create scheduling context
+        let _ctx = IoSchedulingContext::new(file_size, self.config.base_buffer_size)
+            .with_sequential(matches!(pattern, AccessPattern::Sequential))
+            .with_media(media);
+
+        // Get base buffer size from core scheduler
+        let permit_wait = Duration::from_millis(10); // Default wait time
+        let is_sequential = matches!(pattern, AccessPattern::Sequential);
+        let core_strategy = self.scheduler.calculate_strategy(file_size, permit_wait, is_sequential);
+        let base_size = core_strategy.buffer_size;
+
+        // Apply multi-factor adjustments
+        let adjusted_size = self.apply_adjustments(base_size, media, pattern, load, concurrent);
+
+        // Record metrics
+        io_metrics::record_io_scheduler_decision(adjusted_size, load.as_str(), pattern.as_str());
+
+        adjusted_size.min(self.config.max_buffer_size)
+    }
+
+    fn apply_adjustments(
+        &self,
+        base_size: usize,
+        media: StorageMedia,
+        pattern: AccessPattern,
+        load: IoLoadLevel,
+        concurrent: usize,
+    ) -> usize {
+        let mut size = base_size;
+
+        // Media adjustment
+        size = match media {
+            StorageMedia::Nvme => (size as f64 * 1.5) as usize,
+            StorageMedia::Ssd => (size as f64 * 1.2) as usize,
+            StorageMedia::Hdd => size,
+            _ => size,
+        };
+
+        // Pattern adjustment
+        size = match pattern {
+            AccessPattern::Sequential => (size as f64 * 1.5) as usize,
+            AccessPattern::Random => (size as f64 * 0.5) as usize,
+            _ => size,
+        };
+
+        // Load adjustment
+        size = match load {
+            IoLoadLevel::Low => (size as f64 * 1.2) as usize,
+            IoLoadLevel::Medium => size,
+            IoLoadLevel::High => (size as f64 * 0.7) as usize,
+            IoLoadLevel::Critical => (size as f64 * 0.5) as usize,
+        };
+
+        // Concurrency adjustment
+        if concurrent > 10 {
+            size = (size as f64 * 0.8) as usize;
+        }
+
+        size
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &SchedulerConfig {
+        &self.config
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_scheduler_config() {
+        let config = SchedulerConfig::default();
+        assert!(config.base_buffer_size < config.max_buffer_size);
+    }
+
+    #[test]
+    fn test_scheduler_manager() {
+        let manager = SchedulerManager::new(1024, 4096, 512, 2048);
+        let priority = manager.get_priority(100);
+        assert!(priority.is_high());
+    }
+
+    #[test]
+    fn test_io_strategy() {
+        let manager = SchedulerManager::new(1024, 4096, 512, 2048);
+        let strategy = manager.create_strategy();
+
+        let size = strategy.calculate_buffer_size(1024 * 1024, StorageMedia::Ssd, AccessPattern::Sequential, IoLoadLevel::Low, 1);
+
+        assert!(size > 0);
+    }
+}

crates/concurrency/src/timeout.rs

@@ -0,0 +1,150 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Timeout management for operations
+
+use rustfs_io_core::{TimeoutError, calculate_adaptive_timeout};
+use std::time::{Duration, Instant};
+use tokio_util::sync::CancellationToken;
+
+/// Timeout configuration
+#[derive(Debug, Clone)]
+pub struct TimeoutConfig {
+    /// Default timeout duration
+    pub default_timeout: Duration,
+    /// Maximum timeout duration
+    pub max_timeout: Duration,
+    /// Enable dynamic timeout calculation
+    pub enable_dynamic: bool,
+}
+
+impl Default for TimeoutConfig {
+    fn default() -> Self {
+        Self {
+            default_timeout: Duration::from_secs(30),
+            max_timeout: Duration::from_secs(300),
+            enable_dynamic: true,
+        }
+    }
+}
+
+/// Timeout manager
+pub struct TimeoutManager {
+    config: TimeoutConfig,
+}
+
+impl TimeoutManager {
+    /// Create a new timeout manager
+    pub fn new(default_timeout: Duration, max_timeout: Duration, enable_dynamic: bool) -> Self {
+        Self {
+            config: TimeoutConfig {
+                default_timeout,
+                max_timeout,
+                enable_dynamic,
+            },
+        }
+    }
+
+    /// Get the configuration
+    pub fn config(&self) -> &TimeoutConfig {
+        &self.config
+    }
+
+    /// Calculate timeout for a given size
+    pub fn calculate_timeout(&self, size: u64, _history: &[Duration]) -> Duration {
+        if !self.config.enable_dynamic {
+            return self.config.default_timeout;
+        }
+
+        calculate_adaptive_timeout(self.config.default_timeout, None, 0, size).min(self.config.max_timeout)
+    }
+
+    /// Wrap an operation with timeout control
+    pub async fn wrap_operation<F, T, E>(&self, operation: F, timeout: Option<Duration>) -> Result<T, TimeoutError>
+    where
+        F: std::future::Future<Output = Result<T, E>>,
+        E: Into<TimeoutError>,
+    {
+        let timeout = timeout.unwrap_or(self.config.default_timeout);
+
+        match tokio::time::timeout(timeout, operation).await {
+            Ok(Ok(result)) => Ok(result),
+            Ok(Err(e)) => Err(e.into()),
+            Err(_) => Err(TimeoutError::TimedOut(timeout)),
+        }
+    }
+
+    /// Create a timeout guard for manual timeout control
+    pub fn create_guard(&self, timeout: Option<Duration>) -> TimeoutGuard {
+        TimeoutGuard::new(timeout.unwrap_or(self.config.default_timeout))
+    }
+}
+
+/// Timeout guard for manual timeout control
+pub struct TimeoutGuard {
+    timeout: Duration,
+    start: Instant,
+    cancel_token: CancellationToken,
+}
+
+impl TimeoutGuard {
+    fn new(timeout: Duration) -> Self {
+        Self {
+            timeout,
+            start: Instant::now(),
+            cancel_token: CancellationToken::new(),
+        }
+    }
+
+    /// Check if timeout has elapsed
+    pub fn is_timed_out(&self) -> bool {
+        self.start.elapsed() > self.timeout
+    }
+
+    /// Get remaining time
+    pub fn remaining(&self) -> Duration {
+        self.timeout.saturating_sub(self.start.elapsed())
+    }
+
+    /// Get the cancellation token
+    pub fn cancel_token(&self) -> CancellationToken {
+        self.cancel_token.clone()
+    }
+
+    /// Cancel the operation
+    pub fn cancel(&self) {
+        self.cancel_token.cancel();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_timeout_config() {
+        let config = TimeoutConfig::default();
+        assert!(config.default_timeout < config.max_timeout);
+    }
+
+    #[tokio::test]
+    async fn test_wrap_operation_success() {
+        let manager = TimeoutManager::new(Duration::from_secs(5), Duration::from_secs(10), true);
+
+        let result = manager.wrap_operation(async { Ok::<_, TimeoutError>(42) }, None).await;
+
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), 42);
+    }
+}

crates/config/Cargo.toml

@@ -37,3 +37,6 @@ constants = ["dep:const-str"]
 notify = ["dep:const-str", "constants"]
 observability = ["constants"]
 opa = ["constants"]
+
+[lib]
+doctest = false

crates/config/src/audit/mqtt.rs

@@ -23,9 +23,15 @@ pub const ENV_AUDIT_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_AUDIT_MQTT_RECONNECT
 pub const ENV_AUDIT_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_AUDIT_MQTT_KEEP_ALIVE_INTERVAL";
 pub const ENV_AUDIT_MQTT_QUEUE_DIR: &str = "RUSTFS_AUDIT_MQTT_QUEUE_DIR";
 pub const ENV_AUDIT_MQTT_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_MQTT_QUEUE_LIMIT";
+pub const ENV_AUDIT_MQTT_TLS_POLICY: &str = "RUSTFS_AUDIT_MQTT_TLS_POLICY";
+pub const ENV_AUDIT_MQTT_TLS_CA: &str = "RUSTFS_AUDIT_MQTT_TLS_CA";
+pub const ENV_AUDIT_MQTT_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_MQTT_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_MQTT_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_MQTT_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_MQTT_TLS_TRUST_LEAF_AS_CA: &str = "RUSTFS_AUDIT_MQTT_TLS_TRUST_LEAF_AS_CA";
+pub const ENV_AUDIT_MQTT_WS_PATH_ALLOWLIST: &str = "RUSTFS_AUDIT_MQTT_WS_PATH_ALLOWLIST";
 
 /// A list of all valid configuration keys for an MQTT target.
-pub const ENV_AUDIT_MQTT_KEYS: &[&str; 10] = &[
+pub const ENV_AUDIT_MQTT_KEYS: &[&str; 16] = &[
     ENV_AUDIT_MQTT_ENABLE,
     ENV_AUDIT_MQTT_BROKER,
     ENV_AUDIT_MQTT_TOPIC,
@@ -36,6 +42,12 @@ pub const ENV_AUDIT_MQTT_KEYS: &[&str; 10] = &[
     ENV_AUDIT_MQTT_KEEP_ALIVE_INTERVAL,
     ENV_AUDIT_MQTT_QUEUE_DIR,
     ENV_AUDIT_MQTT_QUEUE_LIMIT,
+    ENV_AUDIT_MQTT_TLS_POLICY,
+    ENV_AUDIT_MQTT_TLS_CA,
+    ENV_AUDIT_MQTT_TLS_CLIENT_CERT,
+    ENV_AUDIT_MQTT_TLS_CLIENT_KEY,
+    ENV_AUDIT_MQTT_TLS_TRUST_LEAF_AS_CA,
+    ENV_AUDIT_MQTT_WS_PATH_ALLOWLIST,
 ];
 
 /// A list of all valid configuration keys for an MQTT target.
@@ -50,5 +62,11 @@ pub const AUDIT_MQTT_KEYS: &[&str] = &[
     crate::MQTT_KEEP_ALIVE_INTERVAL,
     crate::MQTT_QUEUE_DIR,
     crate::MQTT_QUEUE_LIMIT,
+    crate::MQTT_TLS_POLICY,
+    crate::MQTT_TLS_CA,
+    crate::MQTT_TLS_CLIENT_CERT,
+    crate::MQTT_TLS_CLIENT_KEY,
+    crate::MQTT_TLS_TRUST_LEAF_AS_CA,
+    crate::MQTT_WS_PATH_ALLOWLIST,
     crate::COMMENT_KEY,
 ];

crates/config/src/constants/app.rs

@@ -131,9 +131,64 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable for server access key.
+pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
+
+/// Environment variable for server access key file.
+pub const ENV_RUSTFS_ACCESS_KEY_FILE: &str = "RUSTFS_ACCESS_KEY_FILE";
+
+/// Environment variable for server root user.
+pub const ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
+
+/// Environment variable for server secret key.
+pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
+
+/// Environment variable for server secret key file.
+pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
+
+/// Environment variable for server root password.
+pub const ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
+
+/// Environment variable for server OBS endpoint.
+pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
+
+/// Environment variable for console server enable.
+pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
+
+/// Environment variable for console server address.
+pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
+
 /// Environment variable for server tls path.
 pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
 
+/// Environment variable for server KMS enable.
+pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
+
+/// Default KMS enable for server-side encryption
+/// This is the default value for enabling KMS encryption for server-side encryption.
+/// Default value: false
+pub const DEFAULT_KMS_ENABLE: bool = false;
+
+/// Environment variable for server KMS backend.
+pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
+
+/// Environment variable for Vault Transit mount path.
+pub const ENV_RUSTFS_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
+
+/// Default KMS backend for server-side encryption
+/// This is the default KMS backend for server-side encryption.
+/// Default value: local
+pub const DEFAULT_KMS_BACKEND: &str = "local";
+
+/// Environment variable for selecting the buffer profile used for adaptive buffer sizing.
+pub const ENV_RUSTFS_BUFFER_PROFILE: &str = "RUSTFS_BUFFER_PROFILE";
+
+/// Default buffer profile for adaptive buffer sizing
+/// This is the default buffer profile for adaptive buffer sizing.
+/// It is used to identify the workload profile for adaptive buffer sizing.
+/// Default value: GeneralPurpose
+pub const DEFAULT_BUFFER_PROFILE: &str = "GeneralPurpose";
+
 /// Default value for the server TLS path if `ENV_RUSTFS_TLS_PATH` is not set.
 pub const DEFAULT_RUSTFS_TLS_PATH: &str = "";
 

crates/config/src/constants/capacity.rs

@@ -0,0 +1,155 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Capacity calculation configuration constants
+
+// ============================================================================
+// Environment Variable Names
+// ============================================================================
+
+/// Environment variable for scheduled update interval
+pub const ENV_CAPACITY_SCHEDULED_INTERVAL: &str = "RUSTFS_CAPACITY_SCHEDULED_INTERVAL";
+
+/// Environment variable for write trigger delay
+pub const ENV_CAPACITY_WRITE_TRIGGER_DELAY: &str = "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY";
+
+/// Environment variable for write frequency threshold
+pub const ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD: &str = "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD";
+
+/// Environment variable for fast update threshold
+pub const ENV_CAPACITY_FAST_UPDATE_THRESHOLD: &str = "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD";
+
+/// Environment variable for max files threshold
+pub const ENV_CAPACITY_MAX_FILES_THRESHOLD: &str = "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD";
+
+/// Environment variable for statistics timeout
+pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
+
+/// Environment variable for sample rate
+pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
+
+/// Environment variable for following symbolic links during capacity calculation
+pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
+
+/// Environment variable for maximum symlink follow depth
+pub const ENV_CAPACITY_MAX_SYMLINK_DEPTH: &str = "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH";
+
+/// Environment variable for enabling dynamic timeout calculation
+pub const ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: &str = "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT";
+
+/// Environment variable for minimum capacity calculation timeout
+pub const ENV_CAPACITY_MIN_TIMEOUT: &str = "RUSTFS_CAPACITY_MIN_TIMEOUT";
+
+/// Environment variable for maximum capacity calculation timeout
+pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
+
+/// Environment variable for progress stall detection timeout
+pub const ENV_CAPACITY_STALL_TIMEOUT: &str = "RUSTFS_CAPACITY_STALL_TIMEOUT";
+
+// ============================================================================
+// Default Values
+// ============================================================================
+
+/// Scheduled update interval in seconds
+/// Default: 120 seconds (2 minutes)
+pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 120;
+
+/// Write trigger delay in seconds
+/// Default: 5 seconds
+pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 5;
+
+/// Write frequency threshold (writes per minute)
+/// Default: 5 writes/minute
+pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 5;
+
+/// Fast update threshold in seconds
+/// Default: 30 seconds
+pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 30;
+
+/// Maximum files threshold for sampling
+/// Default: 200,000 files
+pub const DEFAULT_MAX_FILES_THRESHOLD: usize = 200_000;
+
+/// Statistics timeout in seconds
+/// Default: 3 seconds
+pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
+
+/// Sampling rate (1 in every N files)
+/// Default: 200
+pub const DEFAULT_SAMPLE_RATE: usize = 200;
+
+/// Follow symbolic links during capacity calculation
+/// Default: false (disabled for safety)
+pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
+
+/// Maximum symlink follow depth
+/// Default: 3 levels
+pub const DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH: u8 = 3;
+
+/// Enable dynamic timeout calculation based on directory characteristics
+/// Default: true (enabled)
+pub const DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT: bool = true;
+
+/// Minimum capacity calculation timeout in seconds
+/// Default: 2 seconds
+pub const DEFAULT_CAPACITY_MIN_TIMEOUT_SECS: u64 = 2;
+
+/// Maximum capacity calculation timeout in seconds
+/// Default: 15 seconds
+pub const DEFAULT_CAPACITY_MAX_TIMEOUT_SECS: u64 = 15;
+
+/// Progress stall detection timeout in seconds
+/// Default: 20 seconds
+pub const DEFAULT_CAPACITY_STALL_TIMEOUT_SECS: u64 = 20;
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_env_var_names() {
+        assert_eq!(ENV_CAPACITY_SCHEDULED_INTERVAL, "RUSTFS_CAPACITY_SCHEDULED_INTERVAL");
+        assert_eq!(ENV_CAPACITY_WRITE_TRIGGER_DELAY, "RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY");
+        assert_eq!(ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, "RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_FAST_UPDATE_THRESHOLD, "RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
+        assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
+        assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
+        assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
+        assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
+        assert_eq!(ENV_CAPACITY_STALL_TIMEOUT, "RUSTFS_CAPACITY_STALL_TIMEOUT");
+    }
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, 120);
+        assert_eq!(DEFAULT_WRITE_TRIGGER_DELAY_SECS, 5);
+        assert_eq!(DEFAULT_WRITE_FREQUENCY_THRESHOLD, 5);
+        assert_eq!(DEFAULT_FAST_UPDATE_THRESHOLD_SECS, 30);
+        assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
+        assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
+        assert_eq!(DEFAULT_SAMPLE_RATE, 200);
+        assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
+        assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2);
+        assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15);
+        assert_eq!(DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, 20);
+    }
+}

crates/config/src/constants/mod.rs

@@ -14,6 +14,7 @@
 
 pub(crate) mod app;
 pub(crate) mod body_limits;
+pub(crate) mod capacity;
 pub(crate) mod compress;
 pub(crate) mod console;
 pub(crate) mod env;
@@ -28,3 +29,5 @@ pub(crate) mod runtime;
 pub(crate) mod scanner;
 pub(crate) mod targets;
 pub(crate) mod tls;
+pub(crate) mod workload;
+pub(crate) mod zero_copy;

crates/config/src/constants/object.rs

@@ -12,62 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/// Environment variable name to toggle object-level in-memory caching.
-///
-/// - Purpose: Enable or disable the object-level in-memory cache (moka).
-/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config.
-/// - Semantics: When enabled, the system keeps fully-read objects in memory to reduce backend requests; when disabled, reads bypass the object cache.
-/// - Example: `export RUSTFS_OBJECT_CACHE_ENABLE=true`
-/// - Note: Evaluate together with `RUSTFS_OBJECT_CACHE_CAPACITY_MB`, TTL/TTI and concurrency thresholds to balance memory usage and throughput.
-pub const ENV_OBJECT_CACHE_ENABLE: &str = "RUSTFS_OBJECT_CACHE_ENABLE";
-
-/// Environment variable name that specifies the object cache capacity in megabytes.
-///
-/// - Purpose: Set the maximum total capacity of the object cache (in MB).
-/// - Unit: MB (1 MB = 1_048_576 bytes).
-/// - Valid values: any positive integer (0 may indicate disabled or alternative handling).
-/// - Semantics: When the moka cache reaches this capacity, eviction policies will remove entries; tune according to available memory and object size distribution.
-/// - Example: `export RUSTFS_OBJECT_CACHE_CAPACITY_MB=512`
-/// - Note: Actual memory usage will be slightly higher due to object headers and indexing overhead.
-pub const ENV_OBJECT_CACHE_CAPACITY_MB: &str = "RUSTFS_OBJECT_CACHE_CAPACITY_MB";
-
-/// Environment variable name for maximum object size eligible for caching in megabytes.
-///
-/// - Purpose: Define the upper size limit for individual objects to be considered for caching.
-/// - Unit: MB (1 MB = 1_048_576 bytes).
-/// - Valid values: any positive integer; objects larger than this size will not be cached.
-/// - Semantics: Prevents caching of excessively large objects that could monopolize cache capacity; tune based on typical object size distribution.
-/// - Example: `export RUSTFS_OBJECT_CACHE_MAX_OBJECT_SIZE_MB=50`
-/// - Note: Setting this too low may reduce cache effectiveness; setting it too high may lead to inefficient memory usage.
-pub const ENV_OBJECT_CACHE_MAX_OBJECT_SIZE_MB: &str = "RUSTFS_OBJECT_CACHE_MAX_OBJECT_SIZE_MB";
-
-/// Environment variable name for object cache TTL (time-to-live) in seconds.
-///
-/// - Purpose: Specify the maximum lifetime of a cached entry from the moment it is written.
-/// - Unit: seconds (u64).
-/// - Semantics: TTL acts as a hard upper bound; entries older than TTL are considered expired and removed by periodic cleanup.
-/// - Example: `export RUSTFS_OBJECT_CACHE_TTL_SECS=300`
-/// - Note: TTL and TTI both apply; either policy can cause eviction.
-pub const ENV_OBJECT_CACHE_TTL_SECS: &str = "RUSTFS_OBJECT_CACHE_TTL_SECS";
-
-/// Environment variable name for object cache TTI (time-to-idle) in seconds.
-///
-/// - Purpose: Specify how long an entry may remain in cache without being accessed before it is evicted.
-/// - Unit: seconds (u64).
-/// - Semantics: TTI helps remove one-time or infrequently used entries; frequent accesses reset idle timers but do not extend beyond TTL unless additional logic exists.
-/// - Example: `export RUSTFS_OBJECT_CACHE_TTI_SECS=120`
-/// - Note: Works together with TTL to keep the cache populated with actively used objects.
-pub const ENV_OBJECT_CACHE_TTI_SECS: &str = "RUSTFS_OBJECT_CACHE_TTI_SECS";
-
-/// Environment variable name for threshold of "hot" object hit count used to extend life.
-///
-/// - Purpose: Define a hit-count threshold to mark objects as "hot" so they may be treated preferentially near expiration.
-/// - Valid values: positive integer (usize).
-/// - Semantics: Objects reaching this hit count can be considered for relaxed eviction to avoid thrashing hot items.
-/// - Example: `export RUSTFS_OBJECT_HOT_MIN_HITS_TO_EXTEND=5`
-/// - Note: This is an optional enhancement and requires cache-layer statistics and extension logic to take effect.
-pub const ENV_OBJECT_HOT_MIN_HITS_TO_EXTEND: &str = "RUSTFS_OBJECT_HOT_MIN_HITS_TO_EXTEND";
-
 /// Environment variable name for high concurrency threshold used in adaptive buffering.
 ///
 /// - Purpose: When concurrent request count exceeds this threshold, the system enters a "high concurrency" optimization mode to reduce per-request buffer sizes.
@@ -94,24 +38,6 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
 /// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
 pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";
 
-/// Default: object caching is disabled.
-///
-/// - Semantics: Safe default to avoid unexpected memory usage or cache consistency concerns when not explicitly enabled.
-/// - Default is set to false (disabled).
-pub const DEFAULT_OBJECT_CACHE_ENABLE: bool = false;
-
-/// Default object cache capacity in MB.
-///
-/// - Default: 100 MB (can be overridden by `RUSTFS_OBJECT_CACHE_CAPACITY_MB`).
-/// - Note: Choose a conservative default to reduce memory pressure in development/testing.
-pub const DEFAULT_OBJECT_CACHE_CAPACITY_MB: u64 = 100;
-
-/// Default maximum object size eligible for caching in MB.
-///
-/// - Default: 10 MB (can be overridden by `RUSTFS_OBJECT_CACHE_MAX_OBJECT_SIZE_MB`).
-/// - Note: Balances caching effectiveness with memory usage.
-pub const DEFAULT_OBJECT_CACHE_MAX_OBJECT_SIZE_MB: usize = 10;
-
 /// Maximum concurrent requests before applying aggressive optimization.
 ///
 /// When concurrent requests exceed this threshold (>8), the system switches to
@@ -141,33 +67,6 @@ pub const DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: usize = 4;
 /// Default is set to 64 concurrent reads.
 pub const DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS: usize = 64;
 
-/// Time-to-live for cached objects (5 minutes = 300 seconds).
-///
-/// After this duration, cached objects are automatically expired by Moka's
-/// background cleanup process, even if they haven't been accessed. This prevents
-/// stale data from consuming cache capacity indefinitely.
-///
-/// Default is set to 300 seconds.
-pub const DEFAULT_OBJECT_CACHE_TTL_SECS: u64 = 300;
-
-/// Time-to-idle for cached objects (2 minutes = 120 seconds).
-///
-/// Objects that haven't been accessed for this duration are automatically evicted,
-/// even if their TTL hasn't expired. This ensures cache is populated with actively
-/// used objects and clears out one-time reads efficiently.
-///
-/// Default is set to 120 seconds.
-pub const DEFAULT_OBJECT_CACHE_TTI_SECS: u64 = 120;
-
-/// Minimum hit count to extend object lifetime beyond TTL.
-///
-/// "Hot" objects that have been accessed at least this many times are treated
-/// specially - they can survive longer in cache even as they approach TTL expiration.
-/// This prevents frequently accessed objects from being evicted prematurely.
-///
-/// Default is set to 5 hits.
-pub const DEFAULT_OBJECT_HOT_MIN_HITS_TO_EXTEND: usize = 5;
-
 /// Skip bitrot hash verification on GetObject reads.
 ///
 /// When enabled, GetObject reads skip the per-shard hash
@@ -180,3 +79,425 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
+
+// =============================================================================
+// Concurrent Request Fix - Timeout and Backpressure Configuration
+// =============================================================================
+
+/// Environment variable for GetObject request timeout in seconds.
+///
+/// When a GetObject request exceeds this duration, it will be cancelled
+/// and return a 504 Gateway Timeout error. This prevents requests from
+/// hanging indefinitely due to deadlocks or resource exhaustion.
+///
+/// Default: 30 seconds (can be overridden by `RUSTFS_OBJECT_GET_TIMEOUT`).
+/// Set to 0 to disable timeout (not recommended for production).
+pub const ENV_OBJECT_GET_TIMEOUT: &str = "RUSTFS_OBJECT_GET_TIMEOUT";
+
+/// Default GetObject request timeout in seconds.
+///
+/// This value balances between allowing large object transfers to complete
+/// and preventing indefinite hangs. For 20-26MB objects with concurrent
+/// range reads, 30 seconds should be sufficient under normal conditions.
+pub const DEFAULT_OBJECT_GET_TIMEOUT: u64 = 30;
+
+/// Environment variable for disk read operation timeout in seconds.
+///
+/// Individual disk read operations that exceed this duration will be
+/// cancelled and treated as failures. This helps detect slow or hung
+/// disks without waiting indefinitely.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DISK_READ_TIMEOUT`).
+pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT";
+
+/// Default disk read timeout in seconds.
+pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
+
+/// Environment variable for minimum GetObject timeout in seconds.
+///
+/// When dynamic timeout calculation is enabled, this is the minimum timeout
+/// that will be used regardless of object size. This prevents excessively
+/// short timeouts for very small objects.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_MIN_TIMEOUT`).
+pub const ENV_OBJECT_MIN_TIMEOUT: &str = "RUSTFS_OBJECT_MIN_TIMEOUT";
+
+/// Default minimum GetObject timeout: 5 seconds.
+pub const DEFAULT_OBJECT_MIN_TIMEOUT: u64 = 5;
+
+/// Environment variable for maximum GetObject timeout in seconds.
+///
+/// When dynamic timeout calculation is enabled, this is the maximum timeout
+/// that will be used regardless of object size. This prevents excessively
+/// long timeouts for very large objects.
+///
+/// Default: 300 seconds (5 minutes, can be overridden by `RUSTFS_OBJECT_MAX_TIMEOUT`).
+pub const ENV_OBJECT_MAX_TIMEOUT: &str = "RUSTFS_OBJECT_MAX_TIMEOUT";
+
+/// Default maximum GetObject timeout: 300 seconds (5 minutes).
+pub const DEFAULT_OBJECT_MAX_TIMEOUT: u64 = 300;
+
+/// Environment variable for default bytes per second for timeout estimation.
+///
+/// This value is used to estimate timeout duration based on object size when
+/// dynamic timeout calculation is enabled. The timeout is calculated as:
+/// (object_size / bytes_per_second) * buffer_factor
+///
+/// Default: 1048576 (1 MB/s, can be overridden by `RUSTFS_OBJECT_BYTES_PER_SECOND`).
+pub const ENV_OBJECT_BYTES_PER_SECOND: &str = "RUSTFS_OBJECT_BYTES_PER_SECOND";
+
+/// Default bytes per second for timeout estimation: 1 MB/s.
+pub const DEFAULT_OBJECT_BYTES_PER_SECOND: u64 = 1024 * 1024;
+
+/// Environment variable to enable dynamic timeout calculation.
+///
+/// When enabled, timeout is calculated based on object size and transfer speed
+/// rather than using a fixed timeout value. This provides better timeout
+/// handling for objects of varying sizes.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_DYNAMIC_TIMEOUT_ENABLE`).
+pub const ENV_OBJECT_DYNAMIC_TIMEOUT_ENABLE: &str = "RUSTFS_OBJECT_DYNAMIC_TIMEOUT_ENABLE";
+
+/// Default: dynamic timeout calculation is enabled.
+pub const DEFAULT_OBJECT_DYNAMIC_TIMEOUT_ENABLE: bool = true;
+
+/// Environment variable for duplex pipe buffer size in bytes.
+///
+/// The duplex pipe connects the disk read task to the HTTP response stream.
+/// A larger buffer reduces backpressure but increases memory usage.
+/// For large objects (20-26MB), a 4MB buffer provides good throughput.
+///
+/// Default: 4194304 (4 MB, can be overridden by `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE`).
+/// Minimum recommended: 1048576 (1 MB).
+pub const ENV_OBJECT_DUPLEX_BUFFER_SIZE: &str = "RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE";
+
+/// Default duplex buffer size: 4 MB.
+///
+/// This is 4x larger than the original 1 MB buffer, providing better
+/// handling of large objects and reducing backpressure-related hangs.
+pub const DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE: usize = 4 * 1024 * 1024;
+
+/// Environment variable for I/O buffer size in bytes.
+///
+/// This controls the buffer size used for individual I/O operations.
+/// A larger buffer improves throughput for sequential reads but may
+/// increase latency for small random reads.
+///
+/// Default: 131072 (128 KB, can be overridden by `RUSTFS_OBJECT_IO_BUFFER_SIZE`).
+pub const ENV_OBJECT_IO_BUFFER_SIZE: &str = "RUSTFS_OBJECT_IO_BUFFER_SIZE";
+
+/// Default I/O buffer size: 128 KB.
+pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024;
+
+/// Environment variable to enable/disable lock optimization.
+///
+/// When enabled, read locks are released immediately after metadata
+/// is read, rather than being held for the entire data transfer.
+/// This significantly reduces lock contention under high concurrency.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`).
+pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE";
+
+/// Default: lock optimization is enabled.
+pub const DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE: bool = true;
+
+/// Environment variable to enable/disable priority-based I/O scheduling.
+///
+/// When enabled, smaller requests (< 1MB) are given higher priority
+/// than larger requests (> 10MB), preventing "starvation" of small
+/// requests by large ones.
+///
+/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE`).
+pub const ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE: &str = "RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE";
+
+/// Default: priority scheduling is enabled.
+pub const DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE: bool = true;
+
+/// Environment variable to enable/disable deadlock detection.
+///
+/// When enabled, the system monitors active requests and detects
+/// potential deadlock situations (circular lock wait chains).
+/// This has some performance overhead and is intended for debugging.
+///
+/// Default: false (disabled, can be overridden by `RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE`).
+pub const ENV_OBJECT_DEADLOCK_DETECTION_ENABLE: &str = "RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE";
+
+/// Default: deadlock detection is disabled for performance.
+pub const DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE: bool = false;
+
+/// Environment variable for deadlock detection check interval in seconds.
+///
+/// How often the deadlock detector analyzes the lock wait graph.
+/// More frequent checks detect deadlocks faster but use more CPU.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL`).
+pub const ENV_OBJECT_DEADLOCK_CHECK_INTERVAL: &str = "RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL";
+
+/// Default deadlock check interval: 5 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_CHECK_INTERVAL: u64 = 5;
+
+/// Environment variable for deadlock detection hang threshold in seconds.
+///
+/// Requests that have been running longer than this threshold are
+/// considered "potentially hung" and included in deadlock analysis.
+///
+/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD`).
+pub const ENV_OBJECT_DEADLOCK_HANG_THRESHOLD: &str = "RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD";
+
+/// Default hang threshold: 10 seconds.
+pub const DEFAULT_OBJECT_DEADLOCK_HANG_THRESHOLD: u64 = 10;
+
+/// Environment variable for backpressure high watermark percentage.
+///
+/// When buffer usage exceeds this percentage, the system enters
+/// "high watermark" state and may apply backpressure to producers.
+///
+/// Default: 80 (80%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK";
+
+/// Default high watermark: 80%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK: u32 = 80;
+
+/// Environment variable for backpressure low watermark percentage.
+///
+/// When buffer usage drops below this percentage after being in
+/// high watermark state, backpressure is released.
+///
+/// Default: 50 (50%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK`).
+pub const ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK";
+
+/// Default low watermark: 50%.
+pub const DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK: u32 = 50;
+
+/// Environment variable for lock acquisition timeout in seconds.
+///
+/// When a lock cannot be acquired within this duration, the operation
+/// will fail with a timeout error. This prevents indefinite waiting
+/// for locks that may never be released due to deadlocks.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`).
+pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT";
+
+/// Default lock acquisition timeout: 5 seconds.
+pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
+
+// ============================================================================
+// I/O priority scheduling configuration
+// ============================================================================
+
+/// Environment variable for I/O high priority size threshold in bytes.
+///
+/// Requests smaller than this threshold are classified as high priority.
+/// High priority requests are processed first to prevent starvation of small requests.
+///
+/// Default: 1048576 (1 MB, can be overridden by `RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD";
+
+/// Default high priority size threshold: 1 MB.
+pub const DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: usize = 1024 * 1024;
+
+/// Environment variable for I/O low priority size threshold in bytes.
+///
+/// Requests larger than this threshold are classified as low priority.
+/// Low priority requests are processed last to avoid blocking small requests.
+///
+/// Default: 10485760 (10 MB, can be overridden by `RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD`).
+pub const ENV_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD";
+
+/// Default low priority size threshold: 10 MB.
+pub const DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: usize = 10 * 1024 * 1024;
+
+/// Environment variable for high priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the high priority queue.
+///
+/// Default: 32 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_HIGH_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY";
+
+/// Default high priority queue capacity: 32.
+pub const DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY: usize = 32;
+
+/// Environment variable for normal priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the normal priority queue.
+///
+/// Default: 64 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_NORMAL_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY";
+
+/// Default normal priority queue capacity: 64.
+pub const DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY: usize = 64;
+
+/// Environment variable for low priority queue capacity.
+///
+/// Maximum number of requests that can be queued in the low priority queue.
+///
+/// Default: 16 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY`).
+pub const ENV_OBJECT_IO_QUEUE_LOW_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY";
+
+/// Default low priority queue capacity: 16.
+pub const DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY: usize = 16;
+
+/// Environment variable for starvation prevention check interval in milliseconds.
+///
+/// How often the system checks for starving low-priority requests.
+/// When a low-priority request has been waiting longer than the starvation threshold,
+/// it is promoted to normal priority.
+///
+/// Default: 100 ms (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL`).
+pub const ENV_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: &str = "RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL";
+
+/// Default starvation prevention interval: 100 ms.
+pub const DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: u64 = 100;
+
+/// Environment variable for starvation threshold in seconds.
+///
+/// Maximum time a low-priority request can wait before being promoted to normal priority.
+/// This prevents indefinite starvation of low-priority requests.
+///
+/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS`).
+pub const ENV_OBJECT_IO_STARVATION_THRESHOLD_SECS: &str = "RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS";
+
+/// Default starvation threshold: 5 seconds.
+pub const DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS: u64 = 5;
+
+/// Environment variable for load sampling window size.
+///
+/// Number of recent samples used to calculate I/O load metrics.
+///
+/// Default: 100 samples (can be overridden by `RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW`).
+pub const ENV_OBJECT_IO_LOAD_SAMPLE_WINDOW: &str = "RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW";
+
+/// Default load sampling window: 100 samples.
+pub const DEFAULT_OBJECT_IO_LOAD_SAMPLE_WINDOW: usize = 100;
+
+/// Environment variable for high load wait time threshold in milliseconds.
+///
+/// When average wait time exceeds this threshold, the system is considered to be under high load.
+///
+/// Default: 50 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS";
+
+/// Default high load threshold: 50 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: u64 = 50;
+
+/// Environment variable for low load wait time threshold in milliseconds.
+///
+/// When average wait time is below this threshold, the system is considered to be under low load.
+///
+/// Default: 10 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS`).
+pub const ENV_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS";
+
+/// Default low load threshold: 10 ms.
+pub const DEFAULT_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: u64 = 10;
+
+/// Environment variable for enabling storage media detection for adaptive I/O scheduling.
+///
+/// When disabled, the scheduler falls back to `Unknown` storage media unless an explicit
+/// override is provided.
+///
+/// Default: true (can be overridden by `RUSTFS_OBJECT_IO_STORAGE_DETECTION_ENABLE`).
+pub const ENV_OBJECT_IO_STORAGE_DETECTION_ENABLE: &str = "RUSTFS_OBJECT_IO_STORAGE_DETECTION_ENABLE";
+
+/// Default storage media detection setting: enabled.
+pub const DEFAULT_OBJECT_IO_STORAGE_DETECTION_ENABLE: bool = true;
+
+/// Environment variable for overriding detected storage media.
+///
+/// Supported values: `nvme`, `ssd`, `hdd`, `unknown`.
+/// Empty value means auto-detect or fallback to `Unknown`.
+///
+/// Default: empty string (can be overridden by `RUSTFS_OBJECT_IO_STORAGE_MEDIA_OVERRIDE`).
+pub const ENV_OBJECT_IO_STORAGE_MEDIA_OVERRIDE: &str = "RUSTFS_OBJECT_IO_STORAGE_MEDIA_OVERRIDE";
+
+/// Default storage media override: no override.
+pub const DEFAULT_OBJECT_IO_STORAGE_MEDIA_OVERRIDE: &str = "";
+
+/// Environment variable for access-pattern history size.
+///
+/// Controls how many recent offset/length observations are used to classify
+/// sequential, random, or mixed reads.
+///
+/// Default: 8 (can be overridden by `RUSTFS_OBJECT_IO_PATTERN_HISTORY_SIZE`).
+pub const ENV_OBJECT_IO_PATTERN_HISTORY_SIZE: &str = "RUSTFS_OBJECT_IO_PATTERN_HISTORY_SIZE";
+
+/// Default access-pattern history size: 8 samples.
+pub const DEFAULT_OBJECT_IO_PATTERN_HISTORY_SIZE: usize = 8;
+
+/// Environment variable for sequential access step tolerance in bytes.
+///
+/// Small gaps between adjacent reads within this tolerance are still treated as sequential.
+///
+/// Default: 131072 bytes (128 KiB, can be overridden by `RUSTFS_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES`).
+pub const ENV_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES: &str = "RUSTFS_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES";
+
+/// Default sequential step tolerance: 128 KiB.
+pub const DEFAULT_OBJECT_IO_SEQUENTIAL_STEP_TOLERANCE_BYTES: u64 = 128 * 1024;
+
+/// Environment variable for bandwidth EMA beta.
+///
+/// Lower values react faster to recent throughput changes while higher values smooth
+/// short-term fluctuations more aggressively.
+///
+/// Default: 0.1 (can be overridden by `RUSTFS_OBJECT_IO_BANDWIDTH_EMA_BETA`).
+pub const ENV_OBJECT_IO_BANDWIDTH_EMA_BETA: &str = "RUSTFS_OBJECT_IO_BANDWIDTH_EMA_BETA";
+
+/// Default bandwidth EMA beta: 0.1.
+pub const DEFAULT_OBJECT_IO_BANDWIDTH_EMA_BETA: f64 = 0.1;
+
+/// Environment variable for the low bandwidth threshold in bytes per second.
+///
+/// Observed throughput below this value causes the scheduler to be more conservative
+/// with buffer growth and read-ahead.
+///
+/// Default: 67108864 bytes/sec (64 MiB/s, can be overridden by `RUSTFS_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS`).
+pub const ENV_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS: &str = "RUSTFS_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS";
+
+/// Default low bandwidth threshold: 64 MiB/s.
+pub const DEFAULT_OBJECT_IO_BANDWIDTH_LOW_THRESHOLD_BPS: u64 = 64 * 1024 * 1024;
+
+/// Environment variable for the high bandwidth threshold in bytes per second.
+///
+/// Observed throughput above this value allows the scheduler to be more aggressive
+/// for sequential workloads.
+///
+/// Default: 536870912 bytes/sec (512 MiB/s, can be overridden by `RUSTFS_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS`).
+pub const ENV_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS: &str = "RUSTFS_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS";
+
+/// Default high bandwidth threshold: 512 MiB/s.
+pub const DEFAULT_OBJECT_IO_BANDWIDTH_HIGH_THRESHOLD_BPS: u64 = 512 * 1024 * 1024;
+
+/// Environment variable for NVMe buffer cap in bytes.
+///
+/// Sequential reads on NVMe can scale up to this buffer cap.
+///
+/// Default: 2097152 bytes (2 MiB, can be overridden by `RUSTFS_OBJECT_IO_NVME_BUFFER_CAP`).
+pub const ENV_OBJECT_IO_NVME_BUFFER_CAP: &str = "RUSTFS_OBJECT_IO_NVME_BUFFER_CAP";
+
+/// Default NVMe buffer cap: 2 MiB.
+pub const DEFAULT_OBJECT_IO_NVME_BUFFER_CAP: usize = 2 * 1024 * 1024;
+
+/// Environment variable for SSD buffer cap in bytes.
+///
+/// Default: 1048576 bytes (1 MiB, can be overridden by `RUSTFS_OBJECT_IO_SSD_BUFFER_CAP`).
+pub const ENV_OBJECT_IO_SSD_BUFFER_CAP: &str = "RUSTFS_OBJECT_IO_SSD_BUFFER_CAP";
+
+/// Default SSD buffer cap: 1 MiB.
+pub const DEFAULT_OBJECT_IO_SSD_BUFFER_CAP: usize = 1024 * 1024;
+
+/// Environment variable for HDD buffer cap in bytes.
+///
+/// Default: 524288 bytes (512 KiB, can be overridden by `RUSTFS_OBJECT_IO_HDD_BUFFER_CAP`).
+pub const ENV_OBJECT_IO_HDD_BUFFER_CAP: &str = "RUSTFS_OBJECT_IO_HDD_BUFFER_CAP";
+
+/// Default HDD buffer cap: 512 KiB.
+pub const DEFAULT_OBJECT_IO_HDD_BUFFER_CAP: usize = 512 * 1024;
+
+/// Environment variable for disabling read-ahead under random or mixed access with concurrency.
+///
+/// When concurrent requests reach this threshold, random-heavy workloads stop using read-ahead.
+///
+/// Default: 4 (can be overridden by `RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY`).
+pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY";
+
+/// Default read-ahead disable concurrency threshold: 4.
+pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;

crates/config/src/constants/oidc.rs

@@ -24,6 +24,7 @@ pub const OIDC_CLAIM_PREFIX: &str = "claim_prefix";
 pub const OIDC_ROLE_POLICY: &str = "role_policy";
 pub const OIDC_DISPLAY_NAME: &str = "display_name";
 pub const OIDC_GROUPS_CLAIM: &str = "groups_claim";
+pub const OIDC_ROLES_CLAIM: &str = "roles_claim";
 pub const OIDC_EMAIL_CLAIM: &str = "email_claim";
 pub const OIDC_USERNAME_CLAIM: &str = "username_claim";
 
@@ -40,11 +41,12 @@ pub const ENV_IDENTITY_OPENID_CLAIM_PREFIX: &str = "RUSTFS_IDENTITY_OPENID_CLAIM
 pub const ENV_IDENTITY_OPENID_ROLE_POLICY: &str = "RUSTFS_IDENTITY_OPENID_ROLE_POLICY";
 pub const ENV_IDENTITY_OPENID_DISPLAY_NAME: &str = "RUSTFS_IDENTITY_OPENID_DISPLAY_NAME";
 pub const ENV_IDENTITY_OPENID_GROUPS_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM";
+pub const ENV_IDENTITY_OPENID_ROLES_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_ROLES_CLAIM";
 pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM";
 pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
 
 /// List of all environment variable keys for an OIDC provider.
-pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 15] = &[
     ENV_IDENTITY_OPENID_ENABLE,
     ENV_IDENTITY_OPENID_CONFIG_URL,
     ENV_IDENTITY_OPENID_CLIENT_ID,
@@ -57,6 +59,7 @@ pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
     ENV_IDENTITY_OPENID_ROLE_POLICY,
     ENV_IDENTITY_OPENID_DISPLAY_NAME,
     ENV_IDENTITY_OPENID_GROUPS_CLAIM,
+    ENV_IDENTITY_OPENID_ROLES_CLAIM,
     ENV_IDENTITY_OPENID_EMAIL_CLAIM,
     ENV_IDENTITY_OPENID_USERNAME_CLAIM,
 ];
@@ -75,6 +78,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
     OIDC_ROLE_POLICY,
     OIDC_DISPLAY_NAME,
     OIDC_GROUPS_CLAIM,
+    OIDC_ROLES_CLAIM,
     OIDC_EMAIL_CLAIM,
     OIDC_USERNAME_CLAIM,
     crate::COMMENT_KEY,
@@ -84,6 +88,8 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
 pub const OIDC_DEFAULT_SCOPES: &str = "openid,profile,email";
 pub const OIDC_DEFAULT_CLAIM_NAME: &str = "groups";
 pub const OIDC_DEFAULT_GROUPS_CLAIM: &str = "groups";
+/// Empty means do not merge a secondary claim into groups (legacy behavior). Set to e.g. `roles` to opt in.
+pub const OIDC_DEFAULT_ROLES_CLAIM: &str = "";
 pub const OIDC_DEFAULT_EMAIL_CLAIM: &str = "email";
 pub const OIDC_DEFAULT_USERNAME_CLAIM: &str = "preferred_username";
 

crates/config/src/constants/runtime.rs

@@ -27,6 +27,16 @@ pub const ENV_RNG_SEED: &str = "RUSTFS_RUNTIME_RNG_SEED";
 /// Event polling interval
 pub const ENV_EVENT_INTERVAL: &str = "RUSTFS_RUNTIME_EVENT_INTERVAL";
 
+// Dial9 Tokio Telemetry Configuration
+pub const ENV_RUNTIME_DIAL9_ENABLED: &str = "RUSTFS_RUNTIME_DIAL9_ENABLED";
+pub const ENV_RUNTIME_DIAL9_OUTPUT_DIR: &str = "RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR";
+pub const ENV_RUNTIME_DIAL9_FILE_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_FILE_PREFIX";
+pub const ENV_RUNTIME_DIAL9_MAX_FILE_SIZE: &str = "RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE";
+pub const ENV_RUNTIME_DIAL9_ROTATION_COUNT: &str = "RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT";
+pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
+pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
+pub const ENV_RUNTIME_DIAL9_SAMPLING_RATE: &str = "RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE";
+
 // Default values for Tokio runtime
 pub const DEFAULT_WORKER_THREADS: usize = 16;
 pub const DEFAULT_MAX_BLOCKING_THREADS: usize = 1024;
@@ -40,6 +50,15 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
 pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
 pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 
+// Dial9 Tokio Telemetry Default values
+pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
+pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";
+pub const DEFAULT_RUNTIME_DIAL9_FILE_PREFIX: &str = "rustfs-tokio";
+pub const DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE: u64 = 100 * 1024 * 1024; // 100MB
+pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10;
+pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling
+// Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option<String>)
+
 /// Threshold for small object seek support in megabytes.
 ///
 /// When an object is smaller than this size, rustfs will provide seek support.

crates/config/src/constants/targets.rs

@@ -34,3 +34,16 @@ pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval";
 pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
 pub const MQTT_QUEUE_DIR: &str = "queue_dir";
 pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";
+pub const MQTT_TLS_POLICY: &str = "tls_policy";
+pub const MQTT_TLS_CA: &str = "tls_ca";
+pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
+pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
+pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
+
+/// Environment variable controlling whether target queue files are Snappy-compressed.
+/// Applies to both notify and audit target queue stores.
+pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
+
+/// Queue-store compression is enabled by default to reduce disk footprint.
+pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;

crates/config/src/constants/tls.rs

@@ -84,3 +84,69 @@ pub const ENV_SERVER_MTLS_ENABLE: &str = "RUSTFS_SERVER_MTLS_ENABLE";
 /// By default, RustFS server mTLS is disabled.
 /// To change this behavior, set the environment variable RUSTFS_SERVER_MTLS_ENABLE=1
 pub const DEFAULT_SERVER_MTLS_ENABLE: bool = false;
+
+// ── HTTP Transport Tuning Parameters ──
+
+/// Environment variable for HTTP/2 initial stream window size (bytes)
+/// Default: 4194304 (4 MB)
+pub const ENV_H2_INITIAL_STREAM_WINDOW_SIZE: &str = "RUSTFS_H2_INITIAL_STREAM_WINDOW_SIZE";
+pub const DEFAULT_H2_INITIAL_STREAM_WINDOW_SIZE: u32 = 4 * 1024 * 1024; // 4 MB
+
+/// Environment variable for HTTP/2 initial connection window size (bytes)
+/// Default: 8388608 (8 MB)
+pub const ENV_H2_INITIAL_CONN_WINDOW_SIZE: &str = "RUSTFS_H2_INITIAL_CONN_WINDOW_SIZE";
+pub const DEFAULT_H2_INITIAL_CONN_WINDOW_SIZE: u32 = 8 * 1024 * 1024; // 8 MB
+
+/// Environment variable for HTTP/2 max frame size (bytes)
+/// Range: 16384 (16 KB) to 16777216 (16 MB) per RFC 7540
+/// Default: 524288 (512 KB)
+pub const ENV_H2_MAX_FRAME_SIZE: &str = "RUSTFS_H2_MAX_FRAME_SIZE";
+pub const DEFAULT_H2_MAX_FRAME_SIZE: u32 = 512 * 1024; // 512 KB
+
+/// Environment variable for HTTP/2 max header list size (bytes)
+/// Default: 65536 (64 KB)
+pub const ENV_H2_MAX_HEADER_LIST_SIZE: &str = "RUSTFS_H2_MAX_HEADER_LIST_SIZE";
+pub const DEFAULT_H2_MAX_HEADER_LIST_SIZE: u32 = 64 * 1024; // 64 KB
+
+/// Environment variable for HTTP/2 max concurrent streams
+/// Default: 2048
+pub const ENV_H2_MAX_CONCURRENT_STREAMS: &str = "RUSTFS_H2_MAX_CONCURRENT_STREAMS";
+pub const DEFAULT_H2_MAX_CONCURRENT_STREAMS: u32 = 2048;
+
+/// Environment variable for HTTP/2 keep-alive interval (seconds)
+/// Default: 20
+pub const ENV_H2_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_H2_KEEP_ALIVE_INTERVAL";
+pub const DEFAULT_H2_KEEP_ALIVE_INTERVAL: u64 = 20;
+
+/// Environment variable for HTTP/2 keep-alive timeout (seconds)
+/// Default: 10
+pub const ENV_H2_KEEP_ALIVE_TIMEOUT: &str = "RUSTFS_H2_KEEP_ALIVE_TIMEOUT";
+pub const DEFAULT_H2_KEEP_ALIVE_TIMEOUT: u64 = 10;
+
+/// Environment variable for HTTP/1.1 header read timeout (seconds)
+/// Default: 5
+pub const ENV_HTTP1_HEADER_READ_TIMEOUT: &str = "RUSTFS_HTTP1_HEADER_READ_TIMEOUT";
+pub const DEFAULT_HTTP1_HEADER_READ_TIMEOUT: u64 = 5;
+
+/// Environment variable for HTTP/1.1 max buffer size (bytes)
+/// Default: 65536 (64 KB)
+pub const ENV_HTTP1_MAX_BUF_SIZE: &str = "RUSTFS_HTTP1_MAX_BUF_SIZE";
+pub const DEFAULT_HTTP1_MAX_BUF_SIZE: usize = 64 * 1024; // 64 KB
+
+// ── TLS Hot Reload Parameters ──
+
+/// Environment variable to enable TLS certificate hot reload
+/// Default: false
+/// To enable, set the environment variable RUSTFS_TLS_RELOAD_ENABLE=1
+pub const ENV_TLS_RELOAD_ENABLE: &str = "RUSTFS_TLS_RELOAD_ENABLE";
+
+/// Default value for TLS certificate hot reload
+/// By default, RustFS does not reload TLS certificates automatically.
+pub const DEFAULT_TLS_RELOAD_ENABLE: bool = false;
+
+/// Environment variable for TLS certificate reload interval (seconds)
+/// Default: 30 seconds. Minimum: 5 seconds.
+pub const ENV_TLS_RELOAD_INTERVAL: &str = "RUSTFS_TLS_RELOAD_INTERVAL";
+
+/// Default interval for TLS certificate reload check
+pub const DEFAULT_TLS_RELOAD_INTERVAL: u64 = 30;

crates/config/src/constants/workload.rs

@@ -0,0 +1,70 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Workload profile buffer configuration constants.
+//!
+//! This module defines environment variable keys and default values for
+//! custom buffer profile configuration.
+
+use crate::{KI_B, MI_B};
+
+/// Environment variable for minimum buffer size
+/// Default: 64KB (65536 bytes)
+pub const ENV_RUSTFS_BUFFER_MIN_SIZE: &str = "RUSTFS_BUFFER_MIN_SIZE";
+
+/// Environment variable for maximum buffer size
+/// Default: 1MB (1048576 bytes)
+pub const ENV_RUSTFS_BUFFER_MAX_SIZE: &str = "RUSTFS_BUFFER_MAX_SIZE";
+
+/// Environment variable for default buffer size (used when file size is unknown)
+/// Default: 256KB (262144 bytes)
+pub const ENV_RUSTFS_BUFFER_DEFAULT_SIZE: &str = "RUSTFS_BUFFER_DEFAULT_SIZE";
+
+/// Default minimum buffer size: 64KB
+pub const DEFAULT_BUFFER_MIN_SIZE: usize = 64 * KI_B;
+
+/// Default maximum buffer size: 1MB
+pub const DEFAULT_BUFFER_MAX_SIZE: usize = MI_B;
+
+/// Default buffer size for unknown file size: 256KB
+pub const DEFAULT_BUFFER_UNKNOWN_SIZE: usize = 256 * KI_B;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::MI_B;
+
+    #[test]
+    fn test_default_values() {
+        assert_eq!(DEFAULT_BUFFER_MIN_SIZE, 65536); // 64KB
+        assert_eq!(DEFAULT_BUFFER_MAX_SIZE, 1048576); // 1MB
+        assert_eq!(DEFAULT_BUFFER_UNKNOWN_SIZE, 262144); // 256KB
+    }
+
+    #[test]
+    fn test_constants() {
+        assert_eq!(KI_B, 1024);
+        assert_eq!(MI_B, 1024 * 1024);
+        assert_eq!(64 * KI_B, DEFAULT_BUFFER_MIN_SIZE);
+        assert_eq!(MI_B, DEFAULT_BUFFER_MAX_SIZE);
+        assert_eq!(256 * KI_B, DEFAULT_BUFFER_UNKNOWN_SIZE);
+    }
+
+    #[test]
+    fn test_env_var_names() {
+        assert_eq!(ENV_RUSTFS_BUFFER_MIN_SIZE, "RUSTFS_BUFFER_MIN_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_MAX_SIZE, "RUSTFS_BUFFER_MAX_SIZE");
+        assert_eq!(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, "RUSTFS_BUFFER_DEFAULT_SIZE");
+    }
+}

crates/config/src/constants/zero_copy.rs

@@ -0,0 +1,105 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Zero-copy I/O configuration constants.
+//!
+//! This module defines environment variables and default values for zero-copy
+//! read operations, which use memory mapping (mmap) to avoid data copying.
+
+// =============================================================================
+// Zero-Copy Configuration
+// =============================================================================
+
+/// Environment variable for zero-copy read enable.
+///
+/// When enabled, uses mmap (Unix) or optimized reads for zero-copy data access.
+/// This reduces memory copies from 3-4 to 1, lowering CPU usage by 20-30%
+/// and improving P95 latency by 15-25%.
+///
+/// - Purpose: Enable or disable zero-copy read operations
+/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config
+/// - Semantics: When enabled, uses mmap on Unix systems for memory-mapped file reads;
+///   falls back to regular I/O on non-Unix platforms or when mmap fails
+/// - Example: `export RUSTFS_OBJECT_ZERO_COPY_ENABLE=true`
+/// - Note: Zero-copy is safe for all workloads and provides significant performance
+///   benefits with minimal risk. Disable only if mmap-related issues are encountered.
+pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
+
+/// Default: zero-copy reads are enabled.
+///
+/// Zero-copy uses memory mapping (mmap) on Unix systems to avoid data copying
+/// between kernel and user space. This provides:
+/// - Reduced memory copies: from 3-4 copies to 1 copy
+/// - Lower CPU usage: 20-30% reduction expected
+/// - Improved latency P95: 15-25% reduction expected
+/// - Increased throughput: 10-20% improvement expected
+///
+/// On non-Unix platforms or when mmap fails, the system automatically falls back
+/// to regular I/O without errors.
+pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true;
+
+// =============================================================================
+// Direct I/O Configuration
+// =============================================================================
+
+/// Environment variable for Direct I/O enable (Linux only).
+///
+/// When enabled, uses O_DIRECT flag to bypass OS page cache for large files.
+/// This is only beneficial for specific workloads (databases, large sequential reads).
+///
+/// - Purpose: Enable or disable Direct I/O for large file operations
+/// - Acceptable values: `"true"` / `"false"` (case-insensitive) or a boolean typed config
+/// - Semantics: When enabled, files larger than the threshold will use O_DIRECT flag;
+///   this bypasses the OS page cache and transfers data directly between disk and application
+/// - Example: `export RUSTFS_OBJECT_DIRECT_IO_ENABLE=true`
+/// - Note: Direct I/O is disabled by default because it's only beneficial for specific
+///   use cases. For most workloads, the OS page cache provides better performance.
+pub const ENV_OBJECT_DIRECT_IO_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_ENABLE";
+
+/// Default: Direct I/O is disabled.
+///
+/// Direct I/O is disabled by default because it's only beneficial for specific use cases:
+/// - Large file transfers (>128MB)
+/// - Databases with their own cache
+/// - Applications requiring predictable I/O latency
+///
+/// For most workloads, the OS page cache provides better performance through:
+/// - Read-ahead caching
+/// - Write buffering
+/// - Multi-use caching (same data cached for multiple operations)
+pub const DEFAULT_OBJECT_DIRECT_IO_ENABLE: bool = false;
+
+/// Environment variable for Direct I/O minimum file size threshold.
+///
+/// Files smaller than this size will use regular I/O even if Direct I/O is enabled.
+/// This avoids the overhead of Direct I/O for small files where the OS page cache
+/// is more effective.
+///
+/// - Purpose: Set the minimum file size for Direct I/O operations
+/// - Unit: Bytes
+/// - Valid values: any positive integer (default: 134,217,728 bytes = 128 MB)
+/// - Semantics: Only files larger than this threshold will use Direct I/O when enabled;
+///   smaller files use regular buffered I/O
+/// - Example: `export RUSTFS_OBJECT_DIRECT_IO_THRESHOLD=268435456`
+/// - Note: The default threshold of 128MB balances the overhead of Direct I/O setup
+///   against the benefits of bypassing the page cache for large files.
+pub const ENV_OBJECT_DIRECT_IO_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_THRESHOLD";
+
+/// Default Direct I/O threshold: 128 MB.
+///
+/// Only files larger than 128MB will use Direct I/O when enabled.
+/// Smaller files benefit from OS page cache.
+///
+/// Formula: 128 * 1024 * 1024 = 134,217,728 bytes
+pub const DEFAULT_OBJECT_DIRECT_IO_THRESHOLD: usize = 128 * 1024 * 1024;

crates/config/src/lib.rs

@@ -19,6 +19,8 @@ pub use constants::app::*;
 #[cfg(feature = "constants")]
 pub use constants::body_limits::*;
 #[cfg(feature = "constants")]
+pub use constants::capacity::*;
+#[cfg(feature = "constants")]
 pub use constants::compress::*;
 #[cfg(feature = "constants")]
 pub use constants::console::*;
@@ -45,6 +47,10 @@ pub use constants::targets::*;
 #[cfg(feature = "constants")]
 pub use constants::tls::*;
 #[cfg(feature = "constants")]
+pub use constants::workload::*;
+#[cfg(feature = "constants")]
+pub use constants::zero_copy::*;
+#[cfg(feature = "constants")]
 pub mod oidc {
     pub use super::constants::oidc::*;
 }

crates/config/src/notify/mqtt.rs

@@ -24,6 +24,12 @@ pub const NOTIFY_MQTT_KEYS: &[&str] = &[
     crate::MQTT_KEEP_ALIVE_INTERVAL,
     crate::MQTT_QUEUE_DIR,
     crate::MQTT_QUEUE_LIMIT,
+    crate::MQTT_TLS_POLICY,
+    crate::MQTT_TLS_CA,
+    crate::MQTT_TLS_CLIENT_CERT,
+    crate::MQTT_TLS_CLIENT_KEY,
+    crate::MQTT_TLS_TRUST_LEAF_AS_CA,
+    crate::MQTT_WS_PATH_ALLOWLIST,
     crate::COMMENT_KEY,
 ];
 
@@ -38,8 +44,14 @@ pub const ENV_NOTIFY_MQTT_RECONNECT_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_RECONNE
 pub const ENV_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL";
 pub const ENV_NOTIFY_MQTT_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_DIR";
 pub const ENV_NOTIFY_MQTT_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MQTT_QUEUE_LIMIT";
+pub const ENV_NOTIFY_MQTT_TLS_POLICY: &str = "RUSTFS_NOTIFY_MQTT_TLS_POLICY";
+pub const ENV_NOTIFY_MQTT_TLS_CA: &str = "RUSTFS_NOTIFY_MQTT_TLS_CA";
+pub const ENV_NOTIFY_MQTT_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_MQTT_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_MQTT_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_MQTT_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_MQTT_TLS_TRUST_LEAF_AS_CA: &str = "RUSTFS_NOTIFY_MQTT_TLS_TRUST_LEAF_AS_CA";
+pub const ENV_NOTIFY_MQTT_WS_PATH_ALLOWLIST: &str = "RUSTFS_NOTIFY_MQTT_WS_PATH_ALLOWLIST";
 
-pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 10] = &[
+pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 16] = &[
     ENV_NOTIFY_MQTT_ENABLE,
     ENV_NOTIFY_MQTT_BROKER,
     ENV_NOTIFY_MQTT_TOPIC,
@@ -50,4 +62,10 @@ pub const ENV_NOTIFY_MQTT_KEYS: &[&str; 10] = &[
     ENV_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL,
     ENV_NOTIFY_MQTT_QUEUE_DIR,
     ENV_NOTIFY_MQTT_QUEUE_LIMIT,
+    ENV_NOTIFY_MQTT_TLS_POLICY,
+    ENV_NOTIFY_MQTT_TLS_CA,
+    ENV_NOTIFY_MQTT_TLS_CLIENT_CERT,
+    ENV_NOTIFY_MQTT_TLS_CLIENT_KEY,
+    ENV_NOTIFY_MQTT_TLS_TRUST_LEAF_AS_CA,
+    ENV_NOTIFY_MQTT_WS_PATH_ALLOWLIST,
 ];

crates/credentials/src/credentials.rs

@@ -204,10 +204,11 @@ pub fn gen_secret_key(length: usize) -> std::io::Result<String> {
     let mut key = vec![0u8; URL_SAFE_NO_PAD.estimated_decoded_length(length)];
     rng.fill_bytes(&mut key);
 
+    // URL_SAFE_NO_PAD uses "-" and "_" instead of "+" and "/", so "/" never
+    // appears in the output. The .replace("/", "+") was a dead no-op.
     let encoded = URL_SAFE_NO_PAD.encode_to_string(&key);
-    let key_str = encoded.replace("/", "+");
 
-    Ok(key_str)
+    Ok(encoded)
 }
 
 /// Get the RPC authentication token from environment variable
@@ -447,7 +448,7 @@ mod tests {
             // Initialize
             let test_ak = "test_access_key".to_string();
             let test_sk = "test_secret_key_123456".to_string();
-            init_global_action_credentials(Some(test_ak.clone()), Some(test_sk.clone())).ok();
+            init_global_action_credentials(Some(test_ak), Some(test_sk)).ok();
         }
 
         // Verify the state after initialization
@@ -473,6 +474,16 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_gen_secret_key_uses_url_safe_base64_without_padding() {
+        let key = gen_secret_key(32).expect("secret key should generate");
+
+        assert_eq!(key.len(), 32);
+        assert!(!key.contains('/'));
+        assert!(!key.contains('+'));
+        assert!(!key.contains('='));
+    }
+
     #[test]
     fn test_masked_debug() {
         // Test None

crates/crypto/Cargo.toml

@@ -56,3 +56,6 @@ crypto = [
     "dep:rand",
     "dep:sha2",
 ]
+
+[lib]
+doctest = false

crates/e2e_test/Cargo.toml

@@ -30,6 +30,7 @@ ftps = []
 [dependencies]
 rustfs-ecstore.workspace = true
 rustfs-common.workspace = true
+rustfs-rio.workspace = true
 flatbuffers.workspace = true
 futures.workspace = true
 rustfs-lock.workspace = true
@@ -40,23 +41,38 @@ serde_json.workspace = true
 tonic = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
+clap = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
 bytes.workspace = true
 serial_test = { workspace = true }
 aws-sdk-s3.workspace = true
 aws-config = { workspace = true }
+aws-smithy-http-client.workspace = true
+async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
+flate2.workspace = true
+http.workspace = true
+http-body.workspace = true
+http-body-util.workspace = true
 reqwest = { workspace = true }
+rustfs-signer.workspace = true
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 uuid = { workspace = true }
+urlencoding.workspace = true
+walkdir.workspace = true
 base64 = { workspace = true }
 rand = { workspace = true }
 chrono = { workspace = true }
 md5 = { workspace = true }
 sha2 = { workspace = true }
+astral-tokio-tar = { workspace = true }
+s3s.workspace = true
+zstd.workspace = true
+time.workspace = true
 suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
 rcgen.workspace = true
 anyhow.workspace = true
-rustls.workspace = true
+rustls.workspace = true
+zip.workspace = true

crates/e2e_test/src/anonymous_access_test.rs

@@ -16,7 +16,7 @@
 //! Verifies that anonymous access works correctly with bucket policies
 //! when PublicAccessBlock configuration is missing or explicitly set.
 
-use crate::common::{RustFSTestEnvironment, init_logging};
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::types::PublicAccessBlockConfiguration;
 use serial_test::serial;
 use tracing::info;
@@ -67,7 +67,7 @@ async fn anonymous_get_object(
     key: &str,
 ) -> Result<reqwest::Response, reqwest::Error> {
     let url = format!("{}/{}/{}", env.url, bucket_name, key);
-    reqwest::Client::new().get(&url).send().await
+    local_http_client().get(&url).send().await
 }
 
 /// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it

crates/e2e_test/src/archive_download_integrity_test.rs

@@ -0,0 +1,875 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
+    use aws_sdk_s3::Client as S3Client;
+    use aws_sdk_s3::error::ProvideErrorMetadata;
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+    use http::header::{CONTENT_TYPE, HOST};
+    use reqwest::StatusCode;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::{pre_sign_v4, sign_v4};
+    use s3s::Body;
+    use serial_test::serial;
+    use sha2::{Digest, Sha256};
+    use std::error::Error;
+    use std::io::{Cursor, Write};
+    use std::process::Command;
+    use time::OffsetDateTime;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+    use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
+
+    const ARCHIVE_TEST_BUCKET: &str = "archive-download-integrity";
+    const MULTIPART_ARCHIVE_TEST_BUCKET: &str = "archive-multipart-integrity";
+    const MULTIPART_PART_SIZE: usize = 5 * 1024 * 1024;
+
+    fn build_zip_bytes(files: &[(&str, &[u8])]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let cursor = Cursor::new(Vec::new());
+        let mut zip = ZipWriter::new(cursor);
+        let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
+
+        for (name, content) in files {
+            zip.start_file(*name, options)?;
+            zip.write_all(content)?;
+        }
+
+        Ok(zip.finish()?.into_inner())
+    }
+
+    fn random_bytes(size: usize) -> Vec<u8> {
+        (0..size).map(|idx| (idx % 251) as u8).collect()
+    }
+
+    async fn start_rustfs_server_with_env(
+        env: &mut RustFSTestEnvironment,
+        extra_env: &[(&str, &str)],
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let binary_path = rustfs_binary_path();
+        let mut command = Command::new(&binary_path);
+        command.env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
+        for (key, value) in extra_env {
+            command.env(key, value);
+        }
+
+        let process = command
+            .args([
+                "--address",
+                &env.address,
+                "--access-key",
+                &env.access_key,
+                "--secret-key",
+                &env.secret_key,
+                &env.temp_dir,
+            ])
+            .spawn()?;
+
+        env.process = Some(process);
+        env.wait_for_server_ready().await?;
+        Ok(())
+    }
+
+    async fn presigned_get_request_with_accept_encoding(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        accept_encoding: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let signed = pre_sign_v4(
+            http::Request::builder()
+                .method(http::Method::GET)
+                .uri(uri)
+                .header(HOST, authority)
+                .body(Body::empty())?,
+            access_key,
+            secret_key,
+            "",
+            "us-east-1",
+            600,
+            OffsetDateTime::now_utc(),
+        );
+
+        let client = reqwest::Client::builder()
+            .no_proxy()
+            .no_gzip()
+            .no_brotli()
+            .no_zstd()
+            .no_deflate()
+            .build()?;
+
+        Ok(client
+            .get(signed.uri().to_string())
+            .header("Accept-Encoding", accept_encoding)
+            .send()
+            .await?)
+    }
+
+    fn find_header_terminator(buf: &[u8]) -> Option<usize> {
+        buf.windows(4).position(|window| window == b"\r\n\r\n")
+    }
+
+    async fn read_proxy_request(stream: &mut tokio::net::TcpStream) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let mut buffer = Vec::new();
+        let mut chunk = [0_u8; 4096];
+
+        loop {
+            let read = stream.read(&mut chunk).await?;
+            if read == 0 {
+                return Err("proxy request ended before headers were fully received".into());
+            }
+            buffer.extend_from_slice(&chunk[..read]);
+            if find_header_terminator(&buffer).is_some() {
+                return Ok(());
+            }
+        }
+    }
+
+    async fn spawn_reverse_proxy_to_presigned_url(
+        target_url: String,
+    ) -> Result<(String, tokio::task::JoinHandle<Result<(), Box<dyn Error + Send + Sync>>>), Box<dyn Error + Send + Sync>> {
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let address = listener.local_addr()?;
+        let proxy_url = format!("http://{address}/");
+
+        let handle = tokio::spawn(async move {
+            let (mut downstream, _) = listener.accept().await?;
+            read_proxy_request(&mut downstream).await?;
+
+            let upstream_response: Result<reqwest::Response, reqwest::Error> = local_http_client().get(&target_url).send().await;
+            let (status, body, content_type) = match upstream_response {
+                Ok(response) => {
+                    let status = response.status();
+                    let content_type = response
+                        .headers()
+                        .get("content-type")
+                        .and_then(|value| value.to_str().ok())
+                        .map(str::to_string);
+                    match response.bytes().await {
+                        Ok(body) => (status, body.to_vec(), content_type),
+                        Err(err) => {
+                            let body = format!("upstream body read failed: {err}").into_bytes();
+                            (StatusCode::BAD_GATEWAY, body, Some("text/plain".to_string()))
+                        }
+                    }
+                }
+                Err(err) => {
+                    let body = format!("upstream request failed: {err}").into_bytes();
+                    (StatusCode::BAD_GATEWAY, body, Some("text/plain".to_string()))
+                }
+            };
+
+            let mut response_head = format!("HTTP/1.1 {}\r\ncontent-length: {}\r\nconnection: close\r\n", status, body.len());
+            if let Some(content_type) = content_type {
+                response_head.push_str(&format!("content-type: {content_type}\r\n"));
+            }
+            response_head.push_str("\r\n");
+
+            downstream.write_all(response_head.as_bytes()).await?;
+            downstream.write_all(&body).await?;
+            downstream.shutdown().await?;
+            Ok(())
+        });
+
+        Ok((proxy_url, handle))
+    }
+
+    async fn signed_put_request_with_headers(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        body: Vec<u8>,
+        content_type: &str,
+        content_encoding: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::PUT)
+            .uri(uri)
+            .header(HOST, authority)
+            .header(CONTENT_TYPE, content_type)
+            .header("content-encoding", content_encoding)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+        let signed = sign_v4(request, body.len() as i64, access_key, secret_key, "", "us-east-1");
+
+        let client = reqwest::Client::builder().no_proxy().build()?;
+        let mut builder = client.put(url).body(body);
+        for (name, value) in signed.headers() {
+            builder = builder.header(name, value);
+        }
+
+        Ok(builder.send().await?)
+    }
+
+    async fn assert_archive_object_content_encoding(
+        client: &S3Client,
+        bucket: &str,
+        key: &str,
+        expected_content_encoding: Option<&str>,
+        expected_body: &[u8],
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let head_resp = client.head_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(head_resp.content_encoding(), expected_content_encoding);
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(get_resp.content_encoding(), expected_content_encoding);
+        let body = get_resp.body.collect().await?.into_bytes();
+        assert_eq!(body.as_ref(), expected_body);
+
+        Ok(())
+    }
+
+    async fn complete_archive_multipart_upload_with_content_encoding(
+        client: &S3Client,
+        bucket: &str,
+        key: &str,
+        content_encoding: Option<&str>,
+    ) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let payload = random_bytes(MULTIPART_PART_SIZE + 256 * 1024);
+        let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
+        assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
+
+        let mut create_builder = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .content_type("application/zip");
+        if let Some(content_encoding) = content_encoding {
+            create_builder = create_builder.content_encoding(content_encoding);
+        }
+        let create_output = create_builder.send().await?;
+        let upload_id = create_output.upload_id().expect("multipart upload id");
+
+        let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
+        let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
+
+        let upload_part_1 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(first_part))
+            .send()
+            .await?;
+
+        let upload_part_2 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(2)
+            .body(ByteStream::from(second_part))
+            .send()
+            .await?;
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload_part_1.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload_part_2.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await?;
+
+        Ok(zip_bytes)
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
+        let response =
+            signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip")
+                .await?;
+
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        let head_resp = client
+            .head_object()
+            .bucket(ARCHIVE_TEST_BUCKET)
+            .key("bundle.zip")
+            .send()
+            .await?;
+        assert_eq!(head_resp.content_encoding(), Some("gzip"));
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
+        let response =
+            signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip")
+                .await?;
+
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+        let body = response.text().await?;
+        assert!(
+            body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
+            "unexpected error body: {body}"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked-gzip.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked,gzip",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-aws-chunked-gzip.zip",
+            Some("gzip"),
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-strict-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked-gzip.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes,
+            "application/zip",
+            "aws-chunked,gzip",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+        let body = response.text().await?;
+        assert!(
+            body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
+            "unexpected error body: {body}"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_download_roundtrip_with_http_compression_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(
+            &mut env,
+            &[
+                ("RUSTFS_COMPRESS_ENABLE", "on"),
+                ("RUSTFS_COMPRESS_MIME_TYPES", "text/*,application/json,application/zip"),
+                ("RUSTFS_COMPRESS_MIN_SIZE", "1"),
+            ],
+        )
+        .await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = build_zip_bytes(&[
+            ("docs/readme.txt", b"archive-download-integrity"),
+            ("docs/notes.txt", b"response-compression-must-not-alter-zip-bytes"),
+        ])?;
+        let expected_sha256 = Sha256::digest(&zip_bytes);
+
+        client
+            .put_object()
+            .bucket(ARCHIVE_TEST_BUCKET)
+            .key("bundle.zip")
+            .content_type("application/zip")
+            .body(ByteStream::from(zip_bytes.clone()))
+            .send()
+            .await?;
+
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
+        let response =
+            presigned_get_request_with_accept_encoding(&object_url, &env.access_key, &env.secret_key, "gzip, br, zstd").await?;
+
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(
+            response
+                .headers()
+                .get("content-encoding")
+                .and_then(|value| value.to_str().ok()),
+            None,
+            "archive download must not be HTTP-compressed"
+        );
+
+        let downloaded = response.bytes().await?;
+        assert_eq!(
+            downloaded.as_ref(),
+            zip_bytes.as_slice(),
+            "downloaded archive bytes must match uploaded bytes"
+        );
+        assert_eq!(
+            Sha256::digest(downloaded.as_ref()).as_slice(),
+            expected_sha256.as_slice(),
+            "archive SHA256 mismatch"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_roundtrip_preserves_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server_without_cleanup(vec![]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let payload = random_bytes(MULTIPART_PART_SIZE + 512 * 1024);
+        let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
+        assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
+        let expected_sha256 = Sha256::digest(&zip_bytes);
+
+        let create_output = client
+            .create_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .content_type("application/zip")
+            .send()
+            .await?;
+        let upload_id = create_output.upload_id().expect("multipart upload id");
+
+        let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
+        let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
+
+        let upload_part_1 = client
+            .upload_part()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(first_part))
+            .send()
+            .await?;
+
+        let upload_part_2 = client
+            .upload_part()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .upload_id(upload_id)
+            .part_number(2)
+            .body(ByteStream::from(second_part))
+            .send()
+            .await?;
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload_part_1.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload_part_2.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await?;
+
+        let downloaded = client
+            .get_object()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-bundle.zip")
+            .send()
+            .await?
+            .body
+            .collect()
+            .await?
+            .into_bytes();
+
+        assert_eq!(
+            downloaded.as_ref(),
+            zip_bytes.as_slice(),
+            "multipart archive bytes must match uploaded bytes"
+        );
+        assert_eq!(
+            Sha256::digest(downloaded.as_ref()).as_slice(),
+            expected_sha256.as_slice(),
+            "multipart archive SHA256 mismatch"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-aws-chunked-gzip.zip",
+            Some("aws-chunked,gzip"),
+        )
+        .await?;
+        assert_archive_object_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-aws-chunked-gzip.zip",
+            Some("gzip"),
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-strict-aws-chunked.zip",
+            Some("aws-chunked"),
+        )
+        .await?;
+        assert_archive_object_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-strict-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let create_result = client
+            .create_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-strict-aws-chunked-gzip.zip")
+            .content_type("application/zip")
+            .content_encoding("aws-chunked,gzip")
+            .send()
+            .await;
+        let err = create_result.expect_err("strict mode should reject effective archive content encoding");
+        assert_eq!(err.code(), Some("InvalidArgument"));
+        assert!(
+            err.message().is_some_and(|message| {
+                message.contains("Content-Encoding") && message.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING")
+            }),
+            "unexpected error metadata: code={:?}, message={:?}",
+            err.code(),
+            err.message()
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let payload = random_bytes(MULTIPART_PART_SIZE + 768 * 1024);
+        let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
+        assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
+
+        let create_output = client
+            .create_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("presigned-multipart-bundle.zip")
+            .content_type("application/zip")
+            .send()
+            .await?;
+        let upload_id = create_output.upload_id().expect("multipart upload id");
+
+        let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
+        let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
+
+        let upload_part_1 = client
+            .upload_part()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("presigned-multipart-bundle.zip")
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(first_part))
+            .send()
+            .await?;
+
+        let upload_part_2 = client
+            .upload_part()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("presigned-multipart-bundle.zip")
+            .upload_id(upload_id)
+            .part_number(2)
+            .body(ByteStream::from(second_part))
+            .send()
+            .await?;
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload_part_1.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload_part_2.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("presigned-multipart-bundle.zip")
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await?;
+
+        let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, "presigned-multipart-bundle.zip");
+        let direct_response =
+            presigned_get_request_with_accept_encoding(&object_url, &env.access_key, &env.secret_key, "identity").await?;
+        assert_eq!(direct_response.status(), StatusCode::OK);
+        assert_eq!(
+            direct_response
+                .headers()
+                .get("content-length")
+                .and_then(|value| value.to_str().ok())
+                .and_then(|value| value.parse::<usize>().ok()),
+            Some(zip_bytes.len())
+        );
+        let direct_body = direct_response.bytes().await?;
+        assert_eq!(direct_body.len(), zip_bytes.len());
+        assert_eq!(direct_body.as_ref(), zip_bytes.as_slice());
+
+        let signed = pre_sign_v4(
+            http::Request::builder()
+                .method(http::Method::GET)
+                .uri(object_url.parse::<http::Uri>()?)
+                .header(
+                    HOST,
+                    object_url
+                        .parse::<http::Uri>()?
+                        .authority()
+                        .ok_or("request URL missing authority")?
+                        .to_string(),
+                )
+                .body(Body::empty())?,
+            &env.access_key,
+            &env.secret_key,
+            "",
+            "us-east-1",
+            600,
+            OffsetDateTime::now_utc(),
+        );
+        let (proxy_url, proxy_handle) = spawn_reverse_proxy_to_presigned_url(signed.uri().to_string()).await?;
+        let proxied_response: reqwest::Response = local_http_client().get(&proxy_url).send().await?;
+        assert_eq!(proxied_response.status(), StatusCode::OK);
+        assert_eq!(
+            proxied_response
+                .headers()
+                .get("content-length")
+                .and_then(|value| value.to_str().ok())
+                .and_then(|value| value.parse::<usize>().ok()),
+            Some(zip_bytes.len())
+        );
+        let proxied_body: bytes::Bytes = proxied_response.bytes().await?;
+        assert_eq!(proxied_body.len(), zip_bytes.len());
+        assert_eq!(proxied_body.as_ref(), zip_bytes.as_slice());
+
+        proxy_handle.await??;
+        env.stop_server();
+        Ok(())
+    }
+}

crates/e2e_test/src/bucket_logging_test.rs

@@ -0,0 +1,544 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end tests for S3 dummy-compat bucket APIs.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::error::ProvideErrorMetadata;
+    use aws_sdk_s3::types::{
+        AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
+        RequestPaymentConfiguration, WebsiteConfiguration,
+    };
+    use serial_test::serial;
+    use std::path::PathBuf;
+    use std::process::Command;
+    use tracing::info;
+
+    fn awscurl_binary_path() -> PathBuf {
+        std::env::var_os("AWSCURL_PATH")
+            .map(PathBuf::from)
+            .unwrap_or_else(|| PathBuf::from("awscurl"))
+    }
+
+    fn awscurl_available() -> bool {
+        Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
+    }
+
+    fn execute_s3_awscurl(
+        method: &str,
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+        let output = Command::new(awscurl_binary_path())
+            .args([
+                "--service",
+                "s3",
+                "--region",
+                "us-east-1",
+                "--access_key",
+                access_key,
+                "--secret_key",
+                secret_key,
+                "-i",
+                "-X",
+                method,
+                url,
+            ])
+            .output()?;
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
+        }
+        Ok(String::from_utf8_lossy(&output.stdout).to_string())
+    }
+
+    fn parse_status(raw: &str) -> Option<u16> {
+        raw.lines()
+            .filter_map(|line| {
+                if line.starts_with("HTTP/") {
+                    line.split_whitespace().nth(1)?.parse::<u16>().ok()
+                } else {
+                    None
+                }
+            })
+            .next_back()
+    }
+
+    fn parse_body(raw: &str) -> String {
+        if let Some(pos) = raw.rfind("\r\n\r\n") {
+            return raw[pos + 4..].to_string();
+        }
+        if let Some(pos) = raw.rfind("\n\n") {
+            return raw[pos + 2..].to_string();
+        }
+        String::new()
+    }
+
+    fn parse_headers(raw: &str) -> String {
+        let start = raw.rfind("HTTP/").unwrap_or(0);
+        let tail = &raw[start..];
+        if let Some(pos) = tail.find("\r\n\r\n") {
+            return tail[..pos].to_string();
+        }
+        if let Some(pos) = tail.find("\n\n") {
+            return tail[..pos].to_string();
+        }
+        tail.to_string()
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_compatibility_endpoints() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "test-get-bucket-logging";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        let result = client.get_bucket_logging().bucket(bucket).send().await;
+        assert!(
+            result.is_ok(),
+            "GetBucketLogging should return success for existing bucket, got: {:?}",
+            result.err()
+        );
+
+        let output = result.unwrap();
+        assert!(
+            output.logging_enabled().is_none(),
+            "Default GetBucketLogging should return empty logging configuration"
+        );
+
+        let put_logging = client
+            .put_bucket_logging()
+            .bucket(bucket)
+            .bucket_logging_status(
+                BucketLoggingStatus::builder()
+                    .logging_enabled(
+                        LoggingEnabled::builder()
+                            .target_bucket(bucket)
+                            .target_prefix("logs/")
+                            .build()
+                            .expect("failed to build LoggingEnabled"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_logging.is_ok(),
+            "PutBucketLogging should return success for existing bucket, got: {:?}",
+            put_logging.err()
+        );
+
+        let output_after_put = client
+            .get_bucket_logging()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketLogging should succeed after PutBucketLogging");
+        let logging_after_put = output_after_put
+            .logging_enabled()
+            .expect("GetBucketLogging should return persisted logging_enabled");
+        assert_eq!(
+            logging_after_put.target_bucket(),
+            bucket,
+            "GetBucketLogging should preserve target bucket"
+        );
+        assert_eq!(
+            logging_after_put.target_prefix(),
+            "logs/",
+            "GetBucketLogging should preserve target prefix"
+        );
+
+        let accelerate = client
+            .get_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketAccelerateConfiguration should succeed");
+        assert!(
+            accelerate.status().is_none(),
+            "Default GetBucketAccelerateConfiguration should return empty status"
+        );
+
+        let payment = client
+            .get_bucket_request_payment()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketRequestPayment should succeed");
+        assert_eq!(
+            payment.payer().map(|p| p.as_str()),
+            Some("BucketOwner"),
+            "GetBucketRequestPayment should return BucketOwner by default"
+        );
+
+        let put_accelerate = client
+            .put_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .accelerate_configuration(
+                AccelerateConfiguration::builder()
+                    .status(BucketAccelerateStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_accelerate.is_ok(),
+            "PutBucketAccelerateConfiguration should return success for existing bucket, got: {:?}",
+            put_accelerate.err()
+        );
+
+        let put_request_payment = client
+            .put_bucket_request_payment()
+            .bucket(bucket)
+            .request_payment_configuration(
+                RequestPaymentConfiguration::builder()
+                    .payer(Payer::Requester)
+                    .build()
+                    .expect("failed to build RequestPaymentConfiguration"),
+            )
+            .send()
+            .await;
+        assert!(
+            put_request_payment.is_ok(),
+            "PutBucketRequestPayment should return success for existing bucket, got: {:?}",
+            put_request_payment.err()
+        );
+
+        let accelerate_after_put = client
+            .get_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketAccelerateConfiguration should succeed after put");
+        assert_eq!(
+            accelerate_after_put.status().map(|s| s.as_str()),
+            Some("Suspended"),
+            "GetBucketAccelerateConfiguration should preserve put status"
+        );
+
+        let payment_after_put = client
+            .get_bucket_request_payment()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketRequestPayment should succeed after put");
+        assert_eq!(
+            payment_after_put.payer().map(|p| p.as_str()),
+            Some("Requester"),
+            "GetBucketRequestPayment should preserve put payer"
+        );
+
+        let put_website = client
+            .put_bucket_website()
+            .bucket(bucket)
+            .website_configuration(
+                WebsiteConfiguration::builder()
+                    .index_document(
+                        IndexDocument::builder()
+                            .suffix("index.html")
+                            .build()
+                            .expect("failed to build IndexDocument"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_website.is_ok(),
+            "PutBucketWebsite should return success for existing bucket, got: {:?}",
+            put_website.err()
+        );
+
+        let website = client.get_bucket_website().bucket(bucket).send().await;
+        assert!(website.is_ok(), "GetBucketWebsite should return persisted website configuration");
+        let website_output = website.unwrap();
+        assert_eq!(
+            website_output.index_document().map(|doc| doc.suffix()),
+            Some("index.html"),
+            "GetBucketWebsite should preserve index document suffix"
+        );
+
+        client
+            .delete_bucket_website()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("DeleteBucketWebsite should return success");
+
+        let website_after_delete = client.get_bucket_website().bucket(bucket).send().await;
+        assert!(
+            website_after_delete.is_err(),
+            "GetBucketWebsite should return NoSuchWebsiteConfiguration after deletion"
+        );
+        let website_err = website_after_delete.err().unwrap();
+        let website_code = website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(website_code, Some("NoSuchWebsiteConfiguration")),
+            "Unexpected GetBucketWebsite error code: {:?}, err: {:?}",
+            website_code,
+            website_err
+        );
+
+        env.stop_server();
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let missing_bucket = "test-dummy-bucket-missing";
+
+        let get_logging = client.get_bucket_logging().bucket(missing_bucket).send().await;
+        assert!(get_logging.is_err(), "GetBucketLogging should fail for missing bucket");
+        let get_logging_err = get_logging.err().unwrap();
+        let get_logging_code = get_logging_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_logging_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketLogging error code: {:?}, err: {:?}",
+            get_logging_code,
+            get_logging_err
+        );
+
+        let put_logging = client
+            .put_bucket_logging()
+            .bucket(missing_bucket)
+            .bucket_logging_status(BucketLoggingStatus::builder().build())
+            .send()
+            .await;
+        assert!(put_logging.is_err(), "PutBucketLogging should fail for missing bucket");
+        let put_logging_err = put_logging.err().unwrap();
+        let put_logging_code = put_logging_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_logging_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketLogging error code: {:?}, err: {:?}",
+            put_logging_code,
+            put_logging_err
+        );
+
+        let get_accelerate = client
+            .get_bucket_accelerate_configuration()
+            .bucket(missing_bucket)
+            .send()
+            .await;
+        assert!(get_accelerate.is_err(), "GetBucketAccelerateConfiguration should fail for missing bucket");
+        let get_accelerate_err = get_accelerate.err().unwrap();
+        let get_accelerate_code = get_accelerate_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_accelerate_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketAccelerateConfiguration error code: {:?}, err: {:?}",
+            get_accelerate_code,
+            get_accelerate_err
+        );
+
+        let get_request_payment = client.get_bucket_request_payment().bucket(missing_bucket).send().await;
+        assert!(get_request_payment.is_err(), "GetBucketRequestPayment should fail for missing bucket");
+        let get_request_payment_err = get_request_payment.err().unwrap();
+        let get_request_payment_code = get_request_payment_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_request_payment_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketRequestPayment error code: {:?}, err: {:?}",
+            get_request_payment_code,
+            get_request_payment_err
+        );
+
+        let put_accelerate = client
+            .put_bucket_accelerate_configuration()
+            .bucket(missing_bucket)
+            .accelerate_configuration(
+                AccelerateConfiguration::builder()
+                    .status(BucketAccelerateStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(put_accelerate.is_err(), "PutBucketAccelerateConfiguration should fail for missing bucket");
+        let put_accelerate_err = put_accelerate.err().unwrap();
+        let put_accelerate_code = put_accelerate_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_accelerate_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketAccelerateConfiguration error code: {:?}, err: {:?}",
+            put_accelerate_code,
+            put_accelerate_err
+        );
+
+        let put_request_payment = client
+            .put_bucket_request_payment()
+            .bucket(missing_bucket)
+            .request_payment_configuration(
+                RequestPaymentConfiguration::builder()
+                    .payer(Payer::BucketOwner)
+                    .build()
+                    .expect("failed to build RequestPaymentConfiguration"),
+            )
+            .send()
+            .await;
+        assert!(put_request_payment.is_err(), "PutBucketRequestPayment should fail for missing bucket");
+        let put_request_payment_err = put_request_payment.err().unwrap();
+        let put_request_payment_code = put_request_payment_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_request_payment_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketRequestPayment error code: {:?}, err: {:?}",
+            put_request_payment_code,
+            put_request_payment_err
+        );
+
+        let put_website = client
+            .put_bucket_website()
+            .bucket(missing_bucket)
+            .website_configuration(
+                WebsiteConfiguration::builder()
+                    .index_document(
+                        IndexDocument::builder()
+                            .suffix("index.html")
+                            .build()
+                            .expect("failed to build IndexDocument"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(put_website.is_err(), "PutBucketWebsite should fail for missing bucket");
+        let put_website_err = put_website.err().unwrap();
+        let put_website_code = put_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_website_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketWebsite error code: {:?}, err: {:?}",
+            put_website_code,
+            put_website_err
+        );
+
+        let get_website = client.get_bucket_website().bucket(missing_bucket).send().await;
+        assert!(get_website.is_err(), "GetBucketWebsite should fail for missing bucket");
+        let get_website_err = get_website.err().unwrap();
+        let get_website_code = get_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_website_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketWebsite error code: {:?}, err: {:?}",
+            get_website_code,
+            get_website_err
+        );
+
+        let delete_website = client.delete_bucket_website().bucket(missing_bucket).send().await;
+        assert!(delete_website.is_err(), "DeleteBucketWebsite should fail for missing bucket");
+        let delete_website_err = delete_website.err().unwrap();
+        let delete_website_code = delete_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(delete_website_code, Some("NoSuchBucket")),
+            "Unexpected DeleteBucketWebsite error code: {:?}, err: {:?}",
+            delete_website_code,
+            delete_website_err
+        );
+
+        env.stop_server();
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_endpoints_http_contracts() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket API HTTP contracts");
+        if !awscurl_available() {
+            info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
+            return;
+        }
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "test-dummy-bucket-http-contracts";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketLogging HTTP request failed");
+        assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
+        let logging_body = parse_body(&logging_raw);
+        assert!(
+            logging_body.contains("<BucketLoggingStatus"),
+            "GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
+        );
+
+        let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketAccelerateConfiguration HTTP request failed");
+        assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
+        let accel_body = parse_body(&accel_raw);
+        assert!(
+            accel_body.contains("<AccelerateConfiguration"),
+            "GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
+        );
+
+        let payment_raw =
+            execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
+                .expect("GetBucketRequestPayment HTTP request failed");
+        assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
+        let payment_body = parse_body(&payment_raw);
+        assert!(
+            payment_body.contains("<Payer>BucketOwner</Payer>"),
+            "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
+        );
+
+        let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketWebsite HTTP request failed");
+        assert_eq!(
+            parse_status(&website_raw),
+            Some(404),
+            "GetBucketWebsite should return 404 when website config is absent"
+        );
+        let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
+        assert!(
+            website_content_type.contains("content-type:") && website_content_type.contains("xml"),
+            "GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
+        );
+        let website_body = parse_body(&website_raw);
+        assert!(
+            website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
+            "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
+        );
+
+        let delete_raw =
+            execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+                .expect("DeleteBucketWebsite HTTP request failed");
+        assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
+
+        env.stop_server();
+    }
+}

crates/e2e_test/src/bucket_policy_check_test.rs

@@ -54,6 +54,10 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
 #[serial]
 async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if !crate::common::awscurl_available() {
+        info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
+        return Ok(());
+    }
     info!("Starting test_bucket_policy_authenticated_user...");
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/checksum_upload_test.rs

@@ -20,8 +20,9 @@ mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
     use base64::Engine;
+    use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
     use tracing::info;
@@ -57,6 +58,12 @@ mod tests {
         base64::engine::general_purpose::STANDARD.encode(digest.as_slice())
     }
 
+    fn checksum_crc64nvme_base64(body: &[u8]) -> String {
+        Checksum::new_from_data(RioChecksumType::CRC64_NVME, body)
+            .expect("crc64nvme checksum")
+            .encoded
+    }
+
     /// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
     #[tokio::test]
     #[serial]
@@ -226,4 +233,135 @@ mod tests {
         );
         info!("PASSED: MultipartUpload with checksum and GetObject content match");
     }
+
+    /// Regression test for issue #2282:
+    /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
+    #[tokio::test]
+    #[serial]
+    async fn test_crc64nvme_matches_between_put_object_and_multipart_upload() {
+        init_logging();
+        info!("TEST: CRC64NVME matches between direct PutObject and multipart upload");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-crc64nvme-multipart-match";
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        const PART_SIZE: usize = 6 * 1024 * 1024;
+        let part1: Vec<u8> = (0..PART_SIZE).map(|i| (i % 251) as u8).collect();
+        let part2: Vec<u8> = (0..PART_SIZE).map(|i| ((i + 17) % 251) as u8).collect();
+        let content: Vec<u8> = part1.iter().chain(part2.iter()).copied().collect();
+
+        let direct_key = "crc64nvme-direct.bin";
+        let multipart_key = "crc64nvme-multipart.bin";
+        let full_checksum = checksum_crc64nvme_base64(&content);
+        let part1_checksum = checksum_crc64nvme_base64(&part1);
+        let part2_checksum = checksum_crc64nvme_base64(&part2);
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(direct_key)
+            .body(ByteStream::from(content.clone()))
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .checksum_crc64_nvme(full_checksum.clone())
+            .send()
+            .await
+            .expect("Failed to put direct object with CRC64NVME");
+
+        let create_result = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .send()
+            .await
+            .expect("Failed to create multipart upload");
+
+        let upload_id = create_result.upload_id().expect("No upload_id").to_string();
+
+        let upload1 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(&upload_id)
+            .part_number(1)
+            .body(ByteStream::from(part1.clone()))
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .checksum_crc64_nvme(part1_checksum)
+            .send()
+            .await
+            .expect("Failed to upload multipart part 1");
+
+        let upload2 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(&upload_id)
+            .part_number(2)
+            .body(ByteStream::from(part2.clone()))
+            .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme)
+            .checksum_crc64_nvme(part2_checksum)
+            .send()
+            .await
+            .expect("Failed to upload multipart part 2");
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload1.e_tag().expect("No etag for part 1"))
+                    .checksum_crc64_nvme(upload1.checksum_crc64_nvme().expect("No CRC64NVME for part 1"))
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload2.e_tag().expect("No etag for part 2"))
+                    .checksum_crc64_nvme(upload2.checksum_crc64_nvme().expect("No CRC64NVME for part 2"))
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(multipart_key)
+            .upload_id(&upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await
+            .expect("Failed to complete multipart upload");
+
+        let direct_head = client
+            .head_object()
+            .bucket(bucket)
+            .key(direct_key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("Failed to head direct object");
+
+        let multipart_head = client
+            .head_object()
+            .bucket(bucket)
+            .key(multipart_key)
+            .checksum_mode(ChecksumMode::Enabled)
+            .send()
+            .await
+            .expect("Failed to head multipart object");
+
+        assert_eq!(
+            direct_head.checksum_crc64_nvme(),
+            Some(full_checksum.as_str()),
+            "Direct object should report the uploaded full-object CRC64NVME"
+        );
+        assert_eq!(
+            multipart_head.checksum_crc64_nvme(),
+            Some(full_checksum.as_str()),
+            "Multipart object should report the same full-object CRC64NVME as direct upload"
+        );
+    }
 }

crates/e2e_test/src/common.rs

@@ -23,7 +23,11 @@
 
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
-use std::path::PathBuf;
+use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use reqwest::Client as HttpClient;
+use std::ffi::OsStr;
+use std::fs as stdfs;
+use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use std::sync::Once;
 use std::time::Duration;
@@ -32,11 +36,29 @@ use tokio::net::TcpStream;
 use tokio::time::sleep;
 use tracing::{error, info, warn};
 use uuid::Uuid;
+use walkdir::WalkDir;
 
 // Common constants for all E2E tests
 pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
 pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
 pub const TEST_BUCKET: &str = "e2e-test-bucket";
+
+fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
+    let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
+    let mut config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(endpoint_url)
+        .force_path_style(true)
+        .behavior_version_latest();
+
+    if endpoint_url.starts_with("http://") {
+        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
+    }
+
+    config.build()
+}
+
 pub fn workspace_root() -> PathBuf {
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     path.pop(); // e2e_test
@@ -44,29 +66,125 @@ pub fn workspace_root() -> PathBuf {
     path
 }
 
+pub fn local_http_client() -> HttpClient {
+    HttpClient::builder()
+        .no_proxy()
+        .build()
+        .expect("failed to build local reqwest client")
+}
+
 /// Resolve the RustFS binary relative to the workspace.
-/// Always builds the binary to ensure it's up to date.
 pub fn rustfs_binary_path() -> PathBuf {
+    rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())
+}
+
+/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
+pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
     if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
         return PathBuf::from(path);
     }
 
-    // Always build the binary to ensure it's up to date
-    info!("Building RustFS binary to ensure it's up to date...");
-    build_rustfs_binary();
-
     let mut binary_path = workspace_root();
     binary_path.push("target");
     let profile_dir = if cfg!(debug_assertions) { "debug" } else { "release" };
     binary_path.push(profile_dir);
     binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
 
+    let features_match = binary_features_match(&binary_path, requested_features);
+    let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
+    let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match;
+    if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) {
+        if source_is_newer {
+            warn!(
+                "RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds",
+                binary_path
+            );
+        }
+        info!("Using existing RustFS binary at {:?}", binary_path);
+        return binary_path;
+    }
+
+    info!("Building RustFS binary to ensure it's up to date...");
+    build_rustfs_binary(requested_features);
+
     info!("Using RustFS binary at {:?}", binary_path);
     binary_path
 }
 
+fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool {
+    let Ok(binary_meta) = std::fs::metadata(binary_path) else {
+        return true;
+    };
+    let Ok(binary_modified) = binary_meta.modified() else {
+        return true;
+    };
+
+    let workspace = workspace_root();
+    let watch_roots = [
+        workspace.join("Cargo.toml"),
+        workspace.join("Cargo.lock"),
+        workspace.join("rustfs"),
+        workspace.join("crates"),
+    ];
+
+    watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path))
+}
+
+fn running_inside_e2e_test_binary() -> bool {
+    std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test")
+}
+
+fn requested_rustfs_build_features() -> Option<String> {
+    std::env::var("RUSTFS_BUILD_FEATURES")
+        .ok()
+        .map(|value| value.trim().to_string())
+        .filter(|value| !value.is_empty())
+}
+
+fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf {
+    binary_path.with_extension("features")
+}
+
+fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool {
+    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
+    let recorded = stdfs::read_to_string(stamp_path).ok().map(|value| value.trim().to_string());
+
+    match requested_features {
+        Some(features) => recorded.as_deref() == Some(features),
+        None => recorded.as_deref().is_none_or(str::is_empty),
+    }
+}
+
+fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool {
+    if path.is_file() {
+        return std::fs::metadata(path)
+            .and_then(|meta| meta.modified())
+            .map(|modified| modified > binary_modified)
+            .unwrap_or(false);
+    }
+
+    if !path.is_dir() {
+        return false;
+    }
+
+    WalkDir::new(path)
+        .into_iter()
+        .filter_entry(|entry| {
+            let name = entry.file_name();
+            name != OsStr::new("target") && name != OsStr::new(".git")
+        })
+        .filter_map(Result::ok)
+        .filter(|entry| entry.file_type().is_file())
+        .any(|entry| {
+            std::fs::metadata(entry.path())
+                .and_then(|meta| meta.modified())
+                .map(|modified| modified > binary_modified)
+                .unwrap_or(false)
+        })
+}
+
 /// Build the RustFS binary using cargo
-fn build_rustfs_binary() {
+fn build_rustfs_binary(requested_features: Option<&str>) {
     let workspace = workspace_root();
     info!("Building RustFS binary from workspace: {:?}", workspace);
 
@@ -81,11 +199,8 @@ fn build_rustfs_binary() {
     let mut cmd = Command::new("cargo");
     cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);
 
-    // Read features from environment variable for e2e tests
-    if let Ok(features) = std::env::var("RUSTFS_BUILD_FEATURES")
-        && !features.is_empty()
-    {
-        cmd.arg("--features").arg(&features);
+    if let Some(features) = requested_features {
+        cmd.arg("--features").arg(features);
         info!("Building with features: {}", features);
     }
 
@@ -105,6 +220,15 @@ fn build_rustfs_binary() {
         panic!("Failed to build RustFS binary. Error: {stderr}");
     }
 
+    let mut binary_path = workspace;
+    binary_path.push("target");
+    binary_path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
+    binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
+    let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
+    if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
+        warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
+    }
+
     info!("✅ RustFS binary built successfully");
 }
 
@@ -114,6 +238,17 @@ fn awscurl_binary_path() -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("awscurl"))
 }
 
+pub fn awscurl_available() -> bool {
+    let path = awscurl_binary_path();
+    if path.components().count() > 1 || path.is_absolute() {
+        return path.is_file();
+    }
+
+    std::env::var_os("PATH")
+        .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
+        .unwrap_or(false)
+}
+
 // Global initialization
 static INIT: Once = Once::new();
 
@@ -183,24 +318,22 @@ impl RustFSTestEnvironment {
 
     /// Kill any existing RustFS processes
     pub async fn cleanup_existing_processes(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        info!("Cleaning up any existing RustFS processes");
-        let binary_path = rustfs_binary_path();
-        let binary_name = binary_path.to_string_lossy();
-        let output = Command::new("pkill").args(["-f", &binary_name]).output();
+        info!("Cleaning up any existing RustFS processes for {}", self.address);
 
-        if let Ok(output) = output
-            && output.status.success()
-        {
-            info!("Killed existing RustFS processes: {}", binary_name);
-            sleep(Duration::from_millis(1000)).await;
+        for pattern in [&self.address, &self.temp_dir] {
+            let output = Command::new("pkill").args(["-f", pattern]).output();
+
+            if let Ok(output) = output
+                && output.status.success()
+            {
+                info!("Killed existing RustFS processes matching: {}", pattern);
+                sleep(Duration::from_millis(250)).await;
+            }
         }
         Ok(())
     }
 
-    /// Start RustFS server with basic configuration
-    pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        self.cleanup_existing_processes().await?;
-
+    fn build_start_args<'a>(&'a self, extra_args: Vec<&'a str>) -> Vec<&'a str> {
         let mut args = vec![
             "--address",
             &self.address,
@@ -210,16 +343,29 @@ impl RustFSTestEnvironment {
             &self.secret_key,
         ];
 
-        // Add extra arguments
         args.extend(extra_args);
-
-        // Add temp directory as the last argument
         args.push(&self.temp_dir);
+        args
+    }
+
+    async fn start_rustfs_server_inner(
+        &mut self,
+        extra_args: Vec<&str>,
+        cleanup_existing: bool,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        if cleanup_existing {
+            self.cleanup_existing_processes().await?;
+        }
+
+        let args = self.build_start_args(extra_args);
 
         info!("Starting RustFS server with args: {:?}", args);
 
         let binary_path = rustfs_binary_path();
-        let process = Command::new(&binary_path).args(&args).spawn()?;
+        let process = Command::new(&binary_path)
+            .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+            .args(&args)
+            .spawn()?;
 
         self.process = Some(process);
 
@@ -229,18 +375,40 @@ impl RustFSTestEnvironment {
         Ok(())
     }
 
-    /// Wait for RustFS server to be ready by checking TCP connectivity
+    /// Start RustFS server with basic configuration
+    pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner(extra_args, true).await
+    }
+
+    /// Start RustFS server without cleaning up other running RustFS processes.
+    ///
+    /// This is useful for tests that need multiple independent RustFS instances
+    /// alive at the same time.
+    pub async fn start_rustfs_server_without_cleanup(
+        &mut self,
+        extra_args: Vec<&str>,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner(extra_args, false).await
+    }
+
+    /// Wait for RustFS server to be ready.
+    ///
+    /// A listening TCP port is not sufficient here: the process may accept
+    /// connections before the S3 stack is fully initialized, which causes
+    /// early requests to fail intermittently. Treat readiness as "S3 API
+    /// responds successfully" instead.
     pub async fn wait_for_server_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         info!("Waiting for RustFS server to be ready on {}", self.address);
+        let client = self.create_s3_client();
 
-        for i in 0..30 {
-            if TcpStream::connect(&self.address).await.is_ok() {
+        for i in 0..60 {
+            if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
                 info!("✅ RustFS server is ready after {} attempts", i + 1);
                 return Ok(());
             }
 
-            if i == 29 {
-                return Err("RustFS server failed to become ready within 30 seconds".into());
+            if i == 59 {
+                return Err("RustFS server failed to become ready within 60 seconds".into());
             }
 
             sleep(Duration::from_secs(1)).await;
@@ -251,16 +419,7 @@ impl RustFSTestEnvironment {
 
     /// Create an AWS S3 client configured for this RustFS instance
     pub fn create_s3_client(&self) -> Client {
-        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "e2e-test");
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&self.url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-
-        Client::from_conf(config)
+        Client::from_conf(build_test_s3_config(&self.url, &self.access_key, &self.secret_key, "e2e-test"))
     }
 
     /// Create test bucket
@@ -304,18 +463,18 @@ impl Drop for RustFSTestEnvironment {
     }
 }
 
-/// Utility function to execute awscurl commands
-pub async fn execute_awscurl(
+async fn execute_awscurl_with_service(
     url: &str,
     method: &str,
     body: Option<&str>,
     access_key: &str,
     secret_key: &str,
+    service: &str,
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
     let mut args = vec![
         "--fail-with-body",
         "--service",
-        "s3",
+        service,
         "--region",
         "us-east-1",
         "--access_key",
@@ -331,7 +490,64 @@ pub async fn execute_awscurl(
         args.extend(&["-d", body_content]);
     }
 
-    info!("Executing awscurl: {} {}", method, url);
+    info!("Executing awscurl: {} {} (service={})", method, url, service);
+    let awscurl_path = awscurl_binary_path();
+    let output = Command::new(&awscurl_path).args(&args).output()?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
+    }
+
+    let response = String::from_utf8_lossy(&output.stdout).to_string();
+    Ok(response)
+}
+
+/// Utility function to execute awscurl commands (SigV4 service `s3` for admin APIs).
+pub async fn execute_awscurl(
+    url: &str,
+    method: &str,
+    body: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    execute_awscurl_with_service(url, method, body, access_key, secret_key, "s3").await
+}
+
+/// `POST` with SigV4 `--service sts` and explicit `Content-Type: application/x-www-form-urlencoded`.
+///
+/// RustFS `AssumeRole` is handled on `POST /` by the admin router; `is_match` requires this
+/// content type so s3s routes to the custom handler instead of `Unknown operation`.
+pub async fn awscurl_post_sts_form_urlencoded(
+    url: &str,
+    body: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    let args = vec![
+        "--fail-with-body",
+        "--service",
+        "sts",
+        "--region",
+        "us-east-1",
+        "--access_key",
+        access_key,
+        "--secret_key",
+        secret_key,
+        "-H",
+        "Content-Type: application/x-www-form-urlencoded",
+        "-X",
+        "POST",
+        url,
+        "-d",
+        body,
+    ];
+
+    info!(
+        "Executing awscurl: POST {} (service=sts, Content-Type=application/x-www-form-urlencoded)",
+        url
+    );
     let awscurl_path = awscurl_binary_path();
     let output = Command::new(&awscurl_path).args(&args).output()?;
 
@@ -406,6 +622,7 @@ pub struct RustFSTestClusterEnvironment {
     pub temp_dir: String,
     pub access_key: String,
     pub secret_key: String,
+    pub extra_env: Vec<(String, String)>,
 }
 
 impl RustFSTestClusterEnvironment {
@@ -454,9 +671,19 @@ impl RustFSTestClusterEnvironment {
             temp_dir,
             access_key: DEFAULT_ACCESS_KEY.to_string(),
             secret_key: DEFAULT_SECRET_KEY.to_string(),
+            extra_env: Vec::new(),
         })
     }
 
+    /// Add an extra environment variable applied to every cluster node process.
+    pub fn set_env<K, V>(&mut self, key: K, value: V)
+    where
+        K: Into<String>,
+        V: Into<String>,
+    {
+        self.extra_env.push((key.into(), value.into()));
+    }
+
     /// Build the volumes argument string for RustFS binary (internal helper method).
     ///
     /// Concatenates the address and data directory of all cluster nodes into a single string
@@ -487,14 +714,20 @@ impl RustFSTestClusterEnvironment {
         for (i, node) in self.nodes.iter_mut().enumerate() {
             info!("Starting cluster node {} on {}", i, node.address);
 
-            let process = Command::new(&binary_path)
+            let mut command = Command::new(&binary_path);
+            command
                 .env("RUSTFS_VOLUMES", &volumes_arg)
                 .env("RUSTFS_ADDRESS", &node.address)
                 .env("RUSTFS_ACCESS_KEY", &self.access_key)
                 .env("RUSTFS_SECRET_KEY", &self.secret_key)
                 .env("RUSTFS_CONSOLE_ENABLE", "false")
-                .current_dir(&node.data_dir)
-                .spawn()?;
+                .env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
+
+            for (key, value) in &self.extra_env {
+                command.env(key, value);
+            }
+
+            let process = command.current_dir(&node.data_dir).spawn()?;
 
             node.process = Some(process);
         }
@@ -503,7 +736,9 @@ impl RustFSTestClusterEnvironment {
             self.wait_for_node_ready(&node.address, i).await?;
         }
 
-        self.wait_for_service_ready().await?;
+        for node_idx in 0..self.nodes.len() {
+            self.wait_for_node_service_ready(node_idx).await?;
+        }
 
         Ok(())
     }
@@ -523,17 +758,17 @@ impl RustFSTestClusterEnvironment {
         Err(format!("Node {} failed to become ready", idx).into())
     }
 
-    /// Wait for the entire cluster's S3-compatible service to be ready (internal helper method).
+    /// Wait for a specific node's S3-compatible service to be ready (internal helper method).
     ///
-    /// Verifies service availability by calling the S3 `list_buckets` API, retries up to 120 times
-    /// with a 1-second interval between attempts. Fails if the API call remains unsuccessful after all retries.
-    async fn wait_for_service_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let client = self.create_s3_client(0)?;
+    /// Verifies service availability by calling the S3 `list_buckets` API against the requested node,
+    /// retries up to 120 times with a 1-second interval between attempts.
+    async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let client = self.create_s3_client(node_idx)?;
 
         for attempt in 0..120 {
             match client.list_buckets().send().await {
                 Ok(_) => {
-                    info!("Cluster service ready after {} attempts", attempt + 1);
+                    info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);
                     return Ok(());
                 }
                 Err(_) => {
@@ -541,7 +776,8 @@ impl RustFSTestClusterEnvironment {
                 }
             }
         }
-        Err("Cluster service failed to become ready".into())
+
+        Err(format!("Cluster node {} service failed to become ready", node_idx).into())
     }
 
     /// Create an S3 client configured to communicate with a specific cluster node.
@@ -562,15 +798,12 @@ impl RustFSTestClusterEnvironment {
         if node_idx >= self.nodes.len() {
             return Err("node_idx is invalid".into());
         }
-        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "cluster-test");
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&self.nodes[node_idx].url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-        Ok(Client::from_conf(config))
+        Ok(Client::from_conf(build_test_s3_config(
+            &self.nodes[node_idx].url,
+            &self.access_key,
+            &self.secret_key,
+            "cluster-test",
+        )))
     }
 
     /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector.

crates/e2e_test/src/content_encoding_test.rs

@@ -138,4 +138,63 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding,
+    /// only the effective encoding should roundtrip through GET/HEAD.
+    #[tokio::test]
+    #[serial]
+    async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() {
+        init_logging();
+        info!("aws-chunked,gzip should persist only gzip");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "content-encoding-aws-chunked-gzip-test";
+        let key = "streamed/object.txt";
+        let content = b"streaming upload body with effective gzip encoding";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .content_type("text/plain")
+            .content_encoding("aws-chunked,gzip")
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PUT failed");
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
+        assert_eq!(
+            get_resp.content_encoding(),
+            Some("gzip"),
+            "GET must return only the effective content encoding after aws-chunked is stripped"
+        );
+        let body = get_resp.body.collect().await.unwrap().into_bytes();
+        assert_eq!(body.as_ref(), content, "Body content mismatch");
+
+        let head_resp = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("HEAD failed");
+        assert_eq!(
+            head_resp.content_encoding(),
+            Some("gzip"),
+            "HEAD must return only the effective content encoding after aws-chunked is stripped"
+        );
+
+        env.stop_server();
+    }
 }

crates/e2e_test/src/delete_objects_versioning_test.rs

@@ -16,15 +16,13 @@
 //! "In a versioned Bucket, DeleteMarkers are not appearing straight after
 //! a delete_objects is called."
 //!
-//! Root cause: `delete_versions_internal` wrote new xl.meta to disk via
-//! `write_all_private` without invalidating the `GlobalFileCache`. Subsequent
-//! calls to `read_metadata` returned the stale cached xl.meta (without the
-//! delete marker), making `list_object_versions` show the old version as
-//! `IsLatest=true` rather than the new delete marker.
+//! Root cause: metadata updates could become temporarily invisible to
+//! `list_object_versions`, so the old version was still reported as
+//! `IsLatest=true` instead of the newly-created delete marker.
 //!
-//! Fix: `write_all_private` now calls `get_global_file_cache().invalidate()`
-//! after every successful write, and `rename_data` also invalidates the cache
-//! for the destination path after the atomic rename.
+//! Fix: metadata write, delete, and rename paths now make the updated
+//! `xl.meta` immediately visible, and the old file-cache shortcut has been
+//! removed from the read path.
 
 #[cfg(test)]
 mod tests {

crates/e2e_test/src/existing_object_tag_policy_test.rs

@@ -0,0 +1,364 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! E2E: `s3:ExistingObjectTag` with **IAM identity policy**, **bucket policy**, and **STS AssumeRole
+//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
+//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
+
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
+};
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Tag, Tagging};
+use aws_sdk_s3::{Client, Config};
+use serial_test::serial;
+use tracing::info;
+use uuid::Uuid;
+
+fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
+    let open = format!("<{tag}>");
+    let close = format!("</{tag}>");
+    let start = xml.find(&open)? + open.len();
+    let end = xml[start..].find(&close)? + start;
+    Some(xml[start..end].to_string())
+}
+
+fn parse_assume_role_credentials(xml: &str) -> Result<(String, String, String), Box<dyn std::error::Error + Send + Sync>> {
+    let ak = extract_xml_tag(xml, "AccessKeyId").ok_or("missing AccessKeyId in AssumeRole response")?;
+    let sk = extract_xml_tag(xml, "SecretAccessKey").ok_or("missing SecretAccessKey in AssumeRole response")?;
+    let token = extract_xml_tag(xml, "SessionToken").ok_or("missing SessionToken in AssumeRole response")?;
+    Ok((ak, sk, token))
+}
+
+async fn assume_role_with_session_policy(
+    env: &RustFSTestEnvironment,
+    parent_ak: &str,
+    parent_sk: &str,
+    session_policy_json: &str,
+) -> Result<(String, String, String), Box<dyn std::error::Error + Send + Sync>> {
+    let policy_enc = urlencoding::encode(session_policy_json);
+    let body = format!("Action=AssumeRole&Version=2011-06-15&DurationSeconds=3600&Policy={}", policy_enc);
+    let url = format!("{}/", env.url.trim_end_matches('/'));
+    let xml = awscurl_post_sts_form_urlencoded(&url, &body, parent_ak, parent_sk).await?;
+    parse_assume_role_credentials(&xml)
+}
+
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    password: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string();
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_add_canned_policy(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    policy_json: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_attach_policy_to_user(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    username: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+        env.url, policy_name, username
+    );
+    awscurl_put(&url, "", &env.access_key, &env.secret_key).await?;
+    Ok(())
+}
+
+async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) {
+    let url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, username);
+    let _ = awscurl_delete(&url, &env.access_key, &env.secret_key).await;
+}
+
+async fn admin_remove_policy(env: &RustFSTestEnvironment, policy_name: &str) {
+    let url = format!("{}/rustfs/admin/v3/remove-canned-policy?name={}", env.url, policy_name);
+    let _ = awscurl_delete(&url, &env.access_key, &env.secret_key).await;
+}
+
+async fn put_object_with_tagging_str(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    data: &[u8],
+    tagging: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(data.to_vec()))
+        .tagging(tagging)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn put_object_tag_kv(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    tag_key: &str,
+    tag_value: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let tag = Tag::builder()
+        .key(tag_key)
+        .value(tag_value)
+        .build()
+        .map_err(|e| format!("Tag build: {e}"))?;
+    let tagging = Tagging::builder()
+        .tag_set(tag)
+        .build()
+        .map_err(|e| format!("Tagging build: {e}"))?;
+    client
+        .put_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .tagging(tagging)
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
+    let _ = admin.delete_object().bucket(bucket).key(key).send().await;
+    let _ = admin.delete_bucket().bucket(bucket).send().await;
+}
+
+/// IAM identity policy: GetObject allowed only when `s3:ExistingObjectTag/security` == `public`.
+#[tokio::test]
+#[serial]
+async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let user = format!("e2eiamtag-{suffix}");
+    let user_secret = "longSecretKeyForTest123!";
+    let policy_name = format!("e2e-iam-tag-pol-{suffix}");
+    let bucket = format!("e2e-iam-tag-bkt-{suffix}");
+    let key = "tagged-object.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &user, user_secret).await?;
+
+    let policy_doc = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    admin_add_canned_policy(&env, &policy_name, &policy_doc).await?;
+    admin_attach_policy_to_user(&env, &policy_name, &user).await?;
+
+    admin.create_bucket().bucket(&bucket).send().await?;
+    put_object_with_tagging_str(&admin, &bucket, key, b"hello-iam-tag", "security=public").await?;
+
+    let uclient = user_client(&env, &user, user_secret);
+    let out = uclient.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = out.body.collect().await?;
+
+    put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
+    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
+    assert!(
+        denied.is_err(),
+        "GetObject must be denied when ExistingObjectTag no longer matches IAM policy"
+    );
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &user).await;
+    admin_remove_policy(&env, &policy_name).await;
+
+    info!("test_e2e_iam_policy_existing_object_tag_get_object passed");
+    Ok(())
+}
+
+/// Bucket policy: same `ExistingObjectTag` condition; user has no canned IAM policy attached.
+#[tokio::test]
+#[serial]
+async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let user = format!("e2ebptag-{suffix}");
+    let user_secret = "longSecretKeyForTest456!";
+    let bucket = format!("e2e-bp-tag-bkt-{suffix}");
+    let key = "obj.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &user, user_secret).await?;
+
+    admin.create_bucket().bucket(&bucket).send().await?;
+
+    let deny_before = user_client(&env, &user, user_secret)
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .send()
+        .await;
+    assert!(deny_before.is_err(), "without bucket policy, user must be denied");
+
+    let bp = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Principal": { "AWS": [user.clone()] },
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    admin.put_bucket_policy().bucket(&bucket).policy(&bp).send().await?;
+    put_object_with_tagging_str(&admin, &bucket, key, b"data", "security=public").await?;
+
+    let uclient = user_client(&env, &user, user_secret);
+    let ok = uclient.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = ok.body.collect().await?;
+
+    put_object_tag_kv(&admin, &bucket, key, "security", "private").await?;
+    let denied = uclient.get_object().bucket(&bucket).key(key).send().await;
+    assert!(denied.is_err(), "GetObject must fail when tag no longer satisfies bucket policy");
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &user).await;
+
+    info!("test_e2e_bucket_policy_existing_object_tag_get_object passed");
+    Ok(())
+}
+
+/// STS `AssumeRole` with inline `Policy` (session policy): GetObject only when `ExistingObjectTag/security` is `public`.
+#[tokio::test]
+#[serial]
+async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    if !awscurl_available() {
+        info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
+        return Ok(());
+    }
+
+    let suffix = Uuid::new_v4();
+    let parent = format!("e2e-sts-par-{suffix}");
+    let parent_secret = "longSecretKeyForParentSts99!";
+    let policy_readwrite = format!("e2e-sts-rw-{suffix}");
+    let bucket = format!("e2e-sts-tag-bkt-{suffix}");
+    let key = "sts-obj.txt";
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let admin = env.create_s3_client();
+    admin_create_user(&env, &parent, parent_secret).await?;
+
+    let rw = serde_json::to_string(&serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": ["arn:aws:s3:::*"]
+        }]
+    }))?;
+    admin_add_canned_policy(&env, &policy_readwrite, &rw).await?;
+    admin_attach_policy_to_user(&env, &policy_readwrite, &parent).await?;
+
+    let parent_client = user_client(&env, &parent, parent_secret);
+    parent_client.create_bucket().bucket(&bucket).send().await?;
+    put_object_with_tagging_str(&parent_client, &bucket, key, b"sts-e2e-data", "security=public").await?;
+
+    let session_policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{}/*", bucket)],
+            "Condition": { "StringEquals": { "s3:ExistingObjectTag/security": "public" } }
+        }]
+    })
+    .to_string();
+
+    let (ak, sk, token) = assume_role_with_session_policy(&env, &parent, parent_secret, &session_policy).await?;
+
+    let session_client = sts_session_client(&env, &ak, &sk, &token);
+    let ok = session_client.get_object().bucket(&bucket).key(key).send().await?;
+    let _ = ok.body.collect().await?;
+
+    put_object_tag_kv(&parent_client, &bucket, key, "security", "private").await?;
+    let denied = session_client.get_object().bucket(&bucket).key(key).send().await;
+    assert!(
+        denied.is_err(),
+        "session policy must deny GetObject when ExistingObjectTag no longer matches"
+    );
+
+    cleanup_bucket_and_object(&admin, &bucket, key).await;
+    admin_remove_user(&env, &parent).await;
+    admin_remove_policy(&env, &policy_readwrite).await;
+
+    info!("test_e2e_sts_assume_role_session_policy_existing_object_tag passed");
+    Ok(())
+}

crates/e2e_test/src/kms/common.rs

@@ -22,11 +22,13 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities
 
-use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging};
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
+};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
-use base64::Engine;
+use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
 use serde_json;
 use std::process::{Child, Command};
 use std::time::Duration;
@@ -44,6 +46,7 @@ pub const VAULT_ADDRESS: &str = "127.0.0.1:8200";
 pub const VAULT_TOKEN: &str = "dev-root-token";
 pub const VAULT_TRANSIT_PATH: &str = "transit";
 pub const VAULT_KEY_NAME: &str = "rustfs-master-key";
+pub const ENV_TEST_VAULT_BIN: &str = "RUSTFS_TEST_VAULT_BIN";
 
 /// Initialize tracing for KMS tests with KMS-specific log levels
 pub fn init_logging() {
@@ -51,6 +54,19 @@ pub fn init_logging() {
     // Additional KMS-specific logging configuration can be added here if needed
 }
 
+pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
+    if awscurl_available() {
+        return false;
+    }
+
+    info!("Skipping {} because awscurl is not available in PATH", test_name);
+    true
+}
+
+pub fn sse_customer_key_md5_base64(key: &str) -> String {
+    BASE64.encode(md5::compute(key).0)
+}
+
 // KMS-specific helper functions
 /// Configure KMS backend via admin API
 pub async fn configure_kms(
@@ -133,10 +149,10 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
         "usage": "EncryptDecrypt",
         "status": "Active",
         "metadata": HashMap::<String, String>::new(),
-        "created_at": chrono::Utc::now().to_rfc3339(),
+        "created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
         "rotated_at": serde_json::Value::Null,
         "created_by": "e2e-test",
-        "encrypted_key_material": key_data.to_vec(),
+        "encrypted_key_material": BASE64.encode(key_data),
         "nonce": Vec::<u8>::new()
     });
 
@@ -155,7 +171,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
 
     let test_key = "01234567890123456789012345678901"; // 32-byte key
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let test_data = b"Hello, KMS SSE-C World!";
     let object_key = "test-sse-c-object";
 
@@ -272,6 +288,10 @@ pub async fn test_kms_key_management(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
+        return Ok(());
+    }
+
     info!("Testing KMS key management APIs");
 
     // Test CreateKey
@@ -324,8 +344,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
     let wrong_key = "98765432109876543210987654321098";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
     let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
     let test_data = b"Test data for error scenarios";
     let object_key = "test-error-object";
 
@@ -366,6 +386,10 @@ pub struct VaultTestEnvironment {
 }
 
 impl VaultTestEnvironment {
+    fn resolve_vault_binary() -> String {
+        std::env::var(ENV_TEST_VAULT_BIN).unwrap_or_else(|_| "vault".to_string())
+    }
+
     /// Create a new Vault test environment
     pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
         let base_env = RustFSTestEnvironment::new().await?;
@@ -380,7 +404,8 @@ impl VaultTestEnvironment {
     pub async fn start_vault(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         info!("Starting Vault server in development mode");
 
-        let vault_process = Command::new("vault")
+        let vault_binary = Self::resolve_vault_binary();
+        let vault_process = Command::new(&vault_binary)
             .args([
                 "server",
                 "-dev",
@@ -406,7 +431,7 @@ impl VaultTestEnvironment {
             let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
             if port_check {
                 // Additional check by making a health request
-                if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await
+                if let Ok(response) = local_http_client().get(format!("{VAULT_URL}/v1/sys/health")).send().await
                     && response.status().is_success()
                 {
                     info!("Vault server is ready after {} seconds", i);
@@ -426,7 +451,7 @@ impl VaultTestEnvironment {
 
     /// Setup Vault transit secrets engine
     pub async fn setup_vault_transit(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let client = reqwest::Client::new();
+        let client = local_http_client();
 
         info!("Enabling Vault transit secrets engine");
 
@@ -471,10 +496,10 @@ impl VaultTestEnvironment {
         self.base_env.start_rustfs_server(Vec::new()).await
     }
 
-    /// Configure Vault KMS backend
-    pub async fn configure_vault_kms(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    /// Configure Vault Transit KMS backend
+    pub async fn configure_vault_transit_kms(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let kms_config = serde_json::json!({
-            "backend_type": "vault",
+            "backend_type": "VaultTransit",
             "address": VAULT_URL,
             "auth_method": {
                 "Token": {
@@ -482,8 +507,6 @@ impl VaultTestEnvironment {
                 }
             },
             "mount_path": VAULT_TRANSIT_PATH,
-            "kv_mount": "secret",
-            "key_path_prefix": "rustfs/kms/keys",
             "default_key_id": VAULT_KEY_NAME,
             "skip_tls_verify": true
         })
@@ -687,7 +710,7 @@ pub async fn test_multipart_upload_with_config(
 /// Create a standard SSE-C encryption configuration for testing
 pub fn create_sse_c_config() -> EncryptionType {
     let key = "01234567890123456789012345678901"; // 32-byte key
-    let key_md5 = format!("{:x}", md5::compute(key));
+    let key_md5 = sse_customer_key_md5_base64(key);
     EncryptionType::SSEC {
         key: key.to_string(),
         key_md5,
@@ -768,7 +791,7 @@ impl LocalKMSTestEnvironment {
 
         // Configure KMS with the default key in one step
         let kms_config = serde_json::json!({
-            "backend_type": "local",
+            "backend_type": "Local",
             "key_dir": self.kms_keys_dir,
             "file_permissions": 0o600,
             "default_key_id": default_key_id

crates/e2e_test/src/kms/encryption_metadata_test.rs

@@ -26,27 +26,20 @@ use serial_test::serial;
 use std::collections::{HashMap, VecDeque};
 use tracing::info;
 
-fn assert_encryption_metadata(metadata: &HashMap<String, String>, expected_size: usize) {
+fn assert_managed_encryption_metadata_hidden(metadata: Option<&HashMap<String, String>>) {
+    let Some(metadata) = metadata else { return };
+
     for key in [
         "x-rustfs-encryption-key",
         "x-rustfs-encryption-iv",
         "x-rustfs-encryption-context",
         "x-rustfs-encryption-original-size",
     ] {
-        assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present");
         assert!(
-            !metadata.get(key).unwrap().is_empty(),
-            "managed encryption metadata '{key}' should not be empty"
+            !metadata.contains_key(key),
+            "managed encryption metadata '{key}' should not be exposed to clients"
         );
     }
-
-    let size_value = metadata
-        .get("x-rustfs-encryption-original-size")
-        .expect("managed encryption metadata should include original size");
-    let parsed_size: usize = size_value
-        .parse()
-        .expect("x-rustfs-encryption-original-size should be numeric");
-    assert_eq!(parsed_size, expected_size, "recorded original size should match uploaded payload length");
 }
 
 fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &str, plaintext: &[u8]) {
@@ -142,10 +135,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
         "head_object should advertise SSE-S3"
     );
 
-    let metadata = head
-        .metadata()
-        .expect("head_object should return managed encryption metadata");
-    assert_encryption_metadata(metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     assert_storage_encrypted(std::path::Path::new(&kms_env.base_env.temp_dir), TEST_BUCKET, key, payload);
 
@@ -210,10 +200,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "source object should maintain the configured KMS key id"
     );
-    let source_metadata = head_source
-        .metadata()
-        .expect("source object should include managed encryption metadata");
-    assert_encryption_metadata(source_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_source.metadata());
 
     let dest_key = "metadata-sse-kms-object-copy";
     let copy_source = format!("{TEST_BUCKET}/{source_key}");
@@ -238,10 +225,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "copied object should keep the default KMS key id"
     );
-    let dest_metadata = head_dest
-        .metadata()
-        .expect("copied object should include managed encryption metadata");
-    assert_encryption_metadata(dest_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_dest.metadata());
 
     let copied_body = s3_client
         .get_object()
@@ -358,10 +342,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
         "multipart object should retain bucket default KMS key"
     );
 
-    assert_encryption_metadata(
-        head.metadata().expect("multipart head_object should expose managed metadata"),
-        combined.len(),
-    );
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     // Data returned to clients should decrypt back to original payload
     let fetched = s3_client

crates/e2e_test/src/kms/kms_comprehensive_test.rs

@@ -19,9 +19,9 @@
 //! complex workflows.
 
 use super::common::{
-    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, test_all_multipart_encryption_types,
-    test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption, test_sse_kms_encryption,
-    test_sse_s3_encryption,
+    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, sse_customer_key_md5_base64,
+    test_all_multipart_encryption_types, test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
@@ -149,8 +149,8 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     // Test different SSE-C keys to ensure isolation
     let key1 = "01234567890123456789012345678901";
     let key2 = "98765432109876543210987654321098";
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let config1 = MultipartTestConfig::new(
         "isolation-test-key1",
@@ -183,7 +183,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     info!("🔒 Verify key isolation");
     let wrong_key = "11111111111111111111111111111111";
     let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
 
     // Try to read file encrypted with key1 using wrong key
     let wrong_read_result = s3_client

crates/e2e_test/src/kms/kms_edge_cases_test.rs

@@ -21,7 +21,7 @@
 //! - Concurrent encryption operations
 //! - Security validation tests
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use aws_sdk_s3::types::ServerSideEncryption;
 use base64::Engine;
@@ -72,7 +72,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
     info!("📤 Testing SSE-C with zero-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "zero-byte-sse-c";
 
     let _put_response_c = s3_client
@@ -166,7 +166,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
     info!("📤 Testing SSE-C with single-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "single-byte-sse-c";
 
     s3_client
@@ -334,7 +334,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
     info!("🔍 Testing access to SSE-C object without key");
 
     // First upload a valid SSE-C object
-    let valid_key_md5 = format!("{:x}", compute(valid_key));
+    let valid_key_md5 = sse_customer_key_md5_base64(valid_key);
     s3_client
         .put_object()
         .bucket(TEST_BUCKET)
@@ -421,7 +421,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
                     // SSE-C
                     let key = format!("testkey{i:026}"); // 32-byte key
                     let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
-                    let key_md5 = format!("{:x}", compute(&key));
+                    let key_md5 = sse_customer_key_md5_base64(&key);
 
                     client
                         .put_object()
@@ -493,8 +493,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
 
     let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
     let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     // Upload same data with different keys
     s3_client

crates/e2e_test/src/kms/kms_local_test.rs

@@ -19,7 +19,10 @@
 //! - S3 object upload/download with SSE-S3, SSE-KMS, SSE-C encryption
 //! - Complete encryption/decryption lifecycle
 
-use super::common::{LocalKMSTestEnvironment, get_kms_status, test_kms_key_management, test_sse_c_encryption};
+use super::common::{
+    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    test_kms_key_management, test_sse_c_encryption,
+};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{error, info};
@@ -28,6 +31,9 @@ use tracing::{error, info};
 #[serial]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Local KMS End-to-End Test");
 
     // Create LocalKMS test environment
@@ -140,8 +146,8 @@ async fn test_local_kms_key_isolation() {
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Data encrypted with key 1";
     let data2 = b"Data encrypted with key 2";
@@ -562,7 +568,7 @@ async fn test_multipart_upload_with_sse_c(
     // SSE-C encryption key
     let encryption_key = "01234567890123456789012345678901";
     let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
-    let key_md5 = format!("{:x}", md5::compute(encryption_key));
+    let key_md5 = sse_customer_key_md5_base64(encryption_key);
 
     // Generate test data
     let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 3) % 256) as u8).collect();

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -19,14 +19,14 @@
 //! multipart upload behaviour.
 
 use crate::common::{TEST_BUCKET, init_logging};
-use md5::compute;
 use serial_test::serial;
 use tokio::time::{Duration, sleep};
 use tracing::{error, info};
 
 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, start_kms, test_all_multipart_encryption_types, test_error_scenarios,
-    test_kms_key_management, test_sse_c_encryption, test_sse_kms_encryption, test_sse_s3_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 
 /// Helper that brings up Vault, configures RustFS, and starts the KMS service.
@@ -42,7 +42,7 @@ impl VaultKmsTestContext {
         env.setup_vault_transit().await?;
 
         env.start_rustfs_for_vault().await?;
-        env.configure_vault_kms().await?;
+        env.configure_vault_transit_kms().await?;
 
         start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
 
@@ -65,6 +65,9 @@ impl VaultKmsTestContext {
 #[serial]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
 
     let context = VaultKmsTestContext::new().await?;
@@ -118,6 +121,9 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
+        return Ok(());
+    }
     info!("Starting Vault KMS SSE-C key isolation test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -133,8 +139,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Vault data encrypted with key 1";
     let data2 = b"Vault data encrypted with key 2";
@@ -202,6 +208,9 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[serial]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
+        return Ok(());
+    }
     info!("Starting Vault KMS large file SSE-S3 test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -264,6 +273,9 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
+        return Ok(());
+    }
     info!("Starting Vault KMS multipart upload encryption suite");
 
     let context = VaultKmsTestContext::new().await?;
@@ -292,6 +304,9 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[serial]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
+        return Ok(());
+    }
     info!("Starting Vault KMS key operations test (CRUD)");
 
     let context = VaultKmsTestContext::new().await?;

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -21,7 +21,7 @@
 //! 3. Test the saving and reading of encrypted metadata
 //! 4. Test the complete sharded upload encryption process
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{debug, info};
@@ -504,7 +504,7 @@ async fn test_multipart_encryption_type(
     let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
         let key = "01234567890123456789012345678901";
         let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
-        let key_md5 = format!("{:x}", md5::compute(key));
+        let key_md5 = sse_customer_key_md5_base64(key);
         (Some(key_b64), Some(key_md5))
     } else {
         (None, None)

crates/e2e_test/src/lib.rs

@@ -40,6 +40,10 @@ mod quota_test;
 #[cfg(test)]
 mod bucket_policy_check_test;
 
+/// IAM / bucket / STS session policy with `s3:ExistingObjectTag` conditions (E2E).
+#[cfg(test)]
+mod existing_object_tag_policy_test;
+
 // Regression tests for Issue #2036: anonymous access with PublicAccessBlock
 #[cfg(test)]
 mod anonymous_access_test;
@@ -52,6 +56,9 @@ mod special_chars_test;
 #[cfg(test)]
 mod content_encoding_test;
 
+#[cfg(test)]
+mod archive_download_integrity_test;
+
 // ListObjectsV2 pagination test (Issue #1596)
 #[cfg(test)]
 mod list_objects_v2_pagination_test;
@@ -66,6 +73,19 @@ mod compression_test;
 // Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
 #[cfg(test)]
 mod delete_objects_versioning_test;
+
+// Regression test for Issue #2252: ListObjectVersions misses newest version after put -> delete -> put
+#[cfg(test)]
+mod list_object_versions_regression_test;
+
+// versions&metadata=true extension regression test
+#[cfg(test)]
+mod list_object_versions_metadata_extension_test;
+
+// list-type=2&metadata=true extension regression test
+#[cfg(test)]
+mod list_objects_v2_metadata_extension_test;
+
 #[cfg(test)]
 mod protocols;
 
@@ -83,3 +103,25 @@ mod checksum_upload_test;
 // Group deletion tests
 #[cfg(test)]
 mod group_delete_test;
+
+// S3 dummy-compat bucket API tests
+#[cfg(test)]
+mod bucket_logging_test;
+
+// Multipart control API auth regression tests
+#[cfg(test)]
+mod multipart_auth_test;
+
+#[cfg(test)]
+mod stale_multipart_cleanup_cluster_test;
+
+// Object lambda end-to-end regression tests
+#[cfg(test)]
+mod object_lambda_test;
+
+// Replication extension end-to-end regression tests
+#[cfg(test)]
+mod replication_extension_test;
+
+#[cfg(test)]
+mod snowball_auto_extract_test;

crates/e2e_test/src/list_object_versions_metadata_extension_test.rs

@@ -0,0 +1,122 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end regression test for the versions metadata extension:
+//! `GET /{bucket}?versions&metadata=true`
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use http::header::HOST;
+    use reqwest::StatusCode;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
+    use serial_test::serial;
+    use std::error::Error;
+    use tracing::info;
+
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_metadata_extension_returns_metadata_tags_and_internal()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        info!("🧪 TEST: versions&metadata=true returns versions metadata extensions");
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "test-list-versions-metadata-ext";
+        let key = "versions/metadata-object.txt";
+
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await?;
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .metadata("project", "alpha")
+            .metadata("owner", "ops")
+            .tagging("env=test&project=alpha")
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"metadata extension body"))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}?versions&metadata=true&prefix={}", env.url, bucket, urlencoding::encode(key));
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(response.status(), StatusCode::OK, "versions&metadata=true should succeed");
+
+        let body = response.text().await?;
+        info!("versions&metadata=true response body: {}", body);
+
+        assert!(body.contains("<ListVersionsResult"), "expected ListVersionsResult XML, got: {body}");
+        assert!(body.contains("<Version>"), "expected at least one Version entry, got: {body}");
+        assert!(body.contains("<UserMetadata>"), "expected UserMetadata extension, got: {body}");
+        assert!(
+            body.contains("<project>alpha</project>"),
+            "expected stripped user metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<owner>ops</owner>"),
+            "expected second metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<UserTags>env=test&amp;project=alpha</UserTags>"),
+            "expected UserTags extension in response, got: {body}"
+        );
+        assert!(body.contains("<Internal>"), "expected Internal extension, got: {body}");
+        assert!(body.contains("<K>1</K>"), "expected Internal/K in response, got: {body}");
+        assert!(body.contains("<M>0</M>"), "expected Internal/M in response, got: {body}");
+
+        Ok(())
+    }
+}

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -0,0 +1,182 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for Issue #2252:
+//! "ListObjectVersions misses the newest version after create -> delete -> create."
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use serial_test::serial;
+    use tracing::info;
+
+    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
+        env.create_s3_client()
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_immediately_returns_latest_put_after_delete_marker() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns the newest version immediately after put -> delete -> put");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-object-versions-2252";
+        let key = "test-prefix/test-object.txt";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Enabled)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to enable versioning");
+
+        let first_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"first version"))
+            .send()
+            .await
+            .expect("Failed to put first object version");
+        let first_version_id = first_put
+            .version_id()
+            .map(str::to_string)
+            .expect("First put should return a version_id");
+
+        let delete_resp = client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("Failed to create delete marker");
+        let delete_marker_version_id = delete_resp
+            .version_id()
+            .map(str::to_string)
+            .expect("DeleteObject should return a delete marker version_id");
+
+        let second_put = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"second version"))
+            .send()
+            .await
+            .expect("Failed to put second object version");
+        let second_version_id = second_put
+            .version_id()
+            .map(str::to_string)
+            .expect("Second put should return a version_id");
+
+        let first_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions immediately after second put");
+        let second_listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("Failed to list object versions a second time");
+
+        let first_versions = first_listing.versions().to_vec();
+        let first_delete_markers = first_listing.delete_markers().to_vec();
+        let second_versions = second_listing.versions().to_vec();
+        let second_delete_markers = second_listing.delete_markers().to_vec();
+
+        info!(
+            "First listing: {} versions, {} delete markers",
+            first_versions.len(),
+            first_delete_markers.len()
+        );
+        info!(
+            "Second listing: {} versions, {} delete markers",
+            second_versions.len(),
+            second_delete_markers.len()
+        );
+
+        assert_eq!(
+            first_versions.len(),
+            2,
+            "First ListObjectVersions call should return both object versions immediately (regression #2252)"
+        );
+        assert_eq!(
+            first_delete_markers.len(),
+            1,
+            "First ListObjectVersions call should return the delete marker immediately (regression #2252)"
+        );
+        assert_eq!(
+            second_versions.len(),
+            2,
+            "Second ListObjectVersions call should still return both object versions"
+        );
+        assert_eq!(
+            second_delete_markers.len(),
+            1,
+            "Second ListObjectVersions call should still return the delete marker"
+        );
+
+        let first_latest_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(second_version_id.as_str()))
+            .expect("First listing should include the newest object version");
+        assert_eq!(
+            first_latest_version.is_latest(),
+            Some(true),
+            "Newest object version should be latest on the first listing"
+        );
+
+        let first_original_version = first_versions
+            .iter()
+            .find(|version| version.version_id() == Some(first_version_id.as_str()))
+            .expect("First listing should include the original object version");
+        assert_eq!(
+            first_original_version.is_latest(),
+            Some(false),
+            "Original object version should no longer be latest"
+        );
+
+        let first_delete_marker = first_delete_markers
+            .iter()
+            .find(|marker| marker.version_id() == Some(delete_marker_version_id.as_str()))
+            .expect("First listing should include the delete marker");
+        assert_eq!(
+            first_delete_marker.is_latest(),
+            Some(false),
+            "Delete marker should no longer be latest after the second put"
+        );
+    }
+}

crates/e2e_test/src/list_objects_duplicates_test.rs

@@ -132,4 +132,70 @@ mod tests {
         // Stop the RustFS server to ensure proper cleanup
         env.stop_server();
     }
+
+    /// Test ensuring that ListObjectsV2 returns unique keys when an explicit directory marker
+    /// exists under the requested prefix and delimiter is not provided.
+    ///
+    /// Bug Reference: Issue #2439
+    /// When both "marker/subdir/" and "marker/subdir/file.txt" exist, listing with
+    /// Prefix="marker/" must not duplicate "marker/subdir/file.txt" in Contents.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_unique_contents_with_explicit_directory_markers() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 should return unique keys with explicit directory markers");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-unique-contents";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        for (key, body) in [
+            ("marker/", ByteStream::from_static(b"")),
+            ("marker/subdir/", ByteStream::from_static(b"")),
+            ("marker/file.txt", ByteStream::from_static(b"content")),
+            ("marker/subdir/file.txt", ByteStream::from_static(b"nested")),
+        ] {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(body)
+                .send()
+                .await
+                .unwrap_or_else(|err| panic!("Failed to create test object {key}: {err}"));
+        }
+
+        let result = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix("marker/")
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        let keys: Vec<String> = result
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(ToOwned::to_owned))
+            .collect();
+
+        info!("Contents: {:?}", keys);
+
+        assert_eq!(
+            keys,
+            vec![
+                "marker/".to_string(),
+                "marker/file.txt".to_string(),
+                "marker/subdir/".to_string(),
+                "marker/subdir/file.txt".to_string(),
+            ]
+        );
+        assert_eq!(result.key_count(), Some(4));
+
+        env.stop_server();
+    }
 }

crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs

@@ -0,0 +1,110 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end regression test for the objects metadata extension:
+//! `GET /{bucket}?list-type=2&metadata=true`
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+    use http::header::HOST;
+    use reqwest::StatusCode;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
+    use serial_test::serial;
+    use std::error::Error;
+    use tracing::info;
+
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_metadata_extension_returns_metadata_tags_and_internal()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        info!("🧪 TEST: list-type=2&metadata=true returns object metadata extensions");
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "test-list-objects-v2-metadata-ext";
+        let key = "objects/metadata-object.txt";
+
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .metadata("project", "alpha")
+            .metadata("owner", "ops")
+            .tagging("env=test&project=alpha")
+            .body(aws_sdk_s3::primitives::ByteStream::from_static(b"metadata extension body"))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}?list-type=2&metadata=true&prefix={}", env.url, bucket, urlencoding::encode(key));
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(response.status(), StatusCode::OK, "list-type=2&metadata=true should succeed");
+
+        let body = response.text().await?;
+        info!("list-type=2&metadata=true response body: {}", body);
+
+        assert!(body.contains("<ListBucketResult"), "expected ListBucketResult XML, got: {body}");
+        assert!(body.contains("<Contents>"), "expected at least one Contents entry, got: {body}");
+        assert!(body.contains("<UserMetadata>"), "expected UserMetadata extension, got: {body}");
+        assert!(
+            body.contains("<project>alpha</project>"),
+            "expected stripped user metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<owner>ops</owner>"),
+            "expected second metadata key in response, got: {body}"
+        );
+        assert!(
+            body.contains("<UserTags>env=test&amp;project=alpha</UserTags>"),
+            "expected UserTags extension in response, got: {body}"
+        );
+        assert!(body.contains("<Internal>"), "expected Internal extension, got: {body}");
+        assert!(body.contains("<K>1</K>"), "expected Internal/K in response, got: {body}");
+        assert!(body.contains("<M>0</M>"), "expected Internal/M in response, got: {body}");
+
+        Ok(())
+    }
+}

crates/e2e_test/src/object_lock/object_lock_test.rs

@@ -25,8 +25,11 @@
 //! - Default bucket retention is applied to new objects
 
 use super::common::*;
+use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{Delete, ObjectIdentifier, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
+use aws_sdk_s3::types::{
+    CompletedMultipartUpload, CompletedPart, Delete, ObjectIdentifier, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
+};
 use serial_test::serial;
 use tracing::info;
 
@@ -37,6 +40,45 @@ fn init_logging() {
         .try_init();
 }
 
+async fn put_bucket_deny_policy(
+    client: &Client,
+    bucket: &str,
+    sid: &str,
+    action: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Sid": sid,
+            "Effect": "Deny",
+            "Principal": "*",
+            "Action": action,
+            "Resource": format!("arn:aws:s3:::{}/*", bucket)
+        }]
+    })
+    .to_string();
+
+    client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
+    Ok(())
+}
+
+fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
+    let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
+    aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
+        .expect("retention timestamp should parse")
+}
+
+fn assert_access_denied<T, E: std::fmt::Debug>(result: Result<T, E>, context: &str) {
+    let err = match result {
+        Ok(_) => panic!("{context}"),
+        Err(err) => format!("{err:?}"),
+    };
+    assert!(
+        err.contains("AccessDenied") || err.to_lowercase().contains("access denied"),
+        "{context}: expected AccessDenied, got: {err}"
+    );
+}
+
 // ============================================================================
 // DeleteObject Tests
 // ============================================================================
@@ -149,6 +191,57 @@ async fn test_delete_object_allowed_by_governance_with_bypass() {
     info!("✅ Test passed: GOVERNANCE retention allows deletion with bypass");
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_object_creates_delete_marker_for_retained_current_version() {
+    init_logging();
+    info!("🧪 Test: DeleteObject creates delete marker for retained current version");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-retention-delete-marker";
+    let key = "retained-object";
+    let data = b"test data for retained current version";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+
+    let retain_until = future_retain_until(30);
+    let retained_version_id =
+        put_object_with_retention(&client, bucket, key, data, ObjectLockRetentionMode::Governance, retain_until)
+            .await
+            .unwrap();
+
+    let delete_marker_output = client.delete_object().bucket(bucket).key(key).send().await.unwrap();
+    assert_eq!(delete_marker_output.delete_marker(), Some(true));
+
+    let delete_marker_version_id = delete_marker_output
+        .version_id()
+        .expect("delete marker should have a version id")
+        .to_string();
+
+    let protected_delete = delete_object_with_bypass(&client, bucket, key, Some(&retained_version_id), false).await;
+    assert!(protected_delete.is_err(), "Retained version should still reject direct deletion");
+
+    delete_object_with_bypass(&client, bucket, key, Some(&delete_marker_version_id), false)
+        .await
+        .unwrap();
+
+    let still_protected = delete_object_with_bypass(&client, bucket, key, Some(&retained_version_id), false).await;
+    assert!(
+        still_protected.is_err(),
+        "Retained version should remain protected after delete marker removal"
+    );
+
+    delete_object_with_bypass(&client, bucket, key, Some(&retained_version_id), true)
+        .await
+        .unwrap();
+
+    info!("✅ Test passed: Delete marker is allowed while retained version stays protected");
+}
+
 #[tokio::test]
 #[serial]
 async fn test_delete_object_blocked_by_legal_hold() {
@@ -182,6 +275,42 @@ async fn test_delete_object_blocked_by_legal_hold() {
     info!("✅ Test passed: Legal Hold blocks deletion");
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_object_allowed_with_legal_hold_off() {
+    init_logging();
+    info!("🧪 Test: DeleteObject allowed with Legal Hold OFF");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-legal-hold-off-delete";
+    let key = "legal-hold-off-object";
+    let data = b"test data for legal hold off";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+
+    let version_id = put_object_with_legal_hold(&client, bucket, key, data, ObjectLockLegalHoldStatus::Off)
+        .await
+        .unwrap();
+
+    let delete_result = delete_object_with_bypass(&client, bucket, key, Some(&version_id), false).await;
+    assert!(delete_result.is_ok(), "Delete should succeed when legal hold is OFF");
+
+    let head_result = client
+        .head_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await;
+    assert!(head_result.is_err(), "Object should be deleted when legal hold is OFF");
+
+    info!("✅ Test passed: Legal Hold OFF allows deletion");
+}
+
 #[tokio::test]
 #[serial]
 async fn test_delete_object_after_legal_hold_removed() {
@@ -216,6 +345,737 @@ async fn test_delete_object_after_legal_hold_removed() {
     info!("✅ Test passed: Deletion succeeds after Legal Hold removal");
 }
 
+#[tokio::test]
+#[serial]
+async fn test_get_object_legal_hold_returns_updated_status() {
+    init_logging();
+    info!("🧪 Test: GetObjectLegalHold returns updated status");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-get-legal-hold";
+    let key = "legal-hold-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let version_id = put_object_with_legal_hold(&client, bucket, key, b"test data", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let on_hold = client
+        .get_object_legal_hold()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(
+        on_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("ON")
+    );
+
+    put_object_legal_hold(&client, bucket, key, Some(&version_id), ObjectLockLegalHoldStatus::Off)
+        .await
+        .unwrap();
+
+    let off_hold = client
+        .get_object_legal_hold()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(
+        off_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("OFF")
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_retention_returns_configured_values() {
+    init_logging();
+    info!("🧪 Test: GetObjectRetention returns configured values");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-get-retention";
+    let key = "retained-object";
+    let retain_until = future_retain_until(30);
+    let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let version_id =
+        put_object_with_retention(&client, bucket, key, b"test data", ObjectLockRetentionMode::Governance, retain_until)
+            .await
+            .unwrap();
+
+    let retention = client
+        .get_object_retention()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await
+        .unwrap();
+    let retention = retention.retention().expect("retention should be present");
+
+    assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
+    assert_eq!(
+        retention
+            .retain_until_date()
+            .expect("retain_until_date should be present")
+            .fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)
+            .unwrap(),
+        retain_until_expected
+    );
+}
+
+// ============================================================================
+// Put/Copy/Multipart Legal Hold Tests
+// ============================================================================
+
+#[tokio::test]
+#[serial]
+async fn test_put_object_overwrite_blocked_by_legal_hold() {
+    init_logging();
+    info!("🧪 Test: PutObject overwrite blocked by Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-put-overwrite-legal-hold";
+    let key = "overwrite-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+
+    put_object_with_legal_hold(&client, bucket, key, b"locked-body", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let overwrite_result = client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(b"replacement-body".to_vec()))
+        .send()
+        .await;
+
+    assert!(overwrite_result.is_err(), "PutObject overwrite should fail while legal hold is ON");
+
+    let error_str = format!("{:?}", overwrite_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("legal") || error_str.to_lowercase().contains("hold"),
+        "overwrite error should mention legal hold, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_copy_object_applies_requested_legal_hold() {
+    init_logging();
+    info!("🧪 Test: CopyObject applies requested Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-copy-object-legal-hold";
+    let src_key = "src-object";
+    let dst_key = "dst-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    client
+        .copy_object()
+        .copy_source(format!("{bucket}/{src_key}"))
+        .bucket(bucket)
+        .key(dst_key)
+        .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+        .send()
+        .await
+        .unwrap();
+
+    let legal_hold = client
+        .get_object_legal_hold()
+        .bucket(bucket)
+        .key(dst_key)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(
+        legal_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("ON")
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_copy_object_overwrite_blocked_by_legal_hold() {
+    init_logging();
+    info!("🧪 Test: CopyObject overwrite blocked by Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-copy-overwrite-legal-hold";
+    let src_key = "src-object";
+    let dst_key = "dst-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_object_with_legal_hold(&client, bucket, dst_key, b"locked-destination", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let copy_result = client
+        .copy_object()
+        .copy_source(format!("{bucket}/{src_key}"))
+        .bucket(bucket)
+        .key(dst_key)
+        .send()
+        .await;
+
+    assert!(
+        copy_result.is_err(),
+        "CopyObject overwrite should fail while destination legal hold is ON"
+    );
+
+    let error_str = format!("{:?}", copy_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("legal") || error_str.to_lowercase().contains("hold"),
+        "copy overwrite error should mention legal hold, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_create_multipart_upload_applies_requested_legal_hold() {
+    init_logging();
+    info!("🧪 Test: CreateMultipartUpload applies requested Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-legal-hold";
+    let key = "multipart-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+        .send()
+        .await
+        .unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await
+        .unwrap();
+
+    let legal_hold = client.get_object_legal_hold().bucket(bucket).key(key).send().await.unwrap();
+
+    assert_eq!(
+        legal_hold
+            .legal_hold()
+            .and_then(|value| value.status())
+            .map(|value| value.as_str()),
+        Some("ON")
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_create_multipart_upload_blocked_by_compliance_retention() {
+    init_logging();
+    info!("🧪 Test: CreateMultipartUpload blocked by COMPLIANCE retention");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-create-compliance";
+    let key = "protected-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    put_object_with_retention(
+        &client,
+        bucket,
+        key,
+        b"locked-destination",
+        ObjectLockRetentionMode::Compliance,
+        future_retain_until(30),
+    )
+    .await
+    .unwrap();
+
+    let create_result = client.create_multipart_upload().bucket(bucket).key(key).send().await;
+
+    assert!(
+        create_result.is_err(),
+        "CreateMultipartUpload should fail while destination is under active COMPLIANCE retention"
+    );
+
+    let error_str = format!("{:?}", create_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("retention") || error_str.to_lowercase().contains("compliance"),
+        "multipart create error should mention retention, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_delete_completed_multipart_object_blocked_by_legal_hold() {
+    init_logging();
+    info!("🧪 Test: Delete completed multipart object blocked by Legal Hold");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-delete-legal-hold";
+    let key = "multipart-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+        .send()
+        .await
+        .unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_output = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await
+        .unwrap();
+
+    let version_id = complete_output.version_id().expect("multipart object should be versioned");
+    let delete_result = delete_object_with_bypass(&client, bucket, key, Some(version_id), false).await;
+    assert!(delete_result.is_err(), "Delete should fail for multipart object protected by legal hold");
+}
+
+#[tokio::test]
+#[serial]
+async fn test_delete_completed_multipart_object_blocked_by_retention() {
+    init_logging();
+    info!("🧪 Test: Delete completed multipart object blocked by retention");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-multipart-delete-retention";
+    let key = "multipart-object";
+    let retain_until = retention_timestamp(30);
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Compliance)
+        .object_lock_retain_until_date(retain_until)
+        .send()
+        .await
+        .unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_output = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await
+        .unwrap();
+
+    let version_id = complete_output.version_id().expect("multipart object should be versioned");
+    let delete_result = delete_object_with_bypass(&client, bucket, key, Some(version_id), false).await;
+    assert!(delete_result.is_err(), "Delete should fail for multipart object protected by retention");
+}
+
+#[tokio::test]
+#[serial]
+async fn test_complete_multipart_upload_blocked_when_legal_hold_added_after_create() {
+    init_logging();
+    info!("🧪 Test: CompleteMultipartUpload blocked when Legal Hold appears after MPU creation");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-complete-multipart-legal-hold";
+    let key = "multipart-race-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_object_with_legal_hold(&client, bucket, key, b"locked-current-version", ObjectLockLegalHoldStatus::On)
+        .await
+        .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_result = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await;
+
+    assert!(complete_result.is_err(), "CompleteMultipartUpload should fail once legal hold is enabled");
+
+    let error_str = format!("{:?}", complete_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("legal") || error_str.to_lowercase().contains("hold"),
+        "complete error should mention legal hold, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_complete_multipart_upload_blocked_when_compliance_retention_added_after_create() {
+    init_logging();
+    info!("🧪 Test: CompleteMultipartUpload blocked when COMPLIANCE retention appears after MPU creation");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-complete-multipart-compliance";
+    let key = "multipart-race-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
+
+    let upload_id = create_output.upload_id().unwrap();
+    let upload_part_output = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(b"multipart-body".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_object_with_retention(
+        &client,
+        bucket,
+        key,
+        b"locked-current-version",
+        ObjectLockRetentionMode::Compliance,
+        future_retain_until(30),
+    )
+    .await
+    .unwrap();
+
+    let completed_upload = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .e_tag(upload_part_output.e_tag().unwrap_or_default())
+                .build(),
+        )
+        .build();
+
+    let complete_result = client
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(upload_id)
+        .multipart_upload(completed_upload)
+        .send()
+        .await;
+
+    assert!(
+        complete_result.is_err(),
+        "CompleteMultipartUpload should fail once COMPLIANCE retention is enabled"
+    );
+
+    let error_str = format!("{:?}", complete_result.unwrap_err());
+    assert!(
+        error_str.to_lowercase().contains("retention") || error_str.to_lowercase().contains("compliance"),
+        "complete error should mention retention, got: {error_str}"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_write_paths_require_put_object_legal_hold_permission() {
+    init_logging();
+    info!("🧪 Test: write paths require PutObjectLegalHold permission");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-legal-hold-permissions";
+    let src_key = "src-object";
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_bucket_deny_policy(&client, bucket, "DenyPutObjectLegalHold", "s3:PutObjectLegalHold")
+        .await
+        .unwrap();
+
+    assert_access_denied(
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("put-target")
+            .body(ByteStream::from(b"put-body".to_vec()))
+            .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+            .send()
+            .await,
+        "PutObject with legal hold should require s3:PutObjectLegalHold",
+    );
+
+    assert_access_denied(
+        client
+            .copy_object()
+            .copy_source(format!("{bucket}/{src_key}"))
+            .bucket(bucket)
+            .key("copy-target")
+            .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+            .send()
+            .await,
+        "CopyObject with legal hold should require s3:PutObjectLegalHold",
+    );
+
+    assert_access_denied(
+        client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key("multipart-target")
+            .object_lock_legal_hold_status(ObjectLockLegalHoldStatus::On)
+            .send()
+            .await,
+        "CreateMultipartUpload with legal hold should require s3:PutObjectLegalHold",
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn test_write_paths_require_put_object_retention_permission() {
+    init_logging();
+    info!("🧪 Test: write paths require PutObjectRetention permission");
+
+    let mut env = ObjectLockTestEnvironment::new().await.unwrap();
+    env.start_rustfs().await.unwrap();
+
+    let bucket = "test-retention-permissions";
+    let src_key = "src-object";
+    let retain_until = retention_timestamp(30);
+
+    env.create_object_lock_bucket(bucket).await.unwrap();
+
+    let client = env.s3_client();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(src_key)
+        .body(ByteStream::from(b"copy-source".to_vec()))
+        .send()
+        .await
+        .unwrap();
+
+    put_bucket_deny_policy(&client, bucket, "DenyPutObjectRetention", "s3:PutObjectRetention")
+        .await
+        .unwrap();
+
+    assert_access_denied(
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("put-target")
+            .body(ByteStream::from(b"put-body".to_vec()))
+            .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
+            .object_lock_retain_until_date(retain_until)
+            .send()
+            .await,
+        "PutObject with retention should require s3:PutObjectRetention",
+    );
+
+    assert_access_denied(
+        client
+            .copy_object()
+            .copy_source(format!("{bucket}/{src_key}"))
+            .bucket(bucket)
+            .key("copy-target")
+            .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
+            .object_lock_retain_until_date(retain_until)
+            .send()
+            .await,
+        "CopyObject with retention should require s3:PutObjectRetention",
+    );
+
+    assert_access_denied(
+        client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key("multipart-target")
+            .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
+            .object_lock_retain_until_date(retain_until)
+            .send()
+            .await,
+        "CreateMultipartUpload with retention should require s3:PutObjectRetention",
+    );
+}
+
 // ============================================================================
 // DeleteObjects (Batch Delete) Tests
 // ============================================================================

crates/e2e_test/src/policy/policy_variables_test.rs

@@ -14,7 +14,7 @@
 
 //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
 
-use crate::common::{awscurl_put, init_logging};
+use crate::common::{awscurl_delete, awscurl_put, init_logging};
 use crate::policy::test_env::PolicyTestEnvironment;
 use aws_sdk_s3::primitives::ByteStream;
 use serial_test::serial;
@@ -113,11 +113,11 @@ async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, po
 
     // Remove user
     let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, username);
-    let _ = awscurl_put(&remove_user_url, "", &env.access_key, &env.secret_key).await;
+    let _ = awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await;
 
     // Remove policy
     let remove_policy_url = format!("{}/rustfs/admin/v3/remove-canned-policy?name={}", env.url, policy_name);
-    let _ = awscurl_put(&remove_policy_url, "", &env.access_key, &env.secret_key).await;
+    let _ = awscurl_delete(&remove_policy_url, &env.access_key, &env.secret_key).await;
 }
 
 /// Test AWS policy variables with single-value scenarios

crates/e2e_test/src/protocols/ftps_core.rs

@@ -14,11 +14,13 @@
 
 //! Core FTPS tests
 
-use crate::common::rustfs_binary_path;
+use crate::common::rustfs_binary_path_with_features;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use rcgen::generate_simple_self_signed;
-use rustls::{ClientConfig, RootCertStore, pki_types::CertificateDer, pki_types::pem::PemObject};
+use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
+use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
+use rustls::{ClientConfig, DigitallySignedStruct, Error as RustlsError, SignatureScheme};
 use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -31,6 +33,46 @@ use tracing::info;
 const FTPS_PORT: u16 = 9021;
 const FTPS_ADDRESS: &str = "127.0.0.1:9021";
 
+#[derive(Debug)]
+struct AcceptAnyServerCertVerifier;
+
+impl ServerCertVerifier for AcceptAnyServerCertVerifier {
+    fn verify_server_cert(
+        &self,
+        _end_entity: &CertificateDer<'_>,
+        _intermediates: &[CertificateDer<'_>],
+        _server_name: &ServerName<'_>,
+        _ocsp_response: &[u8],
+        _now: UnixTime,
+    ) -> Result<ServerCertVerified, RustlsError> {
+        Ok(ServerCertVerified::assertion())
+    }
+
+    fn verify_tls12_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn verify_tls13_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+        rustls::crypto::aws_lc_rs::default_provider()
+            .signature_verification_algorithms
+            .supported_schemes()
+    }
+}
+
 /// Test FTPS: put, ls, mkdir, rmdir, delete operations
 pub async fn test_ftps_core_operations() -> Result<()> {
     let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
@@ -58,7 +100,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
 
     // Start server manually
     info!("Starting FTPS server on {}", FTPS_ADDRESS);
-    let binary_path = rustfs_binary_path();
+    let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav"));
     let mut server_process = Command::new(&binary_path)
         .env("RUSTFS_FTPS_ENABLE", "true")
         .env("RUSTFS_FTPS_ADDRESS", FTPS_ADDRESS)
@@ -78,19 +120,9 @@ pub async fn test_ftps_core_operations() -> Result<()> {
             .install_default()
             .map_err(|e| anyhow::anyhow!("Failed to install crypto provider: {:?}", e))?;
 
-        // Create a simple rustls config that accepts any certificate for testing
-        let mut root_store = RootCertStore::empty();
-        // Add the self-signed certificate to the trust store for e2e
-        // Note: In a real environment, you'd use proper root certificates
-        let cert_pem = default_cert.cert.pem();
-        let cert_der = CertificateDer::pem_reader_iter(&mut Cursor::new(cert_pem))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?;
-
-        root_store.add_parsable_certificates(cert_der);
-
         let config = ClientConfig::builder()
-            .with_root_certificates(root_store)
+            .dangerous()
+            .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
             .with_no_client_auth();
 
         // Wrap in suppaftp's RustlsConnector

crates/e2e_test/src/protocols/webdav_core.rs

@@ -14,7 +14,7 @@
 
 //! Core WebDAV tests
 
-use crate::common::rustfs_binary_path;
+use crate::common::rustfs_binary_path_with_features;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use base64::Engine;
@@ -47,7 +47,7 @@ pub async fn test_webdav_core_operations() -> Result<()> {
 
     // Start server manually
     info!("Starting WebDAV server on {}", WEBDAV_ADDRESS);
-    let binary_path = rustfs_binary_path();
+    let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav"));
     let mut server_process = Command::new(&binary_path)
         .env("RUSTFS_WEBDAV_ENABLE", "true")
         .env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)

crates/e2e_test/src/quota_test.rs

@@ -17,6 +17,15 @@ use aws_sdk_s3::Client;
 use serial_test::serial;
 use tracing::{debug, info};
 
+fn skip_without_awscurl() -> bool {
+    if crate::common::awscurl_available() {
+        return false;
+    }
+
+    info!("Skipping quota test because awscurl is not available");
+    true
+}
+
 /// Test environment setup for quota tests
 pub struct QuotaTestEnv {
     pub env: RustFSTestEnvironment,
@@ -233,6 +242,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         // Create test bucket
@@ -269,6 +281,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -302,6 +317,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -336,6 +354,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -372,6 +393,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -403,6 +427,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -441,6 +468,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         // Create two buckets in the same environment
@@ -479,6 +509,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -512,6 +545,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -575,6 +611,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -628,6 +667,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -671,6 +713,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -727,6 +772,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;