fix(ecstore): log missing local paths during physical disk independence checks (#2680)

Co-authored-by: houseme <housemecn@gmail.com>

.docker/observability/README.md

@@ -13,6 +13,9 @@ The stack is composed of the following best-in-class open-source components:
 - **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
 - **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
 
+By default, this stack uses Tempo in single-binary mode and does not require Kafka/Redpanda.
+If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rustfs.yml` together with `docker-compose-tempo-ha-override.yml`.
+
 ## Architecture
 
 1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
@@ -46,6 +49,15 @@ Run the following command to start the entire stack:
 docker compose up -d
 ```
 
+### High Availability Tempo
+
+The default `docker-compose.yml` is the single-node stack.
+If you need the Kafka-backed HA Tempo configuration, start it with:
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### Access Dashboards
 
 | Service        | URL                                              | Credentials       | Description                    |

.docker/observability/README_ZH.md

@@ -13,6 +13,9 @@
 - **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
 - **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
 
+默认情况下，这套技术栈使用 Tempo 单二进制模式，不依赖 Kafka/Redpanda。
+如果需要基于 Kafka 的 HA Tempo 路径，请使用 `docker-compose-example-for-rustfs.yml` 配合 `docker-compose-tempo-ha-override.yml`。
+
 ## 架构
 
 1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
@@ -46,6 +49,15 @@
 docker compose up -d
 ```
 
+### Tempo 高可用模式
+
+默认的 `docker-compose.yml` 对应单机栈。
+如果需要基于 Kafka 的 HA Tempo 配置，请使用：
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### 访问仪表盘
 
 | 服务 | URL | 凭据 | 描述 |

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -85,10 +85,8 @@ services:
     networks:
       - otel-network
     restart: unless-stopped
-    depends_on:
-      - redpanda
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
@@ -205,7 +203,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/docker-compose.yml

@@ -16,10 +16,8 @@ services:
 
   # --- Tracing ---
   tempo:
-    image: grafana/tempo:2.10.3
+    image: grafana/tempo:2.10.5
     container_name: tempo
-    depends_on:
-      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -33,31 +31,11 @@ services:
       - otel-network
     restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
-  redpanda:
-    image: redpandadata/redpanda:latest
-    ports:
-      - "9092:9092" # Kafka API for clients
-    command: >
-      redpanda start --overprovisioned
-      --mode=dev-container
-      --kafka-addr=PLAINTEXT://0.0.0.0:9092
-      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
-
-  redpanda-console:
-    image: docker.redpanda.com/redpandadata/console:latest
-    environment:
-      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
-    volumes:
-      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
-    ports:
-      - "8080:8080"
-    depends_on:
-      - redpanda
 
   vulture:
     image: grafana/tempo-vulture:latest
@@ -168,7 +146,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/grafana/provisioning/datasources.yaml

@@ -1,97 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    uid: prometheus
-    access: proxy
-    orgId: 1
-    url: http://prometheus:9090
-    isDefault: true
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      exemplarTraceIdDestinations:
-        - name: trace_id
-          datasourceUid: tempo
-
-  - name: Tempo
-    type: tempo
-    uid: tempo
-    access: proxy
-    orgId: 1
-    url: http://tempo:3200
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      serviceMap:
-        datasourceUid: prometheus
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false
-      tracesToMetrics:
-        datasourceUid: prometheus
-        tags: [ { key: 'service.name' }, { key: 'job' } ]
-        queries:
-          - name: 'Service-Level Latency'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
-          - name: 'Service-Level Calls'
-            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
-          - name: 'Service-Level Errors'
-            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
-      nodeGraph:
-        enabled: true
-
-  - name: Loki
-    type: loki
-    uid: loki
-    orgId: 1
-    url: http://loki:3100
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      derivedFields:
-        - datasourceUid: tempo
-          matcherRegex: 'trace_id=(\w+)'
-          name: 'TraceID'
-          url: '$${__value.raw}'
-
-  - name: Jaeger
-    type: jaeger
-    uid: jaeger
-    url: http://jaeger:16686
-    access: proxy
-    isDefault: false
-    editable: false
-    jsonData:
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false

.docker/observability/tempo.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-partition_ring_live_store: true
 stream_over_http_enabled: true
 
 server:
@@ -33,33 +32,17 @@ distributor:
           endpoint: "0.0.0.0:4317"
         http:
           endpoint: "0.0.0.0:4318"
-  #log_received_spans:
-  #    enabled: true
-  #  log_discarded_spans:
-  #    enabled: true
 
-backend_scheduler:
-  provider:
-    compaction:
-      compaction:
-        block_retention: 1h
-
-backend_worker:
-  backend_scheduler_addr: localhost:3200
-  compaction:
-    block_retention: 1h
-  ring:
-    kvstore:
-      store: memberlist
-
-querier:
-  query_live_store: true
+ingester:
+  max_block_duration: 5m
 
 metrics_generator:
   registry:
     external_labels:
       source: tempo
       cluster: docker-compose
+  traces_storage:
+    path: /var/tempo/generator/traces
   storage:
     path: /var/tempo/generator/wal
     remote_write:
@@ -85,15 +68,5 @@ overrides:
       processors: [ "span-metrics", "service-graphs", "local-blocks" ]
       generate_native_histograms: both
 
-ingest:
-  enabled: true
-  kafka:
-    address: redpanda:9092
-    topic: tempo-ingest
-
-block_builder:
-  consume_cycle_duration: 30s
-
 usage_report:
   reporting_enabled: false
-

AGENTS.md

@@ -6,24 +6,39 @@ Use the nearest subdirectory `AGENTS.md` for path-specific guidance.
 ## Rule Precedence
 
 1. System/developer instructions.
-2. This file (global defaults).
-3. The nearest `AGENTS.md` in the current path (more specific scope wins).
+2. Current user/task instructions.
+3. The nearest `AGENTS.md` in the current path.
+4. This file (global defaults).
 
 If repo-level instructions conflict, follow the nearest file and keep behavior aligned with CI.
 
+## Execution Discipline
+
+- Read the relevant existing code, tests, and local guidance before changing behavior.
+- State assumptions when they affect the implementation or verification path.
+- If a task has multiple plausible interpretations, list the options briefly and choose the narrowest reasonable path; ask when the ambiguity would make the change risky.
+- For multi-step work, keep the plan minimal and tied to verifiable outcomes.
+- Avoid redundant file reads, repeated commands, and unnecessary exploratory work once enough context is available.
+- A good result is a minimal diff with clear assumptions, no over-engineering, and independent verification.
+
 ## Communication and Language
 
 - Respond in the same language used by the requester.
 - Keep source code, comments, commit messages, and PR title/body in English.
+- Be concise. Avoid sycophantic openers, closing fluff, and verbose status reporting.
 
 ## Change Style for Existing Logic
 
 - Prefer direct, local code over extracting one-off helpers.
 - Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow.
+- Solve only the requested problem; do not add speculative features, configurability, or adjacent improvements.
+- Prefer editing existing code over rewriting files or reshaping unrelated logic.
+- Modify only what is required and remove only artifacts introduced by your own changes.
 - Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths.
 - Do not refactor existing code only to make it easier to unit test.
 - Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths.
 - Keep code elegant, concise, and direct. Prefer minimal, readable implementations over over-engineering and excessive abstraction. Use comments to clarify non-obvious intent and invariants, not to compensate for unclear code.
+- Mention unrelated issues when useful, but do not fix them as part of a narrow task.
 
 ## Constant and String Usage
 
@@ -44,6 +59,8 @@ Reference the source files above instead.
 
 ## Verification Before PR
 
+Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion.
+
 For code changes, run and pass the following before opening a PR:
 
 ```bash
@@ -69,6 +86,7 @@ Do not open a PR with code changes when the required checks fail.
 - Use `N/A` for non-applicable template sections.
 - Include verification commands in the PR description.
 - When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
+- Do not include the literal sequence `\n` in any GitHub issue, pull request, or discussion comment.
 - After fixing code review comments or CI findings, always mark corresponding review
   comments/threads as resolved before returning to the user.
 - In handling review comments, confirm the underlying issue before changing code.

ARCHITECTURE.md

@@ -86,7 +86,7 @@ Crates are organized in a dependency DAG with 9 depth levels (0 = leaf, 8 = top)
 ```
 Depth 0 — LEAF (no internal deps):
   appauth, checksums, config, credentials, crypto, io-metrics,
-  madmin, mcp, s3-common, workers, zip
+  madmin, s3-common, workers, zip
 
 Depth 1:
   io-core (→ io-metrics)
@@ -195,7 +195,6 @@ Depth 8 — TOP:
 | `trusted-proxies` | 4.0K | Trusted proxy / IP forwarding |
 | `zip` | 986 | ZIP archive support for bulk downloads |
 | `workers` | 136 | Simple worker abstraction |
-| `mcp` | 2.0K | Model Context Protocol server (AI tooling) |
 
 ## Architecture Invariants
 
@@ -224,7 +223,7 @@ Depth 8 — TOP:
 6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
    not bare `Error`).
    - ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
-     `mcp` and `heal` use `anyhow` in library code.
+      `heal` use `anyhow` in library code.
 
 ## Known Structural Issues
 
@@ -285,9 +284,9 @@ anyhow::Result<T>             // in library code (OK in tests/CLI)
 
 ### Metrics
 
-- Prometheus-style metrics via `rustfs-metrics` crate
+- Prometheus-style metrics via `rustfs-obs` runtime and schema
 - I/O-specific counters via `rustfs-io-metrics`
-- Registration happens at crate level, collection in `metrics` crate
+- Registration happens at crate level, collection/reporting in `rustfs-obs`
 
 ### Testing
 
@@ -369,7 +368,7 @@ The binary (`main.rs`) boots in this order:
   Add handler in `admin/handlers/`, register in `admin/router.rs`
 
 - **"Where do I add a new metric?"**
-  Define in `crates/metrics/`, register collector, expose via `/minio/v2/metrics`
+  Define descriptor/collector in `crates/obs/src/metrics/`, expose via `/minio/v2/metrics`
 
 ---
 

Cargo.toml

@@ -31,8 +31,6 @@ members = [
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
     "crates/madmin", # Management dashboard and admin API interface
-    "crates/mcp", # MCP server for S3 operations
-    "crates/metrics", # Metrics collection and reporting
     "crates/notify", # Notification system for events
     "crates/obs", # Observability utilities
     "crates/object-capacity", # Capacity scan and refresh core
@@ -40,7 +38,7 @@ members = [
     "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
     "crates/protos", # Protocol buffer definitions
     "crates/rio", # Rust I/O utilities and abstractions
-    "crates/concurrency", # Rust I/O utilities and abstractions
+    "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
     "crates/s3-common", # Common utilities and data structures for S3 compatibility
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -95,8 +93,6 @@ rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
 rustfs-kms = { path = "crates/kms", version = "0.0.5" }
 rustfs-lock = { path = "crates/lock", version = "0.0.5" }
 rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
-rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
-rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
 rustfs-notify = { path = "crates/notify", version = "0.0.5" }
 rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
 rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
@@ -104,6 +100,7 @@ rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
 rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
 rustfs-protos = { path = "crates/protos", version = "0.0.5" }
+rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 rustfs-rio = { path = "crates/rio", version = "0.0.5" }
 rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
 rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
@@ -115,18 +112,19 @@ rustfs-targets = { path = "crates/targets", version = "0.0.5" }
 rustfs-utils = { path = "crates/utils", version = "0.0.5" }
 rustfs-workers = { path = "crates/workers", version = "0.0.5" }
 rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
-rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.41" }
+async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
+async-nats = "0.47.0"
 axum = "0.8.9"
 futures = "0.3.32"
 futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
+pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
 hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
@@ -134,8 +132,9 @@ http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+rustfs-kafka-async = { version = "1.2.0" }
 socket2 = { version = "0.6.3", features = ["all"] }
-tokio = { version = "1.52.0", features = ["fs", "rt-multi-thread", "io-uring"] }
+tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
@@ -154,27 +153,25 @@ flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
 quick-xml = "0.39.2"
-rmcp = { version = "1.4.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
-schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
 argon2 = { version = "0.6.0-rc.8" }
-blake2 = "0.11.0-rc.5"
+blake2 = "0.11.0-rc.6"
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
-pbkdf2 = "0.13.0-rc.10"
+pbkdf2 = "0.13.0"
 rsa = { version = "0.10.0-rc.17" }
-rustls = { version = "0.23.38", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pki-types = "1.14.0"
+rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls-pki-types = "1.14.1"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"
@@ -183,7 +180,7 @@ zeroize = { version = "1.8.2", features = ["derive"] }
 # Time and Date
 chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.23", features = ["serde"] }
+jiff = { version = "0.2.24", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
@@ -192,9 +189,9 @@ arc-swap = "1.9.1"
 astral-tokio-tar = "0.6.0"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.15" }
+aws-config = { version = "1.8.16" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.131.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
@@ -202,7 +199,7 @@ base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.6.0", features = ["derive", "env"] }
+clap = { version = "4.6.1", features = ["derive", "env"] }
 const-str = { version = "1.1.0", features = ["std", "proc"] }
 convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
@@ -210,7 +207,7 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "53.0.0"
+datafusion = "53.1.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
@@ -224,9 +221,9 @@ hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.185"
+libc = "0.2.186"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.11"
+local-ip-address = "0.6.12"
 memmap2 = "0.9.10"
 lz4 = "1.28.1"
 matchit = "0.9.2"
@@ -250,13 +247,13 @@ rayon = "1.12.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
-rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] }
+rumqttc = { package = "rumqttc-next", version = "0.30.0", features = ["websocket"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
-s3s = { git = "https://github.com/rustfs/s3s", rev = "0dd8fcaaa72eda68fafd49c38daea43bb8697558", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
 serial_test = "3.4.0"
-shadow-rs = { version = "1.7.1", default-features = false }
+shadow-rs = { version = "2.0.0", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
@@ -270,14 +267,14 @@ tempfile = "3.27.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
-tracing-appender = "0.2.4"
+tracing-appender = "0.2.5"
 tracing-error = "0.2.1"
 tracing-opentelemetry = "0.32.1"
 tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.23.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
@@ -288,19 +285,19 @@ zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
-dial9-tokio-telemetry = "0.2"
+dial9-tokio-telemetry = "0.3"
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
 opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }
-pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.0.2", features = ["backend-pprof-rs"] }
 
 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }
 unftp-core = "0.1.0"
-suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"
 
 # WebDAV
@@ -320,7 +317,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
 pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] }
 
 [workspace.metadata.cargo-shear]
-ignored = ["rustfs", "rustfs-mcp"]
+ignored = ["rustfs"]
 
 [profile.release]
 opt-level = 3

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM alpine:3.23 AS build
+FROM alpine:3.23.4 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
@@ -54,7 +54,7 @@ RUN set -eux; \
     rm -rf rustfs.zip /build/.tmp || true
 
 
-FROM alpine:3.23
+FROM alpine:3.23.4
 
 ARG RELEASE=latest
 ARG BUILD_DATE
@@ -73,7 +73,7 @@ LABEL name="RustFS" \
       license="Apache-2.0"
 
 RUN apk update && \
-    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
+    apk add --no-cache ca-certificates coreutils curl
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

_typos.toml

@@ -39,6 +39,7 @@ abd = "abd"
 mak = "mak"
 gae = "gae"
 GAE = "GAE"
+thr = "thr"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
 consts = "consts"

crates/audit/Cargo.toml

@@ -41,11 +41,10 @@ serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
-url = { workspace = true }
-rumqttc = { workspace = true }
 
 [dev-dependencies]
 temp-env = { workspace = true }
+url = { workspace = true }
 
 [lints]
 workspace = true

crates/audit/src/factory.rs

@@ -14,28 +14,19 @@
 
 use crate::AuditEntry;
 use async_trait::async_trait;
-use hashbrown::HashSet;
-use rumqttc::QoS;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
-use rustfs_config::{
-    AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
-    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
-    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
-};
+use rustfs_config::AUDIT_DEFAULT_DIR;
+use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
-    error::TargetError,
-    target::{
-        mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
-        webhook::WebhookArgs,
+    config::{
+        build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+        validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
     },
+    error::TargetError,
+    target::TargetType,
 };
-use std::time::Duration;
-use tracing::{debug, warn};
-use url::Url;
+use std::collections::HashSet;
 
 /// Trait for creating targets from configuration
 #[async_trait]
@@ -49,10 +40,6 @@ pub trait TargetFactory: Send + Sync {
     /// Returns a set of valid configuration field names for this target type.
     /// This is used to filter environment variables.
     fn get_valid_fields(&self) -> HashSet<String>;
-
-    /// Returns a set of valid configuration env field names for this target type.
-    /// This is used to filter environment variables.
-    fn get_valid_env_fields(&self) -> HashSet<String>;
 }
 
 /// Factory for creating Webhook targets
@@ -61,71 +48,18 @@ pub struct WebhookTargetFactory;
 #[async_trait]
 impl TargetFactory for WebhookTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        // All config values are now read directly from the merged `config` KVS.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        let parsed_endpoint = endpoint.trim();
-        let endpoint_url = Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let args = WebhookArgs {
-            enable: true, // If we are here, it's already enabled.
-            endpoint: endpoint_url,
-            auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
-            queue_dir: config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(WEBHOOK_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
-            client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
-            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
-            skip_tls_verify: config
-                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
-                .and_then(|v| v.parse::<bool>().ok())
-                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        // Validation also uses the merged `config` KVS directly.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        debug!("endpoint: {}", endpoint);
-        let parsed_endpoint = endpoint.trim();
-        Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let client_cert = config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default();
-        let client_key = config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default();
-
-        if client_cert.is_empty() != client_key.is_empty() {
-            return Err(TargetError::Configuration(
-                "Both client_cert and client_key must be specified together".to_string(),
-            ));
-        }
-
-        let queue_dir = config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string());
-        if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
-            return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
-        }
-
-        Ok(())
+        validate_webhook_config(config, AUDIT_DEFAULT_DIR)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
     }
-
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
-    }
 }
 
 /// Factory for creating MQTT targets
@@ -134,112 +68,73 @@ pub struct MQTTTargetFactory;
 #[async_trait]
 impl TargetFactory for MQTTTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let broker_url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let topic = config
-            .lookup(MQTT_TOPIC)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
-
-        let args = MQTTArgs {
-            enable: true, // Assumed enabled.
-            broker: broker_url,
-            topic,
-            qos: config
-                .lookup(MQTT_QOS)
-                .and_then(|v| v.parse::<u8>().ok())
-                .map(|q| match q {
-                    0 => QoS::AtMostOnce,
-                    1 => QoS::AtLeastOnce,
-                    2 => QoS::ExactlyOnce,
-                    _ => QoS::AtLeastOnce,
-                })
-                .unwrap_or(QoS::AtLeastOnce),
-            username: config.lookup(MQTT_USERNAME).unwrap_or_default(),
-            password: config.lookup(MQTT_PASSWORD).unwrap_or_default(),
-            max_reconnect_interval: config
-                .lookup(MQTT_RECONNECT_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(5)),
-            keep_alive: config
-                .lookup(MQTT_KEEP_ALIVE_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(30)),
-            tls: MQTTTlsConfig::from_values(
-                config.lookup(MQTT_TLS_POLICY).as_deref(),
-                config.lookup(MQTT_TLS_CA).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-                config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-                config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-            )?,
-            queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(MQTT_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let tls = MQTTTlsConfig::from_values(
-            config.lookup(MQTT_TLS_POLICY).as_deref(),
-            config.lookup(MQTT_TLS_CA).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-            config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-            config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-        )?;
-        validate_mqtt_broker_url(&url, &tls)?;
-
-        if config.lookup(MQTT_TOPIC).is_none() {
-            return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
-        }
-
-        if let Some(qos_str) = config.lookup(MQTT_QOS) {
-            let qos = qos_str
-                .parse::<u8>()
-                .map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
-            if qos > 2 {
-                return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
-            }
-        }
-
-        let queue_dir = config.lookup(MQTT_QUEUE_DIR).unwrap_or_default();
-        if !queue_dir.is_empty() {
-            if !std::path::Path::new(&queue_dir).is_absolute() {
-                return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
-            }
-            if let Some(qos_str) = config.lookup(MQTT_QOS)
-                && qos_str == "0"
-            {
-                warn!("Using queue_dir with QoS 0 may result in event loss");
-            }
-        }
-
-        Ok(())
+        validate_mqtt_config(config)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
     }
+}
 
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
+pub struct NATSTargetFactory;
+
+#[async_trait]
+impl TargetFactory for NATSTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_nats_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_NATS_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct PulsarTargetFactory;
+
+#[async_trait]
+impl TargetFactory for PulsarTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_pulsar_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct KafkaTargetFactory;
+
+#[async_trait]
+impl TargetFactory for KafkaTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_kafka_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }

crates/audit/src/global.rs

@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{AuditEntry, AuditResult, AuditSystem};
+use crate::{AuditEntry, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
 use rustfs_ecstore::config::Config;
 use std::sync::{Arc, OnceLock};
 use tracing::{debug, error, trace, warn};
@@ -89,6 +89,15 @@ pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
     with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
 }
 
+/// Returns per-target audit delivery metrics for Prometheus collection.
+pub async fn audit_target_metrics() -> Vec<AuditTargetMetricSnapshot> {
+    if let Some(system) = audit_system() {
+        system.snapshot_target_metrics().await
+    } else {
+        Vec::new()
+    }
+}
+
 /// Check if the global audit system is running
 pub async fn is_audit_system_running() -> bool {
     if let Some(system) = audit_system() {

crates/audit/src/lib.rs

@@ -31,4 +31,4 @@ pub use error::{AuditError, AuditResult};
 pub use global::*;
 pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation};
 pub use registry::AuditRegistry;
-pub use system::AuditSystem;
+pub use system::{AuditSystem, AuditTargetMetricSnapshot};

crates/audit/src/registry.rs

@@ -14,18 +14,19 @@
 
 use crate::{
     AuditEntry, AuditError, AuditResult,
-    factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory},
+    factory::{
+        KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
+    },
 };
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
-use hashbrown::{HashMap, HashSet};
-use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, audit::AUDIT_ROUTE_PREFIX};
+use hashbrown::HashMap;
+use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_targets::arn::TargetID;
-use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
-use std::str::FromStr;
+use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
 use std::sync::Arc;
-use tracing::{debug, error, info, warn};
+use tracing::{error, info};
 
 /// Registry for managing audit targets
 pub struct AuditRegistry {
@@ -52,6 +53,9 @@ impl AuditRegistry {
         // Register built-in factories
         registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
+        registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
+        registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
+        registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
 
         registry
     }
@@ -105,143 +109,22 @@ impl AuditRegistry {
         &self,
         config: &Config,
     ) -> AuditResult<Vec<Box<dyn Target<AuditEntry> + Send + Sync>>> {
-        // Collect only environment variables with the relevant prefix to reduce memory usage
-        let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
-        // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
             info!("Start working on target types...");
-
-            // 2. Prepare the configuration source
-            // 2.1. Get the configuration segment in the file, e.g. 'audit_webhook'
-            let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
-            let file_configs = config.0.get(&section_name).cloned().unwrap_or_default();
-            // 2.2. Get the default configuration for that type
-            let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
-            debug!(?default_cfg, "Get the default configuration");
-
-            // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
-            debug!(?valid_fields, "Get the legitimate configuration fields");
-
-            // 3. Resolve instance IDs and configuration overrides from environment variables
-            let mut instance_ids_from_env = HashSet::new();
-            // 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
-            let enable_prefix =
-                format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
-                    .to_uppercase();
-            for (key, value) in &all_env {
-                if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
-                    && let Some(id) = key.strip_prefix(&enable_prefix)
-                    && !id.is_empty()
-                {
-                    instance_ids_from_env.insert(id.to_lowercase());
-                }
-            }
-
-            // 3.2. Parse all relevant environment variable configurations
-            // 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
-            let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
-            // 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
-            let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
-            for (key, value) in &all_env {
-                if let Some(rest) = key.strip_prefix(&env_prefix) {
-                    // Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
-                    // Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
-                    let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
-
-                    // The first part from the right is INSTANCE_ID
-                    let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
-                    // The remaining part is FIELD_NAME
-                    let field_name_part = parts.next();
-
-                    let (field_name, instance_id) = match field_name_part {
-                        // Case 1: The format is <FIELD_NAME>_<INSTANCE_ID>
-                        // e.g., rest = "ENDPOINT_PRIMARY" -> field_name="ENDPOINT", instance_id="PRIMARY"
-                        Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
-                        // Case 2: The format is <FIELD_NAME> (without INSTANCE_ID)
-                        // e.g., rest = "ENABLE" -> field_name="ENABLE", instance_id="" (Universal configuration `_ DEFAULT_DELIMITER`)
-                        None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
-                    };
-
-                    // *** Optimization point 2: Verify whether the parsed field_name is legal ***
-                    if !field_name.is_empty() && valid_fields.contains(&field_name) {
-                        debug!(
-                            instance_id = %if instance_id.is_empty() { DEFAULT_DELIMITER } else { &instance_id },
-                            %field_name,
-                            %value,
-                            "Parsing to environment variables"
-                        );
-                        env_overrides
-                            .entry(instance_id)
-                            .or_default()
-                            .insert(field_name, value.clone());
-                    } else {
-                        // Ignore illegal field names
-                        warn!(
-                            field_name = %field_name,
-                            "Ignore environment variable fields, not found in the list of valid fields for target type {}",
-                            target_type
-                        );
-                    }
-                }
-            }
-            debug!(?env_overrides, "Complete the environment variable analysis");
-
-            // 4. Determine all instance IDs that need to be processed
-            let mut all_instance_ids: HashSet<String> =
-                file_configs.keys().filter(|k| *k != DEFAULT_DELIMITER).cloned().collect();
-            all_instance_ids.extend(instance_ids_from_env);
-            debug!(?all_instance_ids, "Determine all instance IDs");
-
-            // 5. Merge configurations and create tasks for each instance
-            for id in all_instance_ids {
-                // 5.1. Merge configuration, priority: Environment variables > File instance configuration > File default configuration
-                let mut merged_config = default_cfg.clone();
-                // Instance-specific configuration in application files
-                if let Some(file_instance_cfg) = file_configs.get(&id) {
-                    merged_config.extend(file_instance_cfg.clone());
-                }
-                // Application instance-specific environment variable configuration
-                if let Some(env_instance_cfg) = env_overrides.get(&id) {
-                    // Convert HashMap<String, String> to KVS
-                    let mut kvs_from_env = KVS::new();
-                    for (k, v) in env_instance_cfg {
-                        kvs_from_env.insert(k.clone(), v.clone());
-                    }
-                    merged_config.extend(kvs_from_env);
-                }
-                debug!(instance_id = %id, ?merged_config, "Complete configuration merge");
-
-                // 5.2. Check if the instance is enabled
-                let enabled = merged_config
-                    .lookup(ENABLE_KEY)
-                    .map(|v| {
-                        EnableState::from_str(v.as_str())
-                            .ok()
-                            .map(|s| s.is_enabled())
-                            .unwrap_or(false)
-                    })
-                    .unwrap_or(false);
-
-                if enabled {
-                    info!(instance_id = %id, "Target is enabled, ready to create a task");
-                    // 5.3. Create asynchronous tasks for enabled instances
-                    let tid = id.clone();
-                    let merged_config_arc = Arc::new(merged_config);
-                    tasks.push(async move {
-                        let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (tid, result)
-                    });
-                } else {
-                    info!(instance_id = %id, "Skip disabled target");
-                }
+            for (id, merged_config) in collect_target_configs(config, AUDIT_ROUTE_PREFIX, target_type, &valid_fields) {
+                info!(instance_id = %id, "Target is enabled, ready to create a task");
+                let tid = id.clone();
+                let merged_config_arc = Arc::new(merged_config);
+                tasks.push(async move {
+                    let result = factory.create_target(tid.clone(), &merged_config_arc).await;
+                    (tid, result)
+                });
             }
         }
 
-        // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
         while let Some((id, result)) = tasks.next().await {
             match result {
@@ -317,8 +200,8 @@ impl AuditRegistry {
             }
         }
 
-        if !errors.is_empty() {
-            return Err(AuditError::Target(errors.into_iter().next().unwrap()));
+        if let Some(error) = errors.into_iter().next() {
+            return Err(AuditError::Target(error));
         }
 
         Ok(())

crates/audit/src/system.rs

@@ -24,6 +24,14 @@ use std::sync::Arc;
 use tokio::sync::{Mutex, RwLock, mpsc};
 use tracing::{error, info, warn};
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct AuditTargetMetricSnapshot {
+    pub failed_messages: u64,
+    pub queue_length: u64,
+    pub target_id: String,
+    pub total_messages: u64,
+}
+
 /// State of the audit system
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum AuditSystemState {
@@ -537,8 +545,14 @@ impl AuditSystem {
                                     TargetError::Timeout(_) => {
                                         warn!("Timeout sending to target {}, retrying...", target.id());
                                     }
+                                    TargetError::Dropped(reason) => {
+                                        warn!("Dropped queued payload for target {}: {}", target.id(), reason);
+                                        observability::record_target_failure();
+                                        break;
+                                    }
                                     _ => {
                                         error!("Permanent error for target {}: {}", target.id(), e);
+                                        target.record_final_failure();
                                         observability::record_target_failure();
                                         break;
                                     }
@@ -552,6 +566,7 @@ impl AuditSystem {
 
                     if retries >= MAX_RETRIES && !success {
                         warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id());
+                        target.record_final_failure();
                         observability::record_target_failure();
                     }
                 }
@@ -664,6 +679,25 @@ impl AuditSystem {
         registry.list_target_values()
     }
 
+    /// Returns per-target delivery metrics for Prometheus collection.
+    pub async fn snapshot_target_metrics(&self) -> Vec<AuditTargetMetricSnapshot> {
+        let targets = self.get_target_values().await;
+        let mut snapshots = Vec::with_capacity(targets.len());
+
+        for target in targets {
+            let delivery = target.delivery_snapshot();
+            snapshots.push(AuditTargetMetricSnapshot {
+                failed_messages: delivery.failed_messages,
+                queue_length: delivery.queue_length,
+                target_id: target.id().to_string(),
+                total_messages: delivery.total_messages,
+            });
+        }
+
+        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
+        snapshots
+    }
+
     /// Gets information about a specific target
     ///
     /// # Arguments

crates/common/src/data_usage.rs

@@ -51,14 +51,14 @@ impl AllTierStats {
     pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
         for (tier, st) in tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
     pub fn merge(&mut self, other: AllTierStats) {
         for (tier, st) in other.tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
@@ -673,7 +673,7 @@ impl DataUsageCache {
         let mut leaves = Vec::new();
         let mut remove = total - limit;
         add(self, path, &mut leaves);
-        leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
+        leaves.sort_by_key(|a| a.objects);
 
         while remove > 0 && !leaves.is_empty() {
             let Some(e) = leaves.first() else {

crates/common/src/heal_channel.rs

@@ -18,7 +18,7 @@ use std::{
     fmt::{self, Display},
     sync::OnceLock,
 };
-use tokio::sync::{broadcast, mpsc};
+use tokio::sync::{broadcast, mpsc, oneshot};
 use uuid::Uuid;
 
 pub const HEAL_DELETE_DANGLING: bool = true;
@@ -206,11 +206,59 @@ pub struct HealOpts {
     pub set: Option<usize>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionDropReason {
+    QueueFull,
+    PolicyDropped,
+}
+
+impl HealAdmissionDropReason {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::QueueFull => "queue_full",
+            Self::PolicyDropped => "policy_dropped",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionResult {
+    Accepted,
+    Merged,
+    Full,
+    Dropped(HealAdmissionDropReason),
+}
+
+impl HealAdmissionResult {
+    pub fn result_label(self) -> &'static str {
+        match self {
+            Self::Accepted => "accepted",
+            Self::Merged => "merged",
+            Self::Full => "full",
+            Self::Dropped(_) => "dropped",
+        }
+    }
+
+    pub fn reason_label(self) -> &'static str {
+        match self {
+            Self::Dropped(reason) => reason.as_str(),
+            _ => "none",
+        }
+    }
+
+    pub fn is_admitted(self) -> bool {
+        matches!(self, Self::Accepted | Self::Merged)
+    }
+}
+
 /// Heal channel command type
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub enum HealChannelCommand {
     /// Start a new heal task
-    Start(HealChannelRequest),
+    Start {
+        request: HealChannelRequest,
+        response_tx: oneshot::Sender<Result<HealAdmissionResult, String>>,
+    },
     /// Query heal task status
     Query { heal_path: String, client_token: String },
     /// Cancel heal task
@@ -339,9 +387,22 @@ pub fn subscribe_heal_responses() -> broadcast::Receiver<HealChannelResponse> {
     heal_response_sender().subscribe()
 }
 
+/// Send heal start request and wait for structured admission feedback.
+pub async fn send_heal_request_with_admission(request: HealChannelRequest) -> Result<HealAdmissionResult, String> {
+    let (response_tx, response_rx) = oneshot::channel();
+    send_heal_command(HealChannelCommand::Start { request, response_tx }).await?;
+    response_rx
+        .await
+        .map_err(|e| format!("Failed to receive heal admission response: {e}"))?
+}
+
 /// Send heal start request
 pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> {
-    send_heal_command(HealChannelCommand::Start(request)).await
+    match send_heal_request_with_admission(request).await? {
+        HealAdmissionResult::Accepted | HealAdmissionResult::Merged => Ok(()),
+        HealAdmissionResult::Full => Err("Heal request queue is full".to_string()),
+        HealAdmissionResult::Dropped(reason) => Err(format!("Heal request dropped: {}", reason.as_str())),
+    }
 }
 
 /// Send heal query request
@@ -450,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str)
         }
 
         if let Some(e) = &rule.noncurrent_version_expiration {
-            if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
+            if e.noncurrent_days.is_some() {
                 return true;
             }
             if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
@@ -542,6 +603,19 @@ pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPri
 mod tests {
     use super::*;
 
+    #[test]
+    fn heal_admission_result_labels_are_stable() {
+        assert_eq!(HealAdmissionResult::Accepted.result_label(), "accepted");
+        assert_eq!(HealAdmissionResult::Merged.result_label(), "merged");
+        assert_eq!(HealAdmissionResult::Full.result_label(), "full");
+        assert_eq!(
+            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull).reason_label(),
+            "queue_full"
+        );
+        assert!(HealAdmissionResult::Merged.is_admitted());
+        assert!(!HealAdmissionResult::Full.is_admitted());
+    }
+
     #[tokio::test]
     async fn heal_response_broadcast_reaches_subscriber() {
         let mut receiver = subscribe_heal_responses();

crates/common/src/internode_metrics.rs

@@ -101,11 +101,7 @@ impl InternodeMetrics {
     pub fn snapshot(&self) -> InternodeMetricsSnapshot {
         let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
         let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
-        let dial_avg_time_nanos = if dial_samples_total == 0 {
-            0
-        } else {
-            dial_total_time_nanos / dial_samples_total
-        };
+        let dial_avg_time_nanos = dial_total_time_nanos.checked_div(dial_samples_total).unwrap_or(0);
 
         InternodeMetricsSnapshot {
             sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),

crates/concurrency/Cargo.toml

@@ -26,7 +26,7 @@ thiserror = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]
-tokio = { workspace = true, features = ["full"] }
+tokio = { workspace = true, features = ["test-util","macros","rt-multi-thread"] }
 
 [features]
 default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"]

crates/config/README.md

@@ -53,8 +53,24 @@ Current guidance:
 
 ## Scanner environment aliases
 
+- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
+- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
 - `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
 - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
+- `RUSTFS_SCANNER_IDLE_MODE` (canonical)
+- `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
+
+## Drive timeout environment variables
+
+- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS`
+
+Legacy compatibility fallback:
+- `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
+  This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
 
 ## 📄 License
 

crates/config/src/audit/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Kafka Environment Variables
+pub const ENV_AUDIT_KAFKA_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_ENABLE";
+pub const ENV_AUDIT_KAFKA_BROKERS: &str = "RUSTFS_AUDIT_KAFKA_BROKERS";
+pub const ENV_AUDIT_KAFKA_TOPIC: &str = "RUSTFS_AUDIT_KAFKA_TOPIC";
+pub const ENV_AUDIT_KAFKA_ACKS: &str = "RUSTFS_AUDIT_KAFKA_ACKS";
+pub const ENV_AUDIT_KAFKA_TLS_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_TLS_ENABLE";
+pub const ENV_AUDIT_KAFKA_TLS_CA: &str = "RUSTFS_AUDIT_KAFKA_TLS_CA";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_KAFKA_QUEUE_DIR: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_DIR";
+pub const ENV_AUDIT_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_AUDIT_KAFKA_ENABLE,
+    ENV_AUDIT_KAFKA_BROKERS,
+    ENV_AUDIT_KAFKA_TOPIC,
+    ENV_AUDIT_KAFKA_ACKS,
+    ENV_AUDIT_KAFKA_TLS_ENABLE,
+    ENV_AUDIT_KAFKA_TLS_CA,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_CERT,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_KEY,
+    ENV_AUDIT_KAFKA_QUEUE_DIR,
+    ENV_AUDIT_KAFKA_QUEUE_LIMIT,
+];
+
+/// A list of all valid configuration keys for a Kafka audit target.
+pub const AUDIT_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/mod.rs

@@ -16,10 +16,16 @@
 //! This module defines the configuration for audit systems, including
 //! webhook and MQTT audit-related settings.
 
+mod kafka;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod webhook;
 
+pub use kafka::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use webhook::*;
 
 use crate::DEFAULT_DELIMITER;
@@ -29,8 +35,17 @@ pub const AUDIT_PREFIX: &str = "audit";
 pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
 
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
+pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
 pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
+pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
+pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
 
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
 #[allow(dead_code)]
-pub const AUDIT_SUB_SYSTEMS: &[&str] = &[AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS];
+pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
+    AUDIT_KAFKA_SUB_SYS,
+    AUDIT_MQTT_SUB_SYS,
+    AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_SUB_SYS,
+    AUDIT_WEBHOOK_SUB_SYS,
+];

crates/config/src/audit/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_NATS_ENABLE: &str = "RUSTFS_AUDIT_NATS_ENABLE";
+pub const ENV_AUDIT_NATS_ADDRESS: &str = "RUSTFS_AUDIT_NATS_ADDRESS";
+pub const ENV_AUDIT_NATS_SUBJECT: &str = "RUSTFS_AUDIT_NATS_SUBJECT";
+pub const ENV_AUDIT_NATS_USERNAME: &str = "RUSTFS_AUDIT_NATS_USERNAME";
+pub const ENV_AUDIT_NATS_PASSWORD: &str = "RUSTFS_AUDIT_NATS_PASSWORD";
+pub const ENV_AUDIT_NATS_TOKEN: &str = "RUSTFS_AUDIT_NATS_TOKEN";
+pub const ENV_AUDIT_NATS_CREDENTIALS_FILE: &str = "RUSTFS_AUDIT_NATS_CREDENTIALS_FILE";
+pub const ENV_AUDIT_NATS_TLS_CA: &str = "RUSTFS_AUDIT_NATS_TLS_CA";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
+pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
+pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
+    ENV_AUDIT_NATS_ENABLE,
+    ENV_AUDIT_NATS_ADDRESS,
+    ENV_AUDIT_NATS_SUBJECT,
+    ENV_AUDIT_NATS_USERNAME,
+    ENV_AUDIT_NATS_PASSWORD,
+    ENV_AUDIT_NATS_TOKEN,
+    ENV_AUDIT_NATS_CREDENTIALS_FILE,
+    ENV_AUDIT_NATS_TLS_CA,
+    ENV_AUDIT_NATS_TLS_CLIENT_CERT,
+    ENV_AUDIT_NATS_TLS_CLIENT_KEY,
+    ENV_AUDIT_NATS_TLS_REQUIRED,
+    ENV_AUDIT_NATS_QUEUE_DIR,
+    ENV_AUDIT_NATS_QUEUE_LIMIT,
+];
+
+pub const AUDIT_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_PULSAR_ENABLE: &str = "RUSTFS_AUDIT_PULSAR_ENABLE";
+pub const ENV_AUDIT_PULSAR_BROKER: &str = "RUSTFS_AUDIT_PULSAR_BROKER";
+pub const ENV_AUDIT_PULSAR_TOPIC: &str = "RUSTFS_AUDIT_PULSAR_TOPIC";
+pub const ENV_AUDIT_PULSAR_AUTH_TOKEN: &str = "RUSTFS_AUDIT_PULSAR_AUTH_TOKEN";
+pub const ENV_AUDIT_PULSAR_USERNAME: &str = "RUSTFS_AUDIT_PULSAR_USERNAME";
+pub const ENV_AUDIT_PULSAR_PASSWORD: &str = "RUSTFS_AUDIT_PULSAR_PASSWORD";
+pub const ENV_AUDIT_PULSAR_TLS_CA: &str = "RUSTFS_AUDIT_PULSAR_TLS_CA";
+pub const ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_AUDIT_PULSAR_QUEUE_DIR: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_DIR";
+pub const ENV_AUDIT_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_AUDIT_PULSAR_ENABLE,
+    ENV_AUDIT_PULSAR_BROKER,
+    ENV_AUDIT_PULSAR_TOPIC,
+    ENV_AUDIT_PULSAR_AUTH_TOKEN,
+    ENV_AUDIT_PULSAR_USERNAME,
+    ENV_AUDIT_PULSAR_PASSWORD,
+    ENV_AUDIT_PULSAR_TLS_CA,
+    ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_AUDIT_PULSAR_QUEUE_DIR,
+    ENV_AUDIT_PULSAR_QUEUE_LIMIT,
+];
+
+pub const AUDIT_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/constants/app.rs

@@ -131,6 +131,18 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable to explicitly bypass local physical disk independence checks.
+pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
+
+/// Compatibility alias used by legacy MinIO CI pipelines.
+///
+/// RustFS keeps this alias for backward compatibility only. Prefer
+/// `ENV_UNSAFE_BYPASS_DISK_CHECK` for explicit bypass control.
+pub const ENV_MINIO_CI: &str = "MINIO_CI";
+
+/// Default flag value for bypassing local physical disk independence checks.
+pub const DEFAULT_UNSAFE_BYPASS_DISK_CHECK: bool = false;
+
 /// Environment variable for server access key.
 pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
 

crates/config/src/constants/capacity.rs

@@ -39,6 +39,9 @@ pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
 /// Environment variable for sample rate
 pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
 
+/// Environment variable for metrics logging interval
+pub const ENV_CAPACITY_METRICS_INTERVAL: &str = "RUSTFS_CAPACITY_METRICS_INTERVAL";
+
 /// Environment variable for following symbolic links during capacity calculation
 pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
 
@@ -89,6 +92,10 @@ pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
 /// Default: 200
 pub const DEFAULT_SAMPLE_RATE: usize = 200;
 
+/// Capacity metrics logging interval in seconds
+/// Default: 600 seconds (10 minutes)
+pub const DEFAULT_CAPACITY_METRICS_INTERVAL_SECS: u64 = 600;
+
 /// Follow symbolic links during capacity calculation
 /// Default: false (disabled for safety)
 pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
@@ -130,6 +137,7 @@ mod tests {
         assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
         assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
         assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
+        assert_eq!(ENV_CAPACITY_METRICS_INTERVAL, "RUSTFS_CAPACITY_METRICS_INTERVAL");
         assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
         assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
         assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
@@ -147,6 +155,7 @@ mod tests {
         assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
         assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
         assert_eq!(DEFAULT_SAMPLE_RATE, 200);
+        assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
         assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
         assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2);
         assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15);

crates/config/src/constants/drive.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Legacy global drive timeout fallback.
+/// Deprecated in favor of per-operation drive timeout knobs.
+pub const ENV_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
+
+/// Default timeout in seconds for the legacy global drive timeout fallback.
+pub const DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS: u64 = 30;
+
+/// Timeout for metadata-oriented drive operations such as `read_metadata`.
+pub const ENV_DRIVE_METADATA_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_METADATA_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_METADATA_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `disk_info()` calls on local and remote drives.
+pub const ENV_DRIVE_DISK_INFO_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `list_dir()` style metadata listing operations.
+pub const ENV_DRIVE_LIST_DIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS: u64 = 5;
+
+/// Total timeout for `walk_dir()` operations.
+pub const ENV_DRIVE_WALKDIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
+
+/// Maximum time without forward progress while consuming a `walk_dir()` stream.
+pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
+
+/// Number of consecutive failures before a suspect drive is classified as offline.
+pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
+pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
+
+/// Number of consecutive successful recovery probes before a returning drive is considered online again.
+pub const ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD: &str = "RUSTFS_DRIVE_RETURNING_SUCCESS_THRESHOLD";
+pub const DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD: u64 = 3;
+
+/// Probe interval in seconds while a drive is in the recovery path.
+pub const ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS: &str = "RUSTFS_DRIVE_RETURNING_PROBE_INTERVAL_SECS";
+pub const DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS: u64 = 2;
+
+/// Duration in seconds for classifying a recovered drive as a short offline event.
+pub const ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS: &str = "RUSTFS_DRIVE_OFFLINE_GRACE_PERIOD_SECS";
+pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
+
+/// Duration in seconds after which a recovered drive is classified as long offline.
+pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
+pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;

crates/config/src/constants/env.rs

@@ -26,6 +26,15 @@ pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";
 
+/// Global switch for enabling the audit module.
+pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
+/// Global switch for enabling the notify module.
+pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
+/// Default global audit switch (disabled by default).
+pub const DEFAULT_AUDIT_ENABLE: bool = false;
+/// Default global notify switch (disabled by default).
+pub const DEFAULT_NOTIFY_ENABLE: bool = false;
+
 /// Medium-drawn lines separator
 /// This is used to separate words in environment variable names.
 pub const ENV_WORD_DELIMITER_DASH: &str = "-";
@@ -290,4 +299,10 @@ mod tests {
             assert!(state.is_disabled());
         }
     }
+
+    #[test]
+    fn test_global_audit_notify_switch_constants() {
+        assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
+        assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
+    }
 }

crates/config/src/constants/heal.rs

@@ -56,6 +56,15 @@ pub const ENV_HEAL_TASK_TIMEOUT_SECS: &str = "RUSTFS_HEAL_TASK_TIMEOUT_SECS";
 /// - Note: A higher concurrency limit can speed up healing but may lead to resource contention.
 pub const ENV_HEAL_MAX_CONCURRENT_HEALS: &str = "RUSTFS_HEAL_MAX_CONCURRENT_HEALS";
 
+/// Environment variable name that specifies the maximum number of concurrent heal operations
+/// allowed for a single erasure set.
+///
+/// - Purpose: Prevent one degraded set from consuming all global heal slots.
+/// - Unit: number of operations (usize).
+/// - Valid values: any positive integer.
+/// - Example: `export RUSTFS_HEAL_MAX_CONCURRENT_PER_SET=1`
+pub const ENV_HEAL_MAX_CONCURRENT_PER_SET: &str = "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET";
+
 /// Default value for enabling authentication for heal operations if not specified in the environment variable.
 /// - Value: true (authentication enabled).
 /// - Rationale: Enabling authentication by default enhances security for heal operations.
@@ -86,3 +95,46 @@ pub const DEFAULT_HEAL_TASK_TIMEOUT_SECS: u64 = 300; // 5 minutes
 /// - Rationale: This default concurrency limit helps balance healing speed with resource usage, preventing system overload.
 /// - Adjustments: Users may modify this value via the `RUSTFS_HEAL_MAX_CONCURRENT_HEALS` environment variable based on their system capacity and expected heal workload.
 pub const DEFAULT_HEAL_MAX_CONCURRENT_HEALS: usize = 4;
+
+/// Default maximum number of concurrent heal operations per erasure set.
+///
+/// - Value: 1 concurrent heal operation per set.
+/// - Rationale: Keeps a degraded set from monopolizing the global heal scheduler.
+pub const DEFAULT_HEAL_MAX_CONCURRENT_PER_SET: usize = 1;
+
+/// Environment variable that controls whether low-priority heal requests should merge into
+/// an existing queued request with the same deduplication key.
+pub const ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE: &str = "RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE";
+
+/// Environment variable that allows low-priority heal requests to be dropped when the queue is full.
+pub const ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: &str = "RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL";
+
+/// Environment variable that controls concurrent object heals within a single erasure-set page.
+pub const ENV_HEAL_PAGE_OBJECT_CONCURRENCY: &str = "RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY";
+
+/// Environment variable that toggles notify-driven scheduler wakeups.
+pub const ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: &str = "RUSTFS_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE";
+
+/// Environment variable that toggles per-set bulkhead scheduling.
+pub const ENV_HEAL_SET_BULKHEAD_ENABLE: &str = "RUSTFS_HEAL_SET_BULKHEAD_ENABLE";
+
+/// Environment variable that toggles page-level parallel object healing for erasure-set repair.
+pub const ENV_HEAL_PAGE_PARALLEL_ENABLE: &str = "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE";
+
+/// Default behavior is to merge duplicate low-priority requests.
+pub const DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE: bool = true;
+
+/// Default behavior is to drop low-priority requests instead of blocking when the queue is full.
+pub const DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: bool = true;
+
+/// Default per-page object heal concurrency for erasure-set healing.
+pub const DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY: usize = 8;
+
+/// Default behavior is to keep notify-driven scheduler wakeups enabled.
+pub const DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: bool = true;
+
+/// Default behavior is to keep per-set bulkhead scheduling enabled.
+pub const DEFAULT_HEAL_SET_BULKHEAD_ENABLE: bool = true;
+
+/// Default behavior is to keep erasure-set page parallelism enabled.
+pub const DEFAULT_HEAL_PAGE_PARALLEL_ENABLE: bool = true;

crates/config/src/constants/health.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Enable or disable public `/health` and `/health/ready` endpoints.
+/// When disabled, the routes are not registered and return 404.
+pub const ENV_HEALTH_ENDPOINT_ENABLE: &str = "RUSTFS_HEALTH_ENDPOINT_ENABLE";
+pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
+
+/// Cache TTL for storage readiness runtime-state evaluation (milliseconds).
+/// This reduces storage-layer pressure when probes are called at high frequency.
+pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
+pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
+
+/// Enable minimal health payload mode for GET `/health*` responses.
+/// When enabled, only `status` and `ready` fields are returned.
+pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
+pub const DEFAULT_HEALTH_MINIMAL_RESPONSE_ENABLE: bool = false;

crates/config/src/constants/mod.rs

@@ -17,8 +17,10 @@ pub(crate) mod body_limits;
 pub(crate) mod capacity;
 pub(crate) mod compress;
 pub(crate) mod console;
+pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
+pub(crate) mod health;
 pub(crate) mod object;
 pub(crate) mod oidc;
 pub(crate) mod profiler;

crates/config/src/constants/oidc.rs

@@ -17,6 +17,7 @@ pub const OIDC_CONFIG_URL: &str = "config_url";
 pub const OIDC_CLIENT_ID: &str = "client_id";
 pub const OIDC_CLIENT_SECRET: &str = "client_secret";
 pub const OIDC_SCOPES: &str = "scopes";
+pub const OIDC_OTHER_AUDIENCES: &str = "other_audiences";
 pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
 pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
 pub const OIDC_CLAIM_NAME: &str = "claim_name";
@@ -34,6 +35,7 @@ pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_
 pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
 pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
 pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
+pub const ENV_IDENTITY_OPENID_OTHER_AUDIENCES: &str = "RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
 pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
@@ -46,12 +48,13 @@ pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_
 pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
 
 /// List of all environment variable keys for an OIDC provider.
-pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 15] = &[
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 16] = &[
     ENV_IDENTITY_OPENID_ENABLE,
     ENV_IDENTITY_OPENID_CONFIG_URL,
     ENV_IDENTITY_OPENID_CLIENT_ID,
     ENV_IDENTITY_OPENID_CLIENT_SECRET,
     ENV_IDENTITY_OPENID_SCOPES,
+    ENV_IDENTITY_OPENID_OTHER_AUDIENCES,
     ENV_IDENTITY_OPENID_REDIRECT_URI,
     ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
     ENV_IDENTITY_OPENID_CLAIM_NAME,
@@ -71,6 +74,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
     OIDC_CLIENT_ID,
     OIDC_CLIENT_SECRET,
     OIDC_SCOPES,
+    OIDC_OTHER_AUDIENCES,
     OIDC_REDIRECT_URI,
     OIDC_REDIRECT_URI_DYNAMIC,
     OIDC_CLAIM_NAME,

crates/config/src/constants/proxy.rs

@@ -20,6 +20,11 @@ pub const ENV_TRUSTED_PROXY_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_ENABLED";
 /// Trusted proxy middleware is enabled by default.
 pub const DEFAULT_TRUSTED_PROXY_ENABLED: bool = true;
 
+/// Environment variable to select the trusted proxy implementation.
+pub const ENV_TRUSTED_PROXY_IMPLEMENTATION: &str = "RUSTFS_TRUSTED_PROXY_IMPLEMENTATION";
+/// The simplified implementation is used by default.
+pub const DEFAULT_TRUSTED_PROXY_IMPLEMENTATION: &str = "simple";
+
 /// Environment variable for the proxy validation mode.
 pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDATION_MODE";
 /// Default validation mode is "hop_by_hop".

crates/config/src/constants/scanner.rs

@@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS"
 #[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
 pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
 
+/// Environment variable that specifies the scanner cycle interval in seconds.
+/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour)
+pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE";
+
 /// Environment variable that selects the scanner speed preset.
 /// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
 /// Controls the sleep factor, maximum sleep duration, and cycle interval.
@@ -39,9 +45,24 @@ pub const DEFAULT_SCANNER_SPEED: &str = "default";
 /// - Example: `export RUSTFS_SCANNER_IDLE_MODE=false`
 pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
 
+/// Environment variable that controls scanner cache save timeout in seconds.
+/// The scanner enforces a minimum value of `1`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
+pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
+
 /// Default scanner idle mode.
 pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 
+/// Compatibility flag kept for Patch 3 rollback windows.
+///
+/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
+/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
+pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
+
+/// Default inline scanner heal compatibility mode.
+pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
+
 /// Scanner speed preset controlling throttling behavior.
 ///
 /// Each preset defines three parameters:
@@ -56,6 +77,8 @@ pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 /// | `default` | 2x     | 1 second  | 1 minute       |
 /// | `slow`    | 10x    | 15 seconds| 1 minute       |
 /// | `slowest` | 100x   | 15 seconds| 30 minutes     |
+///
+/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum ScannerSpeed {
     Fastest,

crates/config/src/constants/targets.rs

@@ -40,6 +40,39 @@ pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
 pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
 pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
+pub const KAFKA_BROKERS: &str = "brokers";
+pub const KAFKA_TOPIC: &str = "topic";
+pub const KAFKA_ACKS: &str = "acks";
+pub const KAFKA_QUEUE_DIR: &str = "queue_dir";
+pub const KAFKA_QUEUE_LIMIT: &str = "queue_limit";
+pub const KAFKA_TLS_ENABLE: &str = "tls_enable";
+pub const KAFKA_TLS_CA: &str = "tls_ca";
+pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key";
+
+pub const NATS_ADDRESS: &str = "address";
+pub const NATS_SUBJECT: &str = "subject";
+pub const NATS_USERNAME: &str = "username";
+pub const NATS_PASSWORD: &str = "password";
+pub const NATS_TOKEN: &str = "token";
+pub const NATS_CREDENTIALS_FILE: &str = "credentials_file";
+pub const NATS_TLS_CA: &str = "tls_ca";
+pub const NATS_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
+pub const NATS_TLS_REQUIRED: &str = "tls_required";
+pub const NATS_QUEUE_DIR: &str = "queue_dir";
+pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
+
+pub const PULSAR_BROKER: &str = "broker";
+pub const PULSAR_TOPIC: &str = "topic";
+pub const PULSAR_AUTH_TOKEN: &str = "auth_token";
+pub const PULSAR_USERNAME: &str = "username";
+pub const PULSAR_PASSWORD: &str = "password";
+pub const PULSAR_TLS_CA: &str = "tls_ca";
+pub const PULSAR_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure";
+pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification";
+pub const PULSAR_QUEUE_DIR: &str = "queue_dir";
+pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit";
 
 /// Environment variable controlling whether target queue files are Snappy-compressed.
 /// Applies to both notify and audit target queue stores.

crates/config/src/lib.rs

@@ -25,10 +25,14 @@ pub use constants::compress::*;
 #[cfg(feature = "constants")]
 pub use constants::console::*;
 #[cfg(feature = "constants")]
+pub use constants::drive::*;
+#[cfg(feature = "constants")]
 pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::heal::*;
 #[cfg(feature = "constants")]
+pub use constants::health::*;
+#[cfg(feature = "constants")]
 pub use constants::object::*;
 #[cfg(feature = "constants")]
 pub use constants::profiler::*;

crates/config/src/notify/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// A list of all valid configuration keys for a Kafka target.
+pub const NOTIFY_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+// Kafka Environment Variables
+pub const ENV_NOTIFY_KAFKA_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_ENABLE";
+pub const ENV_NOTIFY_KAFKA_BROKERS: &str = "RUSTFS_NOTIFY_KAFKA_BROKERS";
+pub const ENV_NOTIFY_KAFKA_TOPIC: &str = "RUSTFS_NOTIFY_KAFKA_TOPIC";
+pub const ENV_NOTIFY_KAFKA_ACKS: &str = "RUSTFS_NOTIFY_KAFKA_ACKS";
+pub const ENV_NOTIFY_KAFKA_TLS_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_TLS_ENABLE";
+pub const ENV_NOTIFY_KAFKA_TLS_CA: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CA";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_KAFKA_QUEUE_DIR: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_DIR";
+pub const ENV_NOTIFY_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_NOTIFY_KAFKA_ENABLE,
+    ENV_NOTIFY_KAFKA_BROKERS,
+    ENV_NOTIFY_KAFKA_TOPIC,
+    ENV_NOTIFY_KAFKA_ACKS,
+    ENV_NOTIFY_KAFKA_TLS_ENABLE,
+    ENV_NOTIFY_KAFKA_TLS_CA,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY,
+    ENV_NOTIFY_KAFKA_QUEUE_DIR,
+    ENV_NOTIFY_KAFKA_QUEUE_LIMIT,
+];

crates/config/src/notify/mod.rs

@@ -13,12 +13,18 @@
 // limitations under the License.
 
 mod arn;
+mod kafka;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod store;
 mod webhook;
 
 pub use arn::*;
+pub use kafka::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use store::*;
 pub use webhook::*;
 
@@ -64,9 +70,14 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
 pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
 
 #[allow(dead_code)]
-pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
+pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
+    NOTIFY_KAFKA_SUB_SYS,
+    NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_SUB_SYS,
+    NOTIFY_WEBHOOK_SUB_SYS,
+];
 
-#[allow(dead_code)]
 pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
 pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
 #[allow(dead_code)]
@@ -83,4 +94,5 @@ pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
 pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
 #[allow(dead_code)]
 pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
+pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
 pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";

crates/config/src/notify/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_NATS_ENABLE: &str = "RUSTFS_NOTIFY_NATS_ENABLE";
+pub const ENV_NOTIFY_NATS_ADDRESS: &str = "RUSTFS_NOTIFY_NATS_ADDRESS";
+pub const ENV_NOTIFY_NATS_SUBJECT: &str = "RUSTFS_NOTIFY_NATS_SUBJECT";
+pub const ENV_NOTIFY_NATS_USERNAME: &str = "RUSTFS_NOTIFY_NATS_USERNAME";
+pub const ENV_NOTIFY_NATS_PASSWORD: &str = "RUSTFS_NOTIFY_NATS_PASSWORD";
+pub const ENV_NOTIFY_NATS_TOKEN: &str = "RUSTFS_NOTIFY_NATS_TOKEN";
+pub const ENV_NOTIFY_NATS_CREDENTIALS_FILE: &str = "RUSTFS_NOTIFY_NATS_CREDENTIALS_FILE";
+pub const ENV_NOTIFY_NATS_TLS_CA: &str = "RUSTFS_NOTIFY_NATS_TLS_CA";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
+pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
+pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
+    ENV_NOTIFY_NATS_ENABLE,
+    ENV_NOTIFY_NATS_ADDRESS,
+    ENV_NOTIFY_NATS_SUBJECT,
+    ENV_NOTIFY_NATS_USERNAME,
+    ENV_NOTIFY_NATS_PASSWORD,
+    ENV_NOTIFY_NATS_TOKEN,
+    ENV_NOTIFY_NATS_CREDENTIALS_FILE,
+    ENV_NOTIFY_NATS_TLS_CA,
+    ENV_NOTIFY_NATS_TLS_CLIENT_CERT,
+    ENV_NOTIFY_NATS_TLS_CLIENT_KEY,
+    ENV_NOTIFY_NATS_TLS_REQUIRED,
+    ENV_NOTIFY_NATS_QUEUE_DIR,
+    ENV_NOTIFY_NATS_QUEUE_LIMIT,
+];

crates/config/src/notify/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_PULSAR_ENABLE: &str = "RUSTFS_NOTIFY_PULSAR_ENABLE";
+pub const ENV_NOTIFY_PULSAR_BROKER: &str = "RUSTFS_NOTIFY_PULSAR_BROKER";
+pub const ENV_NOTIFY_PULSAR_TOPIC: &str = "RUSTFS_NOTIFY_PULSAR_TOPIC";
+pub const ENV_NOTIFY_PULSAR_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_PULSAR_AUTH_TOKEN";
+pub const ENV_NOTIFY_PULSAR_USERNAME: &str = "RUSTFS_NOTIFY_PULSAR_USERNAME";
+pub const ENV_NOTIFY_PULSAR_PASSWORD: &str = "RUSTFS_NOTIFY_PULSAR_PASSWORD";
+pub const ENV_NOTIFY_PULSAR_TLS_CA: &str = "RUSTFS_NOTIFY_PULSAR_TLS_CA";
+pub const ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_NOTIFY_PULSAR_QUEUE_DIR: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_DIR";
+pub const ENV_NOTIFY_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_NOTIFY_PULSAR_ENABLE,
+    ENV_NOTIFY_PULSAR_BROKER,
+    ENV_NOTIFY_PULSAR_TOPIC,
+    ENV_NOTIFY_PULSAR_AUTH_TOKEN,
+    ENV_NOTIFY_PULSAR_USERNAME,
+    ENV_NOTIFY_PULSAR_PASSWORD,
+    ENV_NOTIFY_PULSAR_TLS_CA,
+    ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_NOTIFY_PULSAR_QUEUE_DIR,
+    ENV_NOTIFY_PULSAR_QUEUE_LIMIT,
+];

crates/config/src/observability/mod.rs

@@ -22,6 +22,14 @@ pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
 pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT";
 pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT";
 pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT";
+pub const ENV_OBS_ENDPOINT_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_HEADERS";
+pub const ENV_OBS_ENDPOINT_TRACES_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS";
+pub const ENV_OBS_ENDPOINT_METRICS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS";
+pub const ENV_OBS_ENDPOINT_LOGS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS";
+pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
 pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
@@ -108,6 +116,14 @@ mod tests {
         assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT");
         assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT");
         assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT");
+        assert_eq!(ENV_OBS_ENDPOINT_HEADERS, "RUSTFS_OBS_ENDPOINT_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_HEADERS, "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_HEADERS, "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_HEADERS, "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS");
         assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT");
         assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
         assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");

crates/credentials/src/credentials.rs

@@ -20,7 +20,7 @@ use std::collections::HashMap;
 use std::env;
 use std::fmt;
 use std::io::Error;
-use std::sync::OnceLock;
+use std::sync::{LazyLock, OnceLock};
 use time::OffsetDateTime;
 
 /// Global active credentials
@@ -314,6 +314,17 @@ impl fmt::Debug for Credentials {
 }
 
 impl Credentials {
+    /// Returns a reference to this credential's claims, or a shared empty map
+    /// when the credential has no claims attached. Avoids per-call allocation
+    /// at call sites that need an `&HashMap<String, Value>`.
+    pub fn claims_or_empty(&self) -> &HashMap<String, Value> {
+        static EMPTY: LazyLock<HashMap<String, Value>> = LazyLock::new(HashMap::new);
+        match &self.claims {
+            Some(c) => c,
+            None => &EMPTY,
+        }
+    }
+
     pub fn is_expired(&self) -> bool {
         if self.expiration.is_none() {
             return false;

crates/e2e_test/Cargo.toml

@@ -41,7 +41,6 @@ serde_json.workspace = true
 tonic = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
-clap = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
 bytes.workspace = true
@@ -53,8 +52,6 @@ async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
 flate2.workspace = true
 http.workspace = true
-http-body.workspace = true
-http-body-util.workspace = true
 reqwest = { workspace = true }
 rustfs-signer.workspace = true
 tracing = { workspace = true }
@@ -76,3 +73,4 @@ rcgen.workspace = true
 anyhow.workspace = true
 rustls.workspace = true
 zip.workspace = true
+clap.workspace = true

crates/e2e_test/src/archive_download_integrity_test.rs

@@ -218,6 +218,34 @@ mod tests {
         Ok(builder.send().await?)
     }
 
+    async fn signed_get_request_with_headers(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        extra_headers: &[(&str, &str)],
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let mut request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+        for (name, value) in extra_headers {
+            request = request.header(*name, *value);
+        }
+
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut builder = client.get(url);
+        for (name, value) in signed.headers() {
+            builder = builder.header(name, value);
+        }
+
+        Ok(builder.send().await?)
+    }
+
     async fn assert_archive_object_content_encoding(
         client: &S3Client,
         bucket: &str,
@@ -655,6 +683,42 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let key = "multipart-empty-conditional-headers.zip";
+        let zip_bytes =
+            complete_archive_multipart_upload_with_content_encoding(&client, MULTIPART_ARCHIVE_TEST_BUCKET, key, None).await?;
+        let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, key);
+
+        let response = signed_get_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            &[("if-match", ""), ("if-none-match", "")],
+        )
+        .await?;
+        let status = response.status();
+        let body = response.bytes().await?;
+
+        assert_eq!(
+            status,
+            StatusCode::OK,
+            "unexpected multipart GET status {status}, body: {}",
+            String::from_utf8_lossy(body.as_ref())
+        );
+        assert_eq!(body.as_ref(), zip_bytes.as_slice());
+
+        env.stop_server();
+        Ok(())
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()

crates/e2e_test/src/bin/tls_gen.rs

@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod collectors;
-pub mod constants;
-pub mod format;
-mod global;
-mod metrics_type;
+use clap::Parser;
+use e2e_test::tls_gen::{Args, run};
 
-pub use format::report_metrics;
-pub use global::init_metrics_system;
-pub use metrics_type::*;
+fn main() -> anyhow::Result<()> {
+    let out_dir = run(Args::parse())?;
+    println!("Generated RustFS TLS bundle in {}", out_dir.display());
+    Ok(())
+}

crates/e2e_test/src/kms/test_runner.rs

@@ -450,11 +450,7 @@ impl KMSTestSuite {
         if failed > 0 {
             warn!("❌ Failing tests:");
             for result in results.iter().filter(|r| !r.success) {
-                warn!(
-                    "  - {}: {}",
-                    result.test_name,
-                    result.error_message.as_ref().unwrap_or(&"Unknown error".to_string())
-                );
+                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
             }
         }
     }

crates/e2e_test/src/lib.rs

@@ -125,3 +125,5 @@ mod replication_extension_test;
 
 #[cfg(test)]
 mod snowball_auto_extract_test;
+
+pub mod tls_gen;

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -19,6 +19,7 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
     use serial_test::serial;
     use tracing::info;
@@ -179,4 +180,84 @@ mod tests {
             "Delete marker should no longer be latest after the second put"
         );
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_prefix_with_marker_object_returns_children() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns prefix children when a marker object also exists");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-versions-prefix-marker";
+        let marker_key = "data01";
+        let child_keys = [
+            "data01/meta/dump-2026-04-08-053205.json.gz",
+            "data01/meta/dump-2026-04-08-063209.json.gz",
+        ];
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to suspend versioning");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(marker_key)
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to put marker object");
+
+        for key in child_keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(ByteStream::from_static(b"payload"))
+                .send()
+                .await
+                .expect("Failed to put child object");
+        }
+
+        let listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix("data01/")
+            .send()
+            .await
+            .expect("Failed to list object versions by prefix");
+
+        let version_keys: Vec<_> = listing.versions().iter().filter_map(|version| version.key()).collect();
+
+        assert_eq!(
+            version_keys.len(),
+            child_keys.len(),
+            "ListObjectVersions with a trailing slash prefix should include child objects even when the marker object exists"
+        );
+
+        for key in child_keys {
+            assert!(
+                version_keys.contains(&key),
+                "ListObjectVersions(prefix=data01/) should include child object {key}"
+            );
+        }
+    }
 }

crates/e2e_test/src/reliant/conditional_writes.rs

@@ -259,7 +259,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
     let upload_id = initiate_response
         .upload_id()
-        .ok_or(std::io::Error::other("No upload ID returned"))?;
+        .ok_or_else(|| std::io::Error::other("No upload ID returned"))?;
 
     // Upload parts
     for part_number in 1..=num_parts {
@@ -277,7 +277,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
         let part_etag = upload_part_response
             .e_tag()
-            .ok_or(std::io::Error::other("Do not have etag"))?
+            .ok_or_else(|| std::io::Error::other("Do not have etag"))?
             .to_string();
 
         let completed_part = CompletedPart::builder().part_number(part_number).e_tag(part_etag).build();

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -120,11 +120,6 @@ impl LockClient for GrpcLockClient {
             .map_err(|e| LockError::internal(e.to_string()))?
             .into_inner();
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -134,7 +129,8 @@ impl LockClient for GrpcLockClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -50,6 +50,18 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
     }
 }
 
+fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> GenerallyLockResult {
+    if success {
+        GenerallyLockResult {
+            success: true,
+            error_info: None,
+            lock_info: None,
+        }
+    } else {
+        lock_result_from_error(format!("lock not found for release: {lock_id}"))
+    }
+}
+
 /// Minimal NodeService implementation that only supports Lock RPCs
 /// Used for testing distributed lock scenarios with real gRPC
 #[derive(Debug)]
@@ -102,7 +114,7 @@ impl NodeService for MinimalLockNodeService {
                 let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
                 Ok(Response::new(GenerallyLockResponse {
                     success: result.success,
-                    error_info: None,
+                    error_info: result.error,
                     lock_info: lock_info_json,
                 }))
             }
@@ -131,11 +143,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -161,11 +176,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.force_release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -271,10 +289,9 @@ impl NodeService for MinimalLockNodeService {
                 Ok(batch_results) => {
                     for (result_idx, success) in batch_results.into_iter().enumerate() {
                         if let Some(request_idx) = valid_indices.get(result_idx) {
-                            results[*request_idx] = GenerallyLockResult {
-                                success,
-                                error_info: None,
-                                lock_info: None,
+                            results[*request_idx] = match lock_ids.get(result_idx) {
+                                Some(lock_id) => lock_result_from_release(lock_id, success),
+                                None => lock_result_from_error(format!("unlock response index out of range: {result_idx}")),
                             };
                         }
                     }

crates/e2e_test/src/reliant/head_tls_bodyless_test.rs

@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for TLS/HTTP2 `HEAD` responses on missing objects.
+//!
+//! Before the fix, RustFS returned `404` for a missing object but still wrote
+//! the XML error payload on a `HEAD` request. Under HTTP/2 this emitted DATA
+//! frames after the response headers, which clients surfaced as a protocol
+//! error. This test keeps the request at the raw HTTPS layer so it can validate
+//! the final wire-facing behavior rather than SDK-level error mapping.
+
+#![cfg(test)]
+
+use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use http::Version;
+use http::header::HOST;
+use rcgen::generate_simple_self_signed;
+use reqwest::{Certificate, Client, Response, StatusCode};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+use std::path::Path;
+use std::process::Command;
+use tokio::fs;
+use tokio::time::{Duration, sleep};
+use tracing::info;
+
+const ACCESS_KEY: &str = "rustfsadmin";
+const SECRET_KEY: &str = "rustfsadmin";
+const BUCKET: &str = "test-head-tls-bodyless-bucket";
+
+async fn generate_tls_bundle(tls_dir: &Path) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    fs::create_dir_all(tls_dir).await?;
+    let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
+    let cert_pem = cert.cert.pem();
+    let key_pem = cert.signing_key.serialize_pem();
+
+    fs::write(tls_dir.join("rustfs_cert.pem"), cert_pem.as_bytes()).await?;
+    fs::write(tls_dir.join("rustfs_key.pem"), key_pem.as_bytes()).await?;
+
+    Ok(cert_pem.into_bytes())
+}
+
+fn local_https_h2_client(ca_pem: &[u8]) -> Result<Client, Box<dyn Error + Send + Sync>> {
+    let _ca_cert = Certificate::from_pem(ca_pem)?;
+    Ok(Client::builder()
+        .no_proxy()
+        .no_gzip()
+        .no_brotli()
+        .no_zstd()
+        .no_deflate()
+        .danger_accept_invalid_certs(true)
+        .build()?)
+}
+
+async fn signed_empty_request(
+    client: &Client,
+    method: http::Method,
+    url: &str,
+) -> Result<Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let request = http::Request::builder()
+        .method(method.as_str())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+
+    let signed = sign_v4(request, 0, ACCESS_KEY, SECRET_KEY, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        builder = builder.header(name, value);
+    }
+
+    Ok(builder.send().await?)
+}
+
+async fn ensure_bucket_exists(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let bucket_url = format!("{endpoint}/{BUCKET}/");
+    let response = signed_empty_request(client, http::Method::HEAD, &bucket_url).await?;
+
+    if response.status() == StatusCode::OK {
+        return Ok(());
+    }
+
+    let response = signed_empty_request(client, http::Method::PUT, &bucket_url).await?;
+    match response.status() {
+        StatusCode::OK => Ok(()),
+        StatusCode::CONFLICT => Ok(()),
+        status => Err(format!("unexpected bucket setup status: {status}").into()),
+    }
+}
+
+async fn wait_for_tls_server_ready(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let ready_url = format!("{endpoint}/");
+    for _attempt in 0..60 {
+        match signed_empty_request(client, http::Method::GET, &ready_url).await {
+            Ok(response) if response.status().is_success() => return Ok(()),
+            Ok(_) | Err(_) => sleep(Duration::from_millis(500)).await,
+        }
+    }
+
+    Err("RustFS TLS server failed to become ready within 30 seconds".into())
+}
+
+async fn start_tls_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let binary_path = rustfs_binary_path();
+    let mut command = Command::new(&binary_path);
+    command
+        .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .current_dir(&env.temp_dir);
+
+    for key in [
+        "RUSTFS_ADDRESS",
+        "RUSTFS_VOLUMES",
+        "RUSTFS_ACCESS_KEY",
+        "RUSTFS_SECRET_KEY",
+        "RUSTFS_TLS_PATH",
+        "RUSTFS_OBS_LOG_DIRECTORY",
+    ] {
+        command.env_remove(key);
+    }
+
+    let process = command
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .env("RUSTFS_CONSOLE_ENABLE", "false")
+        .args([
+            "--address",
+            &env.address,
+            "--access-key",
+            &env.access_key,
+            "--secret-key",
+            &env.secret_key,
+            &env.temp_dir,
+        ])
+        .spawn()?;
+
+    env.process = Some(process);
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_head_missing_object_over_tls_http2_is_bodyless() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let tls_dir = std::path::PathBuf::from(&env.temp_dir).join("tls");
+    let ca_pem = generate_tls_bundle(&tls_dir).await?;
+    start_tls_rustfs_server(&mut env, &tls_dir).await?;
+
+    let endpoint = format!("https://{}", env.address);
+    let client = local_https_h2_client(&ca_pem)?;
+    wait_for_tls_server_ready(&client, &endpoint).await?;
+    ensure_bucket_exists(&client, &endpoint).await?;
+
+    let missing_key = "head-does-not-exist.txt";
+    let object_url = format!("{endpoint}/{BUCKET}/{missing_key}");
+
+    let get_response = signed_empty_request(&client, http::Method::GET, &object_url).await?;
+    assert_eq!(get_response.status(), StatusCode::NOT_FOUND);
+    let get_version = get_response.version();
+    let get_body = get_response.bytes().await?;
+    let get_body_text = String::from_utf8_lossy(&get_body);
+    assert!(
+        get_body_text.contains("<Code>NoSuchKey</Code>") || get_body_text.contains("<Code>NoSuchObject</Code>"),
+        "GET missing-object error body should expose NoSuchKey/NoSuchObject, got: {}",
+        get_body_text
+    );
+    info!("GET missing object over TLS used {:?} and returned {} bytes", get_version, get_body.len());
+
+    let head_response = signed_empty_request(&client, http::Method::HEAD, &object_url).await?;
+    assert_eq!(head_response.status(), StatusCode::NOT_FOUND);
+    assert_eq!(head_response.version(), Version::HTTP_2, "HEAD regression test must exercise HTTP/2");
+    let head_body = head_response.bytes().await?;
+    assert!(
+        head_body.is_empty(),
+        "HEAD missing-object response must not send body bytes over TLS/HTTP2, got {} bytes: {:?}",
+        head_body.len(),
+        head_body
+    );
+
+    Ok(())
+}

crates/e2e_test/src/reliant/lifecycle.rs

@@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 #[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
+async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
     use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
 
     let client = create_aws_s3_client().await?;
@@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::er
         .build()?;
     let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
 
-    let err = client
+    client
         .put_bucket_lifecycle_configuration()
         .bucket(BUCKET)
         .lifecycle_configuration(lifecycle)
         .send()
-        .await
-        .expect_err("zero-day lifecycle expiration should be rejected");
-
-    let err_msg = format!("{err:?}");
-    assert!(
-        err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"),
-        "unexpected error: {err_msg}"
-    );
+        .await?;
 
     Ok(())
 }

crates/e2e_test/src/reliant/lock.rs

@@ -275,6 +275,41 @@ async fn test_grpc_lock_client_batch_acquire_and_release() {
     handle.abort();
 }
 
+#[tokio::test]
+async fn test_grpc_lock_client_uses_request_lock_id_and_reports_missing_unlock() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+    let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client = GrpcLockClient::new(addr);
+    let request = LockRequest::new(test_resource(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(2));
+
+    let response = grpc_client.acquire_lock(&request).await.expect("gRPC acquire should succeed");
+    let lock_info = response.lock_info.expect("gRPC acquire should include lock info");
+    assert_eq!(lock_info.id, request.lock_id);
+
+    assert!(
+        grpc_client
+            .release(&request.lock_id)
+            .await
+            .expect("gRPC release should succeed"),
+        "release should find the request lock id"
+    );
+
+    let missing_release = grpc_client
+        .release(&request.lock_id)
+        .await
+        .expect_err("second release should report missing lock");
+    assert!(
+        missing_release.to_string().contains("lock not found for release"),
+        "missing release should preserve server error, got: {missing_release}"
+    );
+
+    handle.abort();
+}
+
 #[tokio::test]
 async fn test_distributed_lock_4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes() {
     let manager1 = Arc::new(GlobalLockManager::new());

crates/e2e_test/src/reliant/mod.rs

@@ -17,6 +17,7 @@ mod get_deleted_object_test;
 mod grpc_lock_client;
 mod grpc_lock_server;
 mod head_deleted_object_versioning_test;
+mod head_tls_bodyless_test;
 mod lifecycle;
 mod lock;
 mod node_interact_test;

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -154,7 +154,7 @@ async fn walk_dir() -> Result<(), Box<dyn Error>> {
             match response.next().await {
                 Some(Ok(resp)) => {
                     if !resp.success {
-                        println!("{}", resp.error_info.unwrap_or("".to_string()));
+                        println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
                     }
                     let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
                         .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))

crates/e2e_test/src/replication_extension_test.rs

@@ -12,14 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+};
+use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+use aws_sdk_s3::{Client, Config};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
 use rustfs_madmin::{
-    PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus,
-    SRStatusInfo, SiteReplicationInfo, SyncStatus,
+    AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
+    ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
@@ -30,6 +34,8 @@ use std::error::Error;
 use time::Duration as TimeDuration;
 use tokio::time::{Duration, sleep};
 
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
 #[derive(Debug, Clone, serde::Deserialize)]
 struct ReplicationResetStatusResponse {
     #[serde(rename = "Targets", default)]
@@ -79,6 +85,65 @@ async fn signed_request(
     Ok(request_builder.send().await?)
 }
 
+async fn signed_request_with_session_token(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if !session_token.is_empty() {
+        request = request.header("x-amz-security-token", session_token);
+    }
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(
+        request.body(Body::empty())?,
+        content_len,
+        access_key,
+        secret_key,
+        session_token,
+        "us-east-1",
+    );
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
+    let open = format!("<{tag}>");
+    let close = format!("</{tag}>");
+    let start = xml.find(&open)? + open.len();
+    let end = xml[start..].find(&close)? + start;
+    Some(xml[start..end].to_string())
+}
+
+fn parse_assume_role_credentials(xml: &str) -> Result<(String, String, String), Box<dyn Error + Send + Sync>> {
+    let access_key = extract_xml_tag(xml, "AccessKeyId").ok_or("missing AccessKeyId in AssumeRole response")?;
+    let secret_key = extract_xml_tag(xml, "SecretAccessKey").ok_or("missing SecretAccessKey in AssumeRole response")?;
+    let session_token = extract_xml_tag(xml, "SessionToken").ok_or("missing SessionToken in AssumeRole response")?;
+    Ok((access_key, secret_key, session_token))
+}
+
 async fn set_replication_target(
     source_env: &RustFSTestEnvironment,
     source_bucket: &str,
@@ -190,6 +255,14 @@ async fn put_bucket_replication(
     Ok(())
 }
 
+async fn delete_bucket_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication", env.url);
+    signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
 async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
     let client = env.create_s3_client();
     client
@@ -205,6 +278,143 @@ async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) ->
     Ok(())
 }
 
+fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    let body = serde_json::json!({
+        "secretKey": secret_key,
+        "status": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("create user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_add_canned_policy(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    policy: &serde_json::Value,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(policy.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add canned policy failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_attach_policy_to_user(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    username: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+        env.url, policy_name, username
+    );
+    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("attach policy to user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_update_group_members(
+    env: &RustFSTestEnvironment,
+    group_name: &str,
+    members: &[&str],
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let body = serde_json::json!({
+        "group": group_name,
+        "members": members,
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("update group members failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_attach_policy_to_group(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    group_name: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
+        env.url, policy_name, group_name
+    );
+    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("attach policy to group failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
 async fn run_replication_check(
     env: &RustFSTestEnvironment,
     bucket: &str,
@@ -251,6 +461,182 @@ async fn remove_replication_target_request(
     signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
 }
 
+async fn add_service_account(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    req: &AddServiceAccountReq,
+) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-service-account", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        signer_access_key,
+        signer_secret_key,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add service account failed: {status} {body}").into());
+    }
+
+    let body = response.bytes().await?;
+    let parsed: serde_json::Value = serde_json::from_slice(&body)?;
+    let credentials = parsed
+        .get("credentials")
+        .ok_or("add service account response missing credentials")?;
+    let access_key = credentials
+        .get("accessKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing access key")?
+        .to_string();
+    let secret_key = credentials
+        .get("secretKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing secret key")?
+        .to_string();
+
+    Ok((access_key, secret_key))
+}
+
+async fn add_service_account_with_session_token(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    session_token: &str,
+    req: &AddServiceAccountReq,
+) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-service-account", env.url);
+    let response = signed_request_with_session_token(
+        http::Method::PUT,
+        &url,
+        signer_access_key,
+        signer_secret_key,
+        session_token,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add service account with session token failed: {status} {body}").into());
+    }
+
+    let body = response.bytes().await?;
+    let parsed: serde_json::Value = serde_json::from_slice(&body)?;
+    let credentials = parsed
+        .get("credentials")
+        .ok_or("add service account response missing credentials")?;
+    let access_key = credentials
+        .get("accessKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing access key")?
+        .to_string();
+    let secret_key = credentials
+        .get("secretKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing secret key")?
+        .to_string();
+
+    Ok((access_key, secret_key))
+}
+
+async fn list_service_accounts(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    user: Option<&str>,
+) -> Result<ListServiceAccountsResp, Box<dyn Error + Send + Sync>> {
+    let mut url = format!("{}/rustfs/admin/v3/list-service-accounts", env.url);
+    if let Some(user) = user {
+        url.push_str("?user=");
+        url.push_str(&urlencoding::encode(user));
+    }
+
+    let response = signed_request(http::Method::GET, &url, signer_access_key, signer_secret_key, None, None).await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("list service accounts failed: {status} {body}").into());
+    }
+
+    Ok(response.json().await?)
+}
+
+async fn wait_for_service_accounts(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    user: Option<&str>,
+    expected: &[&str],
+) -> Result<ListServiceAccountsResp, Box<dyn Error + Send + Sync>> {
+    for _ in 0..20 {
+        let resp = list_service_accounts(env, signer_access_key, signer_secret_key, user).await?;
+        let access_keys: Vec<&str> = resp.accounts.iter().map(|account| account.access_key.as_str()).collect();
+        if expected
+            .iter()
+            .all(|expected_key| access_keys.iter().any(|actual| actual == expected_key))
+        {
+            return Ok(resp);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("service accounts did not reach expected keys {expected:?} on {}", env.address).into())
+}
+
+async fn wait_for_object_on_target(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    for _ in 0..40 {
+        match client.get_object().bucket(bucket).key(key).send().await {
+            Ok(output) => {
+                let body = output.body.collect().await?.into_bytes().to_vec();
+                return Ok(body);
+            }
+            Err(err) => {
+                if err.to_string().contains("NoSuchKey") || err.to_string().contains("NotFound") {
+                    sleep(Duration::from_millis(250)).await;
+                    continue;
+                }
+                return Err(err.into());
+            }
+        }
+    }
+
+    Err(format!("object {bucket}/{key} was not replicated in time").into())
+}
+
+async fn wait_for_user_get_object(client: &Client, bucket: &str, key: &str) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    let mut last_error = None;
+    for _ in 0..40 {
+        match client.get_object().bucket(bucket).key(key).send().await {
+            Ok(output) => {
+                let body = output.body.collect().await?.into_bytes().to_vec();
+                return Ok(body);
+            }
+            Err(err) => {
+                last_error = Some(err.to_string());
+                sleep(Duration::from_millis(250)).await;
+            }
+        }
+    }
+
+    Err(format!(
+        "user could not read replicated object {bucket}/{key} in time; last error: {}",
+        last_error.unwrap_or_else(|| "unknown".to_string())
+    )
+    .into())
+}
+
 async fn list_replication_targets_request(
     env: &RustFSTestEnvironment,
     bucket: Option<&str>,
@@ -1084,6 +1470,54 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
     Ok(())
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-delete-config-src";
+    let target_bucket = "replication-delete-config-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    let delete_response = delete_bucket_replication(&source_env, source_bucket).await?;
+    assert!(
+        delete_response.status().is_success(),
+        "unexpected delete status: {}",
+        delete_response.status()
+    );
+
+    let targets_response = list_replication_targets_request(&source_env, Some(source_bucket)).await?;
+    assert_eq!(targets_response.status(), StatusCode::OK);
+    let targets: Vec<serde_json::Value> = targets_response.json().await?;
+    assert!(
+        targets
+            .iter()
+            .all(|target| target.get("arn").and_then(|arn| arn.as_str()) != Some(target_arn.as_str())),
+        "deleted replication config left stale target {target_arn}: {targets:?}"
+    );
+
+    let recreated_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &recreated_arn).await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -1490,3 +1924,449 @@ async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Re
 
     Ok(())
 }
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_object_with_bucket_versioning_real_dual_node() -> TestResult {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-versioned";
+    let key = "hello.txt";
+    let payload = b"site replication should replicate after enabling versioning".to_vec();
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    let replication_response = signed_request(
+        http::Method::GET,
+        &format!("{}/{bucket}?replication", source_env.url),
+        &source_env.access_key,
+        &source_env.secret_key,
+        None,
+        None,
+    )
+    .await?;
+    let replication_status = replication_response.status();
+    let replication_body = replication_response.text().await.unwrap_or_default();
+    assert_eq!(
+        replication_status,
+        StatusCode::OK,
+        "source bucket replication config missing after site replication setup: {replication_body}"
+    );
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-policy-user";
+    let key = "seed.txt";
+    let payload = b"site replication policy-backed user access".to_vec();
+    let policy_name = "site-repl-readonly";
+    let username = "site-repl-user";
+    let secret_key = "site-repl-user-secret-key-123456";
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{bucket}/*")]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                "Resource": [format!("arn:aws:s3:::{bucket}")]
+            }
+        ]
+    });
+    admin_add_canned_policy(&source_env, policy_name, &policy).await?;
+    admin_create_user(&source_env, username, secret_key).await?;
+    admin_attach_policy_to_user(&source_env, policy_name, username).await?;
+
+    let target_user_client = create_user_s3_client(&target_env, username, secret_key);
+    let fetched = wait_for_user_get_object(&target_user_client, bucket, key).await?;
+    assert_eq!(fetched, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_group_policy_backed_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>>
+{
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-policy-group";
+    let key = "seed.txt";
+    let payload = b"site replication group-policy-backed user access".to_vec();
+    let policy_name = "site-repl-group-readonly";
+    let group_name = "site-repl-group";
+    let username = "site-repl-group-user";
+    let secret_key = "site-repl-group-user-secret-key-12";
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{bucket}/*")]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                "Resource": [format!("arn:aws:s3:::{bucket}")]
+            }
+        ]
+    });
+    admin_add_canned_policy(&source_env, policy_name, &policy).await?;
+    admin_create_user(&source_env, username, secret_key).await?;
+    admin_update_group_members(&source_env, group_name, &[username]).await?;
+    admin_attach_policy_to_group(&source_env, policy_name, group_name).await?;
+
+    let target_user_client = create_user_s3_client(&target_env, username, secret_key);
+    let fetched = wait_for_user_get_object(&target_user_client, bucket, key).await?;
+    assert_eq!(fetched, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_multiple_service_accounts_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let first_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-alpha".to_string(),
+        secret_key: "svc-alpha-secret-key-1234567890abcdef".to_string(),
+        name: Some("svc-alpha".to_string()),
+        description: Some("first replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let first = add_service_account(&source_env, &source_env.access_key, &source_env.secret_key, &first_req).await?;
+
+    let target_after_first = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-alpha"],
+    )
+    .await?;
+    assert!(
+        target_after_first
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-alpha"),
+        "target accounts missing svc-alpha: {:?}",
+        target_after_first.accounts
+    );
+
+    let second_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-beta".to_string(),
+        secret_key: "svc-beta-secret-key-1234567890abcdef1".to_string(),
+        name: Some("svc-beta".to_string()),
+        description: Some("second replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let _second = add_service_account(&source_env, &first.0, &first.1, &second_req).await?;
+
+    let target_after_second = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-alpha", "svc-beta"],
+    )
+    .await?;
+    assert!(
+        target_after_second
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-beta"),
+        "target accounts missing svc-beta: {:?}",
+        target_after_second.accounts
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
+    init_logging();
+
+    if !awscurl_available() {
+        eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
+        return Ok(());
+    }
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let assume_role_body = "Action=AssumeRole&Version=2011-06-15&DurationSeconds=3600";
+    let sts_xml = awscurl_post_sts_form_urlencoded(
+        &format!("{}/", source_env.url.trim_end_matches('/')),
+        assume_role_body,
+        &source_env.access_key,
+        &source_env.secret_key,
+    )
+    .await?;
+    let (sts_access_key, sts_secret_key, sts_session_token) = parse_assume_role_credentials(&sts_xml)?;
+
+    let first_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-sts-alpha".to_string(),
+        secret_key: "svc-sts-alpha-secret-key-1234567890".to_string(),
+        name: Some("svc-sts-alpha".to_string()),
+        description: Some("sts-created replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let first =
+        add_service_account_with_session_token(&source_env, &sts_access_key, &sts_secret_key, &sts_session_token, &first_req)
+            .await?;
+
+    let target_after_first = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-sts-alpha"],
+    )
+    .await?;
+    assert!(
+        target_after_first
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-sts-alpha"),
+        "target accounts missing svc-sts-alpha: {:?}",
+        target_after_first.accounts
+    );
+
+    let second_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-sts-beta".to_string(),
+        secret_key: "svc-sts-beta-secret-key-1234567890a".to_string(),
+        name: Some("svc-sts-beta".to_string()),
+        description: Some("second replicated service account from sts-created ak".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let _second = add_service_account(&source_env, &first.0, &first.1, &second_req).await?;
+
+    let target_after_second = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-sts-alpha", "svc-sts-beta"],
+    )
+    .await?;
+    assert!(
+        target_after_second
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-sts-beta"),
+        "target accounts missing svc-sts-beta: {:?}",
+        target_after_second.accounts
+    );
+
+    Ok(())
+}

crates/e2e_test/src/special_chars_test.rs

@@ -26,10 +26,16 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
     use aws_sdk_s3::Client;
     use aws_sdk_s3::primitives::ByteStream;
+    use http::StatusCode;
+    use http::header::HOST;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
     use serial_test::serial;
+    use std::error::Error;
     use tracing::{debug, info};
 
     /// Helper function to create an S3 client for testing
@@ -56,6 +62,30 @@ mod tests {
         }
     }
 
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
     /// Test PUT and GET with space character in path
     ///
     /// This reproduces Part A of the issue:
@@ -274,6 +304,73 @@ mod tests {
         info!("Test completed successfully");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_missing_object_with_trailing_equals_returns_no_such_key() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-missing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let url = format!("{}/{}/path/sitemap.xmlage=", env.url, bucket);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::NOT_FOUND,
+            "missing object key ending with '=' should pass signature validation before object lookup"
+        );
+
+        let body = response.text().await?;
+        assert!(body.contains("<Code>NoSuchKey</Code>"), "expected NoSuchKey XML response, got: {body}");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_existing_object_with_trailing_equals_returns_content() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-existing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let key = "path/sitemap.xmlage=";
+        let content = b"object content for raw signed URL with trailing equals";
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}/{}", env.url, bucket, key);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::OK,
+            "existing object key ending with '=' should pass signature validation and return content"
+        );
+
+        let body = response.bytes().await?;
+        assert_eq!(body.as_ref(), content);
+
+        env.stop_server();
+        Ok(())
+    }
+
     /// Test DELETE operation with special characters
     #[tokio::test]
     #[serial]

crates/e2e_test/src/tls_gen.rs

@@ -0,0 +1,259 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::{Context, Result, bail};
+use clap::Parser;
+use rcgen::{
+    BasicConstraints, CertificateParams, CertifiedIssuer, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose,
+    SanType,
+};
+use std::fs;
+use std::net::IpAddr;
+use std::path::{Path, PathBuf};
+use time::{Duration, OffsetDateTime};
+
+pub const DEFAULT_OUT_DIR: &str = "target/tls";
+pub const OUTPUT_FILES: [&str; 7] = [
+    "rustfs_cert.pem",
+    "rustfs_key.pem",
+    "ca.crt",
+    "public.crt",
+    "client_ca.crt",
+    "client_cert.pem",
+    "client_key.pem",
+];
+
+#[derive(Debug, Parser)]
+#[command(name = "tls_gen", about = "Generate a full RustFS TLS bundle for local TLS and mTLS tests.")]
+pub struct Args {
+    #[arg(long, default_value = DEFAULT_OUT_DIR)]
+    pub out_dir: PathBuf,
+    #[arg(long, default_value_t = 365)]
+    pub days: i64,
+    #[arg(long)]
+    pub force: bool,
+}
+
+pub fn run(args: Args) -> Result<PathBuf> {
+    if args.days <= 0 {
+        bail!("--days must be a positive integer");
+    }
+
+    write_bundle(&args.out_dir, args.force, args.days)?;
+    Ok(args.out_dir)
+}
+
+pub fn ensure_writable(out_dir: &Path, force: bool) -> Result<()> {
+    if force {
+        return Ok(());
+    }
+
+    let existing: Vec<_> = OUTPUT_FILES
+        .iter()
+        .map(|name| out_dir.join(name))
+        .filter(|path| path.exists())
+        .collect();
+
+    if existing.is_empty() {
+        return Ok(());
+    }
+
+    let existing_list = existing
+        .iter()
+        .map(|path| path.file_name().and_then(|name| name.to_str()).unwrap_or("<unknown>"))
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    bail!(
+        "Refusing to overwrite existing files in {}: {}. Re-run with --force to replace them.",
+        out_dir.display(),
+        existing_list
+    )
+}
+
+fn write_bundle(out_dir: &Path, force: bool, days: i64) -> Result<()> {
+    fs::create_dir_all(out_dir).with_context(|| format!("failed to create output directory {}", out_dir.display()))?;
+    ensure_writable(out_dir, force)?;
+
+    let ca_key = generate_private_key()?;
+    let ca = build_ca_certificate(ca_key, days)?;
+
+    let server_key = generate_private_key()?;
+    let server_cert = build_leaf_certificate(
+        &server_key,
+        "localhost",
+        &[SanType::DnsName("localhost".try_into()?)],
+        &[
+            SanType::IpAddress(IpAddr::V4("127.0.0.1".parse()?)),
+            SanType::IpAddress(IpAddr::V6("::1".parse()?)),
+        ],
+        ExtendedKeyUsagePurpose::ServerAuth,
+        &ca,
+        days,
+    )?;
+
+    let client_key = generate_private_key()?;
+    let client_cert = build_leaf_certificate(
+        &client_key,
+        "rustfs-test-client",
+        &[SanType::DnsName("rustfs-test-client".try_into()?)],
+        &[],
+        ExtendedKeyUsagePurpose::ClientAuth,
+        &ca,
+        days,
+    )?;
+
+    let ca_pem = ca.pem();
+    let bundle = [
+        ("rustfs_cert.pem", server_cert.pem()),
+        ("rustfs_key.pem", server_key.serialize_pem()),
+        ("ca.crt", ca_pem.clone()),
+        ("public.crt", ca_pem.clone()),
+        ("client_ca.crt", ca_pem),
+        ("client_cert.pem", client_cert.pem()),
+        ("client_key.pem", client_key.serialize_pem()),
+    ];
+
+    for (name, content) in bundle {
+        fs::write(out_dir.join(name), content).with_context(|| format!("failed to write {}", out_dir.join(name).display()))?;
+    }
+
+    Ok(())
+}
+
+fn build_ca_certificate(signing_key: KeyPair, days: i64) -> Result<CertifiedIssuer<'static, KeyPair>> {
+    let mut params = base_params("RustFS Test CA", days)?;
+    params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+    params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
+
+    CertifiedIssuer::self_signed(params, signing_key).context("failed to create CA certificate")
+}
+
+fn build_leaf_certificate(
+    signing_key: &KeyPair,
+    common_name: &str,
+    dns_names: &[SanType],
+    ip_addresses: &[SanType],
+    usage: ExtendedKeyUsagePurpose,
+    issuer: &CertifiedIssuer<'_, KeyPair>,
+    days: i64,
+) -> Result<rcgen::Certificate> {
+    let mut params = base_params(common_name, days)?;
+    params.is_ca = IsCa::ExplicitNoCa;
+    params.key_usages = vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment];
+    params.extended_key_usages = vec![usage];
+    params.use_authority_key_identifier_extension = true;
+    params.subject_alt_names.extend_from_slice(dns_names);
+    params.subject_alt_names.extend_from_slice(ip_addresses);
+
+    params
+        .signed_by(signing_key, issuer)
+        .with_context(|| format!("failed to create leaf certificate for {common_name}"))
+}
+
+fn base_params(common_name: &str, days: i64) -> Result<CertificateParams> {
+    let mut params = CertificateParams::default();
+    let issued_at = OffsetDateTime::now_utc() - Duration::minutes(5);
+    params.not_before = issued_at;
+    params.not_after = issued_at + Duration::days(days);
+    params.distinguished_name.push(DnType::CountryName, "US");
+    params.distinguished_name.push(DnType::OrganizationName, "RustFS");
+    params.distinguished_name.push(DnType::CommonName, common_name);
+    Ok(params)
+}
+
+fn generate_private_key() -> Result<KeyPair> {
+    KeyPair::generate().context("failed to generate private key")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Args, OUTPUT_FILES, ensure_writable, run};
+    use std::fs;
+    use std::path::{Path, PathBuf};
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn unique_temp_dir() -> PathBuf {
+        let suffix = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("system time must be after unix epoch")
+            .as_nanos();
+        std::env::temp_dir().join(format!("rustfs-tls-gen-{suffix}"))
+    }
+
+    struct TempDir {
+        path: PathBuf,
+    }
+
+    impl TempDir {
+        fn new() -> Self {
+            let path = unique_temp_dir();
+            fs::create_dir_all(&path).expect("temporary directory should be created");
+            Self { path }
+        }
+
+        fn path(&self) -> &Path {
+            &self.path
+        }
+    }
+
+    impl Drop for TempDir {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.path);
+        }
+    }
+
+    #[test]
+    fn run_writes_full_bundle() {
+        let temp_dir = TempDir::new();
+
+        let out_dir = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 365,
+            force: false,
+        })
+        .expect("bundle generation should succeed");
+
+        for name in OUTPUT_FILES {
+            let content = fs::read(out_dir.join(name)).unwrap_or_else(|error| panic!("{name} should exist: {error}"));
+            assert!(!content.is_empty(), "{name} should not be empty");
+        }
+    }
+
+    #[test]
+    fn ensure_writable_rejects_existing_files_without_force() {
+        let temp_dir = TempDir::new();
+        let existing = temp_dir.path().join(OUTPUT_FILES[0]);
+        fs::write(&existing, "existing").expect("existing file should be created");
+
+        let error = ensure_writable(temp_dir.path(), false).expect_err("existing files must be rejected");
+        let message = format!("{error:#}");
+
+        assert!(message.contains("Refusing to overwrite existing files"));
+        assert!(message.contains(OUTPUT_FILES[0]));
+    }
+
+    #[test]
+    fn run_rejects_non_positive_days() {
+        let temp_dir = TempDir::new();
+        let error = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 0,
+            force: false,
+        })
+        .expect_err("non-positive days must fail");
+
+        assert_eq!(format!("{error:#}"), "--days must be a positive integer");
+    }
+}

crates/ecstore/Cargo.toml

@@ -55,6 +55,7 @@ flatbuffers.workspace = true
 futures.workspace = true
 futures-util.workspace = true
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
 serde.workspace = true
 time.workspace = true
 bytesize.workspace = true
@@ -62,6 +63,7 @@ serde_json.workspace = true
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
 s3s.workspace = true
 http.workspace = true
+opentelemetry.workspace = true
 http-body = { workspace = true }
 http-body-util.workspace = true
 url.workspace = true
@@ -86,7 +88,7 @@ hyper.workspace = true
 hyper-util.workspace = true
 hyper-rustls.workspace = true
 rustls.workspace = true
-tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
+tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower.workspace = true
@@ -124,9 +126,10 @@ metrics = { workspace = true }
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true }
 serial_test = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 
 [build-dependencies]
 shadow-rs = { workspace = true, features = ["build", "metadata"] }

crates/ecstore/src/admin_server_info.rs

@@ -162,8 +162,9 @@ pub async fn get_local_server_property() -> ServerProperties {
 
     let mut props = ServerProperties {
         endpoint: addr,
-        uptime: SystemTime::now()
-            .duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
+        uptime: GLOBAL_BOOT_TIME
+            .get()
+            .and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok())
             .unwrap_or_default()
             .as_secs(),
         network,

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -786,7 +786,10 @@ impl BucketTargetSys {
                         && tgt
                             .credentials
                             .as_ref()
-                            .map(|c| c.access_key == target.credentials.as_ref().unwrap_or(&Credentials::default()).access_key)
+                            .map(|c| {
+                                let default_creds = Credentials::default();
+                                c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
+                            })
                             .unwrap_or(false)
                     {
                         return (tgt.arn.clone(), true);
@@ -894,6 +897,41 @@ pub struct RemoveObjectOptions {
     pub replication_validity_check: bool,
 }
 
+fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOptions) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    if opts.force_delete {
+        insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
+    }
+    if opts.governance_bypass {
+        headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
+    }
+
+    if opts.replication_delete_marker {
+        insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
+    }
+
+    if let Some(t) = opts.replication_mtime {
+        insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
+    }
+
+    if !opts.replication_status.is_empty() {
+        headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
+    }
+
+    if let Some(version_id) = version_id {
+        insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
+    }
+
+    if opts.replication_request {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
+    }
+    if opts.replication_validity_check {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
+    }
+
+    headers
+}
+
 #[derive(Debug, Clone)]
 pub struct AdvancedPutOptions {
     pub source_version_id: String,
@@ -1424,39 +1462,15 @@ impl TargetClient {
         version_id: Option<String>,
         opts: RemoveObjectOptions,
     ) -> Result<(), S3ClientError> {
-        let mut headers = HeaderMap::new();
-        if opts.force_delete {
-            insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
-        }
-        if opts.governance_bypass {
-            headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
-        }
-
-        if opts.replication_delete_marker {
-            insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
-        }
-
-        if let Some(t) = opts.replication_mtime {
-            insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
-        }
-
-        if !opts.replication_status.is_empty() {
-            headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
-        }
-
-        if opts.replication_request {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
-        }
-        if opts.replication_validity_check {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
-        }
+        let headers = build_remove_object_headers(version_id.as_deref(), &opts);
+        let api_version_id = if opts.replication_request { None } else { version_id };
 
         match self
             .client
             .delete_object()
             .bucket(bucket)
             .key(object)
-            .set_version_id(version_id)
+            .set_version_id(api_version_id)
             .customize()
             .map_request(move |mut req| {
                 for (k, v) in headers.clone().into_iter() {
@@ -1550,3 +1564,53 @@ impl From<std::io::Error> for BucketTargetError {
 }
 
 impl Error for BucketTargetError {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: true,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert_eq!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(),
+            Some(version_id.as_str()),
+            "replication delete requests must preserve the version id in internal headers"
+        );
+    }
+
+    #[test]
+    fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: false,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(),
+            "delete-marker version purges must not masquerade as delete-marker creations"
+        );
+    }
+}

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -14,8 +14,9 @@
 
 pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
+pub mod core;
 pub mod evaluator;
-pub mod lifecycle;
+pub use self::core as lifecycle;
 pub mod rule;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -1,4 +1,3 @@
-#![allow(unused_imports)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
@@ -109,10 +109,19 @@ impl ObjSweeper {
     }
 
     pub async fn sweep(&self) {
-        let je = self.should_remove_remote_object();
-        if !je.is_none() {
-            let mut expiry_state = GLOBAL_ExpiryState.write().await;
-            expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
+        let Some(je) = self.should_remove_remote_object() else {
+            return;
+        };
+        let hash = je.op_hash();
+        // Grab the sender under a short read lock, then release the lock so we
+        // don't hold it across the async send.
+        let wrkr = GLOBAL_ExpiryState.read().await.get_worker_ch(hash);
+        let Some(wrkr) = wrkr else {
+            GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
+            return;
+        };
+        if wrkr.send(Some(Box::new(je))).await.is_err() {
+            GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
         }
     }
 }

crates/ecstore/src/bucket/replication/config.rs

@@ -152,7 +152,16 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
             }
 
             if obj.op_type == ReplicationType::Delete {
+                if !rule.metadata_replicate(obj) {
+                    return false;
+                }
+
                 if obj.version_id.is_some() {
+                    if obj.delete_marker {
+                        return rule.delete_marker_replication.clone().is_some_and(|d| {
+                            d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
+                        });
+                    }
                     return rule
                         .delete_replication
                         .clone()

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -824,10 +824,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
         let cancel_token = CancellationToken::new();
         resyncer.register_cancel_token(&opts, cancel_token.clone()).await;
         tokio::spawn(async move {
-            resyncer
-                .clone()
-                .resync_bucket(cancel_token, storage, false, opts.clone())
-                .await;
+            Box::pin(resyncer.clone().resync_bucket(cancel_token, storage, false, opts.clone())).await;
             resyncer.clear_cancel_token(&opts).await;
         });
 
@@ -914,7 +911,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
                         };
                         tokio::spawn(async move {
                             resync.register_cancel_token(&opts, ctx.clone()).await;
-                            resync.clone().resync_bucket(ctx, storage, true, opts.clone()).await;
+                            Box::pin(resync.clone().resync_bucket(ctx, storage, true, opts.clone())).await;
                             resync.clear_cancel_token(&opts).await;
                         });
                     }
@@ -959,6 +956,9 @@ pub type DynReplicationPool = dyn ReplicationPoolTrait + Send + Sync;
 /// Trait that abstracts the replication pool operations
 #[async_trait::async_trait]
 pub trait ReplicationPoolTrait: std::fmt::Debug {
+    fn active_workers(&self) -> i32;
+    fn active_mrf_workers(&self) -> i32;
+    fn active_lrg_workers(&self) -> i32;
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo);
     async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo);
     async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
@@ -975,6 +975,18 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
 // Implement the trait for ReplicationPool
 #[async_trait::async_trait]
 impl<S: StorageAPI> ReplicationPoolTrait for ReplicationPool<S> {
+    fn active_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_workers(self)
+    }
+
+    fn active_mrf_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_mrf_workers(self)
+    }
+
+    fn active_lrg_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_lrg_workers(self)
+    }
+
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo) {
         self.queue_replica_task(ri).await;
     }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -34,8 +34,8 @@ use crate::global::get_global_bucket_monitor;
 use crate::set_disk::get_lock_acquire_timeout;
 use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
-use aws_sdk_s3::error::SdkError;
-use aws_sdk_s3::operation::head_object::HeadObjectOutput;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
 use aws_smithy_types::body::SdkBody;
@@ -85,7 +85,8 @@ use tokio::task::JoinSet;
 use tokio::time::Duration as TokioDuration;
 use tokio_util::io::ReaderStream;
 use tokio_util::sync::CancellationToken;
-use tracing::{error, info, instrument, warn};
+use tracing::{debug, error, info, instrument, warn};
+use uuid::Uuid;
 
 pub(crate) const REPLICATION_DIR: &str = ".replication";
 pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -114,6 +115,41 @@ fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &Res
     state.resync_id.is_empty() || opts.resync_id.is_empty() || state.resync_id == opts.resync_id
 }
 
+fn should_count_head_proxy_failure(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
+    if is_not_found || matches!(code, Some("MethodNotAllowed" | "405")) {
+        return false;
+    }
+    !matches!(raw_status, Some(404 | 405))
+}
+
+fn is_head_proxy_failure(err: &SdkError<HeadObjectError>) -> bool {
+    let (is_not_found, code) = err
+        .as_service_error()
+        .map(|service_err| (service_err.is_not_found(), service_err.code()))
+        .unwrap_or((false, None));
+    let raw_status = err.raw_response().map(|resp| resp.status().as_u16());
+    should_count_head_proxy_failure(is_not_found, code, raw_status)
+}
+
+async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) {
+    if let Some(stats) = GLOBAL_REPLICATION_STATS.get() {
+        stats.inc_proxy(bucket, api, is_err).await;
+    }
+}
+
+async fn head_object_with_proxy_stats(
+    source_bucket: &str,
+    target_client: &TargetClient,
+    target_bucket: &str,
+    object: &str,
+    version_id: Option<String>,
+) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    let result = target_client.head_object(target_bucket, object, version_id).await;
+    let is_err = result.as_ref().err().is_some_and(is_head_proxy_failure);
+    record_proxy_request(source_bucket, "HeadObject", is_err).await;
+    result
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ResyncOpts {
     pub bucket: String,
@@ -747,10 +783,15 @@ impl ReplicationResyncer {
 
                     let reset_id = target_client.reset_id.clone();
 
-                    let (size, err) = if let Err(err) = target_client
-                        .head_object(&target_client.bucket, &roi.name, roi.version_id.map(|v| v.to_string()))
-                        .await
-                    {
+                    let head_result = head_object_with_proxy_stats(
+                        &bucket_name,
+                        target_client.as_ref(),
+                        &target_client.bucket,
+                        &roi.name,
+                        roi.version_id.map(|v| v.to_string()),
+                    )
+                    .await;
+                    let (size, err) = if let Err(err) = head_result {
                         if roi.delete_marker {
                             st.replicated_count += 1;
                         } else {
@@ -1271,15 +1312,7 @@ pub async fn check_replicate_delete(
         return ReplicateDecision::default();
     }
 
-    let opts = ObjectOpts {
-        name: dobj.object_name.clone(),
-        ssec: is_ssec_encrypted(&oi.user_defined),
-        user_tags: oi.user_tags.clone(),
-        delete_marker: oi.delete_marker,
-        version_id: dobj.version_id,
-        op_type: ReplicationType::Delete,
-        ..Default::default()
-    };
+    let opts = delete_replication_object_opts(dobj, oi);
 
     let tgt_arns = rcfg.filter_target_arns(&opts);
     let mut dsc = ReplicateDecision::new();
@@ -1331,6 +1364,19 @@ pub async fn check_replicate_delete(
     dsc
 }
 
+fn delete_replication_object_opts(dobj: &ObjectToDelete, oi: &ObjectInfo) -> ObjectOpts {
+    ObjectOpts {
+        name: dobj.object_name.clone(),
+        ssec: is_ssec_encrypted(&oi.user_defined),
+        user_tags: oi.user_tags.clone(),
+        delete_marker: oi.delete_marker,
+        version_id: dobj.version_id,
+        op_type: ReplicationType::Delete,
+        replica: oi.replication_status == ReplicationStatusType::Replica,
+        ..Default::default()
+    }
+}
+
 /// Check if the user-defined metadata contains SSEC encryption headers
 fn is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
     user_defined.contains_key(SSEC_ALGORITHM_HEADER)
@@ -1497,6 +1543,54 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
         }
     };
 
+    if dobj.delete_object.delete_marker
+        && let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
+    {
+        let source_marker_state = storage
+            .get_object_info(
+                &bucket,
+                &dobj.delete_object.object_name,
+                &ObjectOptions {
+                    version_id: Some(delete_marker_version_id.to_string()),
+                    versioned: BucketVersioningSys::prefix_enabled(&bucket, &dobj.delete_object.object_name).await,
+                    version_suspended: BucketVersioningSys::prefix_suspended(&bucket, &dobj.delete_object.object_name).await,
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        match source_marker_state {
+            Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
+            Ok(_) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    "skipping stale delete-marker replication because source version is no longer a delete marker"
+                );
+                return;
+            }
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    "skipping stale delete-marker replication because source version no longer exists"
+                );
+                return;
+            }
+            Err(err) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    error = %err,
+                    "failed to verify source delete-marker state before replication"
+                );
+            }
+        }
+    }
+
     let dsc = match parse_replicate_decision(
         &bucket,
         &dobj
@@ -1529,7 +1623,6 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
             return;
         }
     };
-
     let ns_lock = match storage
         .new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
         .await
@@ -1653,7 +1746,33 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
         }
     }
 
-    let (replication_status, prev_status) = if dobj.delete_object.version_id.is_none() {
+    let is_version_purge = is_version_delete_replication(&dobj.delete_object);
+
+    if should_retry_delete_marker_purge(&dobj.delete_object) {
+        let bucket_clone = bucket.clone();
+        let dobj_clone = dobj.clone();
+        let dsc_clone = dsc.clone();
+        let storage_clone = storage.clone();
+        tokio::spawn(async move {
+            for _ in 0..5 {
+                if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
+                    && source_delete_marker_missing(
+                        &*storage_clone,
+                        &bucket_clone,
+                        &dobj_clone.delete_object.object_name,
+                        delete_marker_version_id,
+                    )
+                    .await
+                {
+                    replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
+                    break;
+                }
+                tokio::time::sleep(TokioDuration::from_secs(1)).await;
+            }
+        });
+    }
+
+    let (replication_status, prev_status) = if !is_version_purge {
         (
             rinfos.replication_status(),
             dobj.delete_object
@@ -1741,6 +1860,65 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
     }
 }
 
+async fn source_delete_marker_missing<S: StorageAPI>(
+    storage: &S,
+    bucket: &str,
+    object_name: &str,
+    delete_marker_version_id: Uuid,
+) -> bool {
+    match storage
+        .get_object_info(
+            bucket,
+            object_name,
+            &ObjectOptions {
+                version_id: Some(delete_marker_version_id.to_string()),
+                versioned: BucketVersioningSys::prefix_enabled(bucket, object_name).await,
+                version_suspended: BucketVersioningSys::prefix_suspended(bucket, object_name).await,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(info) => !info.delete_marker || info.version_id != Some(delete_marker_version_id),
+        Err(err) => is_err_object_not_found(&err) || is_err_version_not_found(&err),
+    }
+}
+
+async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
+    let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
+        return;
+    };
+
+    for tgt_entry in dsc.targets_map.values() {
+        if !tgt_entry.replicate {
+            continue;
+        }
+        if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
+            continue;
+        }
+        let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &tgt_entry.arn).await else {
+            continue;
+        };
+
+        let _ = tgt_client
+            .remove_object(
+                &tgt_client.bucket,
+                &dobj.delete_object.object_name,
+                Some(delete_marker_version_id.to_string()),
+                RemoveObjectOptions {
+                    force_delete: false,
+                    governance_bypass: false,
+                    replication_delete_marker: false,
+                    replication_mtime: dobj.delete_object.delete_marker_mtime,
+                    replication_status: ReplicationStatusType::Replica,
+                    replication_request: true,
+                    replication_validity_check: false,
+                },
+            )
+            .await;
+    }
+}
+
 async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
     let bucket = &dobj.bucket;
     let object_name = &dobj.delete_object.object_name;
@@ -1924,6 +2102,18 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
     }
 }
 
+fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
+    dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
+}
+
+fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool {
+    dobj.delete_marker_version_id.is_some()
+}
+
+fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
+    !is_not_found && !matches!(code, Some("MethodNotAllowed" | "405"))
+}
+
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
     let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
         version_id.to_owned()
@@ -1941,7 +2131,8 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
     rinfo.endpoint = tgt_client.endpoint.clone();
     rinfo.secure = tgt_client.secure;
 
-    if dobj.delete_object.version_id.is_none()
+    let is_version_purge = is_version_delete_replication(&dobj.delete_object);
+    if !is_version_purge
         && rinfo.prev_replication_status == ReplicationStatusType::Completed
         && dobj.op_type != ReplicationType::ExistingObject
     {
@@ -1949,12 +2140,12 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         return rinfo;
     }
 
-    if dobj.delete_object.version_id.is_some() && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
+    if is_version_purge && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
         return rinfo;
     }
 
     if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
-        if dobj.delete_object.version_id.is_none() {
+        if !is_version_purge {
             rinfo.replication_status = ReplicationStatusType::Failed;
         } else {
             rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -1968,18 +2159,34 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         Some(version_id.to_string())
     };
 
-    if dobj.delete_object.delete_marker_version_id.is_some()
-        && let Err(e) = tgt_client
-            .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
-            .await
-        && let SdkError::ServiceError(service_err) = &e
-        && !service_err.err().is_not_found()
-    {
-        rinfo.replication_status = ReplicationStatusType::Failed;
-        rinfo.error = Some(e.to_string());
-
-        return rinfo;
-    };
+    if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
+        match head_object_with_proxy_stats(
+            &dobj.bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &dobj.delete_object.object_name,
+            version_id.clone(),
+        )
+        .await
+        {
+            Ok(_) => {}
+            Err(e) => {
+                let non_retryable = matches!(
+                    &e,
+                    SdkError::ServiceError(service_err)
+                        if is_retryable_delete_replication_head_error(
+                            service_err.err().is_not_found(),
+                            service_err.err().code(),
+                        )
+                );
+                if non_retryable {
+                    rinfo.replication_status = ReplicationStatusType::Failed;
+                    rinfo.error = Some(e.to_string());
+                    return rinfo;
+                }
+            }
+        }
+    }
 
     match tgt_client
         .remove_object(
@@ -1989,7 +2196,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
             RemoveObjectOptions {
                 force_delete: false,
                 governance_bypass: false,
-                replication_delete_marker: dobj.delete_object.delete_marker_version_id.is_some(),
+                replication_delete_marker: dobj.delete_object.delete_marker,
                 replication_mtime: dobj.delete_object.delete_marker_mtime,
                 replication_status: ReplicationStatusType::Replica,
                 replication_request: true,
@@ -1999,15 +2206,32 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         .await
     {
         Ok(_) => {
-            if dobj.delete_object.version_id.is_none() {
+            debug!(
+                bucket = tgt_client.bucket,
+                object = dobj.delete_object.object_name,
+                version_id = ?version_id,
+                delete_marker = dobj.delete_object.delete_marker,
+                is_version_purge,
+                "replicate_delete_to_target succeeded"
+            );
+            if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Completed;
             } else {
                 rinfo.version_purge_status = VersionPurgeStatusType::Complete;
             }
         }
         Err(e) => {
+            warn!(
+                bucket = tgt_client.bucket,
+                object = dobj.delete_object.object_name,
+                version_id = ?version_id,
+                delete_marker = dobj.delete_object.delete_marker,
+                is_version_purge,
+                error = %e,
+                "replicate_delete_to_target failed"
+            );
             rinfo.error = Some(e.to_string());
-            if dobj.delete_object.version_id.is_none() {
+            if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Failed;
             } else {
                 rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -2292,9 +2516,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         }
 
         let mut replication_action = replication_action;
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2351,7 +2580,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
 
         if let Some(err) = if is_multipart {
             drop(gr);
-            replicate_object_with_multipart(MultipartReplicationContext {
+            let result = replicate_object_with_multipart(MultipartReplicationContext {
                 storage: storage.clone(),
                 cli: tgt_client.clone(),
                 src_bucket: &bucket,
@@ -2362,16 +2591,18 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 arn: &rinfo.arn,
                 put_opts,
             })
-            .await
-            .err()
+            .await;
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            result.err()
         } else {
             gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
             let byte_stream = async_read_to_bytestream(gr.stream);
-            tgt_client
+            let result = tgt_client
                 .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                 .await
-                .map_err(|e| std::io::Error::other(e.to_string()))
-                .err()
+                .map_err(|e| std::io::Error::other(e.to_string()));
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            result.err()
         } {
             rinfo.replication_status = ReplicationStatusType::Failed;
             rinfo.error = Some(err.to_string());
@@ -2511,9 +2742,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             warn!("failed to set replication tagging directive header: {err}");
         }
 
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2633,7 +2869,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
 
             if let Some(err) = if is_multipart {
                 drop(gr);
-                replicate_object_with_multipart(MultipartReplicationContext {
+                let result = replicate_object_with_multipart(MultipartReplicationContext {
                     storage: storage.clone(),
                     cli: tgt_client.clone(),
                     src_bucket: &bucket,
@@ -2644,16 +2880,18 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     arn: &rinfo.arn,
                     put_opts,
                 })
-                .await
-                .err()
+                .await;
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                result.err()
             } else {
                 gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
                 let byte_stream = async_read_to_bytestream(gr.stream);
-                tgt_client
+                let result = tgt_client
                     .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                     .await
-                    .map_err(|e| std::io::Error::other(e.to_string()))
-                    .err()
+                    .map_err(|e| std::io::Error::other(e.to_string()));
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                result.err()
             } {
                 rinfo.replication_status = ReplicationStatusType::Failed;
                 rinfo.error = Some(err.to_string());
@@ -3450,6 +3688,153 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_delete_replication_object_opts_marks_replica_deletes() {
+        let dobj = ObjectToDelete {
+            object_name: "obj".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            replication_status: ReplicationStatusType::Replica,
+            ..Default::default()
+        };
+
+        let opts = delete_replication_object_opts(&dobj, &oi);
+
+        assert!(
+            opts.replica,
+            "replica deletes must preserve replica status for downstream ReplicaModifications rules"
+        );
+        assert_eq!(opts.version_id, dobj.version_id);
+        assert_eq!(opts.name, dobj.object_name);
+        assert_eq!(opts.op_type, ReplicationType::Delete);
+    }
+
+    #[test]
+    fn test_delete_replication_object_opts_keeps_non_replica_deletes_local() {
+        let dobj = ObjectToDelete {
+            object_name: "obj".to_string(),
+            ..Default::default()
+        };
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            replication_status: ReplicationStatusType::Completed,
+            ..Default::default()
+        };
+
+        let opts = delete_replication_object_opts(&dobj, &oi);
+
+        assert!(!opts.replica, "source-originated deletes should not be treated as replica modifications");
+    }
+
+    #[test]
+    fn test_is_version_delete_replication_for_delete_marker_version_purge() {
+        let dobj = DeletedObject {
+            delete_marker: false,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            is_version_delete_replication(&dobj),
+            "delete-marker version purges must be tracked as version purge replication, not delete-marker creation replication"
+        );
+    }
+
+    #[test]
+    fn test_is_version_delete_replication_for_delete_marker_creation() {
+        let dobj = DeletedObject {
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            !is_version_delete_replication(&dobj),
+            "delete-marker creation should remain on the delete-marker replication path"
+        );
+    }
+
+    #[test]
+    fn test_should_retry_delete_marker_purge_for_version_purge() {
+        let dobj = DeletedObject {
+            delete_marker: false,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            should_retry_delete_marker_purge(&dobj),
+            "delete-marker version purge should schedule delayed target cleanup in case the target marker arrives late"
+        );
+    }
+
+    #[test]
+    fn test_should_retry_delete_marker_purge_for_delete_marker_creation() {
+        let dobj = DeletedObject {
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            should_retry_delete_marker_purge(&dobj),
+            "delete-marker creation should keep the late-arrival cleanup path so downstream purges can catch up"
+        );
+    }
+
+    #[test]
+    fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
+        assert!(
+            !is_retryable_delete_replication_head_error(false, Some("405")),
+            "numeric 405 responses should not block delete-marker purge replication"
+        );
+        assert!(
+            !is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")),
+            "MethodNotAllowed responses should not block delete-marker purge replication"
+        );
+        assert!(
+            !is_retryable_delete_replication_head_error(true, Some("NoSuchKey")),
+            "not-found responses should not block delete-marker purge replication"
+        );
+        assert!(
+            is_retryable_delete_replication_head_error(false, Some("AccessDenied")),
+            "unexpected head errors should still fail fast"
+        );
+    }
+
+    #[test]
+    fn test_should_count_head_proxy_failure_ignores_not_found_and_405() {
+        assert!(
+            !should_count_head_proxy_failure(true, Some("NoSuchKey"), Some(404)),
+            "not-found heads are expected when the object has not reached the target yet"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("MethodNotAllowed"), Some(405)),
+            "405 delete-marker probing responses should not be counted as proxy failures"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("405"), Some(405)),
+            "numeric 405 codes must align with MethodNotAllowed semantics"
+        );
+    }
+
+    #[test]
+    fn test_should_count_head_proxy_failure_counts_unexpected_errors() {
+        assert!(
+            should_count_head_proxy_failure(false, Some("AccessDenied"), Some(403)),
+            "non-NotFound and non-405 service errors should be counted as failures"
+        );
+        assert!(
+            should_count_head_proxy_failure(false, None, Some(500)),
+            "raw 5xx head responses should be counted as proxy failures"
+        );
+    }
+
     #[tokio::test]
     async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
         let oi = ObjectInfo {

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -12,18 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::bucket::replication::get_global_replication_pool;
 use crate::error::Error;
 use crate::global::get_global_bucket_monitor;
 use rustfs_filemeta::{ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::interval;
 
+const ROLLING_WINDOW: Duration = Duration::from_secs(60);
+const FAILURE_LAST_HOUR_WINDOW: Duration = Duration::from_secs(60 * 60);
+
 /// Exponential Moving Average with thread-safe interior mutability
 #[derive(Debug)]
 pub struct ExponentialMovingAverage {
@@ -328,6 +332,13 @@ pub struct InQueueStats {
     pub now_count: AtomicI64,
 }
 
+#[derive(Debug, Clone)]
+struct QueueSample {
+    observed_at: Instant,
+    bytes: i64,
+    count: i64,
+}
+
 impl Clone for InQueueStats {
     fn clone(&self) -> Self {
         Self {
@@ -359,9 +370,60 @@ pub struct InQueueMetric {
     pub curr: InQueueStats,
     pub avg: InQueueStats,
     pub max: InQueueStats,
+    pub last_minute: InQueueStats,
+    #[serde(skip)]
+    samples: VecDeque<QueueSample>,
 }
 
 impl InQueueMetric {
+    fn observe(&mut self, observed_at: Instant) {
+        let bytes = self.curr.now_bytes.load(Ordering::Relaxed);
+        let count = self.curr.now_count.load(Ordering::Relaxed);
+
+        self.curr.bytes = bytes;
+        self.curr.count = count;
+        self.samples.push_back(QueueSample {
+            observed_at,
+            bytes,
+            count,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.avg = InQueueStats::default();
+            self.max = InQueueStats::default();
+            self.last_minute = InQueueStats::default();
+            return;
+        }
+
+        let sample_count = self.samples.len() as i64;
+        let total_bytes = self.samples.iter().map(|sample| sample.bytes).sum::<i64>();
+        let total_count = self.samples.iter().map(|sample| sample.count).sum::<i64>();
+        let max_bytes = self.samples.iter().map(|sample| sample.bytes).max().unwrap_or(0);
+        let max_count = self.samples.iter().map(|sample| sample.count).max().unwrap_or(0);
+
+        self.avg.bytes = total_bytes / sample_count;
+        self.avg.count = total_count / sample_count;
+        self.max.bytes = max_bytes;
+        self.max.count = max_count;
+        self.last_minute.bytes = self.avg.bytes;
+        self.last_minute.count = self.avg.count;
+    }
+
+    fn snapshot(&self) -> Self {
+        let mut snapshot = self.clone();
+        snapshot.curr.bytes = snapshot.curr.now_bytes.load(Ordering::Relaxed);
+        snapshot.curr.count = snapshot.curr.now_count.load(Ordering::Relaxed);
+        snapshot
+    }
+
     pub fn merge(&self, other: &InQueueMetric) -> Self {
         Self {
             curr: InQueueStats {
@@ -384,6 +446,12 @@ impl InQueueMetric {
                 count: self.max.count.max(other.max.count),
                 ..Default::default()
             },
+            last_minute: InQueueStats {
+                bytes: self.last_minute.bytes + other.last_minute.bytes,
+                count: self.last_minute.count + other.last_minute.count,
+                ..Default::default()
+            },
+            samples: VecDeque::new(),
         }
     }
 }
@@ -391,8 +459,8 @@ impl InQueueMetric {
 /// Queue cache
 #[derive(Debug, Default)]
 pub struct QueueCache {
-    pub bucket_stats: HashMap<String, InQueueStats>,
-    pub sr_queue_stats: InQueueStats,
+    pub bucket_stats: HashMap<String, InQueueMetric>,
+    pub sr_queue_stats: InQueueMetric,
 }
 
 impl QueueCache {
@@ -401,36 +469,19 @@ impl QueueCache {
     }
 
     pub fn update(&mut self) {
-        // Update queue statistics cache
-        // In actual implementation, this would get latest statistics from queue system
+        let observed_at = Instant::now();
+        self.sr_queue_stats.observe(observed_at);
+        for stats in self.bucket_stats.values_mut() {
+            stats.observe(observed_at);
+        }
     }
 
     pub fn get_bucket_stats(&self, bucket: &str) -> InQueueMetric {
-        if let Some(bucket_stat) = self.bucket_stats.get(bucket) {
-            InQueueMetric {
-                curr: InQueueStats {
-                    bytes: bucket_stat.now_bytes.load(Ordering::Relaxed),
-                    count: bucket_stat.now_count.load(Ordering::Relaxed),
-                    ..Default::default()
-                },
-                avg: InQueueStats::default(), // simplified implementation
-                max: InQueueStats::default(), // simplified implementation
-            }
-        } else {
-            InQueueMetric::default()
-        }
+        self.bucket_stats.get(bucket).map(InQueueMetric::snapshot).unwrap_or_default()
     }
 
     pub fn get_site_stats(&self) -> InQueueMetric {
-        InQueueMetric {
-            curr: InQueueStats {
-                bytes: self.sr_queue_stats.now_bytes.load(Ordering::Relaxed),
-                count: self.sr_queue_stats.now_count.load(Ordering::Relaxed),
-                ..Default::default()
-            },
-            avg: InQueueStats::default(), // simplified implementation
-            max: InQueueStats::default(), // simplified implementation
-        }
+        self.sr_queue_stats.snapshot()
     }
 }
 
@@ -505,11 +556,19 @@ impl ProxyStatsCache {
     }
 }
 
+#[derive(Debug, Clone)]
+struct FailureSample {
+    observed_at: Instant,
+    size: i64,
+}
+
 /// Failure statistics
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct FailStats {
     pub count: i64,
     pub size: i64,
+    #[serde(skip)]
+    recent: VecDeque<FailureSample>,
 }
 
 impl FailStats {
@@ -518,14 +577,42 @@ impl FailStats {
     }
 
     pub fn add_size(&mut self, size: i64, _err: Option<&Error>) {
+        let observed_at = Instant::now();
         self.count += 1;
         self.size += size;
+        self.recent.push_back(FailureSample { observed_at, size });
+        self.prune(observed_at);
+    }
+
+    fn prune(&mut self, observed_at: Instant) {
+        while self
+            .recent
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > FAILURE_LAST_HOUR_WINDOW)
+        {
+            self.recent.pop_front();
+        }
+    }
+
+    pub fn recent_since(&self, window: Duration) -> FailedMetric {
+        let now = Instant::now();
+        let mut count = 0i64;
+        let mut size = 0i64;
+        for sample in self.recent.iter().rev() {
+            if now.duration_since(sample.observed_at) > window {
+                break;
+            }
+            count += 1;
+            size += sample.size;
+        }
+        FailedMetric { count, size }
     }
 
     pub fn merge(&self, other: &FailStats) -> Self {
         Self {
             count: self.count + other.count,
             size: self.size + other.size,
+            recent: VecDeque::new(),
         }
     }
 
@@ -674,6 +761,14 @@ pub struct ActiveWorkerStat {
     pub curr: i32,
     pub max: i32,
     pub avg: f64,
+    #[serde(skip)]
+    samples: VecDeque<WorkerSample>,
+}
+
+#[derive(Debug, Clone)]
+struct WorkerSample {
+    observed_at: Instant,
+    workers: i32,
 }
 
 impl ActiveWorkerStat {
@@ -685,9 +780,31 @@ impl ActiveWorkerStat {
         self.clone()
     }
 
-    pub fn update(&mut self) {
-        // Simulate worker statistics update logic
-        // In actual implementation, this would get current active count from worker pool
+    pub fn update(&mut self, curr: i32) {
+        let observed_at = Instant::now();
+        self.curr = curr;
+        self.samples.push_back(WorkerSample {
+            observed_at,
+            workers: curr,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.max = curr;
+            self.avg = curr as f64;
+            return;
+        }
+
+        self.max = self.samples.iter().map(|sample| sample.workers).max().unwrap_or(curr);
+        let total = self.samples.iter().map(|sample| sample.workers as i64).sum::<i64>();
+        self.avg = total as f64 / self.samples.len() as f64;
     }
 }
 
@@ -740,8 +857,11 @@ impl ReplicationStats {
             let mut interval = interval(Duration::from_secs(2));
             loop {
                 interval.tick().await;
+                let current = get_global_replication_pool()
+                    .map(|pool| pool.active_workers() + pool.active_lrg_workers() + pool.active_mrf_workers())
+                    .unwrap_or(0);
                 let mut workers = workers_clone.lock().await;
-                workers.update();
+                workers.update(current);
             }
         });
 
@@ -832,61 +952,55 @@ impl ReplicationStats {
         let mut rs = ReplStat::new();
 
         match status {
-            ReplicationStatusType::Pending => {
-                if ri.op_type.is_data_replication() && prev_status != status {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        Duration::default(),
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Pending if ri.op_type.is_data_replication() && prev_status != status => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    Duration::default(),
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Completed => {
-                if ri.op_type.is_data_replication() {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        ri.duration,
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Completed if ri.op_type.is_data_replication() => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    ri.duration,
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Failed => {
-                if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        ri.duration,
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Failed
+                if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending =>
+            {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    ri.duration,
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Replica => {
-                if ri.op_type == ReplicationType::Object {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        Duration::default(),
-                        status,
-                        ri.op_type,
-                        String::new(),
-                        false,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Replica if ri.op_type == ReplicationType::Object => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    Duration::default(),
+                    status,
+                    ri.op_type,
+                    String::new(),
+                    false,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
             _ => {}
         }
@@ -931,14 +1045,26 @@ impl ReplicationStats {
     /// Get replication metrics for all buckets
     pub async fn get_all(&self) -> HashMap<String, BucketReplicationStats> {
         let cache = self.cache.read().await;
-        let mut result = HashMap::new();
+        let mut result = HashMap::with_capacity(cache.len());
 
         for (bucket, stats) in cache.iter() {
-            let mut cloned_stats = stats.clone_stats();
-            // Add queue statistics
+            result.insert(bucket.clone(), stats.clone_stats());
+        }
+        drop(cache);
+
+        {
             let q_cache = self.q_cache.lock().await;
-            cloned_stats.q_stat = q_cache.get_bucket_stats(bucket);
-            result.insert(bucket.clone(), cloned_stats);
+            for (bucket, queue_stats) in &q_cache.bucket_stats {
+                let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+                bucket_stats.q_stat = queue_stats.snapshot();
+            }
+        }
+
+        {
+            let p_cache = self.p_cache.lock().await;
+            for bucket in p_cache.bucket_stats.keys() {
+                result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+            }
         }
 
         result
@@ -1120,12 +1246,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        stats.now_count.fetch_add(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_add(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
     }
 
     /// Decrease queue statistics
@@ -1134,12 +1260,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        stats.now_count.fetch_sub(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_sub(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
     }
 
     /// Increase proxy metrics
@@ -1172,6 +1298,51 @@ mod tests {
         assert_eq!(workers.curr, 0);
     }
 
+    #[test]
+    fn test_in_queue_metric_observe_updates_rolling_stats() {
+        let mut metric = InQueueMetric::default();
+        metric.curr.now_bytes.store(128, Ordering::Relaxed);
+        metric.curr.now_count.store(4, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        metric.curr.now_bytes.store(256, Ordering::Relaxed);
+        metric.curr.now_count.store(6, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        assert_eq!(metric.curr.bytes, 256);
+        assert_eq!(metric.curr.count, 6);
+        assert_eq!(metric.max.bytes, 256);
+        assert_eq!(metric.max.count, 6);
+        assert_eq!(metric.last_minute.bytes, 192);
+        assert_eq!(metric.last_minute.count, 5);
+    }
+
+    #[test]
+    fn test_fail_stats_recent_since_tracks_windows() {
+        let mut stats = FailStats::default();
+        stats.add_size(64, None);
+        stats.add_size(32, None);
+
+        let last_minute = stats.recent_since(Duration::from_secs(60));
+        let last_hour = stats.recent_since(Duration::from_secs(60 * 60));
+        assert_eq!(last_minute.count, 2);
+        assert_eq!(last_minute.size, 96);
+        assert_eq!(last_hour.count, 2);
+        assert_eq!(last_hour.size, 96);
+    }
+
+    #[test]
+    fn test_active_worker_stat_update_tracks_rolling_avg_and_max() {
+        let mut stats = ActiveWorkerStat::default();
+        stats.update(2);
+        stats.update(6);
+        stats.update(4);
+
+        assert_eq!(stats.curr, 4);
+        assert_eq!(stats.max, 6);
+        assert_eq!(stats.avg, 4.0);
+    }
+
     #[tokio::test]
     async fn test_delete_bucket_stats() {
         let stats = ReplicationStats::new();
@@ -1224,6 +1395,15 @@ mod tests {
         assert_eq!(stat.replicated_count, 1);
     }
 
+    #[tokio::test]
+    async fn test_get_all_includes_proxy_only_bucket() {
+        let stats = ReplicationStats::new();
+        stats.inc_proxy("proxy-only-bucket", "HeadObject", false).await;
+
+        let all = stats.get_all().await;
+        assert!(all.contains_key("proxy-only-bucket"));
+    }
+
     #[test]
     fn test_sr_stats() {
         let sr_stats = SRStats::new();

crates/ecstore/src/cache_value/metacache_set.rs

@@ -12,12 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::disk_store::get_drive_walkdir_stall_timeout;
 use crate::disk::error::DiskError;
 use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
 use futures::future::join_all;
+use metrics::counter;
 use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
-use std::{future::Future, pin::Pin};
+use std::{future::Future, pin::Pin, time::Duration};
+use tokio::io::AsyncRead;
 use tokio::spawn;
+use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -26,6 +30,21 @@ pub type PartialFn =
     Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 
+#[derive(Debug)]
+enum PeekOutcome {
+    Ready(Option<MetaCacheEntry>),
+    Error(rustfs_filemeta::Error),
+    TimedOut,
+}
+
+async fn peek_with_timeout<R: AsyncRead + Unpin>(reader: &mut MetacacheReader<R>, timeout_duration: Duration) -> PeekOutcome {
+    match timeout(timeout_duration, reader.peek()).await {
+        Ok(Ok(entry)) => PeekOutcome::Ready(entry),
+        Ok(Err(err)) => PeekOutcome::Error(err),
+        Err(_) => PeekOutcome::TimedOut,
+    }
+}
+
 #[derive(Default)]
 pub struct ListPathRawOptions {
     pub disks: Vec<Option<DiskStore>>,
@@ -160,6 +179,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     }
 
     let revjob = spawn(async move {
+        let peek_timeout = get_drive_walkdir_stall_timeout();
         let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
         for _ in 0..readers.len() {
             errs.push(None);
@@ -191,8 +211,8 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                     continue;
                 }
 
-                let entry = match r.peek().await {
-                    Ok(res) => {
+                let entry = match peek_with_timeout(r, peek_timeout).await {
+                    PeekOutcome::Ready(res) => {
                         if let Some(entry) = res {
                             // info!("read entry disk: {}, name: {}", i, entry.name);
                             entry
@@ -203,7 +223,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
-                    Err(err) => {
+                    PeekOutcome::Error(err) => {
                         if err == rustfs_filemeta::Error::Unexpected {
                             at_eof += 1;
                             // warn!("list_path_raw: peek err eof, disk: {}", i);
@@ -236,6 +256,31 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
+                    PeekOutcome::TimedOut => {
+                        has_err += 1;
+                        errs[i] = Some(DiskError::Timeout);
+                        let endpoint = opts
+                            .disks
+                            .get(i)
+                            .and_then(|disk| disk.as_ref().map(|disk| disk.endpoint().to_string()))
+                            .unwrap_or_else(|| "missing".to_string());
+                        counter!(
+                            "rustfs_list_path_raw_stall_total",
+                            "drive" => endpoint.clone()
+                        )
+                        .increment(1);
+                        warn!(
+                            drive = %endpoint,
+                            bucket = %opts.bucket,
+                            path = %opts.path,
+                            timeout_ms = peek_timeout.as_millis(),
+                            "list_path_raw reader peek timed out; excluding drive from current merge"
+                        );
+                        let (detached_rd, write_half) = tokio::io::duplex(1);
+                        drop(write_half);
+                        *r = MetacacheReader::new(detached_rd);
+                        continue;
+                    }
                 };
 
                 // warn!("list_path_raw: loop entry: {:?}, disk: {}", &entry.name, i);
@@ -364,3 +409,42 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     // warn!("list_path_raw: done");
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_filemeta::MetacacheWriter;
+
+    #[tokio::test]
+    async fn peek_with_timeout_times_out_on_silent_reader() {
+        let (_writer, reader) = tokio::io::duplex(64);
+        let mut reader = MetacacheReader::new(reader);
+
+        let outcome = peek_with_timeout(&mut reader, Duration::from_millis(20)).await;
+        assert!(matches!(outcome, PeekOutcome::TimedOut));
+    }
+
+    #[tokio::test]
+    async fn peek_with_timeout_reads_entry_before_deadline() {
+        let (reader, writer) = tokio::io::duplex(256);
+        let mut metacache_reader = MetacacheReader::new(reader);
+
+        tokio::spawn(async move {
+            let mut writer = MetacacheWriter::new(writer);
+            let entry = MetaCacheEntry {
+                name: "bucket/object".to_string(),
+                metadata: vec![1, 2, 3],
+                cached: None,
+                reusable: false,
+            };
+            writer.write(&[entry]).await.expect("entry should be written");
+            writer.close().await.expect("writer should close");
+        });
+
+        let outcome = peek_with_timeout(&mut metacache_reader, Duration::from_secs(1)).await;
+        match outcome {
+            PeekOutcome::Ready(Some(entry)) => assert_eq!(entry.name, "bucket/object"),
+            other => panic!("expected ready entry, got {other:?}"),
+        }
+    }
+}

crates/ecstore/src/client/api_error_response.rs

@@ -100,7 +100,7 @@ pub fn http_resp_to_error_response(
     bucket_name: &str,
     object_name: &str,
 ) -> ErrorResponse {
-    let err_body = String::from_utf8(b).unwrap();
+    let err_body = String::from_utf8_lossy(&b).to_string();
     if h.is_empty() || resp_status.is_client_error() || resp_status.is_server_error() {
         return ErrorResponse {
             status_code: resp_status,
@@ -178,36 +178,46 @@ pub fn http_resp_to_error_response(
                 };
             }
         }
-    } else {
-        err_resp = err_resp_.unwrap();
+    } else if let Ok(parsed_resp) = err_resp_ {
+        err_resp = parsed_resp;
     }
     err_resp.status_code = resp_status;
     if let Some(server_name) = h.get("Server") {
-        err_resp.server = server_name.to_str().expect("err").to_string();
+        if let Ok(server_str) = server_name.to_str() {
+            err_resp.server = server_str.to_string();
+        }
     }
 
-    let code = h.get("x-minio-error-code");
-    if code.is_some() {
-        err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
+    if let Some(code) = h.get("x-minio-error-code") {
+        if let Ok(code_str) = code.to_str() {
+            err_resp.code = S3ErrorCode::Custom(code_str.into());
+        }
     }
-    let desc = h.get("x-minio-error-desc");
-    if desc.is_some() {
-        err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
+    if let Some(desc) = h.get("x-minio-error-desc") {
+        if let Ok(desc_str) = desc.to_str() {
+            err_resp.message = desc_str.trim_matches('"').to_string();
+        }
     }
 
     if err_resp.request_id == "" {
         if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
-            err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
+            if let Ok(request_id_str) = x_amz_request_id.to_str() {
+                err_resp.request_id = request_id_str.to_string();
+            }
         }
     }
     if err_resp.host_id == "" {
         if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
-            err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
+            if let Ok(host_id_str) = x_amz_id_2.to_str() {
+                err_resp.host_id = host_id_str.to_string();
+            }
         }
     }
     if err_resp.region == "" {
         if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
-            err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
+            if let Ok(region_str) = x_amz_bucket_region.to_str() {
+                err_resp.region = region_str.to_string();
+            }
         }
     }
     if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {

crates/ecstore/src/client/api_get_object.rs

@@ -1,4 +1,3 @@
-#![allow(clippy::map_entry)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(clippy::map_entry)]
 #![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]

crates/ecstore/src/client/api_put_object.rs

@@ -137,21 +137,21 @@ impl Default for PutObjectOptions {
 impl PutObjectOptions {
     fn set_match_etag(&mut self, etag: &str) {
         if etag == "*" {
-            self.custom_header
-                .insert("If-Match", HeaderValue::from_str("*").expect("err"));
+            self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
         } else {
-            self.custom_header
-                .insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{}\"", etag)) {
+                self.custom_header.insert("If-Match", etag_value);
+            }
         }
     }
 
     fn set_match_etag_except(&mut self, etag: &str) {
         if etag == "*" {
-            self.custom_header
-                .insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
+            self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
         } else {
-            self.custom_header
-                .insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{etag}\"")) {
+                self.custom_header.insert("If-None-Match", etag_value);
+            }
         }
     }
 
@@ -162,59 +162,75 @@ impl PutObjectOptions {
         if content_type == "" {
             content_type = "application/octet-stream".to_string();
         }
-        header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
+        if let Ok(content_type_value) = HeaderValue::from_str(&content_type) {
+            header.insert("Content-Type", content_type_value);
+        }
 
         if self.content_encoding != "" {
-            header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
+            if let Ok(encoding_value) = HeaderValue::from_str(&self.content_encoding) {
+                header.insert("Content-Encoding", encoding_value);
+            }
         }
         if self.content_disposition != "" {
-            header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
+            if let Ok(disposition_value) = HeaderValue::from_str(&self.content_disposition) {
+                header.insert("Content-Disposition", disposition_value);
+            }
         }
         if self.content_language != "" {
-            header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
+            if let Ok(language_value) = HeaderValue::from_str(&self.content_language) {
+                header.insert("Content-Language", language_value);
+            }
         }
         if self.cache_control != "" {
-            header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
+            if let Ok(cache_value) = HeaderValue::from_str(&self.cache_control) {
+                header.insert("Cache-Control", cache_value);
+            }
         }
 
         if self.expires.unix_timestamp() != 0 {
-            header.insert(
-                "Expires",
-                HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
-            ); //rustfs invalid header
+            if let Ok(expires_str) = self.expires.format(ISO8601_DATEFORMAT) {
+                if let Ok(expires_value) = HeaderValue::from_str(&expires_str) {
+                    header.insert("Expires", expires_value);
+                }
+            }
         }
 
         if self.mode.as_str() != "" {
-            header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
+            if let Ok(mode_value) = HeaderValue::from_str(self.mode.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_MODE, mode_value);
+            }
         }
 
         if self.retain_until_date.unix_timestamp() != 0 {
-            header.insert(
-                X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
-                HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
-            );
+            if let Ok(retain_str) = self.retain_until_date.format(ISO8601_DATEFORMAT) {
+                if let Ok(retain_value) = HeaderValue::from_str(&retain_str) {
+                    header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, retain_value);
+                }
+            }
         }
 
         if self.legalhold.as_str() != "" {
-            header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
+            if let Ok(legalhold_value) = HeaderValue::from_str(self.legalhold.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, legalhold_value);
+            }
         }
 
         if self.storage_class != "" {
-            header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
+            if let Ok(storage_class_value) = HeaderValue::from_str(&self.storage_class) {
+                header.insert(X_AMZ_STORAGE_CLASS, storage_class_value);
+            }
         }
 
         if self.website_redirect_location != "" {
-            header.insert(
-                X_AMZ_WEBSITE_REDIRECT_LOCATION,
-                HeaderValue::from_str(&self.website_redirect_location).expect("err"),
-            );
+            if let Ok(redirect_value) = HeaderValue::from_str(&self.website_redirect_location) {
+                header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, redirect_value);
+            }
         }
 
         if !self.internal.replication_status.as_str().is_empty() {
-            header.insert(
-                X_AMZ_REPLICATION_STATUS,
-                HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
-            );
+            if let Ok(replication_status_value) = HeaderValue::from_str(self.internal.replication_status.as_str()) {
+                header.insert(X_AMZ_REPLICATION_STATUS, replication_status_value);
+            }
         }
 
         for (k, v) in &self.user_metadata {
@@ -360,17 +376,21 @@ impl TransitionClient {
 
             let mut md5_base64: String = "".to_string();
             if opts.send_content_md5 {
-                let mut md5_hasher = self.md5_hasher.lock().unwrap();
-                let hash = md5_hasher.as_mut().expect("err");
-                let hash = hash.hash_encode(&buf[..length]);
-                md5_base64 = base64_encode(hash.as_ref());
+                if let Some(mut md5_hasher) = self.md5_hasher.lock().unwrap().as_mut() {
+                    let hash = md5_hasher.hash_encode(&buf[..length]);
+                    md5_base64 = base64_encode(hash.as_ref());
+                }
             } else {
                 let mut crc = opts.auto_checksum.hasher()?;
                 crc.update(&buf[..length]);
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }

crates/ecstore/src/client/api_put_object_multipart.rs

@@ -127,7 +127,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -309,27 +313,27 @@ impl TransitionClient {
         let h = resp.headers();
         let mut obj_part = ObjectPart {
             checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
-                h_checksum_crc32.to_str().expect("err").to_string()
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
-                h_checksum_crc32c.to_str().expect("err").to_string()
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
-                h_checksum_sha1.to_str().expect("err").to_string()
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
-                h_checksum_sha256.to_str().expect("err").to_string()
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
-                h_checksum_crc64nvme.to_str().expect("err").to_string()
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
@@ -338,7 +342,7 @@ impl TransitionClient {
         obj_part.size = p.size;
         obj_part.part_num = p.part_number;
         obj_part.etag = if let Some(h_etag) = h.get("ETag") {
-            h_etag.to_str().expect("err").trim_matches('"').to_string()
+            h_etag.to_str().unwrap_or("").trim_matches('"').to_string()
         } else {
             "".to_string()
         };
@@ -398,7 +402,7 @@ impl TransitionClient {
             key: complete_multipart_upload_result.key,
             etag: trim_etag(&complete_multipart_upload_result.etag),
             version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
-                h_x_amz_version_id.to_str().expect("err").to_string()
+                h_x_amz_version_id.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },

crates/ecstore/src/client/api_put_object_streaming.rs

@@ -21,6 +21,7 @@
 use bytes::Bytes;
 use futures::future::join_all;
 use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use std::io::Error;
 use std::sync::RwLock;
 use std::{collections::HashMap, sync::Arc};
 use time::{OffsetDateTime, format_description};
@@ -152,7 +153,10 @@ impl TransitionClient {
 
             if opts.send_content_md5 {
                 let mut md5_hasher = self.md5_hasher.lock().unwrap();
-                let md5_hash = md5_hasher.as_mut().expect("err");
+                let md5_hash = match md5_hasher.as_mut() {
+                    Some(hasher) => hasher,
+                    None => return Err(std::io::Error::other("MD5 hasher not initialized")),
+                };
                 let hash = md5_hash.hash_encode(&buf[..length]);
                 md5_base64 = base64_encode(hash.as_ref());
             } else {
@@ -161,7 +165,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -275,11 +283,14 @@ impl TransitionClient {
         for part_number in 1..=total_parts_count {
             let mut buf = Vec::<u8>::new();
             select! {
-                buf = bufs_rx.recv() => {}
+                buf1 = bufs_rx.recv() => {
+                    if let Some(buf1) = buf1 {
+                        buf = buf1;
+                    }
+                }
                 err = err_rx.recv() => {
                     //cancel_token.cancel();
-                    //wg.Wait()
-                    return Err(err.expect("err"));
+                    return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
                 }
                 else => (),
             }
@@ -309,7 +320,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -319,12 +334,19 @@ impl TransitionClient {
             let clone_parts_info = parts_info.clone();
             let clone_upload_id = upload_id.clone();
             let clone_self = self.clone();
+            let err_tx_clone = err_tx.clone();
             futures.push(async move {
                 let mut md5_base64: String = "".to_string();
 
                 if opts.send_content_md5 {
                     let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
-                    let md5_hash = md5_hasher.as_mut().expect("err");
+                    let md5_hash = match md5_hasher.as_mut() {
+                        Some(hasher) => hasher,
+                        None => {
+                            //let _ = err_tx_clone.send(std::io::Error::other("MD5 hasher not initialized")).await;
+                            return Ok::<(), Error>(());
+                        }
+                    };
                     let hash = md5_hash.hash_encode(&buf[..length]);
                     md5_base64 = base64_encode(hash.as_ref());
                 }
@@ -344,12 +366,21 @@ impl TransitionClient {
                     sha256_hex: "".to_string(),
                     trailer: HeaderMap::new(),
                 };
-                let obj_part = clone_self.upload_part(&mut p).await.expect("err");
+                let obj_part = match clone_self.upload_part(&mut p).await {
+                    Ok(part) => part,
+                    Err(err) => {
+                        let _ = err_tx_clone.send(std::io::Error::other(err.to_string())).await;
+                        return Err::<(), Error>(err);
+                    }
+                };
 
-                let mut clone_parts_info = clone_parts_info.write().unwrap();
-                clone_parts_info.entry(part_number).or_insert(obj_part);
+                {
+                    let mut clone_parts_info = clone_parts_info.write().unwrap();
+                    clone_parts_info.entry(part_number).or_insert(obj_part);
+                }
 
-                clone_bufs_tx.send(buf);
+                let _ = clone_bufs_tx.send(buf).await;
+                Ok::<(), Error>(())
             });
 
             total_uploaded_size += length as i64;
@@ -359,7 +390,7 @@ impl TransitionClient {
 
         select! {
             err = err_rx.recv() => {
-                return Err(err.expect("err"));
+                return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
             }
             else => (),
         }
@@ -504,9 +535,10 @@ impl TransitionClient {
         Ok(UploadInfo {
             bucket: bucket_name.to_string(),
             key: object_name.to_string(),
-            etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
+            etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
+
             version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
-                h_x_amz_version_id.to_str().expect("err").to_string()
+                h_x_amz_version_id.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
@@ -514,27 +546,27 @@ impl TransitionClient {
             expiration: exp_time,
             expiration_rule_id: rule_id,
             checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
-                h_checksum_crc32.to_str().expect("err").to_string()
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
-                h_checksum_crc32c.to_str().expect("err").to_string()
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
-                h_checksum_sha1.to_str().expect("err").to_string()
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
-                h_checksum_sha256.to_str().expect("err").to_string()
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
-                h_checksum_crc64nvme.to_str().expect("err").to_string()
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },

crates/ecstore/src/client/api_remove.rs

@@ -25,7 +25,7 @@ use hyper::body::Bytes;
 use rustfs_utils::HashAlgorithm;
 use s3s::S3ErrorCode;
 use s3s::dto::ReplicationStatus;
-use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
+use s3s::header::{X_AMZ_BYPASS_GOVERNANCE_RETENTION, X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
 use serde::Deserialize;
 use std::fmt::Display;
 use std::{
@@ -111,8 +111,9 @@ impl TransitionClient {
             .await?;
 
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.delete(bucket_name);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
         }
         Ok(())
     }
@@ -142,8 +143,9 @@ impl TransitionClient {
             .await?;
 
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.delete(bucket_name);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
         }
 
         Ok(())
@@ -168,7 +170,7 @@ impl TransitionClient {
         let mut headers = HeaderMap::new();
 
         if opts.governance_bypass {
-            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
+            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true")); //amzBypassGovernance
         }
 
         let resp = self
@@ -197,13 +199,12 @@ impl TransitionClient {
         Ok(RemoveObjectResult {
             object_name: object_name.to_string(),
             object_version_id: opts.version_id,
-            delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
+            delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
             delete_marker_version_id: resp
                 .headers()
-                .get("x-amz-version-id")
-                .expect("err")
-                .to_str()
-                .expect("err")
+                .get(X_AMZ_VERSION_ID)
+                .and_then(|v| v.to_str().ok())
+                .unwrap_or_default()
                 .to_string(),
             ..Default::default()
         })
@@ -290,15 +291,15 @@ impl TransitionClient {
                             bucket_name,
                             &object.name,
                             RemoveObjectOptions {
-                                version_id: object.version_id.expect("err").to_string(),
+                                version_id: object.version_id.map(|id| id.to_string()).unwrap_or_default(),
                                 governance_bypass: opts.governance_bypass,
                                 ..Default::default()
                             },
                         )
                         .await?;
                     let remove_result_clone = remove_result.clone();
-                    if !remove_result.err.is_none() {
-                        match to_error_response(&remove_result.err.expect("err")).code {
+                    if let Some(err) = &remove_result.err {
+                        match to_error_response(err).code {
                             S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
                                 continue;
                             }
@@ -326,7 +327,7 @@ impl TransitionClient {
 
             let mut headers = HeaderMap::new();
             if opts.governance_bypass {
-                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
+                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true"));
             }
 
             let remove_bytes = generate_remove_multi_objects_request(&batch);
@@ -423,23 +424,20 @@ impl TransitionClient {
                         request_id: resp
                             .headers()
                             .get("x-amz-request-id")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         host_id: resp
                             .headers()
                             .get("x-amz-id-2")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         region: resp
                             .headers()
                             .get("x-amz-bucket-region")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         ..Default::default()
                     };
@@ -472,10 +470,11 @@ pub struct RemoveObjectError {
 
 impl Display for RemoveObjectError {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        if self.err.is_none() {
-            return write!(f, "unexpected remove object error result");
+        if let Some(err) = &self.err {
+            write!(f, "{}", err.to_string())
+        } else {
+            write!(f, "unexpected remove object error result")
         }
-        write!(f, "{}", self.err.as_ref().expect("err").to_string())
     }
 }
 

crates/ecstore/src/client/bucket_cache.rs

@@ -70,10 +70,10 @@ impl TransitionClient {
 
         let mut location;
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            let ret = bucket_loc_cache.get(bucket_name);
-            if let Some(location) = ret {
-                return Ok(location);
+            if let Ok(bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                if let Some(location) = bucket_loc_cache.get(bucket_name) {
+                    return Ok(location);
+                }
             }
             //location = ret?;
         }
@@ -83,8 +83,9 @@ impl TransitionClient {
         let mut resp = self.doit(req).await?;
         location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.set(bucket_name, &location);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.set(bucket_name, &location);
+            }
         }
         Ok(location)
     }
@@ -108,7 +109,11 @@ impl TransitionClient {
             url_str.push_str("://");
             url_str.push_str(bucket_name);
             url_str.push_str(".");
-            url_str.push_str(target_url.host_str().expect("err"));
+            url_str.push_str(
+                target_url
+                    .host_str()
+                    .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "host is none"))?,
+            );
             url_str.push_str("/?location");
         } else {
             let mut path = bucket_name.to_string();
@@ -135,13 +140,16 @@ impl TransitionClient {
 
         let value;
         {
-            let mut creds_provider = self.creds_provider.lock().unwrap();
-            value = match creds_provider.get_with_context(Some(self.cred_context())) {
-                Ok(v) => v,
-                Err(err) => {
-                    return Err(std::io::Error::other(err));
-                }
-            };
+            if let Ok(mut creds_provider) = self.creds_provider.lock() {
+                value = match creds_provider.get_with_context(Some(self.cred_context())) {
+                    Ok(v) => v,
+                    Err(err) => {
+                        return Err(std::io::Error::other(err));
+                    }
+                };
+            } else {
+                return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
+            }
         }
 
         let mut signer_type = value.signer_type.clone();
@@ -171,8 +179,9 @@ impl TransitionClient {
             content_sha256 = UNSIGNED_PAYLOAD.to_string();
         }
 
-        req.headers_mut()
-            .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
+        if let Ok(content_sha256_value) = content_sha256.parse() {
+            req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
+        }
         let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
         Ok(req)
     }
@@ -228,13 +237,16 @@ async fn process_bucket_location_response(
     }
     let mut location = "".to_string();
     if tier_type == "huaweicloud" {
-        let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(body_vec).unwrap()).unwrap();
-        location = d.location_constraint;
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(d) = quick_xml::de::from_str::<CreateBucketConfiguration>(&body_str) {
+                location = d.location_constraint;
+            }
+        }
     } else {
-        if let Ok(LocationConstraint { field }) =
-            quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(body_vec).unwrap())
-        {
-            location = field;
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&body_str) {
+                location = field;
+            }
         }
     }
     //debug!("location: {}", location);

crates/ecstore/src/client/credentials.rs

@@ -1,4 +1,3 @@
-#![allow(unused_imports)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
@@ -57,7 +57,9 @@ impl<P: Provider + Default> Credentials<P> {
 
     pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
         if self.is_expired() {
-            let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
+            let creds = self.provider.retrieve_with_cred_context(cc.unwrap_or(CredContext {
+                endpoint: "".to_string(),
+            }));
             self.creds = creds;
             self.force_refresh = false;
         }

crates/ecstore/src/client/object_handlers_common.rs

@@ -12,14 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+use tracing::warn;
+
 use crate::bucket::lifecycle::lifecycle;
+use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::store::ECStore;
 use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
+use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
 use rustfs_lock::MAX_DELETE_LIST;
 
-pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
+fn lifecycle_version_delete_replication_state(
+    replicate_decision_str: String,
+    pending_status: Option<String>,
+) -> ReplicationState {
+    ReplicationState {
+        replicate_decision_str,
+        version_purge_status_internal: pending_status.clone(),
+        purge_targets: version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
+        ..Default::default()
+    }
+}
+
+pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
+    let version_suspended = match BucketVersioningSys::get(bucket).await {
+        Ok(vc) => vc.suspended(),
+        Err(err) => {
+            warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup");
+            return;
+        }
+    };
     let mut remaining = to_del;
     loop {
         let mut to_del = remaining;
@@ -29,15 +53,94 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
         } else {
             remaining = &[];
         }
-        let vc = BucketVersioningSys::get(bucket).await.expect("err!");
-        let _deleted_objs = api.delete_objects(
-            bucket,
-            to_del.to_vec(),
-            ObjectOptions {
-                //prefix_enabled_fn:  vc.prefix_enabled(""),
-                version_suspended: vc.suspended(),
+
+        let mut replication_candidates: Vec<Option<ReplicationState>> = Vec::with_capacity(to_del.len());
+        for object in to_del.iter() {
+            let version_id = object.version_id.map(|vid| vid.to_string());
+            let opts = ObjectOptions {
+                version_id: version_id.clone(),
+                versioned: true,
+                version_suspended,
                 ..Default::default()
-            },
-        );
+            };
+            let candidate = match api.get_object_info(bucket, &object.object_name, &opts).await {
+                Ok(info) => {
+                    let dsc = check_replicate_delete(bucket, object, &info, &opts, None).await;
+                    dsc.replicate_any()
+                        .then(|| lifecycle_version_delete_replication_state(dsc.to_string(), dsc.pending_status()))
+                }
+                Err(err) => {
+                    warn!(
+                        bucket,
+                        object = %object.object_name,
+                        version_id = ?version_id,
+                        error = ?err,
+                        "failed to get object info during lifecycle noncurrent version cleanup; skipping delete replication scheduling"
+                    );
+                    None
+                }
+            };
+            replication_candidates.push(candidate);
+        }
+
+        let (mut deleted_objs, errors) = api
+            .delete_objects(
+                bucket,
+                to_del.to_vec(),
+                ObjectOptions {
+                    version_suspended,
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        for (i, deleted_obj) in deleted_objs.iter_mut().enumerate() {
+            if errors.get(i).and_then(|err| err.as_ref()).is_some() {
+                continue;
+            }
+            let Some(replication_state) = replication_candidates.get(i).and_then(|c| c.clone()) else {
+                continue;
+            };
+            deleted_obj.replication_state = Some(replication_state);
+            schedule_replication_delete(DeletedObjectReplicationInfo {
+                delete_object: deleted_obj.clone(),
+                bucket: bucket.to_string(),
+                event_type: REPLICATE_INCOMING_DELETE.to_string(),
+                ..Default::default()
+            })
+            .await;
+        }
+
+        for (i, err) in errors.iter().enumerate() {
+            if let Some(e) = err {
+                let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or("<unknown>");
+                let vid = to_del
+                    .get(i)
+                    .and_then(|o| o.version_id)
+                    .map(|v| v.to_string())
+                    .unwrap_or_default();
+                warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup");
+            }
+        }
+        if remaining.is_empty() {
+            break;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::lifecycle_version_delete_replication_state;
+
+    #[test]
+    fn lifecycle_version_delete_replication_state_tracks_pending_purge_targets() {
+        let state = lifecycle_version_delete_replication_state(
+            "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
+            Some("arn:aws:s3:::target=PENDING;".to_string()),
+        );
+
+        assert_eq!(state.version_purge_status_internal.as_deref(), Some("arn:aws:s3:::target=PENDING;"));
+        assert!(state.purge_targets.contains_key("arn:aws:s3:::target"));
+        assert_eq!(state.replicate_decision_str, "arn:aws:s3:::target=true;false;arn:aws:s3:::target;");
     }
 }

crates/ecstore/src/client/transition_api.rs

@@ -251,9 +251,10 @@ impl TransitionClient {
         };
 
         {
-            let mut md5_hasher = client.md5_hasher.lock().unwrap();
-            if md5_hasher.is_none() {
-                *md5_hasher = Some(HashAlgorithm::Md5);
+            if let Ok(mut md5_hasher) = client.md5_hasher.lock() {
+                if md5_hasher.is_none() {
+                    *md5_hasher = Some(HashAlgorithm::Md5);
+                }
             }
         }
         if client.sha256_hasher.is_none() {
@@ -275,25 +276,30 @@ impl TransitionClient {
     }
 
     fn trace_errors_only_off(&self) {
-        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
-        *trace_errors_only = false;
+        if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
+            *trace_errors_only = false;
+        }
     }
 
     fn trace_off(&self) {
-        let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap();
-        *is_trace_enabled = false;
-        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
-        *trace_errors_only = false;
+        if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
+            *is_trace_enabled = false;
+        }
+        if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
+            *trace_errors_only = false;
+        }
     }
 
     fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
-        let mut endpoint = self.s3_accelerate_endpoint.lock().unwrap();
-        *endpoint = accelerate_endpoint.to_string();
+        if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
+            *endpoint = accelerate_endpoint.to_string();
+        }
     }
 
     fn set_s3_enable_dual_stack(&self, enabled: bool) {
-        let mut dual_stack = self.s3_dual_stack_enabled.lock().unwrap();
-        *dual_stack = enabled;
+        if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
+            *dual_stack = enabled;
+        }
     }
 
     pub fn hash_materials(
@@ -352,7 +358,6 @@ impl TransitionClient {
         let resp;
         let http_client = self.http_client.clone();
         {
-            //let mut http_client = http_client.lock().unwrap();
             req_method = req.method().clone();
             req_uri = req.uri().clone();
             req_headers = req.headers().clone();
@@ -368,7 +373,10 @@ impl TransitionClient {
             return Err(std::io::Error::other(err));
         }
 
-        let resp = resp.unwrap();
+        let resp = match resp {
+            Ok(r) => r,
+            Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
+        };
         debug!("http_resp: {:?}", resp);
 
         //let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
@@ -455,11 +463,13 @@ impl TransitionClient {
                             return Err(std::io::Error::other(err_response));
                         }
                         if metadata.bucket_name != "" {
-                            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-                            let location = bucket_loc_cache.get(&metadata.bucket_name);
-                            if location.is_some() && location.unwrap() != err_response.region {
-                                bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
-                                //continue;
+                            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                                if let Some(location) = bucket_loc_cache.get(&metadata.bucket_name) {
+                                    if location != err_response.region {
+                                        bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
+                                        //continue;
+                                    }
+                                }
                             }
                         } else if err_response.region != metadata.bucket_location {
                             metadata.bucket_location = err_response.region.clone();
@@ -518,8 +528,11 @@ impl TransitionClient {
 
         let value;
         {
-            let mut creds_provider = self.creds_provider.lock().unwrap();
-            value = creds_provider.get_with_context(Some(self.cred_context()))?;
+            if let Ok(mut creds_provider) = self.creds_provider.lock() {
+                value = creds_provider.get_with_context(Some(self.cred_context()))?;
+            } else {
+                return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
+            }
         }
 
         let mut signer_type = value.signer_type.clone();
@@ -548,8 +561,10 @@ impl TransitionClient {
                     )));
                 }
                 let headers = req.headers_mut();
-                for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
-                    headers.insert(k, v.clone());
+                if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
+                    for (k, v) in extra_headers {
+                        headers.insert(k, v.clone());
+                    }
                 }
             }
             if signer_type == SignatureType::SignatureV2 {
@@ -571,18 +586,22 @@ impl TransitionClient {
         self.set_user_agent(&mut req);
 
         for (k, v) in metadata.custom_header.clone() {
-            req.headers_mut().insert(k.expect("err"), v);
+            if let Some(key) = k {
+                req.headers_mut().insert(key, v);
+            }
         }
 
         //req.content_length = metadata.content_length;
         if metadata.content_length <= -1 {
-            let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
-            req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
+            if let Ok(chunked_value) = HeaderValue::from_str(&vec!["chunked"].join(",")) {
+                req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
+            }
         }
 
         if metadata.content_md5_base64.len() > 0 {
-            let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err");
-            req.headers_mut().insert("Content-Md5", md5_value);
+            if let Ok(md5_value) = HeaderValue::from_str(&metadata.content_md5_base64) {
+                req.headers_mut().insert("Content-Md5", md5_value);
+            }
         }
 
         if signer_type == SignatureType::SignatureAnonymous {
@@ -607,8 +626,13 @@ impl TransitionClient {
             } else if metadata.trailer.len() > 0 {
                 sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
             }
-            req.headers_mut()
-                .insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err"));
+            let header_name = "X-Amz-Content-Sha256"
+                .parse::<HeaderName>()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+            let header_value = sha_header
+                .parse()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+            req.headers_mut().insert(header_name, header_value);
 
             req = rustfs_signer::sign_v4_trailer(
                 req,
@@ -636,7 +660,7 @@ impl TransitionClient {
 
     pub fn set_user_agent(&self, req: &mut Request<s3s::Body>) {
         let headers = req.headers_mut();
-        headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
+        headers.insert("User-Agent", HeaderValue::from_static(C_USER_AGENT));
     }
 
     fn make_target_url(
@@ -648,7 +672,10 @@ impl TransitionClient {
         query_values: &HashMap<String, String>,
     ) -> Result<Url, std::io::Error> {
         let scheme = self.endpoint_url.scheme();
-        let host = self.endpoint_url.host().unwrap();
+        let host = self
+            .endpoint_url
+            .host()
+            .ok_or_else(|| std::io::Error::other("Endpoint URL has no host"))?;
         let default_port = if scheme == "https" { 443 } else { 80 };
         let port = self.endpoint_url.port().unwrap_or(default_port);
 
@@ -1155,9 +1182,10 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re
         for (name, value) in h.iter() {
             let header_name = name.as_str().to_lowercase();
             if header_name.starts_with("x-amz-meta-") {
-                let key = header_name.strip_prefix("x-amz-meta-").unwrap().to_string();
-                if let Ok(value_str) = value.to_str() {
-                    meta.insert(key, value_str.to_string());
+                if let Some(key) = header_name.strip_prefix("x-amz-meta-") {
+                    if let Ok(value_str) = value.to_str() {
+                        meta.insert(key.to_string(), value_str.to_string());
+                    }
                 }
             }
         }

crates/ecstore/src/config/audit.rs

@@ -14,11 +14,16 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
-    WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -192,3 +197,204 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_ADDRESS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_SUBJECT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_CREDENTIALS_FILE.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_REQUIRED.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+#[allow(dead_code)]
+pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_AUTH_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
+            value: EnableState::On.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/com.rs

@@ -18,8 +18,14 @@ use crate::error::{Error, Result};
 use crate::global::is_first_cluster_node_local;
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{
+    AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
 use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -34,6 +40,8 @@ const CONFIG_FILE: &str = "config.json";
 
 pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
 
+pub const COMMA_SEPARATED_LISTS: &[&str] = &[rustfs_config::oidc::OIDC_SCOPES, rustfs_config::oidc::OIDC_OTHER_AUDIENCES];
+
 static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
 
 static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
@@ -42,6 +50,84 @@ static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
     h
 });
 
+#[derive(Clone, Copy)]
+struct TargetConfigDescriptor {
+    external_key: &'static str,
+    subsystem_key: &'static str,
+    default_kvs: &'static LazyLock<KVS>,
+    valid_keys: &'static [&'static str],
+}
+
+fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
+    [
+        TargetConfigDescriptor {
+            external_key: "webhook",
+            subsystem_key: NOTIFY_WEBHOOK_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
+            valid_keys: NOTIFY_WEBHOOK_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: NOTIFY_KAFKA_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_KAFKA_KVS,
+            valid_keys: NOTIFY_KAFKA_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "mqtt",
+            subsystem_key: NOTIFY_MQTT_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_MQTT_KVS,
+            valid_keys: NOTIFY_MQTT_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "nats",
+            subsystem_key: NOTIFY_NATS_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_NATS_KVS,
+            valid_keys: NOTIFY_NATS_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "pulsar",
+            subsystem_key: NOTIFY_PULSAR_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_PULSAR_KVS,
+            valid_keys: NOTIFY_PULSAR_KEYS,
+        },
+    ]
+}
+
+fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] {
+    [
+        TargetConfigDescriptor {
+            external_key: "webhook",
+            subsystem_key: AUDIT_WEBHOOK_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
+            valid_keys: AUDIT_WEBHOOK_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: AUDIT_KAFKA_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_KAFKA_KVS,
+            valid_keys: AUDIT_KAFKA_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "mqtt",
+            subsystem_key: AUDIT_MQTT_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_MQTT_KVS,
+            valid_keys: AUDIT_MQTT_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "nats",
+            subsystem_key: AUDIT_NATS_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_NATS_KVS,
+            valid_keys: AUDIT_NATS_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "pulsar",
+            subsystem_key: AUDIT_PULSAR_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_PULSAR_KVS,
+            valid_keys: AUDIT_PULSAR_KEYS,
+        },
+    ]
+}
+
 #[instrument(skip(api))]
 pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
     let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
@@ -192,15 +278,15 @@ fn parse_oidc_scalar_value(key: &str, value: &Value) -> Option<String> {
         }),
         Value::Bool(v) => Some(v.to_string()),
         Value::Number(v) => Some(v.to_string()),
-        Value::Array(values) if key == rustfs_config::oidc::OIDC_SCOPES => {
-            let scopes = values
+        Value::Array(values) if COMMA_SEPARATED_LISTS.contains(&key) => {
+            let values_str = values
                 .iter()
                 .filter_map(Value::as_str)
                 .map(str::trim)
-                .filter(|scope| !scope.is_empty())
+                .filter(|val| !val.is_empty())
                 .collect::<Vec<_>>()
                 .join(",");
-            Some(scopes)
+            Some(values_str)
         }
         Value::Null => None,
         _ => None,
@@ -261,7 +347,7 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
     applied
 }
 
-fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
+fn parse_target_scalar_value(key: &str, value: &Value) -> Option<String> {
     match value {
         Value::String(v) => Some(v.trim().to_string()),
         Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v {
@@ -276,7 +362,7 @@ fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
     }
 }
 
-fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
+fn decode_target_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
     let mut kvs = KVS::new();
 
     for (key, value) in instance {
@@ -284,7 +370,7 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
             continue;
         }
 
-        if let Some(parsed) = parse_notify_scalar_value(key, value) {
+        if let Some(parsed) = parse_target_scalar_value(key, value) {
             kvs.insert(key.clone(), parsed);
         }
     }
@@ -292,21 +378,21 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
     kvs
 }
 
-fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
+fn decode_target_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
     match value {
-        Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)),
+        Value::Object(instance) => Some(decode_target_instance_object(instance, valid_keys)),
         Value::Array(_) => serde_json::from_value::<KVS>(value.clone()).ok(),
         _ => None,
     }
 }
 
-fn is_notify_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
+fn is_target_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
     section
         .iter()
-        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some())
+        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_target_scalar_value(key, value).is_some())
 }
 
-fn apply_external_notify_section(
+fn apply_external_target_section(
     cfg: &mut Config,
     notify_obj: &Map<String, Value>,
     external_key: &str,
@@ -325,8 +411,8 @@ fn apply_external_notify_section(
     let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default();
     let mut applied = false;
 
-    if is_notify_instance_shorthand(section_obj, valid_keys) {
-        let kvs = decode_notify_instance_object(section_obj, valid_keys);
+    if is_target_instance_shorthand(section_obj, valid_keys) {
+        let kvs = decode_target_instance_object(section_obj, valid_keys);
         if !kvs.is_empty() {
             let mut merged = default_kvs.clone();
             merged.extend(kvs);
@@ -337,7 +423,7 @@ fn apply_external_notify_section(
     }
 
     for (raw_instance, value) in section_obj {
-        let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else {
+        let Some(mut kvs) = decode_target_instance_value(value, valid_keys) else {
             continue;
         };
         if kvs.is_empty() {
@@ -363,29 +449,31 @@ fn apply_external_notify_section(
     applied
 }
 
+fn apply_external_target_descriptors(
+    cfg: &mut Config,
+    section_obj: &Map<String, Value>,
+    descriptors: &[TargetConfigDescriptor],
+) -> bool {
+    let mut applied = false;
+    for descriptor in descriptors {
+        applied |= apply_external_target_section(
+            cfg,
+            section_obj,
+            descriptor.external_key,
+            descriptor.subsystem_key,
+            descriptor.default_kvs,
+            descriptor.valid_keys,
+        );
+    }
+    applied
+}
+
 fn apply_external_notify_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
     let Some(Value::Object(notify_obj)) = root.get("notify") else {
         return false;
     };
 
-    let mut applied = false;
-    applied |= apply_external_notify_section(
-        cfg,
-        notify_obj,
-        "webhook",
-        NOTIFY_WEBHOOK_SUB_SYS,
-        &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
-        NOTIFY_WEBHOOK_KEYS,
-    );
-    applied |= apply_external_notify_section(
-        cfg,
-        notify_obj,
-        "mqtt",
-        NOTIFY_MQTT_SUB_SYS,
-        &notify::DEFAULT_NOTIFY_MQTT_KVS,
-        NOTIFY_MQTT_KEYS,
-    );
-    applied
+    apply_external_target_descriptors(cfg, notify_obj, &notify_target_descriptors())
 }
 
 fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
@@ -394,24 +482,7 @@ fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
         return false;
     };
 
-    let mut applied = false;
-    applied |= apply_external_notify_section(
-        cfg,
-        audit_obj,
-        "webhook",
-        AUDIT_WEBHOOK_SUB_SYS,
-        &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
-        AUDIT_WEBHOOK_KEYS,
-    );
-    applied |= apply_external_notify_section(
-        cfg,
-        audit_obj,
-        "mqtt",
-        AUDIT_MQTT_SUB_SYS,
-        &audit::DEFAULT_AUDIT_MQTT_KVS,
-        AUDIT_MQTT_KEYS,
-    );
-    applied
+    apply_external_target_descriptors(cfg, audit_obj, &audit_target_descriptors())
 }
 
 fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
@@ -513,15 +584,15 @@ fn build_oidc_provider_object(kvs: &KVS) -> Map<String, Value> {
             continue;
         }
 
-        if kv.key == rustfs_config::oidc::OIDC_SCOPES {
-            let scopes = kv
+        if COMMA_SEPARATED_LISTS.contains(&kv.key.as_str()) {
+            let values = kv
                 .value
                 .split(',')
                 .map(str::trim)
-                .filter(|scope| !scope.is_empty())
-                .map(|scope| Value::String(scope.to_string()))
+                .filter(|val| !val.is_empty())
+                .map(|val| Value::String(val.to_string()))
                 .collect::<Vec<_>>();
-            provider.insert(kv.key.clone(), Value::Array(scopes));
+            provider.insert(kv.key.clone(), Value::Array(values));
             continue;
         }
 
@@ -537,7 +608,7 @@ fn build_oidc_object(cfg: &Config) -> Map<String, Value> {
     };
 
     let mut providers = subsystem.iter().collect::<Vec<_>>();
-    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    providers.sort_by_key(|(lhs, _)| *lhs);
 
     let mut oidc_obj = Map::new();
     for (instance_key, kvs) in providers {
@@ -572,7 +643,7 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     };
 
     let mut providers = subsystem.iter().collect::<Vec<_>>();
-    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    providers.sort_by_key(|(lhs, _)| *lhs);
 
     let mut oidc_obj = Map::new();
     for (instance_key, kvs) in providers {
@@ -604,12 +675,21 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     oidc_obj
 }
 
-fn is_notify_bool_key(key: &str) -> bool {
-    key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+fn is_target_bool_key(key: &str) -> bool {
+    matches!(
+        key,
+        ENABLE_KEY
+            | rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+            | rustfs_config::KAFKA_TLS_ENABLE
+            | rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
+            | rustfs_config::NATS_TLS_REQUIRED
+            | rustfs_config::PULSAR_TLS_ALLOW_INSECURE
+            | rustfs_config::PULSAR_TLS_HOSTNAME_VERIFICATION
+    )
 }
 
-fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
-    if is_notify_bool_key(key) {
+fn encode_target_scalar_value(key: &str, value: &str) -> Value {
+    if is_target_bool_key(key) {
         if let Ok(state) = value.parse::<EnableState>() {
             return Value::Bool(state.is_enabled());
         }
@@ -630,7 +710,7 @@ fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool {
         .unwrap_or(false)
 }
 
-fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
+fn build_target_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
     let mut instance = Map::new();
 
     for key in valid_keys {
@@ -653,13 +733,13 @@ fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&s
             continue;
         }
 
-        instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value));
+        instance.insert((*key).to_string(), encode_target_scalar_value(key, &effective_value));
     }
 
     instance
 }
 
-fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
+fn merged_target_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
     let mut merged = default_kvs.clone();
     if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) {
         merged.extend(kvs.clone());
@@ -667,7 +747,7 @@ fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS
     merged
 }
 
-fn build_notify_subsystem_object(
+fn build_target_subsystem_object(
     cfg: &Config,
     subsystem_key: &str,
     default_kvs: &KVS,
@@ -677,11 +757,11 @@ fn build_notify_subsystem_object(
         return Map::new();
     };
 
-    let effective_default = merged_notify_default_kvs(subsystem, default_kvs);
+    let effective_default = merged_target_default_kvs(subsystem, default_kvs);
     let mut subsystem_obj = Map::new();
 
     if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) {
-        let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
+        let default_obj = build_target_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
         if !default_obj.is_empty() {
             subsystem_obj.insert("default".to_string(), Value::Object(default_obj));
         }
@@ -691,10 +771,10 @@ fn build_notify_subsystem_object(
         .iter()
         .filter(|(instance_key, _)| instance_key.as_str() != DEFAULT_DELIMITER)
         .collect::<Vec<_>>();
-    instances.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    instances.sort_by_key(|(lhs, _)| *lhs);
 
     for (instance_key, kvs) in instances {
-        let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
+        let instance_obj = build_target_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
         if !instance_obj.is_empty() {
             subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj));
         }
@@ -703,38 +783,43 @@ fn build_notify_subsystem_object(
     subsystem_obj
 }
 
+fn build_target_object(cfg: &Config, descriptors: &[TargetConfigDescriptor]) -> Map<String, Value> {
+    let mut target_obj = Map::new();
+    for descriptor in descriptors {
+        let subsystem_obj =
+            build_target_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
+        if !subsystem_obj.is_empty() {
+            target_obj.insert(descriptor.external_key.to_string(), Value::Object(subsystem_obj));
+        }
+    }
+    target_obj
+}
+
 fn build_notify_object(cfg: &Config) -> Map<String, Value> {
-    let mut notify_obj = Map::new();
-
-    let webhook_obj =
-        build_notify_subsystem_object(cfg, NOTIFY_WEBHOOK_SUB_SYS, &notify::DEFAULT_NOTIFY_WEBHOOK_KVS, NOTIFY_WEBHOOK_KEYS);
-    if !webhook_obj.is_empty() {
-        notify_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
-    }
-
-    let mqtt_obj = build_notify_subsystem_object(cfg, NOTIFY_MQTT_SUB_SYS, &notify::DEFAULT_NOTIFY_MQTT_KVS, NOTIFY_MQTT_KEYS);
-    if !mqtt_obj.is_empty() {
-        notify_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
-    }
-
-    notify_obj
+    build_target_object(cfg, &notify_target_descriptors())
 }
 
 fn build_audit_object(cfg: &Config) -> Map<String, Value> {
-    let mut audit_obj = Map::new();
+    build_target_object(cfg, &audit_target_descriptors())
+}
 
-    let webhook_obj =
-        build_notify_subsystem_object(cfg, AUDIT_WEBHOOK_SUB_SYS, &audit::DEFAULT_AUDIT_WEBHOOK_KVS, AUDIT_WEBHOOK_KEYS);
-    if !webhook_obj.is_empty() {
-        audit_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
+fn sync_rendered_target_object(
+    target_obj: &mut Map<String, Value>,
+    rendered_target: &Map<String, Value>,
+    descriptors: &[TargetConfigDescriptor],
+) {
+    for descriptor in descriptors {
+        match rendered_target.get(descriptor.external_key) {
+            Some(Value::Object(v)) => {
+                target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone()));
+                target_obj.remove(descriptor.subsystem_key);
+            }
+            _ => {
+                target_obj.remove(descriptor.external_key);
+                target_obj.remove(descriptor.subsystem_key);
+            }
+        }
     }
-
-    let mqtt_obj = build_notify_subsystem_object(cfg, AUDIT_MQTT_SUB_SYS, &audit::DEFAULT_AUDIT_MQTT_KVS, AUDIT_MQTT_KEYS);
-    if !mqtt_obj.is_empty() {
-        audit_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
-    }
-
-    audit_obj
 }
 
 fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
@@ -771,67 +856,31 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
         _ => Map::new(),
     };
     let rendered_notify = build_notify_object(cfg);
-    match rendered_notify.get("webhook") {
-        Some(Value::Object(v)) => {
-            notify_obj.insert("webhook".to_string(), Value::Object(v.clone()));
-            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
-        }
-        _ => {
-            notify_obj.remove("webhook");
-            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
-        }
-    }
-    match rendered_notify.get("mqtt") {
-        Some(Value::Object(v)) => {
-            notify_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
-            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
-        }
-        _ => {
-            notify_obj.remove("mqtt");
-            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
-        }
-    }
+    sync_rendered_target_object(&mut notify_obj, &rendered_notify, &notify_target_descriptors());
     if notify_obj.is_empty() {
         root.remove("notify");
     } else {
         root.insert("notify".to_string(), Value::Object(notify_obj));
     }
-    root.remove(NOTIFY_WEBHOOK_SUB_SYS);
-    root.remove(NOTIFY_MQTT_SUB_SYS);
+    for descriptor in notify_target_descriptors() {
+        root.remove(descriptor.subsystem_key);
+    }
 
     let mut logger_obj = match root.remove("logger") {
         Some(Value::Object(v)) => v,
         _ => Map::new(),
     };
     let rendered_audit = build_audit_object(cfg);
-    match rendered_audit.get("webhook") {
-        Some(Value::Object(v)) => {
-            logger_obj.insert("webhook".to_string(), Value::Object(v.clone()));
-            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
-        }
-        _ => {
-            logger_obj.remove("webhook");
-            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
-        }
-    }
-    match rendered_audit.get("mqtt") {
-        Some(Value::Object(v)) => {
-            logger_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
-            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
-        }
-        _ => {
-            logger_obj.remove("mqtt");
-            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
-        }
-    }
+    sync_rendered_target_object(&mut logger_obj, &rendered_audit, &audit_target_descriptors());
     if logger_obj.is_empty() {
         root.remove("logger");
     } else {
         root.insert("logger".to_string(), Value::Object(logger_obj));
     }
     root.remove("audit");
-    root.remove(AUDIT_WEBHOOK_SUB_SYS);
-    root.remove(AUDIT_MQTT_SUB_SYS);
+    for descriptor in audit_target_descriptors() {
+        root.remove(descriptor.subsystem_key);
+    }
 
     Ok(serde_json::to_vec(&Value::Object(root))?)
 }
@@ -1082,8 +1131,8 @@ mod tests {
         ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
     };
     use http::HeaderMap;
-    use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-    use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+    use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
+    use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
     use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
     use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
     use rustfs_filemeta::FileInfo;
@@ -1655,6 +1704,7 @@ mod tests {
               "client_id":"console",
               "client_secret":"secret-value",
               "scopes":["openid","profile","email"],
+              "other_audiences":["aud1", "aud2"],
               "redirect_uri_dynamic":true,
               "display_name":"Default Provider"
             },
@@ -1679,6 +1729,7 @@ mod tests {
         );
         assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "console");
         assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_SCOPES), "openid,profile,email");
+        assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES), "aud1,aud2");
         assert_eq!(default_kvs.get(ENABLE_KEY), EnableState::On.to_string());
 
         let smoke_kvs = cfg
@@ -1715,6 +1766,15 @@ mod tests {
                 "topic":"events",
                 "queue_dir":""
               }
+            },
+            "kafka":{
+              "streaming":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092,127.0.0.1:9093",
+                "topic":"events-kafka",
+                "acks":"all",
+                "tls_enable":true
+              }
             }
           }
         }"#;
@@ -1743,6 +1803,14 @@ mod tests {
             .expect("mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), "");
+
+        let kafka = cfg
+            .get_value(NOTIFY_KAFKA_SUB_SYS, "streaming")
+            .expect("kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092,127.0.0.1:9093");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "events-kafka");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true");
     }
 
     #[test]
@@ -1812,6 +1880,14 @@ mod tests {
                 "broker":"tcp://127.0.0.1:1883",
                 "topic":"audit-events"
               }
+            },
+            "kafka":{
+              "auditlog":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092",
+                "topic":"audit-events-kafka",
+                "acks":"1"
+              }
             }
           }
         }"#;
@@ -1835,6 +1911,12 @@ mod tests {
             .get_value(AUDIT_MQTT_SUB_SYS, "analytics")
             .expect("audit mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
+
+        let kafka = cfg
+            .get_value(AUDIT_KAFKA_SUB_SYS, "auditlog")
+            .expect("audit kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "audit-events-kafka");
     }
 
     #[test]
@@ -1863,6 +1945,7 @@ mod tests {
         );
         default_provider.insert(rustfs_config::oidc::OIDC_CLIENT_ID.to_string(), "console".to_string());
         default_provider.insert(rustfs_config::oidc::OIDC_SCOPES.to_string(), "openid,profile,email".to_string());
+        default_provider.insert(rustfs_config::oidc::OIDC_OTHER_AUDIENCES.to_string(), "aud1,aud2".to_string());
         oidc_section.insert(DEFAULT_DELIMITER.to_string(), default_provider);
         cfg.0.insert(IDENTITY_OPENID_SUB_SYS.to_string(), oidc_section);
 
@@ -1890,6 +1973,13 @@ mod tests {
                 .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
             Some(vec!["openid", "profile", "email"])
         );
+        assert_eq!(
+            default_provider
+                .get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES)
+                .and_then(Value::as_array)
+                .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
+            Some(vec!["aud1", "aud2"])
+        );
         assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
     }
 
@@ -1947,6 +2037,38 @@ mod tests {
         );
         cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = notify::DEFAULT_NOTIFY_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "streaming".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092,127.0.0.1:9093".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_ACKS.to_string(),
+                    value: "all".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_TLS_ENABLE.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(NOTIFY_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let notify = v
@@ -1982,6 +2104,19 @@ mod tests {
             .expect("mqtt target should be encoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883"));
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some(""));
+
+        let kafka = notify
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("streaming"))
+            .and_then(Value::as_object)
+            .expect("kafka target should be encoded");
+        assert_eq!(
+            kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str),
+            Some("127.0.0.1:9092,127.0.0.1:9093")
+        );
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS).and_then(Value::as_str), Some("all"));
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE).and_then(Value::as_bool), Some(true));
     }
 
     #[test]
@@ -2033,6 +2168,28 @@ mod tests {
         );
         cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = audit::DEFAULT_AUDIT_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "audit-events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "auditlog".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(AUDIT_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let logger = v
@@ -2059,6 +2216,14 @@ mod tests {
             .expect("audit mqtt default should be encoded");
         assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
         assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events"));
+
+        let kafka = logger
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("auditlog"))
+            .and_then(Value::as_object)
+            .expect("audit kafka target should be encoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str), Some("127.0.0.1:9092"));
     }
 
     #[test]

crates/ecstore/src/config/mod.rs

@@ -25,8 +25,12 @@ use crate::store::ECStore;
 use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
 use rustfs_config::COMMENT_KEY;
 use rustfs_config::DEFAULT_DELIMITER;
-use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{
+    AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -241,6 +245,12 @@ pub fn init() {
     kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
     kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone());
     kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());
+    kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone());
+    kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
+    kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
+    kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
+    kvs.insert(NOTIFY_KAFKA_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_KAFKA_KVS.clone());
+    kvs.insert(AUDIT_KAFKA_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_KAFKA_KVS.clone());
     kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
 
     // Register all default configurations

crates/ecstore/src/config/notify.rs

@@ -14,11 +14,15 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
-    WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
+    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -171,3 +175,203 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_ADDRESS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_SUBJECT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_CREDENTIALS_FILE.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_REQUIRED.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_AUTH_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
+            value: EnableState::On.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/oidc.rs

@@ -18,8 +18,8 @@ use rustfs_config::{
     oidc::{
         OIDC_CLAIM_NAME, OIDC_CLAIM_PREFIX, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_CONFIG_URL, OIDC_DEFAULT_CLAIM_NAME,
         OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_ROLES_CLAIM, OIDC_DEFAULT_SCOPES,
-        OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME, OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_REDIRECT_URI,
-        OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_ROLES_CLAIM, OIDC_SCOPES, OIDC_USERNAME_CLAIM,
+        OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME, OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_OTHER_AUDIENCES,
+        OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_ROLES_CLAIM, OIDC_SCOPES, OIDC_USERNAME_CLAIM,
     },
 };
 use std::sync::LazyLock;
@@ -52,6 +52,11 @@ pub static DEFAULT_IDENTITY_OPENID_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: OIDC_DEFAULT_SCOPES.to_owned(),
             hidden_if_empty: false,
         },
+        KV {
+            key: OIDC_OTHER_AUDIENCES.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
         KV {
             key: OIDC_REDIRECT_URI.to_owned(),
             value: "".to_owned(),

crates/ecstore/src/disk/disk_store.rs

@@ -16,10 +16,16 @@ use crate::disk::{
     CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
     FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo,
     WalkDirOptions,
+    health_state::{
+        RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
+        get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
+        record_drive_recovery_class, record_drive_runtime_state, record_drive_state_transition,
+    },
     local::{LocalDisk, ScanGuard},
 };
 use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
+use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use std::{
     path::PathBuf,
@@ -40,7 +46,6 @@ const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
 pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
 pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
@@ -51,9 +56,52 @@ lazy_static::lazy_static! {
 }
 
 pub fn get_max_timeout_duration() -> Duration {
-    std::env::var(ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION)
-        .map(|v| Duration::from_secs(v.parse::<u64>().unwrap_or(30)))
-        .unwrap_or(Duration::from_secs(30))
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION,
+        rustfs_config::DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS,
+    ))
+}
+
+fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
+    Duration::from_secs(
+        rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
+            .unwrap_or(default_secs),
+    )
+}
+
+pub fn get_drive_metadata_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_disk_info_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_list_dir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_stall_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+    )
 }
 
 /// DiskHealthTracker tracks the health status of a disk.
@@ -68,6 +116,16 @@ pub struct DiskHealthTracker {
     pub status: AtomicU32,
     /// Atomic number of waiting operations
     pub waiting: AtomicU32,
+    /// Runtime drive health state
+    pub runtime_state: AtomicU32,
+    /// Consecutive failures while transitioning away from online
+    pub consecutive_failures: AtomicU32,
+    /// Consecutive successes while returning online
+    pub consecutive_successes: AtomicU32,
+    /// When the drive first left the online state
+    pub offline_since_unix_secs: AtomicI64,
+    /// Last runtime state transition timestamp
+    pub last_transition_unix_secs: AtomicI64,
 }
 
 impl DiskHealthTracker {
@@ -83,6 +141,11 @@ impl DiskHealthTracker {
             last_started: AtomicI64::new(now),
             status: AtomicU32::new(DISK_HEALTH_OK),
             waiting: AtomicU32::new(0),
+            runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
+            consecutive_failures: AtomicU32::new(0),
+            consecutive_successes: AtomicU32::new(0),
+            offline_since_unix_secs: AtomicI64::new(0),
+            last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
         }
     }
 
@@ -110,12 +173,139 @@ impl DiskHealthTracker {
         self.status.store(DISK_HEALTH_OK, Ordering::Release);
     }
 
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.runtime_state.store(state as u32, Ordering::Release);
+        match state {
+            RuntimeDriveHealthState::Offline => self.set_faulty(),
+            RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => {
+                self.set_ok();
+            }
+        }
+    }
+
     pub fn swap_ok_to_faulty(&self) -> bool {
         self.status
             .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed)
             .is_ok()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire))
+    }
+
+    pub fn offline_duration(&self) -> Option<Duration> {
+        let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
+        if offline_since <= 0 {
+            return None;
+        }
+        let now = current_unix_secs();
+        Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
+    }
+
+    pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let now = current_unix_secs();
+        let next = match current {
+            RuntimeDriveHealthState::Online => {
+                self.consecutive_failures.store(1, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                self.offline_since_unix_secs
+                    .compare_exchange(0, now as i64, Ordering::AcqRel, Ordering::Relaxed)
+                    .ok();
+                RuntimeDriveHealthState::Suspect
+            }
+            RuntimeDriveHealthState::Suspect => {
+                let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
+                if failures >= get_drive_suspect_failure_threshold() {
+                    RuntimeDriveHealthState::Offline
+                } else {
+                    RuntimeDriveHealthState::Suspect
+                }
+            }
+            RuntimeDriveHealthState::Returning => {
+                self.consecutive_failures.store(0, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                RuntimeDriveHealthState::Offline
+            }
+            RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
+        };
+
+        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        self.transition_state(endpoint, current, next, reason);
+        current == RuntimeDriveHealthState::Online
+    }
+
+    pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let next = match current {
+            RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Suspect => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Offline => {
+                self.consecutive_successes.store(1, Ordering::Release);
+                RuntimeDriveHealthState::Returning
+            }
+            RuntimeDriveHealthState::Returning => {
+                let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1;
+                if successes >= get_drive_returning_success_threshold() {
+                    RuntimeDriveHealthState::Online
+                } else {
+                    RuntimeDriveHealthState::Returning
+                }
+            }
+        };
+
+        let became_online = next == RuntimeDriveHealthState::Online;
+        if became_online {
+            self.status.store(DISK_HEALTH_OK, Ordering::Release);
+            self.consecutive_failures.store(0, Ordering::Release);
+            self.consecutive_successes.store(0, Ordering::Release);
+        }
+        self.transition_state(endpoint, current, next, reason);
+        if became_online {
+            self.log_success();
+        }
+        became_online
+    }
+
+    fn transition_state(
+        &self,
+        endpoint: &Endpoint,
+        current: RuntimeDriveHealthState,
+        next: RuntimeDriveHealthState,
+        reason: &'static str,
+    ) {
+        if current == next {
+            return;
+        }
+
+        self.runtime_state.store(next as u32, Ordering::Release);
+        self.last_transition_unix_secs
+            .store(current_unix_secs() as i64, Ordering::Release);
+
+        if matches!(
+            next,
+            RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Offline | RuntimeDriveHealthState::Returning
+        ) && self.offline_since_unix_secs.load(Ordering::Acquire) == 0
+        {
+            self.offline_since_unix_secs
+                .store(current_unix_secs() as i64, Ordering::Release);
+        }
+
+        if next == RuntimeDriveHealthState::Online {
+            if let Some(duration) = self.offline_duration() {
+                record_drive_offline_duration(endpoint, duration);
+                record_drive_recovery_class(classify_drive_recovery(duration));
+            }
+            self.offline_since_unix_secs.store(0, Ordering::Release);
+        } else if let Some(duration) = self.offline_duration() {
+            record_drive_offline_duration(endpoint, duration);
+        }
+
+        record_drive_state_transition(endpoint, current, next, reason);
+        record_drive_runtime_state(endpoint, next);
+    }
+
     /// Increment waiting operations counter
     pub fn increment_waiting(&self) {
         self.waiting.fetch_add(1, Ordering::Relaxed);
@@ -137,6 +327,13 @@ impl DiskHealthTracker {
     }
 }
 
+fn current_unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_secs()
+}
+
 impl Default for DiskHealthTracker {
     fn default() -> Self {
         Self::new()
@@ -186,19 +383,34 @@ impl LocalDiskWrapper {
         let env_health_check =
             rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
 
-        Self {
+        let wrapper = Self {
             disk,
             health: Arc::new(DiskHealthTracker::new()),
             health_check: health_check && env_health_check,
             cancel_token: CancellationToken::new(),
             disk_id: Arc::new(RwLock::new(None)),
-        }
+        };
+        record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
+        wrapper
     }
 
     pub fn get_disk(&self) -> Arc<LocalDisk> {
         self.disk.clone()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        self.health.runtime_state()
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        self.health.offline_duration().map(|duration| duration.as_secs())
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.health.force_runtime_state_for_test(state);
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -219,6 +431,20 @@ impl LocalDiskWrapper {
         self.cancel_token.cancel();
     }
 
+    fn spawn_recovery_monitor_if_needed(&self) {
+        if !self.health_check {
+            return;
+        }
+
+        self.health.increment_waiting();
+        let health = Arc::clone(&self.health);
+        let disk = Arc::clone(&self.disk);
+        let cancel_token = self.cancel_token.clone();
+        tokio::spawn(async move {
+            Self::monitor_disk_status(disk, health, cancel_token).await;
+        });
+    }
+
     /// Monitor disk writability periodically
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         // TODO: config interval
@@ -256,7 +482,9 @@ impl LocalDiskWrapper {
 
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err() && health.swap_ok_to_faulty() {
+                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                        && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
+                    {
                         // Health check failed, disk is considered faulty
                         warn!("health check: failed, disk is considered faulty");
 
@@ -345,9 +573,9 @@ impl LocalDiskWrapper {
 
     /// Monitor disk status and try to bring it back online
     async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        const CHECK_EVERY: Duration = Duration::from_secs(5);
+        let check_every = get_drive_returning_probe_interval();
 
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(check_every);
 
         loop {
             tokio::select! {
@@ -362,12 +590,18 @@ impl LocalDiskWrapper {
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
                     match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
                         Ok(_) => {
+                            let state_before = health.runtime_state();
+                            let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
+                            info!("Disk {} recovery probe succeeded; state={:?}", disk.to_string(), state_before);
+                            if !is_online {
+                                continue;
+                            }
                             info!("Disk {} is back online", disk.to_string());
-                            health.set_ok();
                             health.decrement_waiting();
                             return;
                         }
                         Err(e) => {
+                            health.mark_failure(&disk.endpoint(), "recovery_probe_failed");
                             warn!("Disk {} still faulty: {:?}", disk.to_string(), e);
                         }
                     }
@@ -435,6 +669,19 @@ impl LocalDiskWrapper {
     /// Track disk health for an operation.
     /// This method should wrap disk operations to ensure health checking.
     pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        self.track_disk_health_with_op("unknown", operation, timeout_duration).await
+    }
+
+    pub async fn track_disk_health_with_op<T, F, Fut>(
+        &self,
+        op: &'static str,
+        operation: F,
+        timeout_duration: Duration,
+    ) -> Result<T>
     where
         F: FnOnce() -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
@@ -479,7 +726,21 @@ impl LocalDiskWrapper {
             Err(_) => {
                 // Timeout occurred, mark disk as potentially faulty and decrement waiting counter
                 self.health.decrement_waiting();
-                warn!("disk operation timeout after {:?}", timeout_duration);
+                if self.health.mark_failure(&self.endpoint(), "operation_timeout") {
+                    self.spawn_recovery_monitor_if_needed();
+                }
+                counter!(
+                    "rustfs_drive_op_timeout_total",
+                    "endpoint" => self.endpoint().to_string(),
+                    "op" => op.to_string()
+                )
+                .increment(1);
+                warn!(
+                    endpoint = %self.endpoint(),
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    "Local disk operation timed out"
+                );
                 Err(DiskError::other(format!("disk operation timeout after {timeout_duration:?}")))
             }
         }
@@ -489,8 +750,12 @@ impl LocalDiskWrapper {
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        self.track_disk_health(|| async { self.disk.read_metadata(volume, path).await }, Duration::ZERO)
-            .await
+        self.track_disk_health_with_op(
+            "read_metadata",
+            || async { self.disk.read_metadata(volume, path).await },
+            get_drive_metadata_timeout(),
+        )
+        .await
     }
 
     fn start_scan(&self) -> ScanGuard {
@@ -565,15 +830,22 @@ impl DiskAPI for LocalDiskWrapper {
             return Err(DiskError::FaultyDisk);
         }
 
-        let result = self.disk.disk_info(opts).await?;
+        self.track_disk_health_with_op(
+            "disk_info",
+            || async {
+                let result = self.disk.disk_info(opts).await?;
 
-        if let Some(current_disk_id) = *self.disk_id.read().await
-            && Some(current_disk_id) != result.id
-        {
-            return Err(DiskError::DiskNotFound);
-        };
+                if let Some(current_disk_id) = *self.disk_id.read().await
+                    && Some(current_disk_id) != result.id
+                {
+                    return Err(DiskError::DiskNotFound);
+                };
 
-        Ok(result)
+                Ok(result)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
     }
 
     async fn make_volume(&self, volume: &str) -> Result<()> {
@@ -587,7 +859,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
-        self.track_disk_health(|| async { self.disk.list_volumes().await }, Duration::ZERO)
+        self.track_disk_health_with_op("list_volumes", || async { self.disk.list_volumes().await }, Duration::ZERO)
             .await
     }
 
@@ -602,7 +874,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
-        self.track_disk_health(|| async { self.disk.walk_dir(opts, wr).await }, Duration::ZERO)
+        self.track_disk_health_with_op("walk_dir", || async { self.disk.walk_dir(opts, wr).await }, get_drive_walkdir_timeout())
             .await
     }
 
@@ -710,9 +982,10 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
-        self.track_disk_health(
+        self.track_disk_health_with_op(
+            "list_dir",
             || async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
-            get_max_timeout_duration(),
+            get_drive_list_dir_timeout(),
         )
         .await
     }
@@ -802,3 +1075,74 @@ impl DiskAPI for LocalDiskWrapper {
             .await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::endpoint::Endpoint;
+    use crate::disk::health_state::RuntimeDriveHealthState;
+
+    #[test]
+    fn drive_metadata_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
+                assert_eq!(
+                    get_drive_metadata_timeout(),
+                    Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(17));
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_prefers_canonical_over_legacy() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("7"), || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(7));
+            });
+        });
+    }
+
+    #[test]
+    fn runtime_state_transitions_from_online_to_suspect_then_offline() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+
+        assert!(!health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+        assert!(health.offline_duration().is_some());
+    }
+
+    #[test]
+    fn runtime_state_transitions_back_online_after_recovery_threshold() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        health.mark_failure(&endpoint, "timeout");
+        health.mark_failure(&endpoint, "timeout");
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.offline_duration().is_none());
+    }
+}

crates/ecstore/src/disk/health_state.rs

@@ -0,0 +1,237 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{DiskAPI, DiskStore};
+use crate::disk::endpoint::Endpoint;
+use metrics::{counter, gauge};
+use std::time::Duration;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u32)]
+pub enum RuntimeDriveHealthState {
+    Online = 0,
+    Suspect = 1,
+    Offline = 2,
+    Returning = 3,
+}
+
+impl RuntimeDriveHealthState {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Online => "online",
+            Self::Suspect => "suspect",
+            Self::Offline => "offline",
+            Self::Returning => "returning",
+        }
+    }
+
+    pub fn from_u32(value: u32) -> Self {
+        match value {
+            1 => Self::Suspect,
+            2 => Self::Offline,
+            3 => Self::Returning,
+            _ => Self::Online,
+        }
+    }
+
+    pub fn is_snapshot_eligible(self) -> bool {
+        matches!(self, Self::Online | Self::Suspect | Self::Returning)
+    }
+
+    pub fn is_strictly_online(self) -> bool {
+        matches!(self, Self::Online)
+    }
+
+    pub fn should_probe_for_admin(self) -> bool {
+        matches!(self, Self::Online | Self::Returning)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DriveRecoveryClass {
+    ShortOffline,
+    MediumOffline,
+    LongOffline,
+}
+
+impl DriveRecoveryClass {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::ShortOffline => "short_offline",
+            Self::MediumOffline => "medium_offline",
+            Self::LongOffline => "long_offline",
+        }
+    }
+}
+
+pub fn get_drive_suspect_failure_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_success_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_probe_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_offline_grace_period() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+        rustfs_config::DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+    ))
+}
+
+pub fn get_drive_long_offline_threshold() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+        rustfs_config::DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+    ))
+}
+
+pub fn classify_drive_recovery(duration: Duration) -> DriveRecoveryClass {
+    if duration <= get_drive_offline_grace_period() {
+        DriveRecoveryClass::ShortOffline
+    } else if duration >= get_drive_long_offline_threshold() {
+        DriveRecoveryClass::LongOffline
+    } else {
+        DriveRecoveryClass::MediumOffline
+    }
+}
+
+pub fn record_drive_runtime_state(endpoint: &Endpoint, state: RuntimeDriveHealthState) {
+    let endpoint_label = endpoint.to_string();
+    let pool_label = endpoint.pool_idx.to_string();
+    let set_label = endpoint.set_idx.to_string();
+    let disk_label = endpoint.disk_idx.to_string();
+
+    for candidate in [
+        RuntimeDriveHealthState::Online,
+        RuntimeDriveHealthState::Suspect,
+        RuntimeDriveHealthState::Offline,
+        RuntimeDriveHealthState::Returning,
+    ] {
+        gauge!(
+            "rustfs_drive_runtime_state",
+            "endpoint" => endpoint_label.clone(),
+            "pool" => pool_label.clone(),
+            "set" => set_label.clone(),
+            "disk" => disk_label.clone(),
+            "state" => candidate.as_str().to_string()
+        )
+        .set(if candidate == state { 1.0 } else { 0.0 });
+    }
+}
+
+pub fn record_drive_state_transition(
+    endpoint: &Endpoint,
+    from: RuntimeDriveHealthState,
+    to: RuntimeDriveHealthState,
+    reason: &'static str,
+) {
+    counter!(
+        "rustfs_drive_state_transition_total",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string(),
+        "from" => from.as_str().to_string(),
+        "to" => to.as_str().to_string(),
+        "reason" => reason.to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_recovery_class(class: DriveRecoveryClass) {
+    counter!(
+        "rustfs_drive_recovery_class_total",
+        "class" => class.as_str().to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_offline_duration(endpoint: &Endpoint, duration: Duration) {
+    gauge!(
+        "rustfs_drive_offline_duration_seconds",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string()
+    )
+    .set(duration.as_secs_f64());
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DriveMembershipSnapshot {
+    pub online: Vec<DiskStore>,
+    pub suspect: Vec<DiskStore>,
+    pub returning: Vec<DiskStore>,
+    pub offline: Vec<DiskStore>,
+}
+
+impl DriveMembershipSnapshot {
+    pub fn from_optional_disks(disks: &[Option<DiskStore>]) -> Self {
+        let mut snapshot = Self::default();
+
+        for disk in disks.iter().flatten() {
+            match disk.runtime_state() {
+                RuntimeDriveHealthState::Online => snapshot.online.push(disk.clone()),
+                RuntimeDriveHealthState::Suspect => snapshot.suspect.push(disk.clone()),
+                RuntimeDriveHealthState::Returning => snapshot.returning.push(disk.clone()),
+                RuntimeDriveHealthState::Offline => snapshot.offline.push(disk.clone()),
+            }
+        }
+
+        snapshot
+    }
+
+    pub fn scanner_heal_candidates(&self) -> Vec<DiskStore> {
+        let mut disks = Vec::with_capacity(self.online.len() + self.suspect.len() + self.returning.len());
+        disks.extend(self.online.iter().cloned());
+        disks.extend(self.suspect.iter().cloned());
+        disks.extend(self.returning.iter().cloned());
+        disks
+    }
+
+    pub fn strict_online_candidates(&self) -> Vec<DiskStore> {
+        self.online.clone()
+    }
+
+    pub fn strict_online_local_candidates(&self) -> Vec<DiskStore> {
+        self.online.iter().filter(|disk| disk.is_local()).cloned().collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn runtime_drive_health_state_snapshot_eligibility_matches_membership_policy() {
+        assert!(RuntimeDriveHealthState::Online.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Suspect.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Returning.is_snapshot_eligible());
+        assert!(!RuntimeDriveHealthState::Offline.is_snapshot_eligible());
+    }
+}

crates/ecstore/src/disk/local.rs

@@ -176,7 +176,15 @@ impl LocalDisk {
             let root = root_clone.clone();
             Box::pin(async move {
                 match get_disk_info(root.clone()).await {
-                    Ok((info, root)) => {
+                    Ok((info, is_root_disk)) => {
+                        let physical_device_ids = match rustfs_utils::os::get_physical_device_ids(root.to_string_lossy().as_ref())
+                        {
+                            Ok(ids) => ids,
+                            Err(err) => {
+                                warn!(root = ?root, error = ?err, "failed to resolve physical device ids for disk root");
+                                Vec::new()
+                            }
+                        };
                         let disk_info = DiskInfo {
                             total: info.total,
                             free: info.free,
@@ -186,7 +194,8 @@ impl LocalDisk {
                             major: info.major,
                             minor: info.minor,
                             fs_type: info.fstype,
-                            root_disk: root,
+                            root_disk: is_root_disk,
+                            physical_device_ids,
                             id: disk_id,
                             ..Default::default()
                         };
@@ -1538,7 +1547,7 @@ impl DiskAPI for LocalDisk {
                 volume,
                 path_join_buf(&[
                     path,
-                    &fi.data_dir.map_or("".to_string(), |dir| dir.to_string()),
+                    &fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()),
                     &format!("part.{}", part.number),
                 ])
                 .as_str(),
@@ -1587,7 +1596,7 @@ impl DiskAPI for LocalDisk {
                 self.get_object_path(
                     bucket,
                     path_join_buf(&[
-                        path.parent().unwrap_or(Path::new("")).to_string_lossy().as_ref(),
+                        path.parent().unwrap_or_else(|| Path::new("")).to_string_lossy().as_ref(),
                         &format!("part.{num}"),
                     ])
                     .as_str(),
@@ -1648,7 +1657,7 @@ impl DiskAPI for LocalDisk {
                 volume,
                 path_join_buf(&[
                     path,
-                    &fi.data_dir.map_or("".to_string(), |dir| dir.to_string()),
+                    &fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()),
                     &format!("part.{}", part.number),
                 ])
                 .as_str(),
@@ -2072,6 +2081,7 @@ impl DiskAPI for LocalDisk {
 
         let mut objs_returned = 0;
 
+        let mut skip_current_dir_object = false;
         if opts.base_dir.ends_with(SLASH_SEPARATOR) {
             if let Ok(data) = self
                 .read_metadata(
@@ -2098,7 +2108,7 @@ impl DiskAPI for LocalDisk {
                 if let Ok(meta) = tokio::fs::metadata(fpath).await
                     && meta.is_file()
                 {
-                    return Err(DiskError::FileNotFound);
+                    skip_current_dir_object = true;
                 }
             }
         }
@@ -2109,7 +2119,7 @@ impl DiskAPI for LocalDisk {
             &opts,
             &mut out,
             &mut objs_returned,
-            false,
+            skip_current_dir_object,
         )
         .await?;
 
@@ -2497,7 +2507,7 @@ impl DiskAPI for LocalDisk {
                     let part_path = format!("part.{}", part.number);
                     let part_path = path_join_buf(&[
                         path,
-                        fi.data_dir.map_or("".to_string(), |dir| dir.to_string()).as_str(),
+                        fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(),
                         part_path.as_str(),
                     ]);
                     let part_path = self.get_object_path(volume, part_path.as_str())?;
@@ -2514,7 +2524,7 @@ impl DiskAPI for LocalDisk {
             if inline && fi.shard_file_size(fi.parts[0].actual_size) < DEFAULT_INLINE_BLOCK as i64 {
                 let part_path = path_join_buf(&[
                     path,
-                    fi.data_dir.map_or("".to_string(), |dir| dir.to_string()).as_str(),
+                    fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(),
                     format!("part.{}", fi.parts[0].number).as_str(),
                 ]);
                 let part_path = self.get_object_path(volume, part_path.as_str())?;
@@ -2838,6 +2848,32 @@ mod test {
         assert!(entries.next_entry().await.unwrap().is_some());
     }
 
+    #[tokio::test]
+    async fn cleanup_stale_tmp_objects_keeps_fresh_dirs_and_regular_files() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().unwrap();
+        let tmp = LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_BUCKET);
+        let fresh_dir = tmp.join("fresh").join("data");
+        let regular_file = tmp.join("note.txt");
+        let trash = LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_DELETED_BUCKET);
+
+        fs::create_dir_all(fresh_dir.parent().unwrap()).await.unwrap();
+        fs::create_dir_all(&trash).await.unwrap();
+        fs::write(&fresh_dir, b"temporary").await.unwrap();
+        fs::write(&regular_file, b"keep").await.unwrap();
+
+        LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), Duration::from_secs(60))
+            .await
+            .unwrap();
+
+        assert!(tmp.join("fresh").exists());
+        assert!(regular_file.exists());
+
+        let mut entries = fs::read_dir(&trash).await.unwrap();
+        assert!(entries.next_entry().await.unwrap().is_none());
+    }
+
     #[tokio::test]
     async fn test_scan_dir_includes_nested_object_dirs() {
         use rustfs_filemeta::MetacacheReader;

crates/ecstore/src/disk/mod.rs

@@ -19,6 +19,7 @@ pub mod error_conv;
 pub mod error_reduce;
 pub mod format;
 pub mod fs;
+pub mod health_state;
 pub mod local;
 pub mod os;
 
@@ -33,6 +34,7 @@ pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
 use crate::disk::disk_store::LocalDiskWrapper;
+use crate::disk::health_state::RuntimeDriveHealthState;
 use crate::disk::local::ScanGuard;
 use crate::rpc::RemoteDisk;
 use bytes::Bytes;
@@ -410,6 +412,30 @@ impl DiskAPI for Disk {
     }
 }
 
+impl Disk {
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        match self {
+            Disk::Local(local_disk) => local_disk.runtime_state(),
+            Disk::Remote(remote_disk) => remote_disk.runtime_state(),
+        }
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        match self {
+            Disk::Local(local_disk) => local_disk.offline_duration_secs(),
+            Disk::Remote(remote_disk) => remote_disk.offline_duration_secs(),
+        }
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        match self {
+            Disk::Local(local_disk) => local_disk.force_runtime_state_for_test(state),
+            Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
+        }
+    }
+}
+
 impl Disk {
     /// Enable health monitoring on this disk.
     /// Called after startup format loading completes so that remote peers
@@ -570,6 +596,8 @@ pub struct DiskInfo {
     pub scanning: bool,
     pub endpoint: String,
     pub mount_path: String,
+    /// Leaf physical block devices backing this mount path when available.
+    pub physical_device_ids: Vec<String>,
     pub id: Option<Uuid>,
     pub rotational: bool,
     pub metrics: DiskMetrics,

crates/ecstore/src/endpoints.rs

@@ -17,10 +17,11 @@ use crate::{
     disks_layout::DisksLayout,
     global::global_rustfs_port,
 };
+use rustfs_config::{DEFAULT_UNSAFE_BYPASS_DISK_CHECK, ENV_MINIO_CI, ENV_UNSAFE_BYPASS_DISK_CHECK};
 use rustfs_utils::{XHost, check_local_server_addr, get_host_ip, is_local_host};
 use std::{
-    collections::{HashMap, HashSet, hash_map::Entry},
-    io::{Error, Result},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
+    io::{Error, ErrorKind, Result},
     net::IpAddr,
 };
 use tracing::{error, info, instrument, warn};
@@ -348,6 +349,8 @@ impl PoolEndpointList {
             }
         }
 
+        validate_local_physical_disk_independence(pool_endpoint_list.as_ref())?;
+
         let setup_type = match pool_endpoint_list.as_ref()[0].as_ref()[0].get_type() {
             EndpointType::Path => SetupType::Erasure,
             EndpointType::Url => match unique_args.len() {
@@ -554,7 +557,7 @@ impl EndpointServerPools {
 
         for pool in self.0.iter() {
             for ep in pool.endpoints.as_ref() {
-                let n = node_map.entry(ep.host_port()).or_insert(Node {
+                let n = node_map.entry(ep.host_port()).or_insert_with(|| Node {
                     url: ep.url.clone(),
                     pools: vec![],
                     is_local: ep.is_local,
@@ -645,12 +648,118 @@ impl EndpointServerPools {
     }
 }
 
+fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()> {
+    let mut local_paths = BTreeSet::new();
+    for endpoints in pools {
+        for endpoint in endpoints.as_ref() {
+            if endpoint.is_local {
+                local_paths.insert(endpoint.get_file_path());
+            }
+        }
+    }
+
+    if local_paths.is_empty() {
+        return Ok(());
+    }
+
+    let local_paths = local_paths.into_iter().collect::<Vec<_>>();
+    validate_local_cross_device_mounts(&local_paths)?;
+
+    if local_paths.len() <= 1 {
+        return Ok(());
+    }
+
+    // Compatibility behavior:
+    // - canonical key: RUSTFS_UNSAFE_BYPASS_DISK_CHECK
+    // - legacy CI alias: MINIO_CI
+    // If both are set, `get_env_bool_with_aliases` keeps canonical key precedence.
+    if rustfs_utils::get_env_bool_with_aliases(ENV_UNSAFE_BYPASS_DISK_CHECK, &[ENV_MINIO_CI], DEFAULT_UNSAFE_BYPASS_DISK_CHECK) {
+        warn!(
+            env = ENV_UNSAFE_BYPASS_DISK_CHECK,
+            local_paths = ?local_paths,
+            "Skipping local physical disk independence validation due to explicit environment override",
+        );
+        return Ok(());
+    }
+
+    let mut device_paths = BTreeMap::<String, BTreeSet<String>>::new();
+    let mut missing_paths = Vec::new();
+
+    for path in &local_paths {
+        let canonical = match rustfs_utils::canonicalize(path) {
+            Ok(path) => path,
+            Err(err) if err.kind() == ErrorKind::NotFound => {
+                missing_paths.push(path.clone());
+                continue;
+            }
+            Err(err) => {
+                return Err(Error::other(format!(
+                    "failed to resolve local endpoint path '{path}' for disk validation: {err}"
+                )));
+            }
+        };
+        let canonical_path = canonical.to_string_lossy().into_owned();
+        let device_ids = rustfs_utils::os::get_physical_device_ids(&canonical_path).map_err(|err| {
+            Error::other(format!("failed to inspect physical disk for local endpoint '{canonical_path}': {err}"))
+        })?;
+
+        for device_id in device_ids {
+            device_paths.entry(device_id).or_default().insert(canonical_path.clone());
+        }
+    }
+
+    if !missing_paths.is_empty() {
+        warn!(
+            missing_paths = ?missing_paths,
+            "Excluding non-existent local endpoint paths from physical disk independence validation during endpoint parsing",
+        );
+    }
+
+    let shared_devices = device_paths
+        .into_iter()
+        .filter_map(|(device_id, paths)| {
+            if paths.len() <= 1 {
+                return None;
+            }
+
+            Some((device_id, paths.into_iter().collect::<Vec<_>>()))
+        })
+        .collect::<Vec<_>>();
+
+    if shared_devices.is_empty() {
+        return Ok(());
+    }
+
+    let details = shared_devices
+        .into_iter()
+        .map(|(device_id, paths)| format!("{device_id} => {}", paths.join(", ")))
+        .collect::<Vec<_>>()
+        .join("; ");
+
+    Err(Error::other(format!(
+        "local erasure endpoints must use distinct physical disks; detected shared devices [{details}]. \
+Set {ENV_UNSAFE_BYPASS_DISK_CHECK}=true only for local testing or CI to bypass this safety check"
+    )))
+}
+
+fn validate_local_cross_device_mounts(local_paths: &[String]) -> Result<()> {
+    rustfs_utils::os::check_cross_device_mounts(local_paths)
+        .map_err(|err| Error::other(format!("local endpoint cross-device mount validation failed: {err}")))
+}
+
 #[cfg(test)]
 mod test {
     use rustfs_utils::must_get_local_ips;
 
     use super::*;
+
+    #[cfg(target_os = "linux")]
+    use serial_test::serial;
     use std::path::Path;
+    #[cfg(target_os = "linux")]
+    use temp_env::async_with_vars;
+    #[cfg(target_os = "linux")]
+    use tempfile::tempdir;
 
     #[test]
     fn test_new_endpoints() {
@@ -1412,4 +1521,69 @@ mod test {
             }
         }
     }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn reject_shared_local_physical_disks_by_default() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, None::<&str>)], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let err = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout)
+                .await
+                .unwrap_err();
+
+            let err_text = err.to_string();
+            assert!(err_text.contains("distinct physical disks"), "unexpected error: {err_text}");
+            assert!(err_text.contains(ENV_UNSAFE_BYPASS_DISK_CHECK), "unexpected error: {err_text}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_explicit_env_bypass() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected bypassed disk validation to succeed, got {ret:?}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_minio_ci_alias() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, Some("1"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected MINIO_CI alias to bypass disk validation, got {ret:?}");
+        })
+        .await;
+    }
 }

crates/ecstore/src/erasure_coding/encode.rs

@@ -106,7 +106,7 @@ impl<'a> MultiWriter<'a> {
             self.writers.len(),
             self.errs
                 .iter()
-                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .map(|e| e.as_ref().map_or_else(|| "<nil>".to_string(), |e| e.to_string()))
                 .collect::<Vec<_>>()
                 .join(", ")
         )))
@@ -168,7 +168,7 @@ impl<'a> MultiWriter<'a> {
             self.writers.len(),
             self.errs
                 .iter()
-                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .map(|e| e.as_ref().map_or_else(|| "<nil>".to_string(), |e| e.to_string()))
                 .collect::<Vec<_>>()
                 .join(", ")
         )))

crates/ecstore/src/notification_sys.rs

@@ -786,8 +786,9 @@ where
 
 fn offline_server_properties(host: &str, endpoints: &EndpointServerPools) -> ServerProperties {
     ServerProperties {
-        uptime: SystemTime::now()
-            .duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
+        uptime: GLOBAL_BOOT_TIME
+            .get()
+            .and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok())
             .unwrap_or_default()
             .as_secs(),
         version: get_commit_id(),

crates/ecstore/src/pools.rs

@@ -1405,7 +1405,8 @@ impl ECStore {
 
         let mut fivs = load_decommission_entry_versions(&entry, &bucket, "file_info_versions")?;
 
-        fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time));
+        fivs.versions
+            .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
 
         let mut decommissioned: usize = 0;
         let mut expired: usize = 0;

crates/ecstore/src/rebalance.rs

@@ -1530,7 +1530,8 @@ impl ECStore {
         let mut fivs =
             resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?;
 
-        fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time));
+        fivs.versions
+            .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
 
         let mut rebalanced: usize = 0;
         let mut expired: usize = 0;

crates/ecstore/src/rpc/client.rs

@@ -20,6 +20,8 @@ use std::error::Error;
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
 use tracing::debug;
 
+use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
+
 /// 3. Subsequent calls will attempt fresh connections
 /// 4. If node is still down, connection will fail fast (3s timeout)
 pub async fn node_service_time_out_client(
@@ -55,6 +57,8 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
     fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
         let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET);
         req.metadata_mut().as_mut().extend(headers);
+        inject_trace_context_into_metadata(req.metadata_mut());
+        inject_request_id_into_metadata(req.metadata_mut());
         Ok(req)
     }
 }
@@ -84,3 +88,34 @@ impl tonic::service::Interceptor for TonicInterceptor {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tonic::service::Interceptor;
+
+    #[test]
+    fn test_signature_interceptor_keeps_auth_headers() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        assert!(req.metadata().contains_key("x-rustfs-signature"));
+        assert!(req.metadata().contains_key("x-rustfs-timestamp"));
+    }
+
+    #[test]
+    fn test_signature_interceptor_may_inject_request_id() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let span = tracing::info_span!("grpc-rpc-test-span");
+        let _guard = span.enter();
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        if let Some(v) = req.metadata().get("x-request-id") {
+            assert!(!v.as_encoded_bytes().is_empty());
+        }
+    }
+}