fix(ecstore): log missing local paths during physical disk independence checks (#2680)

Co-authored-by: houseme <housemecn@gmail.com>

.docker/observability/README.md

@@ -13,6 +13,9 @@ The stack is composed of the following best-in-class open-source components:
 - **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
 - **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
 
+By default, this stack uses Tempo in single-binary mode and does not require Kafka/Redpanda.
+If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rustfs.yml` together with `docker-compose-tempo-ha-override.yml`.
+
 ## Architecture
 
 1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
@@ -46,6 +49,15 @@ Run the following command to start the entire stack:
 docker compose up -d
 ```
 
+### High Availability Tempo
+
+The default `docker-compose.yml` is the single-node stack.
+If you need the Kafka-backed HA Tempo configuration, start it with:
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### Access Dashboards
 
 | Service        | URL                                              | Credentials       | Description                    |

.docker/observability/README_ZH.md

@@ -13,6 +13,9 @@
 - **Jaeger** (v1.59.0): 分布式追踪系统（配置为辅助 UI/存储）。
 - **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
 
+默认情况下，这套技术栈使用 Tempo 单二进制模式，不依赖 Kafka/Redpanda。
+如果需要基于 Kafka 的 HA Tempo 路径，请使用 `docker-compose-example-for-rustfs.yml` 配合 `docker-compose-tempo-ha-override.yml`。
+
 ## 架构
 
 1.  **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据（指标、日志、追踪）发送到 **OpenTelemetry Collector**。
@@ -46,6 +49,15 @@
 docker compose up -d
 ```
 
+### Tempo 高可用模式
+
+默认的 `docker-compose.yml` 对应单机栈。
+如果需要基于 Kafka 的 HA Tempo 配置，请使用：
+
+```bash
+docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
+```
+
 ### 访问仪表盘
 
 | 服务 | URL | 凭据 | 描述 |

.docker/observability/docker-compose-example-for-rustfs.yml

@@ -85,10 +85,8 @@ services:
     networks:
       - otel-network
     restart: unless-stopped
-    depends_on:
-      - redpanda
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
@@ -205,7 +203,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/docker-compose.yml

@@ -16,10 +16,8 @@ services:
 
   # --- Tracing ---
   tempo:
-    image: grafana/tempo:2.10.3
+    image: grafana/tempo:2.10.5
     container_name: tempo
-    depends_on:
-      - redpanda
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml:ro
@@ -33,31 +31,11 @@ services:
       - otel-network
     restart: unless-stopped
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
+      test: [ "CMD", "/tempo", "-version" ]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
-  redpanda:
-    image: redpandadata/redpanda:latest
-    ports:
-      - "9092:9092" # Kafka API for clients
-    command: >
-      redpanda start --overprovisioned
-      --mode=dev-container
-      --kafka-addr=PLAINTEXT://0.0.0.0:9092
-      --advertise-kafka-addr=PLAINTEXT://redpanda:9092
-
-  redpanda-console:
-    image: docker.redpanda.com/redpandadata/console:latest
-    environment:
-      - CONFIG_FILEPATH=/etc/redpanda/redpanda-console-config.yaml
-    volumes:
-      - ./redpanda-console.yaml:/etc/redpanda/redpanda-console-config.yaml
-    ports:
-      - "8080:8080"
-    depends_on:
-      - redpanda
 
   vulture:
     image: grafana/tempo-vulture:latest
@@ -168,7 +146,7 @@ services:
       - prometheus
       - loki
     healthcheck:
-      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:13133" ]
+      test: [ "CMD", "/otelcol-contrib", "--version" ]
       interval: 10s
       timeout: 5s
       retries: 3

.docker/observability/grafana/provisioning/datasources.yaml

@@ -1,97 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    uid: prometheus
-    access: proxy
-    orgId: 1
-    url: http://prometheus:9090
-    isDefault: true
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      exemplarTraceIdDestinations:
-        - name: trace_id
-          datasourceUid: tempo
-
-  - name: Tempo
-    type: tempo
-    uid: tempo
-    access: proxy
-    orgId: 1
-    url: http://tempo:3200
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      httpMethod: GET
-      serviceMap:
-        datasourceUid: prometheus
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false
-      tracesToMetrics:
-        datasourceUid: prometheus
-        tags: [ { key: 'service.name' }, { key: 'job' } ]
-        queries:
-          - name: 'Service-Level Latency'
-            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m])) by (le)'
-          - name: 'Service-Level Calls'
-            query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))'
-          - name: 'Service-Level Errors'
-            query: 'sum(rate(traces_spanmetrics_calls_total{status_code="ERROR", $$__tags}[5m]))'
-      nodeGraph:
-        enabled: true
-
-  - name: Loki
-    type: loki
-    uid: loki
-    orgId: 1
-    url: http://loki:3100
-    isDefault: false
-    version: 1
-    editable: false
-    jsonData:
-      derivedFields:
-        - datasourceUid: tempo
-          matcherRegex: 'trace_id=(\w+)'
-          name: 'TraceID'
-          url: '$${__value.raw}'
-
-  - name: Jaeger
-    type: jaeger
-    uid: jaeger
-    url: http://jaeger:16686
-    access: proxy
-    isDefault: false
-    editable: false
-    jsonData:
-      tracesToLogs:
-        datasourceUid: loki
-        tags: [ 'job', 'instance', 'pod', 'namespace', 'service.name' ]
-        mappedTags: [ { key: 'service.name', value: 'app' } ]
-        spanStartTimeShift: '1s'
-        spanEndTimeShift: '-1s'
-        filterByTraceID: true
-        filterBySpanID: false

.docker/observability/tempo.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-partition_ring_live_store: true
 stream_over_http_enabled: true
 
 server:
@@ -33,33 +32,17 @@ distributor:
           endpoint: "0.0.0.0:4317"
         http:
           endpoint: "0.0.0.0:4318"
-  #log_received_spans:
-  #    enabled: true
-  #  log_discarded_spans:
-  #    enabled: true
 
-backend_scheduler:
-  provider:
-    compaction:
-      compaction:
-        block_retention: 1h
-
-backend_worker:
-  backend_scheduler_addr: localhost:3200
-  compaction:
-    block_retention: 1h
-  ring:
-    kvstore:
-      store: memberlist
-
-querier:
-  query_live_store: true
+ingester:
+  max_block_duration: 5m
 
 metrics_generator:
   registry:
     external_labels:
       source: tempo
       cluster: docker-compose
+  traces_storage:
+    path: /var/tempo/generator/traces
   storage:
     path: /var/tempo/generator/wal
     remote_write:
@@ -85,15 +68,5 @@ overrides:
       processors: [ "span-metrics", "service-graphs", "local-blocks" ]
       generate_native_histograms: both
 
-ingest:
-  enabled: true
-  kafka:
-    address: redpanda:9092
-    topic: tempo-ingest
-
-block_builder:
-  consume_cycle_duration: 30s
-
 usage_report:
   reporting_enabled: false
-

Cargo.toml

@@ -38,7 +38,7 @@ members = [
     "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
     "crates/protos", # Protocol buffer definitions
     "crates/rio", # Rust I/O utilities and abstractions
-    "crates/concurrency", # Rust I/O utilities and abstractions
+    "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
     "crates/s3-common", # Common utilities and data structures for S3 compatibility
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -115,14 +115,16 @@ rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.41" }
+async-compression = { version = "0.4.42" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
+async-nats = "0.47.0"
 axum = "0.8.9"
 futures = "0.3.32"
 futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
+pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
 hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
@@ -130,6 +132,7 @@ http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+rustfs-kafka-async = { version = "1.2.0" }
 socket2 = { version = "0.6.3", features = ["all"] }
 tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
@@ -165,10 +168,10 @@ crc-fast = "1.9.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
-pbkdf2 = "0.13.0-rc.10"
+pbkdf2 = "0.13.0"
 rsa = { version = "0.10.0-rc.17" }
-rustls = { version = "0.23.38", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
-rustls-pki-types = "1.14.0"
+rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls-pki-types = "1.14.1"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
 subtle = "2.6"
@@ -177,7 +180,7 @@ zeroize = { version = "1.8.2", features = ["derive"] }
 # Time and Date
 chrono = { version = "0.4.44", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.23", features = ["serde"] }
+jiff = { version = "0.2.24", features = ["serde"] }
 time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
 
 # Utilities and Tools
@@ -186,9 +189,9 @@ arc-swap = "1.9.1"
 astral-tokio-tar = "0.6.0"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.15" }
+aws-config = { version = "1.8.16" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.131.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.7" }
 backtrace = "0.3.76"
@@ -218,9 +221,9 @@ hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.185"
+libc = "0.2.186"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.11"
+local-ip-address = "0.6.12"
 memmap2 = "0.9.10"
 lz4 = "1.28.1"
 matchit = "0.9.2"
@@ -250,7 +253,7 @@ rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
 s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
 serial_test = "3.4.0"
-shadow-rs = { version = "1.7.1", default-features = false }
+shadow-rs = { version = "2.0.0", default-features = false }
 siphasher = "1.0.2"
 smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
@@ -289,12 +292,12 @@ opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rus
 opentelemetry_sdk = { version = "0.31.0" }
 opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
 opentelemetry-stdout = { version = "0.31.0" }
-pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] }
+pyroscope = { version = "2.0.2", features = ["backend-pprof-rs"] }
 
 # FTP and SFTP
 libunftp = { version = "0.23.0", features = ["experimental"] }
 unftp-core = "0.1.0"
-suppaftp = { version = "8.0.2", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
+suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"
 
 # WebDAV

_typos.toml

@@ -39,6 +39,7 @@ abd = "abd"
 mak = "mak"
 gae = "gae"
 GAE = "GAE"
+thr = "thr"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
 consts = "consts"

crates/audit/Cargo.toml

@@ -41,11 +41,10 @@ serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
-url = { workspace = true }
-rumqttc = { workspace = true }
 
 [dev-dependencies]
 temp-env = { workspace = true }
+url = { workspace = true }
 
 [lints]
 workspace = true

crates/audit/src/factory.rs

@@ -14,28 +14,19 @@
 
 use crate::AuditEntry;
 use async_trait::async_trait;
-use hashbrown::HashSet;
-use rumqttc::QoS;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
-use rustfs_config::{
-    AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
-    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
-    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
-};
+use rustfs_config::AUDIT_DEFAULT_DIR;
+use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
-    error::TargetError,
-    target::{
-        mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
-        webhook::WebhookArgs,
+    config::{
+        build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+        validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
     },
+    error::TargetError,
+    target::TargetType,
 };
-use std::time::Duration;
-use tracing::{debug, warn};
-use url::Url;
+use std::collections::HashSet;
 
 /// Trait for creating targets from configuration
 #[async_trait]
@@ -49,10 +40,6 @@ pub trait TargetFactory: Send + Sync {
     /// Returns a set of valid configuration field names for this target type.
     /// This is used to filter environment variables.
     fn get_valid_fields(&self) -> HashSet<String>;
-
-    /// Returns a set of valid configuration env field names for this target type.
-    /// This is used to filter environment variables.
-    fn get_valid_env_fields(&self) -> HashSet<String>;
 }
 
 /// Factory for creating Webhook targets
@@ -61,75 +48,18 @@ pub struct WebhookTargetFactory;
 #[async_trait]
 impl TargetFactory for WebhookTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        // All config values are now read directly from the merged `config` KVS.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        let parsed_endpoint = endpoint.trim();
-        let endpoint_url = Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let args = WebhookArgs {
-            enable: true, // If we are here, it's already enabled.
-            endpoint: endpoint_url,
-            auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
-            queue_dir: config
-                .lookup(WEBHOOK_QUEUE_DIR)
-                .unwrap_or_else(|| AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(WEBHOOK_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
-            client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
-            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
-            skip_tls_verify: config
-                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
-                .and_then(|v| v.parse::<bool>().ok())
-                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        // Validation also uses the merged `config` KVS directly.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        debug!("endpoint: {}", endpoint);
-        let parsed_endpoint = endpoint.trim();
-        Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let client_cert = config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default();
-        let client_key = config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default();
-
-        if client_cert.is_empty() != client_key.is_empty() {
-            return Err(TargetError::Configuration(
-                "Both client_cert and client_key must be specified together".to_string(),
-            ));
-        }
-
-        let queue_dir = config
-            .lookup(WEBHOOK_QUEUE_DIR)
-            .unwrap_or_else(|| AUDIT_DEFAULT_DIR.to_string());
-        if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
-            return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
-        }
-
-        Ok(())
+        validate_webhook_config(config, AUDIT_DEFAULT_DIR)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
     }
-
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
-    }
 }
 
 /// Factory for creating MQTT targets
@@ -138,112 +68,73 @@ pub struct MQTTTargetFactory;
 #[async_trait]
 impl TargetFactory for MQTTTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let broker_url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let topic = config
-            .lookup(MQTT_TOPIC)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
-
-        let args = MQTTArgs {
-            enable: true, // Assumed enabled.
-            broker: broker_url,
-            topic,
-            qos: config
-                .lookup(MQTT_QOS)
-                .and_then(|v| v.parse::<u8>().ok())
-                .map(|q| match q {
-                    0 => QoS::AtMostOnce,
-                    1 => QoS::AtLeastOnce,
-                    2 => QoS::ExactlyOnce,
-                    _ => QoS::AtLeastOnce,
-                })
-                .unwrap_or(QoS::AtLeastOnce),
-            username: config.lookup(MQTT_USERNAME).unwrap_or_default(),
-            password: config.lookup(MQTT_PASSWORD).unwrap_or_default(),
-            max_reconnect_interval: config
-                .lookup(MQTT_RECONNECT_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(5)),
-            keep_alive: config
-                .lookup(MQTT_KEEP_ALIVE_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(30)),
-            tls: MQTTTlsConfig::from_values(
-                config.lookup(MQTT_TLS_POLICY).as_deref(),
-                config.lookup(MQTT_TLS_CA).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-                config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-                config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-            )?,
-            queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or_else(|| AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(MQTT_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let tls = MQTTTlsConfig::from_values(
-            config.lookup(MQTT_TLS_POLICY).as_deref(),
-            config.lookup(MQTT_TLS_CA).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-            config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-            config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-        )?;
-        validate_mqtt_broker_url(&url, &tls)?;
-
-        if config.lookup(MQTT_TOPIC).is_none() {
-            return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
-        }
-
-        if let Some(qos_str) = config.lookup(MQTT_QOS) {
-            let qos = qos_str
-                .parse::<u8>()
-                .map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
-            if qos > 2 {
-                return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
-            }
-        }
-
-        let queue_dir = config.lookup(MQTT_QUEUE_DIR).unwrap_or_default();
-        if !queue_dir.is_empty() {
-            if !std::path::Path::new(&queue_dir).is_absolute() {
-                return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
-            }
-            if let Some(qos_str) = config.lookup(MQTT_QOS)
-                && qos_str == "0"
-            {
-                warn!("Using queue_dir with QoS 0 may result in event loss");
-            }
-        }
-
-        Ok(())
+        validate_mqtt_config(config)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
     }
+}
 
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
+pub struct NATSTargetFactory;
+
+#[async_trait]
+impl TargetFactory for NATSTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_nats_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_NATS_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct PulsarTargetFactory;
+
+#[async_trait]
+impl TargetFactory for PulsarTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_pulsar_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct KafkaTargetFactory;
+
+#[async_trait]
+impl TargetFactory for KafkaTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_kafka_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }

crates/audit/src/registry.rs

@@ -14,18 +14,19 @@
 
 use crate::{
     AuditEntry, AuditError, AuditResult,
-    factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory},
+    factory::{
+        KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
+    },
 };
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
-use hashbrown::{HashMap, HashSet};
-use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, audit::AUDIT_ROUTE_PREFIX};
+use hashbrown::HashMap;
+use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_targets::arn::TargetID;
-use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
-use std::str::FromStr;
+use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
 use std::sync::Arc;
-use tracing::{debug, error, info, warn};
+use tracing::{error, info};
 
 /// Registry for managing audit targets
 pub struct AuditRegistry {
@@ -52,6 +53,9 @@ impl AuditRegistry {
         // Register built-in factories
         registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
+        registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
+        registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
+        registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
 
         registry
     }
@@ -105,143 +109,22 @@ impl AuditRegistry {
         &self,
         config: &Config,
     ) -> AuditResult<Vec<Box<dyn Target<AuditEntry> + Send + Sync>>> {
-        // Collect only environment variables with the relevant prefix to reduce memory usage
-        let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
-        // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
             info!("Start working on target types...");
-
-            // 2. Prepare the configuration source
-            // 2.1. Get the configuration segment in the file, e.g. 'audit_webhook'
-            let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
-            let file_configs = config.0.get(&section_name).cloned().unwrap_or_default();
-            // 2.2. Get the default configuration for that type
-            let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
-            debug!(?default_cfg, "Get the default configuration");
-
-            // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
-            debug!(?valid_fields, "Get the legitimate configuration fields");
-
-            // 3. Resolve instance IDs and configuration overrides from environment variables
-            let mut instance_ids_from_env = HashSet::new();
-            // 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
-            let enable_prefix =
-                format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
-                    .to_uppercase();
-            for (key, value) in &all_env {
-                if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
-                    && let Some(id) = key.strip_prefix(&enable_prefix)
-                    && !id.is_empty()
-                {
-                    instance_ids_from_env.insert(id.to_lowercase());
-                }
-            }
-
-            // 3.2. Parse all relevant environment variable configurations
-            // 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
-            let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
-            // 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
-            let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
-            for (key, value) in &all_env {
-                if let Some(rest) = key.strip_prefix(&env_prefix) {
-                    // Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
-                    // Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
-                    let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
-
-                    // The first part from the right is INSTANCE_ID
-                    let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
-                    // The remaining part is FIELD_NAME
-                    let field_name_part = parts.next();
-
-                    let (field_name, instance_id) = match field_name_part {
-                        // Case 1: The format is <FIELD_NAME>_<INSTANCE_ID>
-                        // e.g., rest = "ENDPOINT_PRIMARY" -> field_name="ENDPOINT", instance_id="PRIMARY"
-                        Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
-                        // Case 2: The format is <FIELD_NAME> (without INSTANCE_ID)
-                        // e.g., rest = "ENABLE" -> field_name="ENABLE", instance_id="" (Universal configuration `_ DEFAULT_DELIMITER`)
-                        None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
-                    };
-
-                    // *** Optimization point 2: Verify whether the parsed field_name is legal ***
-                    if !field_name.is_empty() && valid_fields.contains(&field_name) {
-                        debug!(
-                            instance_id = %if instance_id.is_empty() { DEFAULT_DELIMITER } else { &instance_id },
-                            %field_name,
-                            %value,
-                            "Parsing to environment variables"
-                        );
-                        env_overrides
-                            .entry(instance_id)
-                            .or_default()
-                            .insert(field_name, value.clone());
-                    } else {
-                        // Ignore illegal field names
-                        warn!(
-                            field_name = %field_name,
-                            "Ignore environment variable fields, not found in the list of valid fields for target type {}",
-                            target_type
-                        );
-                    }
-                }
-            }
-            debug!(?env_overrides, "Complete the environment variable analysis");
-
-            // 4. Determine all instance IDs that need to be processed
-            let mut all_instance_ids: HashSet<String> =
-                file_configs.keys().filter(|k| *k != DEFAULT_DELIMITER).cloned().collect();
-            all_instance_ids.extend(instance_ids_from_env);
-            debug!(?all_instance_ids, "Determine all instance IDs");
-
-            // 5. Merge configurations and create tasks for each instance
-            for id in all_instance_ids {
-                // 5.1. Merge configuration, priority: Environment variables > File instance configuration > File default configuration
-                let mut merged_config = default_cfg.clone();
-                // Instance-specific configuration in application files
-                if let Some(file_instance_cfg) = file_configs.get(&id) {
-                    merged_config.extend(file_instance_cfg.clone());
-                }
-                // Application instance-specific environment variable configuration
-                if let Some(env_instance_cfg) = env_overrides.get(&id) {
-                    // Convert HashMap<String, String> to KVS
-                    let mut kvs_from_env = KVS::new();
-                    for (k, v) in env_instance_cfg {
-                        kvs_from_env.insert(k.clone(), v.clone());
-                    }
-                    merged_config.extend(kvs_from_env);
-                }
-                debug!(instance_id = %id, ?merged_config, "Complete configuration merge");
-
-                // 5.2. Check if the instance is enabled
-                let enabled = merged_config
-                    .lookup(ENABLE_KEY)
-                    .map(|v| {
-                        EnableState::from_str(v.as_str())
-                            .ok()
-                            .map(|s| s.is_enabled())
-                            .unwrap_or(false)
-                    })
-                    .unwrap_or(false);
-
-                if enabled {
-                    info!(instance_id = %id, "Target is enabled, ready to create a task");
-                    // 5.3. Create asynchronous tasks for enabled instances
-                    let tid = id.clone();
-                    let merged_config_arc = Arc::new(merged_config);
-                    tasks.push(async move {
-                        let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (tid, result)
-                    });
-                } else {
-                    info!(instance_id = %id, "Skip disabled target");
-                }
+            for (id, merged_config) in collect_target_configs(config, AUDIT_ROUTE_PREFIX, target_type, &valid_fields) {
+                info!(instance_id = %id, "Target is enabled, ready to create a task");
+                let tid = id.clone();
+                let merged_config_arc = Arc::new(merged_config);
+                tasks.push(async move {
+                    let result = factory.create_target(tid.clone(), &merged_config_arc).await;
+                    (tid, result)
+                });
             }
         }
 
-        // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
         while let Some((id, result)) = tasks.next().await {
             match result {
@@ -317,8 +200,8 @@ impl AuditRegistry {
             }
         }
 
-        if !errors.is_empty() {
-            return Err(AuditError::Target(errors.into_iter().next().unwrap()));
+        if let Some(error) = errors.into_iter().next() {
+            return Err(AuditError::Target(error));
         }
 
         Ok(())

crates/config/src/audit/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Kafka Environment Variables
+pub const ENV_AUDIT_KAFKA_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_ENABLE";
+pub const ENV_AUDIT_KAFKA_BROKERS: &str = "RUSTFS_AUDIT_KAFKA_BROKERS";
+pub const ENV_AUDIT_KAFKA_TOPIC: &str = "RUSTFS_AUDIT_KAFKA_TOPIC";
+pub const ENV_AUDIT_KAFKA_ACKS: &str = "RUSTFS_AUDIT_KAFKA_ACKS";
+pub const ENV_AUDIT_KAFKA_TLS_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_TLS_ENABLE";
+pub const ENV_AUDIT_KAFKA_TLS_CA: &str = "RUSTFS_AUDIT_KAFKA_TLS_CA";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_KAFKA_QUEUE_DIR: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_DIR";
+pub const ENV_AUDIT_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_AUDIT_KAFKA_ENABLE,
+    ENV_AUDIT_KAFKA_BROKERS,
+    ENV_AUDIT_KAFKA_TOPIC,
+    ENV_AUDIT_KAFKA_ACKS,
+    ENV_AUDIT_KAFKA_TLS_ENABLE,
+    ENV_AUDIT_KAFKA_TLS_CA,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_CERT,
+    ENV_AUDIT_KAFKA_TLS_CLIENT_KEY,
+    ENV_AUDIT_KAFKA_QUEUE_DIR,
+    ENV_AUDIT_KAFKA_QUEUE_LIMIT,
+];
+
+/// A list of all valid configuration keys for a Kafka audit target.
+pub const AUDIT_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/mod.rs

@@ -16,10 +16,16 @@
 //! This module defines the configuration for audit systems, including
 //! webhook and MQTT audit-related settings.
 
+mod kafka;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod webhook;
 
+pub use kafka::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use webhook::*;
 
 use crate::DEFAULT_DELIMITER;
@@ -29,8 +35,17 @@ pub const AUDIT_PREFIX: &str = "audit";
 pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
 
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
+pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
 pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
+pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
+pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
 
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
 #[allow(dead_code)]
-pub const AUDIT_SUB_SYSTEMS: &[&str] = &[AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS];
+pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
+    AUDIT_KAFKA_SUB_SYS,
+    AUDIT_MQTT_SUB_SYS,
+    AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_SUB_SYS,
+    AUDIT_WEBHOOK_SUB_SYS,
+];

crates/config/src/audit/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_NATS_ENABLE: &str = "RUSTFS_AUDIT_NATS_ENABLE";
+pub const ENV_AUDIT_NATS_ADDRESS: &str = "RUSTFS_AUDIT_NATS_ADDRESS";
+pub const ENV_AUDIT_NATS_SUBJECT: &str = "RUSTFS_AUDIT_NATS_SUBJECT";
+pub const ENV_AUDIT_NATS_USERNAME: &str = "RUSTFS_AUDIT_NATS_USERNAME";
+pub const ENV_AUDIT_NATS_PASSWORD: &str = "RUSTFS_AUDIT_NATS_PASSWORD";
+pub const ENV_AUDIT_NATS_TOKEN: &str = "RUSTFS_AUDIT_NATS_TOKEN";
+pub const ENV_AUDIT_NATS_CREDENTIALS_FILE: &str = "RUSTFS_AUDIT_NATS_CREDENTIALS_FILE";
+pub const ENV_AUDIT_NATS_TLS_CA: &str = "RUSTFS_AUDIT_NATS_TLS_CA";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
+pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
+pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
+    ENV_AUDIT_NATS_ENABLE,
+    ENV_AUDIT_NATS_ADDRESS,
+    ENV_AUDIT_NATS_SUBJECT,
+    ENV_AUDIT_NATS_USERNAME,
+    ENV_AUDIT_NATS_PASSWORD,
+    ENV_AUDIT_NATS_TOKEN,
+    ENV_AUDIT_NATS_CREDENTIALS_FILE,
+    ENV_AUDIT_NATS_TLS_CA,
+    ENV_AUDIT_NATS_TLS_CLIENT_CERT,
+    ENV_AUDIT_NATS_TLS_CLIENT_KEY,
+    ENV_AUDIT_NATS_TLS_REQUIRED,
+    ENV_AUDIT_NATS_QUEUE_DIR,
+    ENV_AUDIT_NATS_QUEUE_LIMIT,
+];
+
+pub const AUDIT_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_PULSAR_ENABLE: &str = "RUSTFS_AUDIT_PULSAR_ENABLE";
+pub const ENV_AUDIT_PULSAR_BROKER: &str = "RUSTFS_AUDIT_PULSAR_BROKER";
+pub const ENV_AUDIT_PULSAR_TOPIC: &str = "RUSTFS_AUDIT_PULSAR_TOPIC";
+pub const ENV_AUDIT_PULSAR_AUTH_TOKEN: &str = "RUSTFS_AUDIT_PULSAR_AUTH_TOKEN";
+pub const ENV_AUDIT_PULSAR_USERNAME: &str = "RUSTFS_AUDIT_PULSAR_USERNAME";
+pub const ENV_AUDIT_PULSAR_PASSWORD: &str = "RUSTFS_AUDIT_PULSAR_PASSWORD";
+pub const ENV_AUDIT_PULSAR_TLS_CA: &str = "RUSTFS_AUDIT_PULSAR_TLS_CA";
+pub const ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_AUDIT_PULSAR_QUEUE_DIR: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_DIR";
+pub const ENV_AUDIT_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_AUDIT_PULSAR_ENABLE,
+    ENV_AUDIT_PULSAR_BROKER,
+    ENV_AUDIT_PULSAR_TOPIC,
+    ENV_AUDIT_PULSAR_AUTH_TOKEN,
+    ENV_AUDIT_PULSAR_USERNAME,
+    ENV_AUDIT_PULSAR_PASSWORD,
+    ENV_AUDIT_PULSAR_TLS_CA,
+    ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_AUDIT_PULSAR_QUEUE_DIR,
+    ENV_AUDIT_PULSAR_QUEUE_LIMIT,
+];
+
+pub const AUDIT_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/constants/app.rs

@@ -131,6 +131,18 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
 /// Environment variable for server volumes.
 pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
 
+/// Environment variable to explicitly bypass local physical disk independence checks.
+pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
+
+/// Compatibility alias used by legacy MinIO CI pipelines.
+///
+/// RustFS keeps this alias for backward compatibility only. Prefer
+/// `ENV_UNSAFE_BYPASS_DISK_CHECK` for explicit bypass control.
+pub const ENV_MINIO_CI: &str = "MINIO_CI";
+
+/// Default flag value for bypassing local physical disk independence checks.
+pub const DEFAULT_UNSAFE_BYPASS_DISK_CHECK: bool = false;
+
 /// Environment variable for server access key.
 pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
 

crates/config/src/constants/capacity.rs

@@ -39,6 +39,9 @@ pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
 /// Environment variable for sample rate
 pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
 
+/// Environment variable for metrics logging interval
+pub const ENV_CAPACITY_METRICS_INTERVAL: &str = "RUSTFS_CAPACITY_METRICS_INTERVAL";
+
 /// Environment variable for following symbolic links during capacity calculation
 pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
 
@@ -89,6 +92,10 @@ pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
 /// Default: 200
 pub const DEFAULT_SAMPLE_RATE: usize = 200;
 
+/// Capacity metrics logging interval in seconds
+/// Default: 600 seconds (10 minutes)
+pub const DEFAULT_CAPACITY_METRICS_INTERVAL_SECS: u64 = 600;
+
 /// Follow symbolic links during capacity calculation
 /// Default: false (disabled for safety)
 pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
@@ -130,6 +137,7 @@ mod tests {
         assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
         assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
         assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
+        assert_eq!(ENV_CAPACITY_METRICS_INTERVAL, "RUSTFS_CAPACITY_METRICS_INTERVAL");
         assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
         assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
         assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
@@ -147,6 +155,7 @@ mod tests {
         assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
         assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
         assert_eq!(DEFAULT_SAMPLE_RATE, 200);
+        assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
         assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
         assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2);
         assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15);

crates/config/src/constants/env.rs

@@ -26,6 +26,15 @@ pub const RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT: bool = false;
 pub const ENABLE_KEY: &str = "enable";
 pub const COMMENT_KEY: &str = "comment";
 
+/// Global switch for enabling the audit module.
+pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
+/// Global switch for enabling the notify module.
+pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
+/// Default global audit switch (disabled by default).
+pub const DEFAULT_AUDIT_ENABLE: bool = false;
+/// Default global notify switch (disabled by default).
+pub const DEFAULT_NOTIFY_ENABLE: bool = false;
+
 /// Medium-drawn lines separator
 /// This is used to separate words in environment variable names.
 pub const ENV_WORD_DELIMITER_DASH: &str = "-";
@@ -290,4 +299,10 @@ mod tests {
             assert!(state.is_disabled());
         }
     }
+
+    #[test]
+    fn test_global_audit_notify_switch_constants() {
+        assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
+        assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
+    }
 }

crates/config/src/constants/health.rs

@@ -0,0 +1,28 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Enable or disable public `/health` and `/health/ready` endpoints.
+/// When disabled, the routes are not registered and return 404.
+pub const ENV_HEALTH_ENDPOINT_ENABLE: &str = "RUSTFS_HEALTH_ENDPOINT_ENABLE";
+pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
+
+/// Cache TTL for storage readiness runtime-state evaluation (milliseconds).
+/// This reduces storage-layer pressure when probes are called at high frequency.
+pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
+pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
+
+/// Enable minimal health payload mode for GET `/health*` responses.
+/// When enabled, only `status` and `ready` fields are returned.
+pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
+pub const DEFAULT_HEALTH_MINIMAL_RESPONSE_ENABLE: bool = false;

crates/config/src/constants/mod.rs

@@ -20,6 +20,7 @@ pub(crate) mod console;
 pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
+pub(crate) mod health;
 pub(crate) mod object;
 pub(crate) mod oidc;
 pub(crate) mod profiler;

crates/config/src/constants/oidc.rs

@@ -17,6 +17,7 @@ pub const OIDC_CONFIG_URL: &str = "config_url";
 pub const OIDC_CLIENT_ID: &str = "client_id";
 pub const OIDC_CLIENT_SECRET: &str = "client_secret";
 pub const OIDC_SCOPES: &str = "scopes";
+pub const OIDC_OTHER_AUDIENCES: &str = "other_audiences";
 pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
 pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
 pub const OIDC_CLAIM_NAME: &str = "claim_name";
@@ -34,6 +35,7 @@ pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_
 pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
 pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
 pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
+pub const ENV_IDENTITY_OPENID_OTHER_AUDIENCES: &str = "RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
 pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
@@ -46,12 +48,13 @@ pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_
 pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
 
 /// List of all environment variable keys for an OIDC provider.
-pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 15] = &[
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 16] = &[
     ENV_IDENTITY_OPENID_ENABLE,
     ENV_IDENTITY_OPENID_CONFIG_URL,
     ENV_IDENTITY_OPENID_CLIENT_ID,
     ENV_IDENTITY_OPENID_CLIENT_SECRET,
     ENV_IDENTITY_OPENID_SCOPES,
+    ENV_IDENTITY_OPENID_OTHER_AUDIENCES,
     ENV_IDENTITY_OPENID_REDIRECT_URI,
     ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
     ENV_IDENTITY_OPENID_CLAIM_NAME,
@@ -71,6 +74,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
     OIDC_CLIENT_ID,
     OIDC_CLIENT_SECRET,
     OIDC_SCOPES,
+    OIDC_OTHER_AUDIENCES,
     OIDC_REDIRECT_URI,
     OIDC_REDIRECT_URI_DYNAMIC,
     OIDC_CLAIM_NAME,

crates/config/src/constants/proxy.rs

@@ -20,6 +20,11 @@ pub const ENV_TRUSTED_PROXY_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_ENABLED";
 /// Trusted proxy middleware is enabled by default.
 pub const DEFAULT_TRUSTED_PROXY_ENABLED: bool = true;
 
+/// Environment variable to select the trusted proxy implementation.
+pub const ENV_TRUSTED_PROXY_IMPLEMENTATION: &str = "RUSTFS_TRUSTED_PROXY_IMPLEMENTATION";
+/// The simplified implementation is used by default.
+pub const DEFAULT_TRUSTED_PROXY_IMPLEMENTATION: &str = "simple";
+
 /// Environment variable for the proxy validation mode.
 pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDATION_MODE";
 /// Default validation mode is "hop_by_hop".

crates/config/src/constants/targets.rs

@@ -40,6 +40,39 @@ pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
 pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
 pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
+pub const KAFKA_BROKERS: &str = "brokers";
+pub const KAFKA_TOPIC: &str = "topic";
+pub const KAFKA_ACKS: &str = "acks";
+pub const KAFKA_QUEUE_DIR: &str = "queue_dir";
+pub const KAFKA_QUEUE_LIMIT: &str = "queue_limit";
+pub const KAFKA_TLS_ENABLE: &str = "tls_enable";
+pub const KAFKA_TLS_CA: &str = "tls_ca";
+pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key";
+
+pub const NATS_ADDRESS: &str = "address";
+pub const NATS_SUBJECT: &str = "subject";
+pub const NATS_USERNAME: &str = "username";
+pub const NATS_PASSWORD: &str = "password";
+pub const NATS_TOKEN: &str = "token";
+pub const NATS_CREDENTIALS_FILE: &str = "credentials_file";
+pub const NATS_TLS_CA: &str = "tls_ca";
+pub const NATS_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
+pub const NATS_TLS_REQUIRED: &str = "tls_required";
+pub const NATS_QUEUE_DIR: &str = "queue_dir";
+pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
+
+pub const PULSAR_BROKER: &str = "broker";
+pub const PULSAR_TOPIC: &str = "topic";
+pub const PULSAR_AUTH_TOKEN: &str = "auth_token";
+pub const PULSAR_USERNAME: &str = "username";
+pub const PULSAR_PASSWORD: &str = "password";
+pub const PULSAR_TLS_CA: &str = "tls_ca";
+pub const PULSAR_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure";
+pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification";
+pub const PULSAR_QUEUE_DIR: &str = "queue_dir";
+pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit";
 
 /// Environment variable controlling whether target queue files are Snappy-compressed.
 /// Applies to both notify and audit target queue stores.

crates/config/src/lib.rs

@@ -31,6 +31,8 @@ pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::heal::*;
 #[cfg(feature = "constants")]
+pub use constants::health::*;
+#[cfg(feature = "constants")]
 pub use constants::object::*;
 #[cfg(feature = "constants")]
 pub use constants::profiler::*;

crates/config/src/notify/kafka.rs

@@ -0,0 +1,53 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// A list of all valid configuration keys for a Kafka target.
+pub const NOTIFY_KAFKA_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::KAFKA_BROKERS,
+    crate::KAFKA_TOPIC,
+    crate::KAFKA_ACKS,
+    crate::KAFKA_TLS_ENABLE,
+    crate::KAFKA_TLS_CA,
+    crate::KAFKA_TLS_CLIENT_CERT,
+    crate::KAFKA_TLS_CLIENT_KEY,
+    crate::KAFKA_QUEUE_DIR,
+    crate::KAFKA_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+// Kafka Environment Variables
+pub const ENV_NOTIFY_KAFKA_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_ENABLE";
+pub const ENV_NOTIFY_KAFKA_BROKERS: &str = "RUSTFS_NOTIFY_KAFKA_BROKERS";
+pub const ENV_NOTIFY_KAFKA_TOPIC: &str = "RUSTFS_NOTIFY_KAFKA_TOPIC";
+pub const ENV_NOTIFY_KAFKA_ACKS: &str = "RUSTFS_NOTIFY_KAFKA_ACKS";
+pub const ENV_NOTIFY_KAFKA_TLS_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_TLS_ENABLE";
+pub const ENV_NOTIFY_KAFKA_TLS_CA: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CA";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_KAFKA_QUEUE_DIR: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_DIR";
+pub const ENV_NOTIFY_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_KAFKA_KEYS: &[&str; 10] = &[
+    ENV_NOTIFY_KAFKA_ENABLE,
+    ENV_NOTIFY_KAFKA_BROKERS,
+    ENV_NOTIFY_KAFKA_TOPIC,
+    ENV_NOTIFY_KAFKA_ACKS,
+    ENV_NOTIFY_KAFKA_TLS_ENABLE,
+    ENV_NOTIFY_KAFKA_TLS_CA,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT,
+    ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY,
+    ENV_NOTIFY_KAFKA_QUEUE_DIR,
+    ENV_NOTIFY_KAFKA_QUEUE_LIMIT,
+];

crates/config/src/notify/mod.rs

@@ -13,12 +13,18 @@
 // limitations under the License.
 
 mod arn;
+mod kafka;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod store;
 mod webhook;
 
 pub use arn::*;
+pub use kafka::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use store::*;
 pub use webhook::*;
 
@@ -64,9 +70,14 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
 pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
 
 #[allow(dead_code)]
-pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
+pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
+    NOTIFY_KAFKA_SUB_SYS,
+    NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_SUB_SYS,
+    NOTIFY_WEBHOOK_SUB_SYS,
+];
 
-#[allow(dead_code)]
 pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
 pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
 #[allow(dead_code)]
@@ -83,4 +94,5 @@ pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
 pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
 #[allow(dead_code)]
 pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
+pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
 pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";

crates/config/src/notify/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_NATS_ENABLE: &str = "RUSTFS_NOTIFY_NATS_ENABLE";
+pub const ENV_NOTIFY_NATS_ADDRESS: &str = "RUSTFS_NOTIFY_NATS_ADDRESS";
+pub const ENV_NOTIFY_NATS_SUBJECT: &str = "RUSTFS_NOTIFY_NATS_SUBJECT";
+pub const ENV_NOTIFY_NATS_USERNAME: &str = "RUSTFS_NOTIFY_NATS_USERNAME";
+pub const ENV_NOTIFY_NATS_PASSWORD: &str = "RUSTFS_NOTIFY_NATS_PASSWORD";
+pub const ENV_NOTIFY_NATS_TOKEN: &str = "RUSTFS_NOTIFY_NATS_TOKEN";
+pub const ENV_NOTIFY_NATS_CREDENTIALS_FILE: &str = "RUSTFS_NOTIFY_NATS_CREDENTIALS_FILE";
+pub const ENV_NOTIFY_NATS_TLS_CA: &str = "RUSTFS_NOTIFY_NATS_TLS_CA";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
+pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
+pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
+    ENV_NOTIFY_NATS_ENABLE,
+    ENV_NOTIFY_NATS_ADDRESS,
+    ENV_NOTIFY_NATS_SUBJECT,
+    ENV_NOTIFY_NATS_USERNAME,
+    ENV_NOTIFY_NATS_PASSWORD,
+    ENV_NOTIFY_NATS_TOKEN,
+    ENV_NOTIFY_NATS_CREDENTIALS_FILE,
+    ENV_NOTIFY_NATS_TLS_CA,
+    ENV_NOTIFY_NATS_TLS_CLIENT_CERT,
+    ENV_NOTIFY_NATS_TLS_CLIENT_KEY,
+    ENV_NOTIFY_NATS_TLS_REQUIRED,
+    ENV_NOTIFY_NATS_QUEUE_DIR,
+    ENV_NOTIFY_NATS_QUEUE_LIMIT,
+];

crates/config/src/notify/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_PULSAR_ENABLE: &str = "RUSTFS_NOTIFY_PULSAR_ENABLE";
+pub const ENV_NOTIFY_PULSAR_BROKER: &str = "RUSTFS_NOTIFY_PULSAR_BROKER";
+pub const ENV_NOTIFY_PULSAR_TOPIC: &str = "RUSTFS_NOTIFY_PULSAR_TOPIC";
+pub const ENV_NOTIFY_PULSAR_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_PULSAR_AUTH_TOKEN";
+pub const ENV_NOTIFY_PULSAR_USERNAME: &str = "RUSTFS_NOTIFY_PULSAR_USERNAME";
+pub const ENV_NOTIFY_PULSAR_PASSWORD: &str = "RUSTFS_NOTIFY_PULSAR_PASSWORD";
+pub const ENV_NOTIFY_PULSAR_TLS_CA: &str = "RUSTFS_NOTIFY_PULSAR_TLS_CA";
+pub const ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_NOTIFY_PULSAR_QUEUE_DIR: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_DIR";
+pub const ENV_NOTIFY_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_NOTIFY_PULSAR_ENABLE,
+    ENV_NOTIFY_PULSAR_BROKER,
+    ENV_NOTIFY_PULSAR_TOPIC,
+    ENV_NOTIFY_PULSAR_AUTH_TOKEN,
+    ENV_NOTIFY_PULSAR_USERNAME,
+    ENV_NOTIFY_PULSAR_PASSWORD,
+    ENV_NOTIFY_PULSAR_TLS_CA,
+    ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_NOTIFY_PULSAR_QUEUE_DIR,
+    ENV_NOTIFY_PULSAR_QUEUE_LIMIT,
+];

crates/config/src/observability/mod.rs

@@ -22,6 +22,14 @@ pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
 pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT";
 pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT";
 pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT";
+pub const ENV_OBS_ENDPOINT_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_HEADERS";
+pub const ENV_OBS_ENDPOINT_TRACES_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS";
+pub const ENV_OBS_ENDPOINT_METRICS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS";
+pub const ENV_OBS_ENDPOINT_LOGS_HEADERS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS";
+pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
+pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
 pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
@@ -108,6 +116,14 @@ mod tests {
         assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT");
         assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT");
         assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT");
+        assert_eq!(ENV_OBS_ENDPOINT_HEADERS, "RUSTFS_OBS_ENDPOINT_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_HEADERS, "RUSTFS_OBS_ENDPOINT_TRACES_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_HEADERS, "RUSTFS_OBS_ENDPOINT_METRICS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_HEADERS, "RUSTFS_OBS_ENDPOINT_LOGS_HEADERS");
+        assert_eq!(ENV_OBS_ENDPOINT_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS");
+        assert_eq!(ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS, "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS");
         assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT");
         assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
         assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");

crates/e2e_test/Cargo.toml

@@ -73,3 +73,4 @@ rcgen.workspace = true
 anyhow.workspace = true
 rustls.workspace = true
 zip.workspace = true
+clap.workspace = true

crates/e2e_test/src/bin/tls_gen.rs

@@ -0,0 +1,22 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Parser;
+use e2e_test::tls_gen::{Args, run};
+
+fn main() -> anyhow::Result<()> {
+    let out_dir = run(Args::parse())?;
+    println!("Generated RustFS TLS bundle in {}", out_dir.display());
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -125,3 +125,5 @@ mod replication_extension_test;
 
 #[cfg(test)]
 mod snowball_auto_extract_test;
+
+pub mod tls_gen;

crates/e2e_test/src/list_object_versions_regression_test.rs

@@ -19,6 +19,7 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
     use serial_test::serial;
     use tracing::info;
@@ -179,4 +180,84 @@ mod tests {
             "Delete marker should no longer be latest after the second put"
         );
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_list_object_versions_prefix_with_marker_object_returns_children() {
+        init_logging();
+        info!("🧪 TEST: ListObjectVersions returns prefix children when a marker object also exists");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-list-versions-prefix-marker";
+        let marker_key = "data01";
+        let child_keys = [
+            "data01/meta/dump-2026-04-08-053205.json.gz",
+            "data01/meta/dump-2026-04-08-063209.json.gz",
+        ];
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(
+                VersioningConfiguration::builder()
+                    .status(BucketVersioningStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("Failed to suspend versioning");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(marker_key)
+            .body(ByteStream::from_static(b""))
+            .send()
+            .await
+            .expect("Failed to put marker object");
+
+        for key in child_keys {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .body(ByteStream::from_static(b"payload"))
+                .send()
+                .await
+                .expect("Failed to put child object");
+        }
+
+        let listing = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix("data01/")
+            .send()
+            .await
+            .expect("Failed to list object versions by prefix");
+
+        let version_keys: Vec<_> = listing.versions().iter().filter_map(|version| version.key()).collect();
+
+        assert_eq!(
+            version_keys.len(),
+            child_keys.len(),
+            "ListObjectVersions with a trailing slash prefix should include child objects even when the marker object exists"
+        );
+
+        for key in child_keys {
+            assert!(
+                version_keys.contains(&key),
+                "ListObjectVersions(prefix=data01/) should include child object {key}"
+            );
+        }
+    }
 }

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -120,11 +120,6 @@ impl LockClient for GrpcLockClient {
             .map_err(|e| LockError::internal(e.to_string()))?
             .into_inner();
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -134,7 +129,8 @@ impl LockClient for GrpcLockClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -50,6 +50,18 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
     }
 }
 
+fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> GenerallyLockResult {
+    if success {
+        GenerallyLockResult {
+            success: true,
+            error_info: None,
+            lock_info: None,
+        }
+    } else {
+        lock_result_from_error(format!("lock not found for release: {lock_id}"))
+    }
+}
+
 /// Minimal NodeService implementation that only supports Lock RPCs
 /// Used for testing distributed lock scenarios with real gRPC
 #[derive(Debug)]
@@ -102,7 +114,7 @@ impl NodeService for MinimalLockNodeService {
                 let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
                 Ok(Response::new(GenerallyLockResponse {
                     success: result.success,
-                    error_info: None,
+                    error_info: result.error,
                     lock_info: lock_info_json,
                 }))
             }
@@ -131,11 +143,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -161,11 +176,14 @@ impl NodeService for MinimalLockNodeService {
         };
 
         match self.lock_client.force_release(&args.lock_id).await {
-            Ok(success) => Ok(Response::new(GenerallyLockResponse {
-                success,
-                error_info: None,
-                lock_info: None,
-            })),
+            Ok(success) => {
+                let result = lock_result_from_release(&args.lock_id, success);
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: result.error_info,
+                    lock_info: None,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
@@ -271,10 +289,9 @@ impl NodeService for MinimalLockNodeService {
                 Ok(batch_results) => {
                     for (result_idx, success) in batch_results.into_iter().enumerate() {
                         if let Some(request_idx) = valid_indices.get(result_idx) {
-                            results[*request_idx] = GenerallyLockResult {
-                                success,
-                                error_info: None,
-                                lock_info: None,
+                            results[*request_idx] = match lock_ids.get(result_idx) {
+                                Some(lock_id) => lock_result_from_release(lock_id, success),
+                                None => lock_result_from_error(format!("unlock response index out of range: {result_idx}")),
                             };
                         }
                     }

crates/e2e_test/src/reliant/head_tls_bodyless_test.rs

@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for TLS/HTTP2 `HEAD` responses on missing objects.
+//!
+//! Before the fix, RustFS returned `404` for a missing object but still wrote
+//! the XML error payload on a `HEAD` request. Under HTTP/2 this emitted DATA
+//! frames after the response headers, which clients surfaced as a protocol
+//! error. This test keeps the request at the raw HTTPS layer so it can validate
+//! the final wire-facing behavior rather than SDK-level error mapping.
+
+#![cfg(test)]
+
+use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use http::Version;
+use http::header::HOST;
+use rcgen::generate_simple_self_signed;
+use reqwest::{Certificate, Client, Response, StatusCode};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+use std::path::Path;
+use std::process::Command;
+use tokio::fs;
+use tokio::time::{Duration, sleep};
+use tracing::info;
+
+const ACCESS_KEY: &str = "rustfsadmin";
+const SECRET_KEY: &str = "rustfsadmin";
+const BUCKET: &str = "test-head-tls-bodyless-bucket";
+
+async fn generate_tls_bundle(tls_dir: &Path) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    fs::create_dir_all(tls_dir).await?;
+    let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
+    let cert_pem = cert.cert.pem();
+    let key_pem = cert.signing_key.serialize_pem();
+
+    fs::write(tls_dir.join("rustfs_cert.pem"), cert_pem.as_bytes()).await?;
+    fs::write(tls_dir.join("rustfs_key.pem"), key_pem.as_bytes()).await?;
+
+    Ok(cert_pem.into_bytes())
+}
+
+fn local_https_h2_client(ca_pem: &[u8]) -> Result<Client, Box<dyn Error + Send + Sync>> {
+    let _ca_cert = Certificate::from_pem(ca_pem)?;
+    Ok(Client::builder()
+        .no_proxy()
+        .no_gzip()
+        .no_brotli()
+        .no_zstd()
+        .no_deflate()
+        .danger_accept_invalid_certs(true)
+        .build()?)
+}
+
+async fn signed_empty_request(
+    client: &Client,
+    method: http::Method,
+    url: &str,
+) -> Result<Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let request = http::Request::builder()
+        .method(method.as_str())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+
+    let signed = sign_v4(request, 0, ACCESS_KEY, SECRET_KEY, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        builder = builder.header(name, value);
+    }
+
+    Ok(builder.send().await?)
+}
+
+async fn ensure_bucket_exists(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let bucket_url = format!("{endpoint}/{BUCKET}/");
+    let response = signed_empty_request(client, http::Method::HEAD, &bucket_url).await?;
+
+    if response.status() == StatusCode::OK {
+        return Ok(());
+    }
+
+    let response = signed_empty_request(client, http::Method::PUT, &bucket_url).await?;
+    match response.status() {
+        StatusCode::OK => Ok(()),
+        StatusCode::CONFLICT => Ok(()),
+        status => Err(format!("unexpected bucket setup status: {status}").into()),
+    }
+}
+
+async fn wait_for_tls_server_ready(client: &Client, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let ready_url = format!("{endpoint}/");
+    for _attempt in 0..60 {
+        match signed_empty_request(client, http::Method::GET, &ready_url).await {
+            Ok(response) if response.status().is_success() => return Ok(()),
+            Ok(_) | Err(_) => sleep(Duration::from_millis(500)).await,
+        }
+    }
+
+    Err("RustFS TLS server failed to become ready within 30 seconds".into())
+}
+
+async fn start_tls_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let binary_path = rustfs_binary_path();
+    let mut command = Command::new(&binary_path);
+    command
+        .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .current_dir(&env.temp_dir);
+
+    for key in [
+        "RUSTFS_ADDRESS",
+        "RUSTFS_VOLUMES",
+        "RUSTFS_ACCESS_KEY",
+        "RUSTFS_SECRET_KEY",
+        "RUSTFS_TLS_PATH",
+        "RUSTFS_OBS_LOG_DIRECTORY",
+    ] {
+        command.env_remove(key);
+    }
+
+    let process = command
+        .env("RUSTFS_TLS_PATH", tls_dir)
+        .env("RUSTFS_CONSOLE_ENABLE", "false")
+        .args([
+            "--address",
+            &env.address,
+            "--access-key",
+            &env.access_key,
+            "--secret-key",
+            &env.secret_key,
+            &env.temp_dir,
+        ])
+        .spawn()?;
+
+    env.process = Some(process);
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_head_missing_object_over_tls_http2_is_bodyless() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    let tls_dir = std::path::PathBuf::from(&env.temp_dir).join("tls");
+    let ca_pem = generate_tls_bundle(&tls_dir).await?;
+    start_tls_rustfs_server(&mut env, &tls_dir).await?;
+
+    let endpoint = format!("https://{}", env.address);
+    let client = local_https_h2_client(&ca_pem)?;
+    wait_for_tls_server_ready(&client, &endpoint).await?;
+    ensure_bucket_exists(&client, &endpoint).await?;
+
+    let missing_key = "head-does-not-exist.txt";
+    let object_url = format!("{endpoint}/{BUCKET}/{missing_key}");
+
+    let get_response = signed_empty_request(&client, http::Method::GET, &object_url).await?;
+    assert_eq!(get_response.status(), StatusCode::NOT_FOUND);
+    let get_version = get_response.version();
+    let get_body = get_response.bytes().await?;
+    let get_body_text = String::from_utf8_lossy(&get_body);
+    assert!(
+        get_body_text.contains("<Code>NoSuchKey</Code>") || get_body_text.contains("<Code>NoSuchObject</Code>"),
+        "GET missing-object error body should expose NoSuchKey/NoSuchObject, got: {}",
+        get_body_text
+    );
+    info!("GET missing object over TLS used {:?} and returned {} bytes", get_version, get_body.len());
+
+    let head_response = signed_empty_request(&client, http::Method::HEAD, &object_url).await?;
+    assert_eq!(head_response.status(), StatusCode::NOT_FOUND);
+    assert_eq!(head_response.version(), Version::HTTP_2, "HEAD regression test must exercise HTTP/2");
+    let head_body = head_response.bytes().await?;
+    assert!(
+        head_body.is_empty(),
+        "HEAD missing-object response must not send body bytes over TLS/HTTP2, got {} bytes: {:?}",
+        head_body.len(),
+        head_body
+    );
+
+    Ok(())
+}

crates/e2e_test/src/reliant/lock.rs

@@ -275,6 +275,41 @@ async fn test_grpc_lock_client_batch_acquire_and_release() {
     handle.abort();
 }
 
+#[tokio::test]
+async fn test_grpc_lock_client_uses_request_lock_id_and_reports_missing_unlock() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let local_client: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager));
+
+    let (addr, handle) = spawn_lock_server(local_client).await.expect("Failed to spawn server");
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let grpc_client = GrpcLockClient::new(addr);
+    let request = LockRequest::new(test_resource(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(2));
+
+    let response = grpc_client.acquire_lock(&request).await.expect("gRPC acquire should succeed");
+    let lock_info = response.lock_info.expect("gRPC acquire should include lock info");
+    assert_eq!(lock_info.id, request.lock_id);
+
+    assert!(
+        grpc_client
+            .release(&request.lock_id)
+            .await
+            .expect("gRPC release should succeed"),
+        "release should find the request lock id"
+    );
+
+    let missing_release = grpc_client
+        .release(&request.lock_id)
+        .await
+        .expect_err("second release should report missing lock");
+    assert!(
+        missing_release.to_string().contains("lock not found for release"),
+        "missing release should preserve server error, got: {missing_release}"
+    );
+
+    handle.abort();
+}
+
 #[tokio::test]
 async fn test_distributed_lock_4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes() {
     let manager1 = Arc::new(GlobalLockManager::new());

crates/e2e_test/src/reliant/mod.rs

@@ -17,6 +17,7 @@ mod get_deleted_object_test;
 mod grpc_lock_client;
 mod grpc_lock_server;
 mod head_deleted_object_versioning_test;
+mod head_tls_bodyless_test;
 mod lifecycle;
 mod lock;
 mod node_interact_test;

crates/e2e_test/src/replication_extension_test.rs

@@ -12,14 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+};
+use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+use aws_sdk_s3::{Client, Config};
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
 use rustfs_madmin::{
-    PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus,
-    SRStatusInfo, SiteReplicationInfo, SyncStatus,
+    AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
+    ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
@@ -30,6 +34,8 @@ use std::error::Error;
 use time::Duration as TimeDuration;
 use tokio::time::{Duration, sleep};
 
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
 #[derive(Debug, Clone, serde::Deserialize)]
 struct ReplicationResetStatusResponse {
     #[serde(rename = "Targets", default)]
@@ -79,6 +85,65 @@ async fn signed_request(
     Ok(request_builder.send().await?)
 }
 
+async fn signed_request_with_session_token(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    session_token: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if !session_token.is_empty() {
+        request = request.header("x-amz-security-token", session_token);
+    }
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(
+        request.body(Body::empty())?,
+        content_len,
+        access_key,
+        secret_key,
+        session_token,
+        "us-east-1",
+    );
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
+    let open = format!("<{tag}>");
+    let close = format!("</{tag}>");
+    let start = xml.find(&open)? + open.len();
+    let end = xml[start..].find(&close)? + start;
+    Some(xml[start..end].to_string())
+}
+
+fn parse_assume_role_credentials(xml: &str) -> Result<(String, String, String), Box<dyn Error + Send + Sync>> {
+    let access_key = extract_xml_tag(xml, "AccessKeyId").ok_or("missing AccessKeyId in AssumeRole response")?;
+    let secret_key = extract_xml_tag(xml, "SecretAccessKey").ok_or("missing SecretAccessKey in AssumeRole response")?;
+    let session_token = extract_xml_tag(xml, "SessionToken").ok_or("missing SessionToken in AssumeRole response")?;
+    Ok((access_key, secret_key, session_token))
+}
+
 async fn set_replication_target(
     source_env: &RustFSTestEnvironment,
     source_bucket: &str,
@@ -190,6 +255,14 @@ async fn put_bucket_replication(
     Ok(())
 }
 
+async fn delete_bucket_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication", env.url);
+    signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
 async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
     let client = env.create_s3_client();
     client
@@ -205,6 +278,143 @@ async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) ->
     Ok(())
 }
 
+fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(&env.url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+async fn admin_create_user(
+    env: &RustFSTestEnvironment,
+    username: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
+    let body = serde_json::json!({
+        "secretKey": secret_key,
+        "status": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("create user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_add_canned_policy(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    policy: &serde_json::Value,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(policy.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add canned policy failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_attach_policy_to_user(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    username: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
+        env.url, policy_name, username
+    );
+    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("attach policy to user failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_update_group_members(
+    env: &RustFSTestEnvironment,
+    group_name: &str,
+    members: &[&str],
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
+    let body = serde_json::json!({
+        "group": group_name,
+        "members": members,
+        "isRemove": false,
+        "groupStatus": "enabled"
+    });
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("update group members failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn admin_attach_policy_to_group(
+    env: &RustFSTestEnvironment,
+    policy_name: &str,
+    group_name: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
+        env.url, policy_name, group_name
+    );
+    let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("attach policy to group failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
 async fn run_replication_check(
     env: &RustFSTestEnvironment,
     bucket: &str,
@@ -251,6 +461,182 @@ async fn remove_replication_target_request(
     signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
 }
 
+async fn add_service_account(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    req: &AddServiceAccountReq,
+) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-service-account", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        signer_access_key,
+        signer_secret_key,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add service account failed: {status} {body}").into());
+    }
+
+    let body = response.bytes().await?;
+    let parsed: serde_json::Value = serde_json::from_slice(&body)?;
+    let credentials = parsed
+        .get("credentials")
+        .ok_or("add service account response missing credentials")?;
+    let access_key = credentials
+        .get("accessKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing access key")?
+        .to_string();
+    let secret_key = credentials
+        .get("secretKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing secret key")?
+        .to_string();
+
+    Ok((access_key, secret_key))
+}
+
+async fn add_service_account_with_session_token(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    session_token: &str,
+    req: &AddServiceAccountReq,
+) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/add-service-account", env.url);
+    let response = signed_request_with_session_token(
+        http::Method::PUT,
+        &url,
+        signer_access_key,
+        signer_secret_key,
+        session_token,
+        Some(serde_json::to_vec(req)?),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("add service account with session token failed: {status} {body}").into());
+    }
+
+    let body = response.bytes().await?;
+    let parsed: serde_json::Value = serde_json::from_slice(&body)?;
+    let credentials = parsed
+        .get("credentials")
+        .ok_or("add service account response missing credentials")?;
+    let access_key = credentials
+        .get("accessKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing access key")?
+        .to_string();
+    let secret_key = credentials
+        .get("secretKey")
+        .and_then(|value| value.as_str())
+        .ok_or("add service account response missing secret key")?
+        .to_string();
+
+    Ok((access_key, secret_key))
+}
+
+async fn list_service_accounts(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    user: Option<&str>,
+) -> Result<ListServiceAccountsResp, Box<dyn Error + Send + Sync>> {
+    let mut url = format!("{}/rustfs/admin/v3/list-service-accounts", env.url);
+    if let Some(user) = user {
+        url.push_str("?user=");
+        url.push_str(&urlencoding::encode(user));
+    }
+
+    let response = signed_request(http::Method::GET, &url, signer_access_key, signer_secret_key, None, None).await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("list service accounts failed: {status} {body}").into());
+    }
+
+    Ok(response.json().await?)
+}
+
+async fn wait_for_service_accounts(
+    env: &RustFSTestEnvironment,
+    signer_access_key: &str,
+    signer_secret_key: &str,
+    user: Option<&str>,
+    expected: &[&str],
+) -> Result<ListServiceAccountsResp, Box<dyn Error + Send + Sync>> {
+    for _ in 0..20 {
+        let resp = list_service_accounts(env, signer_access_key, signer_secret_key, user).await?;
+        let access_keys: Vec<&str> = resp.accounts.iter().map(|account| account.access_key.as_str()).collect();
+        if expected
+            .iter()
+            .all(|expected_key| access_keys.iter().any(|actual| actual == expected_key))
+        {
+            return Ok(resp);
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("service accounts did not reach expected keys {expected:?} on {}", env.address).into())
+}
+
+async fn wait_for_object_on_target(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    for _ in 0..40 {
+        match client.get_object().bucket(bucket).key(key).send().await {
+            Ok(output) => {
+                let body = output.body.collect().await?.into_bytes().to_vec();
+                return Ok(body);
+            }
+            Err(err) => {
+                if err.to_string().contains("NoSuchKey") || err.to_string().contains("NotFound") {
+                    sleep(Duration::from_millis(250)).await;
+                    continue;
+                }
+                return Err(err.into());
+            }
+        }
+    }
+
+    Err(format!("object {bucket}/{key} was not replicated in time").into())
+}
+
+async fn wait_for_user_get_object(client: &Client, bucket: &str, key: &str) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+    let mut last_error = None;
+    for _ in 0..40 {
+        match client.get_object().bucket(bucket).key(key).send().await {
+            Ok(output) => {
+                let body = output.body.collect().await?.into_bytes().to_vec();
+                return Ok(body);
+            }
+            Err(err) => {
+                last_error = Some(err.to_string());
+                sleep(Duration::from_millis(250)).await;
+            }
+        }
+    }
+
+    Err(format!(
+        "user could not read replicated object {bucket}/{key} in time; last error: {}",
+        last_error.unwrap_or_else(|| "unknown".to_string())
+    )
+    .into())
+}
+
 async fn list_replication_targets_request(
     env: &RustFSTestEnvironment,
     bucket: Option<&str>,
@@ -1084,6 +1470,54 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
     Ok(())
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-delete-config-src";
+    let target_bucket = "replication-delete-config-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    let delete_response = delete_bucket_replication(&source_env, source_bucket).await?;
+    assert!(
+        delete_response.status().is_success(),
+        "unexpected delete status: {}",
+        delete_response.status()
+    );
+
+    let targets_response = list_replication_targets_request(&source_env, Some(source_bucket)).await?;
+    assert_eq!(targets_response.status(), StatusCode::OK);
+    let targets: Vec<serde_json::Value> = targets_response.json().await?;
+    assert!(
+        targets
+            .iter()
+            .all(|target| target.get("arn").and_then(|arn| arn.as_str()) != Some(target_arn.as_str())),
+        "deleted replication config left stale target {target_arn}: {targets:?}"
+    );
+
+    let recreated_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &recreated_arn).await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -1490,3 +1924,449 @@ async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Re
 
     Ok(())
 }
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_object_with_bucket_versioning_real_dual_node() -> TestResult {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-versioned";
+    let key = "hello.txt";
+    let payload = b"site replication should replicate after enabling versioning".to_vec();
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    let replication_response = signed_request(
+        http::Method::GET,
+        &format!("{}/{bucket}?replication", source_env.url),
+        &source_env.access_key,
+        &source_env.secret_key,
+        None,
+        None,
+    )
+    .await?;
+    let replication_status = replication_response.status();
+    let replication_body = replication_response.text().await.unwrap_or_default();
+    assert_eq!(
+        replication_status,
+        StatusCode::OK,
+        "source bucket replication config missing after site replication setup: {replication_body}"
+    );
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-policy-user";
+    let key = "seed.txt";
+    let payload = b"site replication policy-backed user access".to_vec();
+    let policy_name = "site-repl-readonly";
+    let username = "site-repl-user";
+    let secret_key = "site-repl-user-secret-key-123456";
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{bucket}/*")]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                "Resource": [format!("arn:aws:s3:::{bucket}")]
+            }
+        ]
+    });
+    admin_add_canned_policy(&source_env, policy_name, &policy).await?;
+    admin_create_user(&source_env, username, secret_key).await?;
+    admin_attach_policy_to_user(&source_env, policy_name, username).await?;
+
+    let target_user_client = create_user_s3_client(&target_env, username, secret_key);
+    let fetched = wait_for_user_get_object(&target_user_client, bucket, key).await?;
+    assert_eq!(fetched, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_group_policy_backed_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>>
+{
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+    let bucket = "site-repl-policy-group";
+    let key = "seed.txt";
+    let payload = b"site replication group-policy-backed user access".to_vec();
+    let policy_name = "site-repl-group-readonly";
+    let group_name = "site-repl-group";
+    let username = "site-repl-group-user";
+    let secret_key = "site-repl-group-user-secret-key-12";
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(payload.clone()))
+        .send()
+        .await?;
+
+    let replicated = wait_for_object_on_target(&target_client, bucket, key).await?;
+    assert_eq!(replicated, payload);
+
+    let policy = serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetObject"],
+                "Resource": [format!("arn:aws:s3:::{bucket}/*")]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                "Resource": [format!("arn:aws:s3:::{bucket}")]
+            }
+        ]
+    });
+    admin_add_canned_policy(&source_env, policy_name, &policy).await?;
+    admin_create_user(&source_env, username, secret_key).await?;
+    admin_update_group_members(&source_env, group_name, &[username]).await?;
+    admin_attach_policy_to_group(&source_env, policy_name, group_name).await?;
+
+    let target_user_client = create_user_s3_client(&target_env, username, secret_key);
+    let fetched = wait_for_user_get_object(&target_user_client, bucket, key).await?;
+    assert_eq!(fetched, payload);
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_multiple_service_accounts_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let first_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-alpha".to_string(),
+        secret_key: "svc-alpha-secret-key-1234567890abcdef".to_string(),
+        name: Some("svc-alpha".to_string()),
+        description: Some("first replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let first = add_service_account(&source_env, &source_env.access_key, &source_env.secret_key, &first_req).await?;
+
+    let target_after_first = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-alpha"],
+    )
+    .await?;
+    assert!(
+        target_after_first
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-alpha"),
+        "target accounts missing svc-alpha: {:?}",
+        target_after_first.accounts
+    );
+
+    let second_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-beta".to_string(),
+        secret_key: "svc-beta-secret-key-1234567890abcdef1".to_string(),
+        name: Some("svc-beta".to_string()),
+        description: Some("second replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let _second = add_service_account(&source_env, &first.0, &first.1, &second_req).await?;
+
+    let target_after_second = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-alpha", "svc-beta"],
+    )
+    .await?;
+    assert!(
+        target_after_second
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-beta"),
+        "target accounts missing svc-beta: {:?}",
+        target_after_second.accounts
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
+    init_logging();
+
+    if !awscurl_available() {
+        eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
+        return Ok(());
+    }
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
+
+    let assume_role_body = "Action=AssumeRole&Version=2011-06-15&DurationSeconds=3600";
+    let sts_xml = awscurl_post_sts_form_urlencoded(
+        &format!("{}/", source_env.url.trim_end_matches('/')),
+        assume_role_body,
+        &source_env.access_key,
+        &source_env.secret_key,
+    )
+    .await?;
+    let (sts_access_key, sts_secret_key, sts_session_token) = parse_assume_role_credentials(&sts_xml)?;
+
+    let first_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-sts-alpha".to_string(),
+        secret_key: "svc-sts-alpha-secret-key-1234567890".to_string(),
+        name: Some("svc-sts-alpha".to_string()),
+        description: Some("sts-created replicated service account".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let first =
+        add_service_account_with_session_token(&source_env, &sts_access_key, &sts_secret_key, &sts_session_token, &first_req)
+            .await?;
+
+    let target_after_first = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-sts-alpha"],
+    )
+    .await?;
+    assert!(
+        target_after_first
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-sts-alpha"),
+        "target accounts missing svc-sts-alpha: {:?}",
+        target_after_first.accounts
+    );
+
+    let second_req = AddServiceAccountReq {
+        policy: None,
+        target_user: None,
+        access_key: "svc-sts-beta".to_string(),
+        secret_key: "svc-sts-beta-secret-key-1234567890a".to_string(),
+        name: Some("svc-sts-beta".to_string()),
+        description: Some("second replicated service account from sts-created ak".to_string()),
+        expiration: None,
+        comment: None,
+    };
+    let _second = add_service_account(&source_env, &first.0, &first.1, &second_req).await?;
+
+    let target_after_second = wait_for_service_accounts(
+        &target_env,
+        &target_env.access_key,
+        &target_env.secret_key,
+        Some(&source_env.access_key),
+        &["svc-sts-alpha", "svc-sts-beta"],
+    )
+    .await?;
+    assert!(
+        target_after_second
+            .accounts
+            .iter()
+            .any(|account| account.access_key == "svc-sts-beta"),
+        "target accounts missing svc-sts-beta: {:?}",
+        target_after_second.accounts
+    );
+
+    Ok(())
+}

crates/e2e_test/src/tls_gen.rs

@@ -0,0 +1,259 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::{Context, Result, bail};
+use clap::Parser;
+use rcgen::{
+    BasicConstraints, CertificateParams, CertifiedIssuer, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose,
+    SanType,
+};
+use std::fs;
+use std::net::IpAddr;
+use std::path::{Path, PathBuf};
+use time::{Duration, OffsetDateTime};
+
+pub const DEFAULT_OUT_DIR: &str = "target/tls";
+pub const OUTPUT_FILES: [&str; 7] = [
+    "rustfs_cert.pem",
+    "rustfs_key.pem",
+    "ca.crt",
+    "public.crt",
+    "client_ca.crt",
+    "client_cert.pem",
+    "client_key.pem",
+];
+
+#[derive(Debug, Parser)]
+#[command(name = "tls_gen", about = "Generate a full RustFS TLS bundle for local TLS and mTLS tests.")]
+pub struct Args {
+    #[arg(long, default_value = DEFAULT_OUT_DIR)]
+    pub out_dir: PathBuf,
+    #[arg(long, default_value_t = 365)]
+    pub days: i64,
+    #[arg(long)]
+    pub force: bool,
+}
+
+pub fn run(args: Args) -> Result<PathBuf> {
+    if args.days <= 0 {
+        bail!("--days must be a positive integer");
+    }
+
+    write_bundle(&args.out_dir, args.force, args.days)?;
+    Ok(args.out_dir)
+}
+
+pub fn ensure_writable(out_dir: &Path, force: bool) -> Result<()> {
+    if force {
+        return Ok(());
+    }
+
+    let existing: Vec<_> = OUTPUT_FILES
+        .iter()
+        .map(|name| out_dir.join(name))
+        .filter(|path| path.exists())
+        .collect();
+
+    if existing.is_empty() {
+        return Ok(());
+    }
+
+    let existing_list = existing
+        .iter()
+        .map(|path| path.file_name().and_then(|name| name.to_str()).unwrap_or("<unknown>"))
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    bail!(
+        "Refusing to overwrite existing files in {}: {}. Re-run with --force to replace them.",
+        out_dir.display(),
+        existing_list
+    )
+}
+
+fn write_bundle(out_dir: &Path, force: bool, days: i64) -> Result<()> {
+    fs::create_dir_all(out_dir).with_context(|| format!("failed to create output directory {}", out_dir.display()))?;
+    ensure_writable(out_dir, force)?;
+
+    let ca_key = generate_private_key()?;
+    let ca = build_ca_certificate(ca_key, days)?;
+
+    let server_key = generate_private_key()?;
+    let server_cert = build_leaf_certificate(
+        &server_key,
+        "localhost",
+        &[SanType::DnsName("localhost".try_into()?)],
+        &[
+            SanType::IpAddress(IpAddr::V4("127.0.0.1".parse()?)),
+            SanType::IpAddress(IpAddr::V6("::1".parse()?)),
+        ],
+        ExtendedKeyUsagePurpose::ServerAuth,
+        &ca,
+        days,
+    )?;
+
+    let client_key = generate_private_key()?;
+    let client_cert = build_leaf_certificate(
+        &client_key,
+        "rustfs-test-client",
+        &[SanType::DnsName("rustfs-test-client".try_into()?)],
+        &[],
+        ExtendedKeyUsagePurpose::ClientAuth,
+        &ca,
+        days,
+    )?;
+
+    let ca_pem = ca.pem();
+    let bundle = [
+        ("rustfs_cert.pem", server_cert.pem()),
+        ("rustfs_key.pem", server_key.serialize_pem()),
+        ("ca.crt", ca_pem.clone()),
+        ("public.crt", ca_pem.clone()),
+        ("client_ca.crt", ca_pem),
+        ("client_cert.pem", client_cert.pem()),
+        ("client_key.pem", client_key.serialize_pem()),
+    ];
+
+    for (name, content) in bundle {
+        fs::write(out_dir.join(name), content).with_context(|| format!("failed to write {}", out_dir.join(name).display()))?;
+    }
+
+    Ok(())
+}
+
+fn build_ca_certificate(signing_key: KeyPair, days: i64) -> Result<CertifiedIssuer<'static, KeyPair>> {
+    let mut params = base_params("RustFS Test CA", days)?;
+    params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
+    params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
+
+    CertifiedIssuer::self_signed(params, signing_key).context("failed to create CA certificate")
+}
+
+fn build_leaf_certificate(
+    signing_key: &KeyPair,
+    common_name: &str,
+    dns_names: &[SanType],
+    ip_addresses: &[SanType],
+    usage: ExtendedKeyUsagePurpose,
+    issuer: &CertifiedIssuer<'_, KeyPair>,
+    days: i64,
+) -> Result<rcgen::Certificate> {
+    let mut params = base_params(common_name, days)?;
+    params.is_ca = IsCa::ExplicitNoCa;
+    params.key_usages = vec![KeyUsagePurpose::DigitalSignature, KeyUsagePurpose::KeyEncipherment];
+    params.extended_key_usages = vec![usage];
+    params.use_authority_key_identifier_extension = true;
+    params.subject_alt_names.extend_from_slice(dns_names);
+    params.subject_alt_names.extend_from_slice(ip_addresses);
+
+    params
+        .signed_by(signing_key, issuer)
+        .with_context(|| format!("failed to create leaf certificate for {common_name}"))
+}
+
+fn base_params(common_name: &str, days: i64) -> Result<CertificateParams> {
+    let mut params = CertificateParams::default();
+    let issued_at = OffsetDateTime::now_utc() - Duration::minutes(5);
+    params.not_before = issued_at;
+    params.not_after = issued_at + Duration::days(days);
+    params.distinguished_name.push(DnType::CountryName, "US");
+    params.distinguished_name.push(DnType::OrganizationName, "RustFS");
+    params.distinguished_name.push(DnType::CommonName, common_name);
+    Ok(params)
+}
+
+fn generate_private_key() -> Result<KeyPair> {
+    KeyPair::generate().context("failed to generate private key")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Args, OUTPUT_FILES, ensure_writable, run};
+    use std::fs;
+    use std::path::{Path, PathBuf};
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn unique_temp_dir() -> PathBuf {
+        let suffix = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("system time must be after unix epoch")
+            .as_nanos();
+        std::env::temp_dir().join(format!("rustfs-tls-gen-{suffix}"))
+    }
+
+    struct TempDir {
+        path: PathBuf,
+    }
+
+    impl TempDir {
+        fn new() -> Self {
+            let path = unique_temp_dir();
+            fs::create_dir_all(&path).expect("temporary directory should be created");
+            Self { path }
+        }
+
+        fn path(&self) -> &Path {
+            &self.path
+        }
+    }
+
+    impl Drop for TempDir {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.path);
+        }
+    }
+
+    #[test]
+    fn run_writes_full_bundle() {
+        let temp_dir = TempDir::new();
+
+        let out_dir = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 365,
+            force: false,
+        })
+        .expect("bundle generation should succeed");
+
+        for name in OUTPUT_FILES {
+            let content = fs::read(out_dir.join(name)).unwrap_or_else(|error| panic!("{name} should exist: {error}"));
+            assert!(!content.is_empty(), "{name} should not be empty");
+        }
+    }
+
+    #[test]
+    fn ensure_writable_rejects_existing_files_without_force() {
+        let temp_dir = TempDir::new();
+        let existing = temp_dir.path().join(OUTPUT_FILES[0]);
+        fs::write(&existing, "existing").expect("existing file should be created");
+
+        let error = ensure_writable(temp_dir.path(), false).expect_err("existing files must be rejected");
+        let message = format!("{error:#}");
+
+        assert!(message.contains("Refusing to overwrite existing files"));
+        assert!(message.contains(OUTPUT_FILES[0]));
+    }
+
+    #[test]
+    fn run_rejects_non_positive_days() {
+        let temp_dir = TempDir::new();
+        let error = run(Args {
+            out_dir: temp_dir.path().join("tls"),
+            days: 0,
+            force: false,
+        })
+        .expect_err("non-positive days must fail");
+
+        assert_eq!(format!("{error:#}"), "--days must be a positive integer");
+    }
+}

crates/ecstore/Cargo.toml

@@ -55,6 +55,7 @@ flatbuffers.workspace = true
 futures.workspace = true
 futures-util.workspace = true
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
 serde.workspace = true
 time.workspace = true
 bytesize.workspace = true
@@ -62,6 +63,7 @@ serde_json.workspace = true
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
 s3s.workspace = true
 http.workspace = true
+opentelemetry.workspace = true
 http-body = { workspace = true }
 http-body-util.workspace = true
 url.workspace = true
@@ -124,9 +126,10 @@ metrics = { workspace = true }
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 criterion = { workspace = true, features = ["html_reports"] }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true }
 serial_test = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 
 [build-dependencies]
 shadow-rs = { workspace = true, features = ["build", "metadata"] }

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -70,7 +70,7 @@ use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{RwLock, mpsc};
-use tracing::{error, info, warn};
+use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
@@ -413,6 +413,7 @@ impl ExpiryState {
                     let v = v.expect("received None after None check");
                     if v.as_any().is::<ExpiryTask>() {
                         let v = v.as_any().downcast_ref::<ExpiryTask>().expect("ExpiryTask downcast failed");
+                        //debug!("lifecycle expiry worker received task: {:?}", v.obj_info);
                         if !v.obj_info.transitioned_object.status.is_empty() {
                             apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
                         } else {
@@ -1340,8 +1341,8 @@ pub async fn expire_transitioned_object(
         &oi.transitioned_object.tier,
     )
     .await;
-    if ret.is_err() {
-        //transitionLogIf(ctx, err);
+    if let Err(e) = &ret {
+        error!("Failed to delete remote transitioned object {}: {:?}", oi.transitioned_object.name, e);
     }
     mark_delete_opts_skip_decommissioned_on_remote_success(&mut opts, ret.is_ok());
 
@@ -1354,9 +1355,9 @@ pub async fn expire_transitioned_object(
         }
     };
 
-    schedule_lifecycle_replication_delete_if_needed(oi).await;
+    schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
 
-    //defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+    //audit_log_lifecycle(oi, ILMExpiry, tags);
 
     let event_name = if oi.delete_marker {
         EventName::LifecycleExpirationDelete
@@ -1770,7 +1771,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
     let time_ilm = Metrics::time_ilm(lc_event.action);
 
     //debug!("lc_event.action: {:?}", lc_event.action);
-    //debug!("opts: {:?}", opts);
+    debug!("expiry_on_non_transitioned_objects opts: {:?}", opts);
     let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
         Ok(dobj) => dobj,
         Err(e) => {
@@ -1778,7 +1779,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
             return false;
         }
     };
-    schedule_lifecycle_replication_delete_if_needed(oi).await;
+    schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
     //debug!("dobj: {:?}", dobj);
     if dobj.name.is_empty() {
         dobj = oi.clone();
@@ -1819,25 +1820,55 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
     true
 }
 
-async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo) {
-    if !oi.delete_marker || oi.version_id.is_none() {
-        return;
+fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> crate::store_api::DeletedObject {
+    if dobj.delete_marker {
+        return crate::store_api::DeletedObject {
+            object_name: oi.name.clone(),
+            delete_marker: true,
+            delete_marker_version_id: dobj.version_id,
+            delete_marker_mtime: dobj.mod_time.or(oi.mod_time),
+            ..Default::default()
+        };
     }
 
-    let replication_state = lifecycle_delete_replication_state(oi).await;
+    if oi.delete_marker && oi.version_id.is_some() {
+        return crate::store_api::DeletedObject {
+            object_name: oi.name.clone(),
+            delete_marker: false,
+            delete_marker_version_id: oi.version_id,
+            delete_marker_mtime: oi.mod_time,
+            ..Default::default()
+        };
+    }
+
+    crate::store_api::DeletedObject {
+        object_name: oi.name.clone(),
+        delete_marker: false,
+        version_id: oi.version_id,
+        delete_marker_mtime: oi.mod_time,
+        ..Default::default()
+    }
+}
+
+async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo) {
+    let mut delete_object = lifecycle_deleted_object(oi, dobj);
+    let version_id = if delete_object.delete_marker {
+        None
+    } else if delete_object.delete_marker_version_id.is_some() {
+        delete_object.delete_marker_version_id
+    } else {
+        delete_object.version_id
+    };
+
+    let replication_state = lifecycle_delete_replication_state(oi, version_id).await;
     if replication_state.is_none() {
         return;
     }
 
+    delete_object.replication_state = replication_state;
+
     schedule_replication_delete(DeletedObjectReplicationInfo {
-        delete_object: crate::store_api::DeletedObject {
-            object_name: oi.name.clone(),
-            delete_marker_version_id: oi.version_id,
-            delete_marker: false,
-            delete_marker_mtime: oi.mod_time,
-            replication_state,
-            ..Default::default()
-        },
+        delete_object,
         bucket: oi.bucket.clone(),
         event_type: REPLICATE_INCOMING_DELETE.to_string(),
         ..Default::default()
@@ -1845,21 +1876,56 @@ async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo) {
     .await;
 }
 
-async fn lifecycle_delete_replication_state(oi: &ObjectInfo) -> Option<ReplicationState> {
-    if !oi.replication_decision.is_empty() || oi.version_purge_status == VersionPurgeStatusType::Pending {
+fn should_reuse_lifecycle_delete_replication_state(oi: &ObjectInfo, version_delete: bool) -> bool {
+    let state = oi.replication_state();
+    if version_delete {
+        oi.version_purge_status == VersionPurgeStatusType::Pending && !state.purge_targets.is_empty()
+    } else {
+        oi.replication_status == rustfs_filemeta::ReplicationStatusType::Pending && !state.targets.is_empty()
+    }
+}
+
+fn lifecycle_version_purge_state_from_completed_targets(oi: &ObjectInfo) -> Option<ReplicationState> {
+    if oi.replication_status != rustfs_filemeta::ReplicationStatusType::Completed {
+        return None;
+    }
+
+    let targets = oi.replication_state().targets;
+    if targets.is_empty() {
+        return None;
+    }
+
+    let pending_status = targets.keys().map(|arn| format!("{arn}=PENDING;")).collect::<String>();
+
+    Some(ReplicationState {
+        replicate_decision_str: oi.replication_decision.clone(),
+        version_purge_status_internal: Some(pending_status.clone()),
+        purge_targets: rustfs_filemeta::version_purge_statuses_map(&pending_status),
+        ..Default::default()
+    })
+}
+
+async fn lifecycle_delete_replication_state(oi: &ObjectInfo, version_id: Option<Uuid>) -> Option<ReplicationState> {
+    if should_reuse_lifecycle_delete_replication_state(oi, version_id.is_some()) {
         return Some(oi.replication_state());
     }
 
+    if version_id.is_some()
+        && let Some(state) = lifecycle_version_purge_state_from_completed_targets(oi)
+    {
+        return Some(state);
+    }
+
     let dsc = check_replicate_delete(
         &oi.bucket,
         &ObjectToDelete {
             object_name: oi.name.clone(),
-            version_id: oi.version_id,
+            version_id,
             ..Default::default()
         },
         oi,
         &ObjectOptions {
-            version_id: oi.version_id.map(|v| v.to_string()),
+            version_id: version_id.map(|v| v.to_string()),
             versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
             ..Default::default()
         },
@@ -1870,17 +1936,23 @@ async fn lifecycle_delete_replication_state(oi: &ObjectInfo) -> Option<Replicati
         return None;
     }
 
-    Some(replication_state_for_version_delete(dsc))
+    Some(replication_state_for_delete(dsc, version_id.is_some()))
 }
 
-fn replication_state_for_version_delete(dsc: ReplicateDecision) -> ReplicationState {
+fn replication_state_for_delete(dsc: ReplicateDecision, version_delete: bool) -> ReplicationState {
     let pending_status = dsc.pending_status();
-    ReplicationState {
+    let mut state = ReplicationState {
         replicate_decision_str: dsc.to_string(),
-        version_purge_status_internal: pending_status.clone(),
-        purge_targets: rustfs_filemeta::version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
         ..Default::default()
+    };
+    if version_delete {
+        state.version_purge_status_internal = pending_status.clone();
+        state.purge_targets = rustfs_filemeta::version_purge_statuses_map(pending_status.as_deref().unwrap_or_default());
+    } else {
+        state.replication_status_internal = pending_status.clone();
+        state.targets = rustfs_filemeta::replication_statuses_map(pending_status.as_deref().unwrap_or_default());
     }
+    state
 }
 
 pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
@@ -1906,7 +1978,9 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
 mod tests {
     use super::{
         StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
-        mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
+        lifecycle_deleted_object, lifecycle_version_purge_state_from_completed_targets,
+        mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete,
+        should_reuse_lifecycle_delete_replication_state,
     };
     use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
     use crate::bucket::metadata_sys;
@@ -1917,8 +1991,9 @@ mod tests {
     use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
     use crate::store::ECStore;
     use crate::store_api::{
-        BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectOptions, PutObjReader,
+        BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader,
     };
+    use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
     use std::collections::HashMap;
@@ -1960,6 +2035,136 @@ mod tests {
         assert!(opts.skip_decommissioned);
     }
 
+    #[test]
+    fn lifecycle_deleted_object_uses_delete_marker_created_by_expiry() {
+        let source = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "key".to_string(),
+            ..Default::default()
+        };
+        let delete_result = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "key".to_string(),
+            delete_marker: true,
+            version_id: Some(Uuid::new_v4()),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        };
+
+        let deleted = lifecycle_deleted_object(&source, &delete_result);
+
+        assert!(deleted.delete_marker);
+        assert_eq!(deleted.delete_marker_version_id, delete_result.version_id);
+        assert_eq!(deleted.version_id, None);
+        assert_eq!(deleted.object_name, "key");
+    }
+
+    #[test]
+    fn lifecycle_deleted_object_uses_version_id_for_noncurrent_version_purge() {
+        let version_id = Uuid::new_v4();
+        let source = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "key".to_string(),
+            version_id: Some(version_id),
+            ..Default::default()
+        };
+
+        let deleted = lifecycle_deleted_object(&source, &ObjectInfo::default());
+
+        assert!(!deleted.delete_marker);
+        assert_eq!(deleted.version_id, Some(version_id));
+        assert_eq!(deleted.delete_marker_version_id, None);
+    }
+
+    #[test]
+    fn lifecycle_deleted_object_uses_delete_marker_version_for_marker_purge() {
+        let version_id = Uuid::new_v4();
+        let source = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "key".to_string(),
+            delete_marker: true,
+            version_id: Some(version_id),
+            ..Default::default()
+        };
+
+        let deleted = lifecycle_deleted_object(&source, &ObjectInfo::default());
+
+        assert!(!deleted.delete_marker);
+        assert_eq!(deleted.delete_marker_version_id, Some(version_id));
+        assert_eq!(deleted.version_id, None);
+    }
+
+    #[test]
+    fn replication_state_for_delete_uses_replication_targets_for_current_delete() {
+        let arn = "arn:aws:s3:::target-bucket";
+        let mut dsc = ReplicateDecision::default();
+        dsc.set(rustfs_filemeta::ReplicateTargetDecision::new(arn.to_string(), true, false));
+
+        let state = replication_state_for_delete(dsc, false);
+
+        assert_eq!(state.replication_status_internal.as_deref(), Some(format!("{arn}=PENDING;").as_str()));
+        assert!(state.version_purge_status_internal.is_none());
+        assert!(state.targets.contains_key(arn));
+    }
+
+    #[test]
+    fn replication_state_for_delete_uses_purge_targets_for_version_delete() {
+        let arn = "arn:aws:s3:::target-bucket";
+        let mut dsc = ReplicateDecision::default();
+        dsc.set(rustfs_filemeta::ReplicateTargetDecision::new(arn.to_string(), true, false));
+
+        let state = replication_state_for_delete(dsc, true);
+
+        assert_eq!(state.version_purge_status_internal.as_deref(), Some(format!("{arn}=PENDING;").as_str()));
+        assert!(state.replication_status_internal.is_none());
+        assert!(state.purge_targets.contains_key(arn));
+    }
+
+    #[test]
+    fn lifecycle_delete_replication_state_reuses_only_pending_version_purge_state() {
+        let oi = ObjectInfo {
+            version_purge_status: VersionPurgeStatusType::Pending,
+            version_purge_status_internal: Some("arn:aws:s3:::target=PENDING;".to_string()),
+            replication_decision: "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
+            ..Default::default()
+        };
+
+        assert!(should_reuse_lifecycle_delete_replication_state(&oi, true));
+        assert!(!should_reuse_lifecycle_delete_replication_state(&oi, false));
+    }
+
+    #[test]
+    fn lifecycle_delete_replication_state_does_not_reuse_put_replication_for_version_delete() {
+        let oi = ObjectInfo {
+            replication_status: rustfs_filemeta::ReplicationStatusType::Completed,
+            replication_status_internal: Some("arn:aws:s3:::target=COMPLETED;".to_string()),
+            replication_decision: "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
+            ..Default::default()
+        };
+
+        assert!(
+            !should_reuse_lifecycle_delete_replication_state(&oi, true),
+            "version purges must not reuse plain object replication state from prior PUT/delete-marker replication"
+        );
+    }
+
+    #[test]
+    fn lifecycle_version_purge_state_from_completed_targets_derives_pending_purge_targets() {
+        let oi = ObjectInfo {
+            replication_status: rustfs_filemeta::ReplicationStatusType::Completed,
+            replication_status_internal: Some("arn:aws:s3:::target=COMPLETED;".to_string()),
+            replication_decision: "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
+            ..Default::default()
+        };
+
+        let state = lifecycle_version_purge_state_from_completed_targets(&oi)
+            .expect("completed replication targets should be convertible into version-purge targets");
+
+        assert_eq!(state.version_purge_status_internal.as_deref(), Some("arn:aws:s3:::target=PENDING;"));
+        assert!(state.purge_targets.contains_key("arn:aws:s3:::target"));
+        assert_eq!(state.replicate_decision_str, oi.replication_decision);
+    }
+
     static STALE_MULTIPART_TEST_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
 
     async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {

crates/ecstore/src/bucket/lifecycle/core.rs

@@ -439,8 +439,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
     async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime, _newer_noncurrent_versions: usize) -> Event {
         let mut events = Vec::<Event>::new();
         debug!(
-            "eval_inner: object={}, mod_time={:?}, now={:?}, is_latest={}, delete_marker={}",
-            obj.name, obj.mod_time, now, obj.is_latest, obj.delete_marker
+            "eval_inner: object={}, mod_time={:?}, successor_mod_time={:?}, now={:?}, is_latest={}, delete_marker={}",
+            obj.name, obj.mod_time, obj.successor_mod_time, now, obj.is_latest, obj.delete_marker
         );
 
         // Gracefully handle missing mod_time instead of panicking

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -1,4 +1,3 @@
-#![allow(unused_imports)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]

crates/ecstore/src/bucket/replication/config.rs

@@ -152,6 +152,10 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
             }
 
             if obj.op_type == ReplicationType::Delete {
+                if !rule.metadata_replicate(obj) {
+                    return false;
+                }
+
                 if obj.version_id.is_some() {
                     if obj.delete_marker {
                         return rule.delete_marker_replication.clone().is_some_and(|d| {

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -956,6 +956,9 @@ pub type DynReplicationPool = dyn ReplicationPoolTrait + Send + Sync;
 /// Trait that abstracts the replication pool operations
 #[async_trait::async_trait]
 pub trait ReplicationPoolTrait: std::fmt::Debug {
+    fn active_workers(&self) -> i32;
+    fn active_mrf_workers(&self) -> i32;
+    fn active_lrg_workers(&self) -> i32;
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo);
     async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo);
     async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
@@ -972,6 +975,18 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
 // Implement the trait for ReplicationPool
 #[async_trait::async_trait]
 impl<S: StorageAPI> ReplicationPoolTrait for ReplicationPool<S> {
+    fn active_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_workers(self)
+    }
+
+    fn active_mrf_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_mrf_workers(self)
+    }
+
+    fn active_lrg_workers(&self) -> i32 {
+        ReplicationPool::<S>::active_lrg_workers(self)
+    }
+
     async fn queue_replica_task(&self, ri: ReplicateObjectInfo) {
         self.queue_replica_task(ri).await;
     }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -35,7 +35,7 @@ use crate::set_disk::get_lock_acquire_timeout;
 use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
-use aws_sdk_s3::operation::head_object::HeadObjectOutput;
+use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
 use aws_smithy_types::body::SdkBody;
@@ -85,7 +85,7 @@ use tokio::task::JoinSet;
 use tokio::time::Duration as TokioDuration;
 use tokio_util::io::ReaderStream;
 use tokio_util::sync::CancellationToken;
-use tracing::{error, info, instrument, warn};
+use tracing::{debug, error, info, instrument, warn};
 use uuid::Uuid;
 
 pub(crate) const REPLICATION_DIR: &str = ".replication";
@@ -115,6 +115,41 @@ fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &Res
     state.resync_id.is_empty() || opts.resync_id.is_empty() || state.resync_id == opts.resync_id
 }
 
+fn should_count_head_proxy_failure(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
+    if is_not_found || matches!(code, Some("MethodNotAllowed" | "405")) {
+        return false;
+    }
+    !matches!(raw_status, Some(404 | 405))
+}
+
+fn is_head_proxy_failure(err: &SdkError<HeadObjectError>) -> bool {
+    let (is_not_found, code) = err
+        .as_service_error()
+        .map(|service_err| (service_err.is_not_found(), service_err.code()))
+        .unwrap_or((false, None));
+    let raw_status = err.raw_response().map(|resp| resp.status().as_u16());
+    should_count_head_proxy_failure(is_not_found, code, raw_status)
+}
+
+async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) {
+    if let Some(stats) = GLOBAL_REPLICATION_STATS.get() {
+        stats.inc_proxy(bucket, api, is_err).await;
+    }
+}
+
+async fn head_object_with_proxy_stats(
+    source_bucket: &str,
+    target_client: &TargetClient,
+    target_bucket: &str,
+    object: &str,
+    version_id: Option<String>,
+) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
+    let result = target_client.head_object(target_bucket, object, version_id).await;
+    let is_err = result.as_ref().err().is_some_and(is_head_proxy_failure);
+    record_proxy_request(source_bucket, "HeadObject", is_err).await;
+    result
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct ResyncOpts {
     pub bucket: String,
@@ -748,10 +783,15 @@ impl ReplicationResyncer {
 
                     let reset_id = target_client.reset_id.clone();
 
-                    let (size, err) = if let Err(err) = target_client
-                        .head_object(&target_client.bucket, &roi.name, roi.version_id.map(|v| v.to_string()))
-                        .await
-                    {
+                    let head_result = head_object_with_proxy_stats(
+                        &bucket_name,
+                        target_client.as_ref(),
+                        &target_client.bucket,
+                        &roi.name,
+                        roi.version_id.map(|v| v.to_string()),
+                    )
+                    .await;
+                    let (size, err) = if let Err(err) = head_result {
                         if roi.delete_marker {
                             st.replicated_count += 1;
                         } else {
@@ -1272,15 +1312,7 @@ pub async fn check_replicate_delete(
         return ReplicateDecision::default();
     }
 
-    let opts = ObjectOpts {
-        name: dobj.object_name.clone(),
-        ssec: is_ssec_encrypted(&oi.user_defined),
-        user_tags: oi.user_tags.clone(),
-        delete_marker: oi.delete_marker,
-        version_id: dobj.version_id,
-        op_type: ReplicationType::Delete,
-        ..Default::default()
-    };
+    let opts = delete_replication_object_opts(dobj, oi);
 
     let tgt_arns = rcfg.filter_target_arns(&opts);
     let mut dsc = ReplicateDecision::new();
@@ -1332,6 +1364,19 @@ pub async fn check_replicate_delete(
     dsc
 }
 
+fn delete_replication_object_opts(dobj: &ObjectToDelete, oi: &ObjectInfo) -> ObjectOpts {
+    ObjectOpts {
+        name: dobj.object_name.clone(),
+        ssec: is_ssec_encrypted(&oi.user_defined),
+        user_tags: oi.user_tags.clone(),
+        delete_marker: oi.delete_marker,
+        version_id: dobj.version_id,
+        op_type: ReplicationType::Delete,
+        replica: oi.replication_status == ReplicationStatusType::Replica,
+        ..Default::default()
+    }
+}
+
 /// Check if the user-defined metadata contains SSEC encryption headers
 fn is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
     user_defined.contains_key(SSEC_ALGORITHM_HEADER)
@@ -1703,7 +1748,7 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
 
     let is_version_purge = is_version_delete_replication(&dobj.delete_object);
 
-    if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
+    if should_retry_delete_marker_purge(&dobj.delete_object) {
         let bucket_clone = bucket.clone();
         let dobj_clone = dobj.clone();
         let dsc_clone = dsc.clone();
@@ -2061,6 +2106,10 @@ fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
     dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
 }
 
+fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool {
+    dobj.delete_marker_version_id.is_some()
+}
+
 fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
     !is_not_found && !matches!(code, Some("MethodNotAllowed" | "405"))
 }
@@ -2111,9 +2160,14 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
     };
 
     if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
-        match tgt_client
-            .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
-            .await
+        match head_object_with_proxy_stats(
+            &dobj.bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &dobj.delete_object.object_name,
+            version_id.clone(),
+        )
+        .await
         {
             Ok(_) => {}
             Err(e) => {
@@ -2152,6 +2206,14 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         .await
     {
         Ok(_) => {
+            debug!(
+                bucket = tgt_client.bucket,
+                object = dobj.delete_object.object_name,
+                version_id = ?version_id,
+                delete_marker = dobj.delete_object.delete_marker,
+                is_version_purge,
+                "replicate_delete_to_target succeeded"
+            );
             if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Completed;
             } else {
@@ -2159,6 +2221,15 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
             }
         }
         Err(e) => {
+            warn!(
+                bucket = tgt_client.bucket,
+                object = dobj.delete_object.object_name,
+                version_id = ?version_id,
+                delete_marker = dobj.delete_object.delete_marker,
+                is_version_purge,
+                error = %e,
+                "replicate_delete_to_target failed"
+            );
             rinfo.error = Some(e.to_string());
             if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Failed;
@@ -2445,9 +2516,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         }
 
         let mut replication_action = replication_action;
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2504,7 +2580,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
 
         if let Some(err) = if is_multipart {
             drop(gr);
-            replicate_object_with_multipart(MultipartReplicationContext {
+            let result = replicate_object_with_multipart(MultipartReplicationContext {
                 storage: storage.clone(),
                 cli: tgt_client.clone(),
                 src_bucket: &bucket,
@@ -2515,16 +2591,18 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 arn: &rinfo.arn,
                 put_opts,
             })
-            .await
-            .err()
+            .await;
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            result.err()
         } else {
             gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
             let byte_stream = async_read_to_bytestream(gr.stream);
-            tgt_client
+            let result = tgt_client
                 .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                 .await
-                .map_err(|e| std::io::Error::other(e.to_string()))
-                .err()
+                .map_err(|e| std::io::Error::other(e.to_string()));
+            record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+            result.err()
         } {
             rinfo.replication_status = ReplicationStatusType::Failed;
             rinfo.error = Some(err.to_string());
@@ -2664,9 +2742,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             warn!("failed to set replication tagging directive header: {err}");
         }
 
-        match tgt_client
-            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
-            .await
+        match head_object_with_proxy_stats(
+            &bucket,
+            tgt_client.as_ref(),
+            &tgt_client.bucket,
+            &object,
+            self.version_id.map(|v| v.to_string()),
+        )
+        .await
         {
             Ok(oi) => {
                 replication_action = get_replication_action(&object_info, &oi, self.op_type);
@@ -2786,7 +2869,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
 
             if let Some(err) = if is_multipart {
                 drop(gr);
-                replicate_object_with_multipart(MultipartReplicationContext {
+                let result = replicate_object_with_multipart(MultipartReplicationContext {
                     storage: storage.clone(),
                     cli: tgt_client.clone(),
                     src_bucket: &bucket,
@@ -2797,16 +2880,18 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     arn: &rinfo.arn,
                     put_opts,
                 })
-                .await
-                .err()
+                .await;
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                result.err()
             } else {
                 gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
                 let byte_stream = async_read_to_bytestream(gr.stream);
-                tgt_client
+                let result = tgt_client
                     .put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
                     .await
-                    .map_err(|e| std::io::Error::other(e.to_string()))
-                    .err()
+                    .map_err(|e| std::io::Error::other(e.to_string()));
+                record_proxy_request(&bucket, "PutObject", result.is_err()).await;
+                result.err()
             } {
                 rinfo.replication_status = ReplicationStatusType::Failed;
                 rinfo.error = Some(err.to_string());
@@ -3603,6 +3688,49 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_delete_replication_object_opts_marks_replica_deletes() {
+        let dobj = ObjectToDelete {
+            object_name: "obj".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            replication_status: ReplicationStatusType::Replica,
+            ..Default::default()
+        };
+
+        let opts = delete_replication_object_opts(&dobj, &oi);
+
+        assert!(
+            opts.replica,
+            "replica deletes must preserve replica status for downstream ReplicaModifications rules"
+        );
+        assert_eq!(opts.version_id, dobj.version_id);
+        assert_eq!(opts.name, dobj.object_name);
+        assert_eq!(opts.op_type, ReplicationType::Delete);
+    }
+
+    #[test]
+    fn test_delete_replication_object_opts_keeps_non_replica_deletes_local() {
+        let dobj = ObjectToDelete {
+            object_name: "obj".to_string(),
+            ..Default::default()
+        };
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            replication_status: ReplicationStatusType::Completed,
+            ..Default::default()
+        };
+
+        let opts = delete_replication_object_opts(&dobj, &oi);
+
+        assert!(!opts.replica, "source-originated deletes should not be treated as replica modifications");
+    }
+
     #[test]
     fn test_is_version_delete_replication_for_delete_marker_version_purge() {
         let dobj = DeletedObject {
@@ -3631,6 +3759,34 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_should_retry_delete_marker_purge_for_version_purge() {
+        let dobj = DeletedObject {
+            delete_marker: false,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            should_retry_delete_marker_purge(&dobj),
+            "delete-marker version purge should schedule delayed target cleanup in case the target marker arrives late"
+        );
+    }
+
+    #[test]
+    fn test_should_retry_delete_marker_purge_for_delete_marker_creation() {
+        let dobj = DeletedObject {
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            should_retry_delete_marker_purge(&dobj),
+            "delete-marker creation should keep the late-arrival cleanup path so downstream purges can catch up"
+        );
+    }
+
     #[test]
     fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
         assert!(
@@ -3651,6 +3807,34 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_should_count_head_proxy_failure_ignores_not_found_and_405() {
+        assert!(
+            !should_count_head_proxy_failure(true, Some("NoSuchKey"), Some(404)),
+            "not-found heads are expected when the object has not reached the target yet"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("MethodNotAllowed"), Some(405)),
+            "405 delete-marker probing responses should not be counted as proxy failures"
+        );
+        assert!(
+            !should_count_head_proxy_failure(false, Some("405"), Some(405)),
+            "numeric 405 codes must align with MethodNotAllowed semantics"
+        );
+    }
+
+    #[test]
+    fn test_should_count_head_proxy_failure_counts_unexpected_errors() {
+        assert!(
+            should_count_head_proxy_failure(false, Some("AccessDenied"), Some(403)),
+            "non-NotFound and non-405 service errors should be counted as failures"
+        );
+        assert!(
+            should_count_head_proxy_failure(false, None, Some(500)),
+            "raw 5xx head responses should be counted as proxy failures"
+        );
+    }
+
     #[tokio::test]
     async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
         let oi = ObjectInfo {

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -12,18 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::bucket::replication::get_global_replication_pool;
 use crate::error::Error;
 use crate::global::get_global_bucket_monitor;
 use rustfs_filemeta::{ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::interval;
 
+const ROLLING_WINDOW: Duration = Duration::from_secs(60);
+const FAILURE_LAST_HOUR_WINDOW: Duration = Duration::from_secs(60 * 60);
+
 /// Exponential Moving Average with thread-safe interior mutability
 #[derive(Debug)]
 pub struct ExponentialMovingAverage {
@@ -328,6 +332,13 @@ pub struct InQueueStats {
     pub now_count: AtomicI64,
 }
 
+#[derive(Debug, Clone)]
+struct QueueSample {
+    observed_at: Instant,
+    bytes: i64,
+    count: i64,
+}
+
 impl Clone for InQueueStats {
     fn clone(&self) -> Self {
         Self {
@@ -359,9 +370,60 @@ pub struct InQueueMetric {
     pub curr: InQueueStats,
     pub avg: InQueueStats,
     pub max: InQueueStats,
+    pub last_minute: InQueueStats,
+    #[serde(skip)]
+    samples: VecDeque<QueueSample>,
 }
 
 impl InQueueMetric {
+    fn observe(&mut self, observed_at: Instant) {
+        let bytes = self.curr.now_bytes.load(Ordering::Relaxed);
+        let count = self.curr.now_count.load(Ordering::Relaxed);
+
+        self.curr.bytes = bytes;
+        self.curr.count = count;
+        self.samples.push_back(QueueSample {
+            observed_at,
+            bytes,
+            count,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.avg = InQueueStats::default();
+            self.max = InQueueStats::default();
+            self.last_minute = InQueueStats::default();
+            return;
+        }
+
+        let sample_count = self.samples.len() as i64;
+        let total_bytes = self.samples.iter().map(|sample| sample.bytes).sum::<i64>();
+        let total_count = self.samples.iter().map(|sample| sample.count).sum::<i64>();
+        let max_bytes = self.samples.iter().map(|sample| sample.bytes).max().unwrap_or(0);
+        let max_count = self.samples.iter().map(|sample| sample.count).max().unwrap_or(0);
+
+        self.avg.bytes = total_bytes / sample_count;
+        self.avg.count = total_count / sample_count;
+        self.max.bytes = max_bytes;
+        self.max.count = max_count;
+        self.last_minute.bytes = self.avg.bytes;
+        self.last_minute.count = self.avg.count;
+    }
+
+    fn snapshot(&self) -> Self {
+        let mut snapshot = self.clone();
+        snapshot.curr.bytes = snapshot.curr.now_bytes.load(Ordering::Relaxed);
+        snapshot.curr.count = snapshot.curr.now_count.load(Ordering::Relaxed);
+        snapshot
+    }
+
     pub fn merge(&self, other: &InQueueMetric) -> Self {
         Self {
             curr: InQueueStats {
@@ -384,6 +446,12 @@ impl InQueueMetric {
                 count: self.max.count.max(other.max.count),
                 ..Default::default()
             },
+            last_minute: InQueueStats {
+                bytes: self.last_minute.bytes + other.last_minute.bytes,
+                count: self.last_minute.count + other.last_minute.count,
+                ..Default::default()
+            },
+            samples: VecDeque::new(),
         }
     }
 }
@@ -391,8 +459,8 @@ impl InQueueMetric {
 /// Queue cache
 #[derive(Debug, Default)]
 pub struct QueueCache {
-    pub bucket_stats: HashMap<String, InQueueStats>,
-    pub sr_queue_stats: InQueueStats,
+    pub bucket_stats: HashMap<String, InQueueMetric>,
+    pub sr_queue_stats: InQueueMetric,
 }
 
 impl QueueCache {
@@ -401,36 +469,19 @@ impl QueueCache {
     }
 
     pub fn update(&mut self) {
-        // Update queue statistics cache
-        // In actual implementation, this would get latest statistics from queue system
+        let observed_at = Instant::now();
+        self.sr_queue_stats.observe(observed_at);
+        for stats in self.bucket_stats.values_mut() {
+            stats.observe(observed_at);
+        }
     }
 
     pub fn get_bucket_stats(&self, bucket: &str) -> InQueueMetric {
-        if let Some(bucket_stat) = self.bucket_stats.get(bucket) {
-            InQueueMetric {
-                curr: InQueueStats {
-                    bytes: bucket_stat.now_bytes.load(Ordering::Relaxed),
-                    count: bucket_stat.now_count.load(Ordering::Relaxed),
-                    ..Default::default()
-                },
-                avg: InQueueStats::default(), // simplified implementation
-                max: InQueueStats::default(), // simplified implementation
-            }
-        } else {
-            InQueueMetric::default()
-        }
+        self.bucket_stats.get(bucket).map(InQueueMetric::snapshot).unwrap_or_default()
     }
 
     pub fn get_site_stats(&self) -> InQueueMetric {
-        InQueueMetric {
-            curr: InQueueStats {
-                bytes: self.sr_queue_stats.now_bytes.load(Ordering::Relaxed),
-                count: self.sr_queue_stats.now_count.load(Ordering::Relaxed),
-                ..Default::default()
-            },
-            avg: InQueueStats::default(), // simplified implementation
-            max: InQueueStats::default(), // simplified implementation
-        }
+        self.sr_queue_stats.snapshot()
     }
 }
 
@@ -505,11 +556,19 @@ impl ProxyStatsCache {
     }
 }
 
+#[derive(Debug, Clone)]
+struct FailureSample {
+    observed_at: Instant,
+    size: i64,
+}
+
 /// Failure statistics
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct FailStats {
     pub count: i64,
     pub size: i64,
+    #[serde(skip)]
+    recent: VecDeque<FailureSample>,
 }
 
 impl FailStats {
@@ -518,14 +577,42 @@ impl FailStats {
     }
 
     pub fn add_size(&mut self, size: i64, _err: Option<&Error>) {
+        let observed_at = Instant::now();
         self.count += 1;
         self.size += size;
+        self.recent.push_back(FailureSample { observed_at, size });
+        self.prune(observed_at);
+    }
+
+    fn prune(&mut self, observed_at: Instant) {
+        while self
+            .recent
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > FAILURE_LAST_HOUR_WINDOW)
+        {
+            self.recent.pop_front();
+        }
+    }
+
+    pub fn recent_since(&self, window: Duration) -> FailedMetric {
+        let now = Instant::now();
+        let mut count = 0i64;
+        let mut size = 0i64;
+        for sample in self.recent.iter().rev() {
+            if now.duration_since(sample.observed_at) > window {
+                break;
+            }
+            count += 1;
+            size += sample.size;
+        }
+        FailedMetric { count, size }
     }
 
     pub fn merge(&self, other: &FailStats) -> Self {
         Self {
             count: self.count + other.count,
             size: self.size + other.size,
+            recent: VecDeque::new(),
         }
     }
 
@@ -674,6 +761,14 @@ pub struct ActiveWorkerStat {
     pub curr: i32,
     pub max: i32,
     pub avg: f64,
+    #[serde(skip)]
+    samples: VecDeque<WorkerSample>,
+}
+
+#[derive(Debug, Clone)]
+struct WorkerSample {
+    observed_at: Instant,
+    workers: i32,
 }
 
 impl ActiveWorkerStat {
@@ -685,9 +780,31 @@ impl ActiveWorkerStat {
         self.clone()
     }
 
-    pub fn update(&mut self) {
-        // Simulate worker statistics update logic
-        // In actual implementation, this would get current active count from worker pool
+    pub fn update(&mut self, curr: i32) {
+        let observed_at = Instant::now();
+        self.curr = curr;
+        self.samples.push_back(WorkerSample {
+            observed_at,
+            workers: curr,
+        });
+
+        while self
+            .samples
+            .front()
+            .is_some_and(|sample| observed_at.duration_since(sample.observed_at) > ROLLING_WINDOW)
+        {
+            self.samples.pop_front();
+        }
+
+        if self.samples.is_empty() {
+            self.max = curr;
+            self.avg = curr as f64;
+            return;
+        }
+
+        self.max = self.samples.iter().map(|sample| sample.workers).max().unwrap_or(curr);
+        let total = self.samples.iter().map(|sample| sample.workers as i64).sum::<i64>();
+        self.avg = total as f64 / self.samples.len() as f64;
     }
 }
 
@@ -740,8 +857,11 @@ impl ReplicationStats {
             let mut interval = interval(Duration::from_secs(2));
             loop {
                 interval.tick().await;
+                let current = get_global_replication_pool()
+                    .map(|pool| pool.active_workers() + pool.active_lrg_workers() + pool.active_mrf_workers())
+                    .unwrap_or(0);
                 let mut workers = workers_clone.lock().await;
-                workers.update();
+                workers.update(current);
             }
         });
 
@@ -925,14 +1045,26 @@ impl ReplicationStats {
     /// Get replication metrics for all buckets
     pub async fn get_all(&self) -> HashMap<String, BucketReplicationStats> {
         let cache = self.cache.read().await;
-        let mut result = HashMap::new();
+        let mut result = HashMap::with_capacity(cache.len());
 
         for (bucket, stats) in cache.iter() {
-            let mut cloned_stats = stats.clone_stats();
-            // Add queue statistics
+            result.insert(bucket.clone(), stats.clone_stats());
+        }
+        drop(cache);
+
+        {
             let q_cache = self.q_cache.lock().await;
-            cloned_stats.q_stat = q_cache.get_bucket_stats(bucket);
-            result.insert(bucket.clone(), cloned_stats);
+            for (bucket, queue_stats) in &q_cache.bucket_stats {
+                let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+                bucket_stats.q_stat = queue_stats.snapshot();
+            }
+        }
+
+        {
+            let p_cache = self.p_cache.lock().await;
+            for bucket in p_cache.bucket_stats.keys() {
+                result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+            }
         }
 
         result
@@ -1114,12 +1246,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        stats.now_count.fetch_add(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_add(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_add(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
     }
 
     /// Decrease queue statistics
@@ -1128,12 +1260,12 @@ impl ReplicationStats {
         let stats = q_cache
             .bucket_stats
             .entry(bucket.to_string())
-            .or_insert_with(InQueueStats::default);
-        stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        stats.now_count.fetch_sub(1, Ordering::Relaxed);
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
 
-        q_cache.sr_queue_stats.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.now_count.fetch_sub(1, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
     }
 
     /// Increase proxy metrics
@@ -1166,6 +1298,51 @@ mod tests {
         assert_eq!(workers.curr, 0);
     }
 
+    #[test]
+    fn test_in_queue_metric_observe_updates_rolling_stats() {
+        let mut metric = InQueueMetric::default();
+        metric.curr.now_bytes.store(128, Ordering::Relaxed);
+        metric.curr.now_count.store(4, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        metric.curr.now_bytes.store(256, Ordering::Relaxed);
+        metric.curr.now_count.store(6, Ordering::Relaxed);
+        metric.observe(Instant::now());
+
+        assert_eq!(metric.curr.bytes, 256);
+        assert_eq!(metric.curr.count, 6);
+        assert_eq!(metric.max.bytes, 256);
+        assert_eq!(metric.max.count, 6);
+        assert_eq!(metric.last_minute.bytes, 192);
+        assert_eq!(metric.last_minute.count, 5);
+    }
+
+    #[test]
+    fn test_fail_stats_recent_since_tracks_windows() {
+        let mut stats = FailStats::default();
+        stats.add_size(64, None);
+        stats.add_size(32, None);
+
+        let last_minute = stats.recent_since(Duration::from_secs(60));
+        let last_hour = stats.recent_since(Duration::from_secs(60 * 60));
+        assert_eq!(last_minute.count, 2);
+        assert_eq!(last_minute.size, 96);
+        assert_eq!(last_hour.count, 2);
+        assert_eq!(last_hour.size, 96);
+    }
+
+    #[test]
+    fn test_active_worker_stat_update_tracks_rolling_avg_and_max() {
+        let mut stats = ActiveWorkerStat::default();
+        stats.update(2);
+        stats.update(6);
+        stats.update(4);
+
+        assert_eq!(stats.curr, 4);
+        assert_eq!(stats.max, 6);
+        assert_eq!(stats.avg, 4.0);
+    }
+
     #[tokio::test]
     async fn test_delete_bucket_stats() {
         let stats = ReplicationStats::new();
@@ -1218,6 +1395,15 @@ mod tests {
         assert_eq!(stat.replicated_count, 1);
     }
 
+    #[tokio::test]
+    async fn test_get_all_includes_proxy_only_bucket() {
+        let stats = ReplicationStats::new();
+        stats.inc_proxy("proxy-only-bucket", "HeadObject", false).await;
+
+        let all = stats.get_all().await;
+        assert!(all.contains_key("proxy-only-bucket"));
+    }
+
     #[test]
     fn test_sr_stats() {
         let sr_stats = SRStats::new();

crates/ecstore/src/client/api_error_response.rs

@@ -100,7 +100,7 @@ pub fn http_resp_to_error_response(
     bucket_name: &str,
     object_name: &str,
 ) -> ErrorResponse {
-    let err_body = String::from_utf8(b).unwrap();
+    let err_body = String::from_utf8_lossy(&b).to_string();
     if h.is_empty() || resp_status.is_client_error() || resp_status.is_server_error() {
         return ErrorResponse {
             status_code: resp_status,
@@ -178,36 +178,46 @@ pub fn http_resp_to_error_response(
                 };
             }
         }
-    } else {
-        err_resp = err_resp_.unwrap();
+    } else if let Ok(parsed_resp) = err_resp_ {
+        err_resp = parsed_resp;
     }
     err_resp.status_code = resp_status;
     if let Some(server_name) = h.get("Server") {
-        err_resp.server = server_name.to_str().expect("err").to_string();
+        if let Ok(server_str) = server_name.to_str() {
+            err_resp.server = server_str.to_string();
+        }
     }
 
-    let code = h.get("x-minio-error-code");
-    if code.is_some() {
-        err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
+    if let Some(code) = h.get("x-minio-error-code") {
+        if let Ok(code_str) = code.to_str() {
+            err_resp.code = S3ErrorCode::Custom(code_str.into());
+        }
     }
-    let desc = h.get("x-minio-error-desc");
-    if desc.is_some() {
-        err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
+    if let Some(desc) = h.get("x-minio-error-desc") {
+        if let Ok(desc_str) = desc.to_str() {
+            err_resp.message = desc_str.trim_matches('"').to_string();
+        }
     }
 
     if err_resp.request_id == "" {
         if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
-            err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
+            if let Ok(request_id_str) = x_amz_request_id.to_str() {
+                err_resp.request_id = request_id_str.to_string();
+            }
         }
     }
     if err_resp.host_id == "" {
         if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
-            err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
+            if let Ok(host_id_str) = x_amz_id_2.to_str() {
+                err_resp.host_id = host_id_str.to_string();
+            }
         }
     }
     if err_resp.region == "" {
         if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
-            err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
+            if let Ok(region_str) = x_amz_bucket_region.to_str() {
+                err_resp.region = region_str.to_string();
+            }
         }
     }
     if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {

crates/ecstore/src/client/api_get_object.rs

@@ -1,4 +1,3 @@
-#![allow(clippy::map_entry)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(clippy::map_entry)]
 #![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]

crates/ecstore/src/client/api_put_object.rs

@@ -137,21 +137,21 @@ impl Default for PutObjectOptions {
 impl PutObjectOptions {
     fn set_match_etag(&mut self, etag: &str) {
         if etag == "*" {
-            self.custom_header
-                .insert("If-Match", HeaderValue::from_str("*").expect("err"));
+            self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
         } else {
-            self.custom_header
-                .insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{}\"", etag)) {
+                self.custom_header.insert("If-Match", etag_value);
+            }
         }
     }
 
     fn set_match_etag_except(&mut self, etag: &str) {
         if etag == "*" {
-            self.custom_header
-                .insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
+            self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
         } else {
-            self.custom_header
-                .insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{etag}\"")) {
+                self.custom_header.insert("If-None-Match", etag_value);
+            }
         }
     }
 
@@ -162,59 +162,75 @@ impl PutObjectOptions {
         if content_type == "" {
             content_type = "application/octet-stream".to_string();
         }
-        header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
+        if let Ok(content_type_value) = HeaderValue::from_str(&content_type) {
+            header.insert("Content-Type", content_type_value);
+        }
 
         if self.content_encoding != "" {
-            header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
+            if let Ok(encoding_value) = HeaderValue::from_str(&self.content_encoding) {
+                header.insert("Content-Encoding", encoding_value);
+            }
         }
         if self.content_disposition != "" {
-            header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
+            if let Ok(disposition_value) = HeaderValue::from_str(&self.content_disposition) {
+                header.insert("Content-Disposition", disposition_value);
+            }
         }
         if self.content_language != "" {
-            header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
+            if let Ok(language_value) = HeaderValue::from_str(&self.content_language) {
+                header.insert("Content-Language", language_value);
+            }
         }
         if self.cache_control != "" {
-            header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
+            if let Ok(cache_value) = HeaderValue::from_str(&self.cache_control) {
+                header.insert("Cache-Control", cache_value);
+            }
         }
 
         if self.expires.unix_timestamp() != 0 {
-            header.insert(
-                "Expires",
-                HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
-            ); //rustfs invalid header
+            if let Ok(expires_str) = self.expires.format(ISO8601_DATEFORMAT) {
+                if let Ok(expires_value) = HeaderValue::from_str(&expires_str) {
+                    header.insert("Expires", expires_value);
+                }
+            }
         }
 
         if self.mode.as_str() != "" {
-            header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
+            if let Ok(mode_value) = HeaderValue::from_str(self.mode.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_MODE, mode_value);
+            }
         }
 
         if self.retain_until_date.unix_timestamp() != 0 {
-            header.insert(
-                X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
-                HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
-            );
+            if let Ok(retain_str) = self.retain_until_date.format(ISO8601_DATEFORMAT) {
+                if let Ok(retain_value) = HeaderValue::from_str(&retain_str) {
+                    header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, retain_value);
+                }
+            }
         }
 
         if self.legalhold.as_str() != "" {
-            header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
+            if let Ok(legalhold_value) = HeaderValue::from_str(self.legalhold.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, legalhold_value);
+            }
         }
 
         if self.storage_class != "" {
-            header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
+            if let Ok(storage_class_value) = HeaderValue::from_str(&self.storage_class) {
+                header.insert(X_AMZ_STORAGE_CLASS, storage_class_value);
+            }
         }
 
         if self.website_redirect_location != "" {
-            header.insert(
-                X_AMZ_WEBSITE_REDIRECT_LOCATION,
-                HeaderValue::from_str(&self.website_redirect_location).expect("err"),
-            );
+            if let Ok(redirect_value) = HeaderValue::from_str(&self.website_redirect_location) {
+                header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, redirect_value);
+            }
         }
 
         if !self.internal.replication_status.as_str().is_empty() {
-            header.insert(
-                X_AMZ_REPLICATION_STATUS,
-                HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
-            );
+            if let Ok(replication_status_value) = HeaderValue::from_str(self.internal.replication_status.as_str()) {
+                header.insert(X_AMZ_REPLICATION_STATUS, replication_status_value);
+            }
         }
 
         for (k, v) in &self.user_metadata {
@@ -360,17 +376,21 @@ impl TransitionClient {
 
             let mut md5_base64: String = "".to_string();
             if opts.send_content_md5 {
-                let mut md5_hasher = self.md5_hasher.lock().unwrap();
-                let hash = md5_hasher.as_mut().expect("err");
-                let hash = hash.hash_encode(&buf[..length]);
-                md5_base64 = base64_encode(hash.as_ref());
+                if let Some(mut md5_hasher) = self.md5_hasher.lock().unwrap().as_mut() {
+                    let hash = md5_hasher.hash_encode(&buf[..length]);
+                    md5_base64 = base64_encode(hash.as_ref());
+                }
             } else {
                 let mut crc = opts.auto_checksum.hasher()?;
                 crc.update(&buf[..length]);
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }

crates/ecstore/src/client/api_put_object_multipart.rs

@@ -127,7 +127,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -309,27 +313,27 @@ impl TransitionClient {
         let h = resp.headers();
         let mut obj_part = ObjectPart {
             checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
-                h_checksum_crc32.to_str().expect("err").to_string()
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
-                h_checksum_crc32c.to_str().expect("err").to_string()
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
-                h_checksum_sha1.to_str().expect("err").to_string()
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
-                h_checksum_sha256.to_str().expect("err").to_string()
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
-                h_checksum_crc64nvme.to_str().expect("err").to_string()
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
@@ -338,7 +342,7 @@ impl TransitionClient {
         obj_part.size = p.size;
         obj_part.part_num = p.part_number;
         obj_part.etag = if let Some(h_etag) = h.get("ETag") {
-            h_etag.to_str().expect("err").trim_matches('"').to_string()
+            h_etag.to_str().unwrap_or("").trim_matches('"').to_string()
         } else {
             "".to_string()
         };
@@ -398,7 +402,7 @@ impl TransitionClient {
             key: complete_multipart_upload_result.key,
             etag: trim_etag(&complete_multipart_upload_result.etag),
             version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
-                h_x_amz_version_id.to_str().expect("err").to_string()
+                h_x_amz_version_id.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },

crates/ecstore/src/client/api_put_object_streaming.rs

@@ -21,6 +21,7 @@
 use bytes::Bytes;
 use futures::future::join_all;
 use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use std::io::Error;
 use std::sync::RwLock;
 use std::{collections::HashMap, sync::Arc};
 use time::{OffsetDateTime, format_description};
@@ -152,7 +153,10 @@ impl TransitionClient {
 
             if opts.send_content_md5 {
                 let mut md5_hasher = self.md5_hasher.lock().unwrap();
-                let md5_hash = md5_hasher.as_mut().expect("err");
+                let md5_hash = match md5_hasher.as_mut() {
+                    Some(hasher) => hasher,
+                    None => return Err(std::io::Error::other("MD5 hasher not initialized")),
+                };
                 let hash = md5_hash.hash_encode(&buf[..length]);
                 md5_base64 = base64_encode(hash.as_ref());
             } else {
@@ -161,7 +165,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -275,11 +283,14 @@ impl TransitionClient {
         for part_number in 1..=total_parts_count {
             let mut buf = Vec::<u8>::new();
             select! {
-                buf = bufs_rx.recv() => {}
+                buf1 = bufs_rx.recv() => {
+                    if let Some(buf1) = buf1 {
+                        buf = buf1;
+                    }
+                }
                 err = err_rx.recv() => {
                     //cancel_token.cancel();
-                    //wg.Wait()
-                    return Err(err.expect("err"));
+                    return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
                 }
                 else => (),
             }
@@ -309,7 +320,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -319,12 +334,19 @@ impl TransitionClient {
             let clone_parts_info = parts_info.clone();
             let clone_upload_id = upload_id.clone();
             let clone_self = self.clone();
+            let err_tx_clone = err_tx.clone();
             futures.push(async move {
                 let mut md5_base64: String = "".to_string();
 
                 if opts.send_content_md5 {
                     let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
-                    let md5_hash = md5_hasher.as_mut().expect("err");
+                    let md5_hash = match md5_hasher.as_mut() {
+                        Some(hasher) => hasher,
+                        None => {
+                            //let _ = err_tx_clone.send(std::io::Error::other("MD5 hasher not initialized")).await;
+                            return Ok::<(), Error>(());
+                        }
+                    };
                     let hash = md5_hash.hash_encode(&buf[..length]);
                     md5_base64 = base64_encode(hash.as_ref());
                 }
@@ -344,12 +366,21 @@ impl TransitionClient {
                     sha256_hex: "".to_string(),
                     trailer: HeaderMap::new(),
                 };
-                let obj_part = clone_self.upload_part(&mut p).await.expect("err");
+                let obj_part = match clone_self.upload_part(&mut p).await {
+                    Ok(part) => part,
+                    Err(err) => {
+                        let _ = err_tx_clone.send(std::io::Error::other(err.to_string())).await;
+                        return Err::<(), Error>(err);
+                    }
+                };
 
-                let mut clone_parts_info = clone_parts_info.write().unwrap();
-                clone_parts_info.entry(part_number).or_insert(obj_part);
+                {
+                    let mut clone_parts_info = clone_parts_info.write().unwrap();
+                    clone_parts_info.entry(part_number).or_insert(obj_part);
+                }
 
-                clone_bufs_tx.send(buf);
+                let _ = clone_bufs_tx.send(buf).await;
+                Ok::<(), Error>(())
             });
 
             total_uploaded_size += length as i64;
@@ -359,7 +390,7 @@ impl TransitionClient {
 
         select! {
             err = err_rx.recv() => {
-                return Err(err.expect("err"));
+                return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
             }
             else => (),
         }
@@ -504,9 +535,10 @@ impl TransitionClient {
         Ok(UploadInfo {
             bucket: bucket_name.to_string(),
             key: object_name.to_string(),
-            etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
+            etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
+
             version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
-                h_x_amz_version_id.to_str().expect("err").to_string()
+                h_x_amz_version_id.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
@@ -514,27 +546,27 @@ impl TransitionClient {
             expiration: exp_time,
             expiration_rule_id: rule_id,
             checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
-                h_checksum_crc32.to_str().expect("err").to_string()
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
-                h_checksum_crc32c.to_str().expect("err").to_string()
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
-                h_checksum_sha1.to_str().expect("err").to_string()
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
-                h_checksum_sha256.to_str().expect("err").to_string()
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
-                h_checksum_crc64nvme.to_str().expect("err").to_string()
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },

crates/ecstore/src/client/api_remove.rs

@@ -25,7 +25,7 @@ use hyper::body::Bytes;
 use rustfs_utils::HashAlgorithm;
 use s3s::S3ErrorCode;
 use s3s::dto::ReplicationStatus;
-use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
+use s3s::header::{X_AMZ_BYPASS_GOVERNANCE_RETENTION, X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
 use serde::Deserialize;
 use std::fmt::Display;
 use std::{
@@ -111,8 +111,9 @@ impl TransitionClient {
             .await?;
 
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.delete(bucket_name);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
         }
         Ok(())
     }
@@ -142,8 +143,9 @@ impl TransitionClient {
             .await?;
 
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.delete(bucket_name);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
         }
 
         Ok(())
@@ -168,7 +170,7 @@ impl TransitionClient {
         let mut headers = HeaderMap::new();
 
         if opts.governance_bypass {
-            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
+            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true")); //amzBypassGovernance
         }
 
         let resp = self
@@ -197,13 +199,12 @@ impl TransitionClient {
         Ok(RemoveObjectResult {
             object_name: object_name.to_string(),
             object_version_id: opts.version_id,
-            delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
+            delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
             delete_marker_version_id: resp
                 .headers()
-                .get("x-amz-version-id")
-                .expect("err")
-                .to_str()
-                .expect("err")
+                .get(X_AMZ_VERSION_ID)
+                .and_then(|v| v.to_str().ok())
+                .unwrap_or_default()
                 .to_string(),
             ..Default::default()
         })
@@ -290,15 +291,15 @@ impl TransitionClient {
                             bucket_name,
                             &object.name,
                             RemoveObjectOptions {
-                                version_id: object.version_id.expect("err").to_string(),
+                                version_id: object.version_id.map(|id| id.to_string()).unwrap_or_default(),
                                 governance_bypass: opts.governance_bypass,
                                 ..Default::default()
                             },
                         )
                         .await?;
                     let remove_result_clone = remove_result.clone();
-                    if !remove_result.err.is_none() {
-                        match to_error_response(&remove_result.err.expect("err")).code {
+                    if let Some(err) = &remove_result.err {
+                        match to_error_response(err).code {
                             S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
                                 continue;
                             }
@@ -326,7 +327,7 @@ impl TransitionClient {
 
             let mut headers = HeaderMap::new();
             if opts.governance_bypass {
-                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
+                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true"));
             }
 
             let remove_bytes = generate_remove_multi_objects_request(&batch);
@@ -423,23 +424,20 @@ impl TransitionClient {
                         request_id: resp
                             .headers()
                             .get("x-amz-request-id")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         host_id: resp
                             .headers()
                             .get("x-amz-id-2")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         region: resp
                             .headers()
                             .get("x-amz-bucket-region")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         ..Default::default()
                     };
@@ -472,10 +470,11 @@ pub struct RemoveObjectError {
 
 impl Display for RemoveObjectError {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        if self.err.is_none() {
-            return write!(f, "unexpected remove object error result");
+        if let Some(err) = &self.err {
+            write!(f, "{}", err.to_string())
+        } else {
+            write!(f, "unexpected remove object error result")
         }
-        write!(f, "{}", self.err.as_ref().expect("err").to_string())
     }
 }
 

crates/ecstore/src/client/bucket_cache.rs

@@ -70,10 +70,10 @@ impl TransitionClient {
 
         let mut location;
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            let ret = bucket_loc_cache.get(bucket_name);
-            if let Some(location) = ret {
-                return Ok(location);
+            if let Ok(bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                if let Some(location) = bucket_loc_cache.get(bucket_name) {
+                    return Ok(location);
+                }
             }
             //location = ret?;
         }
@@ -83,8 +83,9 @@ impl TransitionClient {
         let mut resp = self.doit(req).await?;
         location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.set(bucket_name, &location);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.set(bucket_name, &location);
+            }
         }
         Ok(location)
     }
@@ -108,7 +109,11 @@ impl TransitionClient {
             url_str.push_str("://");
             url_str.push_str(bucket_name);
             url_str.push_str(".");
-            url_str.push_str(target_url.host_str().expect("err"));
+            url_str.push_str(
+                target_url
+                    .host_str()
+                    .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "host is none"))?,
+            );
             url_str.push_str("/?location");
         } else {
             let mut path = bucket_name.to_string();
@@ -135,13 +140,16 @@ impl TransitionClient {
 
         let value;
         {
-            let mut creds_provider = self.creds_provider.lock().unwrap();
-            value = match creds_provider.get_with_context(Some(self.cred_context())) {
-                Ok(v) => v,
-                Err(err) => {
-                    return Err(std::io::Error::other(err));
-                }
-            };
+            if let Ok(mut creds_provider) = self.creds_provider.lock() {
+                value = match creds_provider.get_with_context(Some(self.cred_context())) {
+                    Ok(v) => v,
+                    Err(err) => {
+                        return Err(std::io::Error::other(err));
+                    }
+                };
+            } else {
+                return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
+            }
         }
 
         let mut signer_type = value.signer_type.clone();
@@ -171,8 +179,9 @@ impl TransitionClient {
             content_sha256 = UNSIGNED_PAYLOAD.to_string();
         }
 
-        req.headers_mut()
-            .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
+        if let Ok(content_sha256_value) = content_sha256.parse() {
+            req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
+        }
         let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
         Ok(req)
     }
@@ -228,13 +237,16 @@ async fn process_bucket_location_response(
     }
     let mut location = "".to_string();
     if tier_type == "huaweicloud" {
-        let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(body_vec).unwrap()).unwrap();
-        location = d.location_constraint;
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(d) = quick_xml::de::from_str::<CreateBucketConfiguration>(&body_str) {
+                location = d.location_constraint;
+            }
+        }
     } else {
-        if let Ok(LocationConstraint { field }) =
-            quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(body_vec).unwrap())
-        {
-            location = field;
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&body_str) {
+                location = field;
+            }
         }
     }
     //debug!("location: {}", location);

crates/ecstore/src/client/credentials.rs

@@ -1,4 +1,3 @@
-#![allow(unused_imports)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
@@ -57,7 +57,9 @@ impl<P: Provider + Default> Credentials<P> {
 
     pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
         if self.is_expired() {
-            let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
+            let creds = self.provider.retrieve_with_cred_context(cc.unwrap_or(CredContext {
+                endpoint: "".to_string(),
+            }));
             self.creds = creds;
             self.force_refresh = false;
         }

crates/ecstore/src/client/object_handlers_common.rs

@@ -16,12 +16,26 @@ use std::sync::Arc;
 use tracing::warn;
 
 use crate::bucket::lifecycle::lifecycle;
+use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::store::ECStore;
 use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
+use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
 use rustfs_lock::MAX_DELETE_LIST;
 
+fn lifecycle_version_delete_replication_state(
+    replicate_decision_str: String,
+    pending_status: Option<String>,
+) -> ReplicationState {
+    ReplicationState {
+        replicate_decision_str,
+        version_purge_status_internal: pending_status.clone(),
+        purge_targets: version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
+        ..Default::default()
+    }
+}
+
 pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
     let version_suspended = match BucketVersioningSys::get(bucket).await {
         Ok(vc) => vc.suspended(),
@@ -39,7 +53,37 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
         } else {
             remaining = &[];
         }
-        let (_deleted_objs, errors) = api
+
+        let mut replication_candidates: Vec<Option<ReplicationState>> = Vec::with_capacity(to_del.len());
+        for object in to_del.iter() {
+            let version_id = object.version_id.map(|vid| vid.to_string());
+            let opts = ObjectOptions {
+                version_id: version_id.clone(),
+                versioned: true,
+                version_suspended,
+                ..Default::default()
+            };
+            let candidate = match api.get_object_info(bucket, &object.object_name, &opts).await {
+                Ok(info) => {
+                    let dsc = check_replicate_delete(bucket, object, &info, &opts, None).await;
+                    dsc.replicate_any()
+                        .then(|| lifecycle_version_delete_replication_state(dsc.to_string(), dsc.pending_status()))
+                }
+                Err(err) => {
+                    warn!(
+                        bucket,
+                        object = %object.object_name,
+                        version_id = ?version_id,
+                        error = ?err,
+                        "failed to get object info during lifecycle noncurrent version cleanup; skipping delete replication scheduling"
+                    );
+                    None
+                }
+            };
+            replication_candidates.push(candidate);
+        }
+
+        let (mut deleted_objs, errors) = api
             .delete_objects(
                 bucket,
                 to_del.to_vec(),
@@ -49,6 +93,24 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
                 },
             )
             .await;
+
+        for (i, deleted_obj) in deleted_objs.iter_mut().enumerate() {
+            if errors.get(i).and_then(|err| err.as_ref()).is_some() {
+                continue;
+            }
+            let Some(replication_state) = replication_candidates.get(i).and_then(|c| c.clone()) else {
+                continue;
+            };
+            deleted_obj.replication_state = Some(replication_state);
+            schedule_replication_delete(DeletedObjectReplicationInfo {
+                delete_object: deleted_obj.clone(),
+                bucket: bucket.to_string(),
+                event_type: REPLICATE_INCOMING_DELETE.to_string(),
+                ..Default::default()
+            })
+            .await;
+        }
+
         for (i, err) in errors.iter().enumerate() {
             if let Some(e) = err {
                 let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or("<unknown>");
@@ -65,3 +127,20 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::lifecycle_version_delete_replication_state;
+
+    #[test]
+    fn lifecycle_version_delete_replication_state_tracks_pending_purge_targets() {
+        let state = lifecycle_version_delete_replication_state(
+            "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
+            Some("arn:aws:s3:::target=PENDING;".to_string()),
+        );
+
+        assert_eq!(state.version_purge_status_internal.as_deref(), Some("arn:aws:s3:::target=PENDING;"));
+        assert!(state.purge_targets.contains_key("arn:aws:s3:::target"));
+        assert_eq!(state.replicate_decision_str, "arn:aws:s3:::target=true;false;arn:aws:s3:::target;");
+    }
+}

crates/ecstore/src/client/transition_api.rs

@@ -251,9 +251,10 @@ impl TransitionClient {
         };
 
         {
-            let mut md5_hasher = client.md5_hasher.lock().unwrap();
-            if md5_hasher.is_none() {
-                *md5_hasher = Some(HashAlgorithm::Md5);
+            if let Ok(mut md5_hasher) = client.md5_hasher.lock() {
+                if md5_hasher.is_none() {
+                    *md5_hasher = Some(HashAlgorithm::Md5);
+                }
             }
         }
         if client.sha256_hasher.is_none() {
@@ -275,25 +276,30 @@ impl TransitionClient {
     }
 
     fn trace_errors_only_off(&self) {
-        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
-        *trace_errors_only = false;
+        if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
+            *trace_errors_only = false;
+        }
     }
 
     fn trace_off(&self) {
-        let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap();
-        *is_trace_enabled = false;
-        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
-        *trace_errors_only = false;
+        if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
+            *is_trace_enabled = false;
+        }
+        if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
+            *trace_errors_only = false;
+        }
     }
 
     fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
-        let mut endpoint = self.s3_accelerate_endpoint.lock().unwrap();
-        *endpoint = accelerate_endpoint.to_string();
+        if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
+            *endpoint = accelerate_endpoint.to_string();
+        }
     }
 
     fn set_s3_enable_dual_stack(&self, enabled: bool) {
-        let mut dual_stack = self.s3_dual_stack_enabled.lock().unwrap();
-        *dual_stack = enabled;
+        if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
+            *dual_stack = enabled;
+        }
     }
 
     pub fn hash_materials(
@@ -352,7 +358,6 @@ impl TransitionClient {
         let resp;
         let http_client = self.http_client.clone();
         {
-            //let mut http_client = http_client.lock().unwrap();
             req_method = req.method().clone();
             req_uri = req.uri().clone();
             req_headers = req.headers().clone();
@@ -368,7 +373,10 @@ impl TransitionClient {
             return Err(std::io::Error::other(err));
         }
 
-        let resp = resp.unwrap();
+        let resp = match resp {
+            Ok(r) => r,
+            Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
+        };
         debug!("http_resp: {:?}", resp);
 
         //let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
@@ -455,11 +463,13 @@ impl TransitionClient {
                             return Err(std::io::Error::other(err_response));
                         }
                         if metadata.bucket_name != "" {
-                            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-                            let location = bucket_loc_cache.get(&metadata.bucket_name);
-                            if location.is_some() && location.unwrap() != err_response.region {
-                                bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
-                                //continue;
+                            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                                if let Some(location) = bucket_loc_cache.get(&metadata.bucket_name) {
+                                    if location != err_response.region {
+                                        bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
+                                        //continue;
+                                    }
+                                }
                             }
                         } else if err_response.region != metadata.bucket_location {
                             metadata.bucket_location = err_response.region.clone();
@@ -518,8 +528,11 @@ impl TransitionClient {
 
         let value;
         {
-            let mut creds_provider = self.creds_provider.lock().unwrap();
-            value = creds_provider.get_with_context(Some(self.cred_context()))?;
+            if let Ok(mut creds_provider) = self.creds_provider.lock() {
+                value = creds_provider.get_with_context(Some(self.cred_context()))?;
+            } else {
+                return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
+            }
         }
 
         let mut signer_type = value.signer_type.clone();
@@ -548,8 +561,10 @@ impl TransitionClient {
                     )));
                 }
                 let headers = req.headers_mut();
-                for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
-                    headers.insert(k, v.clone());
+                if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
+                    for (k, v) in extra_headers {
+                        headers.insert(k, v.clone());
+                    }
                 }
             }
             if signer_type == SignatureType::SignatureV2 {
@@ -571,18 +586,22 @@ impl TransitionClient {
         self.set_user_agent(&mut req);
 
         for (k, v) in metadata.custom_header.clone() {
-            req.headers_mut().insert(k.expect("err"), v);
+            if let Some(key) = k {
+                req.headers_mut().insert(key, v);
+            }
         }
 
         //req.content_length = metadata.content_length;
         if metadata.content_length <= -1 {
-            let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
-            req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
+            if let Ok(chunked_value) = HeaderValue::from_str(&vec!["chunked"].join(",")) {
+                req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
+            }
         }
 
         if metadata.content_md5_base64.len() > 0 {
-            let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err");
-            req.headers_mut().insert("Content-Md5", md5_value);
+            if let Ok(md5_value) = HeaderValue::from_str(&metadata.content_md5_base64) {
+                req.headers_mut().insert("Content-Md5", md5_value);
+            }
         }
 
         if signer_type == SignatureType::SignatureAnonymous {
@@ -607,8 +626,13 @@ impl TransitionClient {
             } else if metadata.trailer.len() > 0 {
                 sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
             }
-            req.headers_mut()
-                .insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err"));
+            let header_name = "X-Amz-Content-Sha256"
+                .parse::<HeaderName>()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+            let header_value = sha_header
+                .parse()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+            req.headers_mut().insert(header_name, header_value);
 
             req = rustfs_signer::sign_v4_trailer(
                 req,
@@ -636,7 +660,7 @@ impl TransitionClient {
 
     pub fn set_user_agent(&self, req: &mut Request<s3s::Body>) {
         let headers = req.headers_mut();
-        headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
+        headers.insert("User-Agent", HeaderValue::from_static(C_USER_AGENT));
     }
 
     fn make_target_url(
@@ -648,7 +672,10 @@ impl TransitionClient {
         query_values: &HashMap<String, String>,
     ) -> Result<Url, std::io::Error> {
         let scheme = self.endpoint_url.scheme();
-        let host = self.endpoint_url.host().unwrap();
+        let host = self
+            .endpoint_url
+            .host()
+            .ok_or_else(|| std::io::Error::other("Endpoint URL has no host"))?;
         let default_port = if scheme == "https" { 443 } else { 80 };
         let port = self.endpoint_url.port().unwrap_or(default_port);
 
@@ -1155,9 +1182,10 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re
         for (name, value) in h.iter() {
             let header_name = name.as_str().to_lowercase();
             if header_name.starts_with("x-amz-meta-") {
-                let key = header_name.strip_prefix("x-amz-meta-").unwrap().to_string();
-                if let Ok(value_str) = value.to_str() {
-                    meta.insert(key, value_str.to_string());
+                if let Some(key) = header_name.strip_prefix("x-amz-meta-") {
+                    if let Ok(value_str) = value.to_str() {
+                        meta.insert(key.to_string(), value_str.to_string());
+                    }
                 }
             }
         }

crates/ecstore/src/config/audit.rs

@@ -14,11 +14,16 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
-    WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
+    WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -192,3 +197,204 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_ADDRESS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_SUBJECT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_CREDENTIALS_FILE.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_REQUIRED.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+#[allow(dead_code)]
+pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_AUTH_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
+            value: EnableState::On.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/com.rs

@@ -18,8 +18,14 @@ use crate::error::{Error, Result};
 use crate::global::is_first_cluster_node_local;
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{
+    AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
 use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -34,6 +40,8 @@ const CONFIG_FILE: &str = "config.json";
 
 pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
 
+pub const COMMA_SEPARATED_LISTS: &[&str] = &[rustfs_config::oidc::OIDC_SCOPES, rustfs_config::oidc::OIDC_OTHER_AUDIENCES];
+
 static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
 
 static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
@@ -42,6 +50,84 @@ static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
     h
 });
 
+#[derive(Clone, Copy)]
+struct TargetConfigDescriptor {
+    external_key: &'static str,
+    subsystem_key: &'static str,
+    default_kvs: &'static LazyLock<KVS>,
+    valid_keys: &'static [&'static str],
+}
+
+fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
+    [
+        TargetConfigDescriptor {
+            external_key: "webhook",
+            subsystem_key: NOTIFY_WEBHOOK_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
+            valid_keys: NOTIFY_WEBHOOK_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: NOTIFY_KAFKA_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_KAFKA_KVS,
+            valid_keys: NOTIFY_KAFKA_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "mqtt",
+            subsystem_key: NOTIFY_MQTT_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_MQTT_KVS,
+            valid_keys: NOTIFY_MQTT_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "nats",
+            subsystem_key: NOTIFY_NATS_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_NATS_KVS,
+            valid_keys: NOTIFY_NATS_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "pulsar",
+            subsystem_key: NOTIFY_PULSAR_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_PULSAR_KVS,
+            valid_keys: NOTIFY_PULSAR_KEYS,
+        },
+    ]
+}
+
+fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] {
+    [
+        TargetConfigDescriptor {
+            external_key: "webhook",
+            subsystem_key: AUDIT_WEBHOOK_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
+            valid_keys: AUDIT_WEBHOOK_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "kafka",
+            subsystem_key: AUDIT_KAFKA_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_KAFKA_KVS,
+            valid_keys: AUDIT_KAFKA_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "mqtt",
+            subsystem_key: AUDIT_MQTT_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_MQTT_KVS,
+            valid_keys: AUDIT_MQTT_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "nats",
+            subsystem_key: AUDIT_NATS_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_NATS_KVS,
+            valid_keys: AUDIT_NATS_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "pulsar",
+            subsystem_key: AUDIT_PULSAR_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_PULSAR_KVS,
+            valid_keys: AUDIT_PULSAR_KEYS,
+        },
+    ]
+}
+
 #[instrument(skip(api))]
 pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
     let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
@@ -192,15 +278,15 @@ fn parse_oidc_scalar_value(key: &str, value: &Value) -> Option<String> {
         }),
         Value::Bool(v) => Some(v.to_string()),
         Value::Number(v) => Some(v.to_string()),
-        Value::Array(values) if key == rustfs_config::oidc::OIDC_SCOPES => {
-            let scopes = values
+        Value::Array(values) if COMMA_SEPARATED_LISTS.contains(&key) => {
+            let values_str = values
                 .iter()
                 .filter_map(Value::as_str)
                 .map(str::trim)
-                .filter(|scope| !scope.is_empty())
+                .filter(|val| !val.is_empty())
                 .collect::<Vec<_>>()
                 .join(",");
-            Some(scopes)
+            Some(values_str)
         }
         Value::Null => None,
         _ => None,
@@ -261,7 +347,7 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
     applied
 }
 
-fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
+fn parse_target_scalar_value(key: &str, value: &Value) -> Option<String> {
     match value {
         Value::String(v) => Some(v.trim().to_string()),
         Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v {
@@ -276,7 +362,7 @@ fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
     }
 }
 
-fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
+fn decode_target_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
     let mut kvs = KVS::new();
 
     for (key, value) in instance {
@@ -284,7 +370,7 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
             continue;
         }
 
-        if let Some(parsed) = parse_notify_scalar_value(key, value) {
+        if let Some(parsed) = parse_target_scalar_value(key, value) {
             kvs.insert(key.clone(), parsed);
         }
     }
@@ -292,21 +378,21 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
     kvs
 }
 
-fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
+fn decode_target_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
     match value {
-        Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)),
+        Value::Object(instance) => Some(decode_target_instance_object(instance, valid_keys)),
         Value::Array(_) => serde_json::from_value::<KVS>(value.clone()).ok(),
         _ => None,
     }
 }
 
-fn is_notify_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
+fn is_target_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
     section
         .iter()
-        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some())
+        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_target_scalar_value(key, value).is_some())
 }
 
-fn apply_external_notify_section(
+fn apply_external_target_section(
     cfg: &mut Config,
     notify_obj: &Map<String, Value>,
     external_key: &str,
@@ -325,8 +411,8 @@ fn apply_external_notify_section(
     let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default();
     let mut applied = false;
 
-    if is_notify_instance_shorthand(section_obj, valid_keys) {
-        let kvs = decode_notify_instance_object(section_obj, valid_keys);
+    if is_target_instance_shorthand(section_obj, valid_keys) {
+        let kvs = decode_target_instance_object(section_obj, valid_keys);
         if !kvs.is_empty() {
             let mut merged = default_kvs.clone();
             merged.extend(kvs);
@@ -337,7 +423,7 @@ fn apply_external_notify_section(
     }
 
     for (raw_instance, value) in section_obj {
-        let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else {
+        let Some(mut kvs) = decode_target_instance_value(value, valid_keys) else {
             continue;
         };
         if kvs.is_empty() {
@@ -363,29 +449,31 @@ fn apply_external_notify_section(
     applied
 }
 
+fn apply_external_target_descriptors(
+    cfg: &mut Config,
+    section_obj: &Map<String, Value>,
+    descriptors: &[TargetConfigDescriptor],
+) -> bool {
+    let mut applied = false;
+    for descriptor in descriptors {
+        applied |= apply_external_target_section(
+            cfg,
+            section_obj,
+            descriptor.external_key,
+            descriptor.subsystem_key,
+            descriptor.default_kvs,
+            descriptor.valid_keys,
+        );
+    }
+    applied
+}
+
 fn apply_external_notify_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
     let Some(Value::Object(notify_obj)) = root.get("notify") else {
         return false;
     };
 
-    let mut applied = false;
-    applied |= apply_external_notify_section(
-        cfg,
-        notify_obj,
-        "webhook",
-        NOTIFY_WEBHOOK_SUB_SYS,
-        &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
-        NOTIFY_WEBHOOK_KEYS,
-    );
-    applied |= apply_external_notify_section(
-        cfg,
-        notify_obj,
-        "mqtt",
-        NOTIFY_MQTT_SUB_SYS,
-        &notify::DEFAULT_NOTIFY_MQTT_KVS,
-        NOTIFY_MQTT_KEYS,
-    );
-    applied
+    apply_external_target_descriptors(cfg, notify_obj, &notify_target_descriptors())
 }
 
 fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
@@ -394,24 +482,7 @@ fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
         return false;
     };
 
-    let mut applied = false;
-    applied |= apply_external_notify_section(
-        cfg,
-        audit_obj,
-        "webhook",
-        AUDIT_WEBHOOK_SUB_SYS,
-        &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
-        AUDIT_WEBHOOK_KEYS,
-    );
-    applied |= apply_external_notify_section(
-        cfg,
-        audit_obj,
-        "mqtt",
-        AUDIT_MQTT_SUB_SYS,
-        &audit::DEFAULT_AUDIT_MQTT_KVS,
-        AUDIT_MQTT_KEYS,
-    );
-    applied
+    apply_external_target_descriptors(cfg, audit_obj, &audit_target_descriptors())
 }
 
 fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
@@ -513,15 +584,15 @@ fn build_oidc_provider_object(kvs: &KVS) -> Map<String, Value> {
             continue;
         }
 
-        if kv.key == rustfs_config::oidc::OIDC_SCOPES {
-            let scopes = kv
+        if COMMA_SEPARATED_LISTS.contains(&kv.key.as_str()) {
+            let values = kv
                 .value
                 .split(',')
                 .map(str::trim)
-                .filter(|scope| !scope.is_empty())
-                .map(|scope| Value::String(scope.to_string()))
+                .filter(|val| !val.is_empty())
+                .map(|val| Value::String(val.to_string()))
                 .collect::<Vec<_>>();
-            provider.insert(kv.key.clone(), Value::Array(scopes));
+            provider.insert(kv.key.clone(), Value::Array(values));
             continue;
         }
 
@@ -604,12 +675,21 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     oidc_obj
 }
 
-fn is_notify_bool_key(key: &str) -> bool {
-    key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+fn is_target_bool_key(key: &str) -> bool {
+    matches!(
+        key,
+        ENABLE_KEY
+            | rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+            | rustfs_config::KAFKA_TLS_ENABLE
+            | rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
+            | rustfs_config::NATS_TLS_REQUIRED
+            | rustfs_config::PULSAR_TLS_ALLOW_INSECURE
+            | rustfs_config::PULSAR_TLS_HOSTNAME_VERIFICATION
+    )
 }
 
-fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
-    if is_notify_bool_key(key) {
+fn encode_target_scalar_value(key: &str, value: &str) -> Value {
+    if is_target_bool_key(key) {
         if let Ok(state) = value.parse::<EnableState>() {
             return Value::Bool(state.is_enabled());
         }
@@ -630,7 +710,7 @@ fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool {
         .unwrap_or(false)
 }
 
-fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
+fn build_target_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
     let mut instance = Map::new();
 
     for key in valid_keys {
@@ -653,13 +733,13 @@ fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&s
             continue;
         }
 
-        instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value));
+        instance.insert((*key).to_string(), encode_target_scalar_value(key, &effective_value));
     }
 
     instance
 }
 
-fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
+fn merged_target_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
     let mut merged = default_kvs.clone();
     if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) {
         merged.extend(kvs.clone());
@@ -667,7 +747,7 @@ fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS
     merged
 }
 
-fn build_notify_subsystem_object(
+fn build_target_subsystem_object(
     cfg: &Config,
     subsystem_key: &str,
     default_kvs: &KVS,
@@ -677,11 +757,11 @@ fn build_notify_subsystem_object(
         return Map::new();
     };
 
-    let effective_default = merged_notify_default_kvs(subsystem, default_kvs);
+    let effective_default = merged_target_default_kvs(subsystem, default_kvs);
     let mut subsystem_obj = Map::new();
 
     if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) {
-        let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
+        let default_obj = build_target_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
         if !default_obj.is_empty() {
             subsystem_obj.insert("default".to_string(), Value::Object(default_obj));
         }
@@ -694,7 +774,7 @@ fn build_notify_subsystem_object(
     instances.sort_by_key(|(lhs, _)| *lhs);
 
     for (instance_key, kvs) in instances {
-        let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
+        let instance_obj = build_target_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
         if !instance_obj.is_empty() {
             subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj));
         }
@@ -703,38 +783,43 @@ fn build_notify_subsystem_object(
     subsystem_obj
 }
 
+fn build_target_object(cfg: &Config, descriptors: &[TargetConfigDescriptor]) -> Map<String, Value> {
+    let mut target_obj = Map::new();
+    for descriptor in descriptors {
+        let subsystem_obj =
+            build_target_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
+        if !subsystem_obj.is_empty() {
+            target_obj.insert(descriptor.external_key.to_string(), Value::Object(subsystem_obj));
+        }
+    }
+    target_obj
+}
+
 fn build_notify_object(cfg: &Config) -> Map<String, Value> {
-    let mut notify_obj = Map::new();
-
-    let webhook_obj =
-        build_notify_subsystem_object(cfg, NOTIFY_WEBHOOK_SUB_SYS, &notify::DEFAULT_NOTIFY_WEBHOOK_KVS, NOTIFY_WEBHOOK_KEYS);
-    if !webhook_obj.is_empty() {
-        notify_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
-    }
-
-    let mqtt_obj = build_notify_subsystem_object(cfg, NOTIFY_MQTT_SUB_SYS, &notify::DEFAULT_NOTIFY_MQTT_KVS, NOTIFY_MQTT_KEYS);
-    if !mqtt_obj.is_empty() {
-        notify_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
-    }
-
-    notify_obj
+    build_target_object(cfg, &notify_target_descriptors())
 }
 
 fn build_audit_object(cfg: &Config) -> Map<String, Value> {
-    let mut audit_obj = Map::new();
+    build_target_object(cfg, &audit_target_descriptors())
+}
 
-    let webhook_obj =
-        build_notify_subsystem_object(cfg, AUDIT_WEBHOOK_SUB_SYS, &audit::DEFAULT_AUDIT_WEBHOOK_KVS, AUDIT_WEBHOOK_KEYS);
-    if !webhook_obj.is_empty() {
-        audit_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
+fn sync_rendered_target_object(
+    target_obj: &mut Map<String, Value>,
+    rendered_target: &Map<String, Value>,
+    descriptors: &[TargetConfigDescriptor],
+) {
+    for descriptor in descriptors {
+        match rendered_target.get(descriptor.external_key) {
+            Some(Value::Object(v)) => {
+                target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone()));
+                target_obj.remove(descriptor.subsystem_key);
+            }
+            _ => {
+                target_obj.remove(descriptor.external_key);
+                target_obj.remove(descriptor.subsystem_key);
+            }
+        }
     }
-
-    let mqtt_obj = build_notify_subsystem_object(cfg, AUDIT_MQTT_SUB_SYS, &audit::DEFAULT_AUDIT_MQTT_KVS, AUDIT_MQTT_KEYS);
-    if !mqtt_obj.is_empty() {
-        audit_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
-    }
-
-    audit_obj
 }
 
 fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
@@ -771,67 +856,31 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
         _ => Map::new(),
     };
     let rendered_notify = build_notify_object(cfg);
-    match rendered_notify.get("webhook") {
-        Some(Value::Object(v)) => {
-            notify_obj.insert("webhook".to_string(), Value::Object(v.clone()));
-            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
-        }
-        _ => {
-            notify_obj.remove("webhook");
-            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
-        }
-    }
-    match rendered_notify.get("mqtt") {
-        Some(Value::Object(v)) => {
-            notify_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
-            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
-        }
-        _ => {
-            notify_obj.remove("mqtt");
-            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
-        }
-    }
+    sync_rendered_target_object(&mut notify_obj, &rendered_notify, &notify_target_descriptors());
     if notify_obj.is_empty() {
         root.remove("notify");
     } else {
         root.insert("notify".to_string(), Value::Object(notify_obj));
     }
-    root.remove(NOTIFY_WEBHOOK_SUB_SYS);
-    root.remove(NOTIFY_MQTT_SUB_SYS);
+    for descriptor in notify_target_descriptors() {
+        root.remove(descriptor.subsystem_key);
+    }
 
     let mut logger_obj = match root.remove("logger") {
         Some(Value::Object(v)) => v,
         _ => Map::new(),
     };
     let rendered_audit = build_audit_object(cfg);
-    match rendered_audit.get("webhook") {
-        Some(Value::Object(v)) => {
-            logger_obj.insert("webhook".to_string(), Value::Object(v.clone()));
-            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
-        }
-        _ => {
-            logger_obj.remove("webhook");
-            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
-        }
-    }
-    match rendered_audit.get("mqtt") {
-        Some(Value::Object(v)) => {
-            logger_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
-            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
-        }
-        _ => {
-            logger_obj.remove("mqtt");
-            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
-        }
-    }
+    sync_rendered_target_object(&mut logger_obj, &rendered_audit, &audit_target_descriptors());
     if logger_obj.is_empty() {
         root.remove("logger");
     } else {
         root.insert("logger".to_string(), Value::Object(logger_obj));
     }
     root.remove("audit");
-    root.remove(AUDIT_WEBHOOK_SUB_SYS);
-    root.remove(AUDIT_MQTT_SUB_SYS);
+    for descriptor in audit_target_descriptors() {
+        root.remove(descriptor.subsystem_key);
+    }
 
     Ok(serde_json::to_vec(&Value::Object(root))?)
 }
@@ -1082,8 +1131,8 @@ mod tests {
         ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
     };
     use http::HeaderMap;
-    use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-    use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+    use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
+    use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
     use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
     use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
     use rustfs_filemeta::FileInfo;
@@ -1655,6 +1704,7 @@ mod tests {
               "client_id":"console",
               "client_secret":"secret-value",
               "scopes":["openid","profile","email"],
+              "other_audiences":["aud1", "aud2"],
               "redirect_uri_dynamic":true,
               "display_name":"Default Provider"
             },
@@ -1679,6 +1729,7 @@ mod tests {
         );
         assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "console");
         assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_SCOPES), "openid,profile,email");
+        assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES), "aud1,aud2");
         assert_eq!(default_kvs.get(ENABLE_KEY), EnableState::On.to_string());
 
         let smoke_kvs = cfg
@@ -1715,6 +1766,15 @@ mod tests {
                 "topic":"events",
                 "queue_dir":""
               }
+            },
+            "kafka":{
+              "streaming":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092,127.0.0.1:9093",
+                "topic":"events-kafka",
+                "acks":"all",
+                "tls_enable":true
+              }
             }
           }
         }"#;
@@ -1743,6 +1803,14 @@ mod tests {
             .expect("mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), "");
+
+        let kafka = cfg
+            .get_value(NOTIFY_KAFKA_SUB_SYS, "streaming")
+            .expect("kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092,127.0.0.1:9093");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "events-kafka");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true");
     }
 
     #[test]
@@ -1812,6 +1880,14 @@ mod tests {
                 "broker":"tcp://127.0.0.1:1883",
                 "topic":"audit-events"
               }
+            },
+            "kafka":{
+              "auditlog":{
+                "enable":true,
+                "brokers":"127.0.0.1:9092",
+                "topic":"audit-events-kafka",
+                "acks":"1"
+              }
             }
           }
         }"#;
@@ -1835,6 +1911,12 @@ mod tests {
             .get_value(AUDIT_MQTT_SUB_SYS, "analytics")
             .expect("audit mqtt target should be decoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
+
+        let kafka = cfg
+            .get_value(AUDIT_KAFKA_SUB_SYS, "auditlog")
+            .expect("audit kafka target should be decoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "audit-events-kafka");
     }
 
     #[test]
@@ -1863,6 +1945,7 @@ mod tests {
         );
         default_provider.insert(rustfs_config::oidc::OIDC_CLIENT_ID.to_string(), "console".to_string());
         default_provider.insert(rustfs_config::oidc::OIDC_SCOPES.to_string(), "openid,profile,email".to_string());
+        default_provider.insert(rustfs_config::oidc::OIDC_OTHER_AUDIENCES.to_string(), "aud1,aud2".to_string());
         oidc_section.insert(DEFAULT_DELIMITER.to_string(), default_provider);
         cfg.0.insert(IDENTITY_OPENID_SUB_SYS.to_string(), oidc_section);
 
@@ -1890,6 +1973,13 @@ mod tests {
                 .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
             Some(vec!["openid", "profile", "email"])
         );
+        assert_eq!(
+            default_provider
+                .get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES)
+                .and_then(Value::as_array)
+                .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
+            Some(vec!["aud1", "aud2"])
+        );
         assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
     }
 
@@ -1947,6 +2037,38 @@ mod tests {
         );
         cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = notify::DEFAULT_NOTIFY_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "streaming".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092,127.0.0.1:9093".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_ACKS.to_string(),
+                    value: "all".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_TLS_ENABLE.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(NOTIFY_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let notify = v
@@ -1982,6 +2104,19 @@ mod tests {
             .expect("mqtt target should be encoded");
         assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883"));
         assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some(""));
+
+        let kafka = notify
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("streaming"))
+            .and_then(Value::as_object)
+            .expect("kafka target should be encoded");
+        assert_eq!(
+            kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str),
+            Some("127.0.0.1:9092,127.0.0.1:9093")
+        );
+        assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS).and_then(Value::as_str), Some("all"));
+        assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE).and_then(Value::as_bool), Some(true));
     }
 
     #[test]
@@ -2033,6 +2168,28 @@ mod tests {
         );
         cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section);
 
+        let mut kafka_default = audit::DEFAULT_AUDIT_KAFKA_KVS.clone();
+        kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "audit-events-kafka".to_string());
+        let mut kafka_section = std::collections::HashMap::new();
+        kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
+        kafka_section.insert(
+            "auditlog".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::KAFKA_BROKERS.to_string(),
+                    value: "127.0.0.1:9092".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(AUDIT_KAFKA_SUB_SYS.to_string(), kafka_section);
+
         let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
         let v: Value = serde_json::from_slice(&out).expect("output should be json");
         let logger = v
@@ -2059,6 +2216,14 @@ mod tests {
             .expect("audit mqtt default should be encoded");
         assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
         assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events"));
+
+        let kafka = logger
+            .get("kafka")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("auditlog"))
+            .and_then(Value::as_object)
+            .expect("audit kafka target should be encoded");
+        assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str), Some("127.0.0.1:9092"));
     }
 
     #[test]

crates/ecstore/src/config/mod.rs

@@ -25,8 +25,12 @@ use crate::store::ECStore;
 use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
 use rustfs_config::COMMENT_KEY;
 use rustfs_config::DEFAULT_DELIMITER;
-use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{
+    AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -241,6 +245,12 @@ pub fn init() {
     kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
     kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone());
     kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());
+    kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone());
+    kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
+    kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
+    kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
+    kvs.insert(NOTIFY_KAFKA_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_KAFKA_KVS.clone());
+    kvs.insert(AUDIT_KAFKA_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_KAFKA_KVS.clone());
     kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
 
     // Register all default configurations

crates/ecstore/src/config/notify.rs

@@ -14,11 +14,15 @@
 
 use crate::config::{KV, KVS};
 use rustfs_config::{
-    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
-    MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
-    WEBHOOK_SKIP_TLS_VERIFY,
+    COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
+    KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
+    MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
+    MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
+    NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
+    PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
+    PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
+    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -171,3 +175,203 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_ADDRESS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_SUBJECT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_CREDENTIALS_FILE.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_REQUIRED.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_AUTH_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
+            value: EnableState::On.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_BROKERS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_ACKS.to_owned(),
+            value: "1".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_ENABLE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: KAFKA_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: KAFKA_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/oidc.rs

@@ -18,8 +18,8 @@ use rustfs_config::{
     oidc::{
         OIDC_CLAIM_NAME, OIDC_CLAIM_PREFIX, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_CONFIG_URL, OIDC_DEFAULT_CLAIM_NAME,
         OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_ROLES_CLAIM, OIDC_DEFAULT_SCOPES,
-        OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME, OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_REDIRECT_URI,
-        OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_ROLES_CLAIM, OIDC_SCOPES, OIDC_USERNAME_CLAIM,
+        OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME, OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_OTHER_AUDIENCES,
+        OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_ROLES_CLAIM, OIDC_SCOPES, OIDC_USERNAME_CLAIM,
     },
 };
 use std::sync::LazyLock;
@@ -52,6 +52,11 @@ pub static DEFAULT_IDENTITY_OPENID_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: OIDC_DEFAULT_SCOPES.to_owned(),
             hidden_if_empty: false,
         },
+        KV {
+            key: OIDC_OTHER_AUDIENCES.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
         KV {
             key: OIDC_REDIRECT_URI.to_owned(),
             value: "".to_owned(),

crates/ecstore/src/disk/local.rs

@@ -176,7 +176,15 @@ impl LocalDisk {
             let root = root_clone.clone();
             Box::pin(async move {
                 match get_disk_info(root.clone()).await {
-                    Ok((info, root)) => {
+                    Ok((info, is_root_disk)) => {
+                        let physical_device_ids = match rustfs_utils::os::get_physical_device_ids(root.to_string_lossy().as_ref())
+                        {
+                            Ok(ids) => ids,
+                            Err(err) => {
+                                warn!(root = ?root, error = ?err, "failed to resolve physical device ids for disk root");
+                                Vec::new()
+                            }
+                        };
                         let disk_info = DiskInfo {
                             total: info.total,
                             free: info.free,
@@ -186,7 +194,8 @@ impl LocalDisk {
                             major: info.major,
                             minor: info.minor,
                             fs_type: info.fstype,
-                            root_disk: root,
+                            root_disk: is_root_disk,
+                            physical_device_ids,
                             id: disk_id,
                             ..Default::default()
                         };
@@ -2072,6 +2081,7 @@ impl DiskAPI for LocalDisk {
 
         let mut objs_returned = 0;
 
+        let mut skip_current_dir_object = false;
         if opts.base_dir.ends_with(SLASH_SEPARATOR) {
             if let Ok(data) = self
                 .read_metadata(
@@ -2098,7 +2108,7 @@ impl DiskAPI for LocalDisk {
                 if let Ok(meta) = tokio::fs::metadata(fpath).await
                     && meta.is_file()
                 {
-                    return Err(DiskError::FileNotFound);
+                    skip_current_dir_object = true;
                 }
             }
         }
@@ -2109,7 +2119,7 @@ impl DiskAPI for LocalDisk {
             &opts,
             &mut out,
             &mut objs_returned,
-            false,
+            skip_current_dir_object,
         )
         .await?;
 

crates/ecstore/src/disk/mod.rs

@@ -596,6 +596,8 @@ pub struct DiskInfo {
     pub scanning: bool,
     pub endpoint: String,
     pub mount_path: String,
+    /// Leaf physical block devices backing this mount path when available.
+    pub physical_device_ids: Vec<String>,
     pub id: Option<Uuid>,
     pub rotational: bool,
     pub metrics: DiskMetrics,

crates/ecstore/src/endpoints.rs

@@ -17,10 +17,11 @@ use crate::{
     disks_layout::DisksLayout,
     global::global_rustfs_port,
 };
+use rustfs_config::{DEFAULT_UNSAFE_BYPASS_DISK_CHECK, ENV_MINIO_CI, ENV_UNSAFE_BYPASS_DISK_CHECK};
 use rustfs_utils::{XHost, check_local_server_addr, get_host_ip, is_local_host};
 use std::{
-    collections::{HashMap, HashSet, hash_map::Entry},
-    io::{Error, Result},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
+    io::{Error, ErrorKind, Result},
     net::IpAddr,
 };
 use tracing::{error, info, instrument, warn};
@@ -348,6 +349,8 @@ impl PoolEndpointList {
             }
         }
 
+        validate_local_physical_disk_independence(pool_endpoint_list.as_ref())?;
+
         let setup_type = match pool_endpoint_list.as_ref()[0].as_ref()[0].get_type() {
             EndpointType::Path => SetupType::Erasure,
             EndpointType::Url => match unique_args.len() {
@@ -645,12 +648,118 @@ impl EndpointServerPools {
     }
 }
 
+fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()> {
+    let mut local_paths = BTreeSet::new();
+    for endpoints in pools {
+        for endpoint in endpoints.as_ref() {
+            if endpoint.is_local {
+                local_paths.insert(endpoint.get_file_path());
+            }
+        }
+    }
+
+    if local_paths.is_empty() {
+        return Ok(());
+    }
+
+    let local_paths = local_paths.into_iter().collect::<Vec<_>>();
+    validate_local_cross_device_mounts(&local_paths)?;
+
+    if local_paths.len() <= 1 {
+        return Ok(());
+    }
+
+    // Compatibility behavior:
+    // - canonical key: RUSTFS_UNSAFE_BYPASS_DISK_CHECK
+    // - legacy CI alias: MINIO_CI
+    // If both are set, `get_env_bool_with_aliases` keeps canonical key precedence.
+    if rustfs_utils::get_env_bool_with_aliases(ENV_UNSAFE_BYPASS_DISK_CHECK, &[ENV_MINIO_CI], DEFAULT_UNSAFE_BYPASS_DISK_CHECK) {
+        warn!(
+            env = ENV_UNSAFE_BYPASS_DISK_CHECK,
+            local_paths = ?local_paths,
+            "Skipping local physical disk independence validation due to explicit environment override",
+        );
+        return Ok(());
+    }
+
+    let mut device_paths = BTreeMap::<String, BTreeSet<String>>::new();
+    let mut missing_paths = Vec::new();
+
+    for path in &local_paths {
+        let canonical = match rustfs_utils::canonicalize(path) {
+            Ok(path) => path,
+            Err(err) if err.kind() == ErrorKind::NotFound => {
+                missing_paths.push(path.clone());
+                continue;
+            }
+            Err(err) => {
+                return Err(Error::other(format!(
+                    "failed to resolve local endpoint path '{path}' for disk validation: {err}"
+                )));
+            }
+        };
+        let canonical_path = canonical.to_string_lossy().into_owned();
+        let device_ids = rustfs_utils::os::get_physical_device_ids(&canonical_path).map_err(|err| {
+            Error::other(format!("failed to inspect physical disk for local endpoint '{canonical_path}': {err}"))
+        })?;
+
+        for device_id in device_ids {
+            device_paths.entry(device_id).or_default().insert(canonical_path.clone());
+        }
+    }
+
+    if !missing_paths.is_empty() {
+        warn!(
+            missing_paths = ?missing_paths,
+            "Excluding non-existent local endpoint paths from physical disk independence validation during endpoint parsing",
+        );
+    }
+
+    let shared_devices = device_paths
+        .into_iter()
+        .filter_map(|(device_id, paths)| {
+            if paths.len() <= 1 {
+                return None;
+            }
+
+            Some((device_id, paths.into_iter().collect::<Vec<_>>()))
+        })
+        .collect::<Vec<_>>();
+
+    if shared_devices.is_empty() {
+        return Ok(());
+    }
+
+    let details = shared_devices
+        .into_iter()
+        .map(|(device_id, paths)| format!("{device_id} => {}", paths.join(", ")))
+        .collect::<Vec<_>>()
+        .join("; ");
+
+    Err(Error::other(format!(
+        "local erasure endpoints must use distinct physical disks; detected shared devices [{details}]. \
+Set {ENV_UNSAFE_BYPASS_DISK_CHECK}=true only for local testing or CI to bypass this safety check"
+    )))
+}
+
+fn validate_local_cross_device_mounts(local_paths: &[String]) -> Result<()> {
+    rustfs_utils::os::check_cross_device_mounts(local_paths)
+        .map_err(|err| Error::other(format!("local endpoint cross-device mount validation failed: {err}")))
+}
+
 #[cfg(test)]
 mod test {
     use rustfs_utils::must_get_local_ips;
 
     use super::*;
+
+    #[cfg(target_os = "linux")]
+    use serial_test::serial;
     use std::path::Path;
+    #[cfg(target_os = "linux")]
+    use temp_env::async_with_vars;
+    #[cfg(target_os = "linux")]
+    use tempfile::tempdir;
 
     #[test]
     fn test_new_endpoints() {
@@ -1412,4 +1521,69 @@ mod test {
             }
         }
     }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn reject_shared_local_physical_disks_by_default() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, None::<&str>)], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let err = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout)
+                .await
+                .unwrap_err();
+
+            let err_text = err.to_string();
+            assert!(err_text.contains("distinct physical disks"), "unexpected error: {err_text}");
+            assert!(err_text.contains(ENV_UNSAFE_BYPASS_DISK_CHECK), "unexpected error: {err_text}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_explicit_env_bypass() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected bypassed disk validation to succeed, got {ret:?}");
+        })
+        .await;
+    }
+
+    #[cfg(target_os = "linux")]
+    #[serial]
+    #[tokio::test]
+    async fn allow_shared_local_physical_disks_with_minio_ci_alias() {
+        async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, Some("1"))], async {
+            let dir = tempdir().unwrap();
+            let disk1 = dir.path().join("disk1");
+            let disk2 = dir.path().join("disk2");
+            std::fs::create_dir_all(&disk1).unwrap();
+            std::fs::create_dir_all(&disk2).unwrap();
+
+            let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
+            let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
+
+            let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
+            assert!(ret.is_ok(), "expected MINIO_CI alias to bypass disk validation, got {ret:?}");
+        })
+        .await;
+    }
 }

crates/ecstore/src/rpc/client.rs

@@ -20,6 +20,8 @@ use std::error::Error;
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
 use tracing::debug;
 
+use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
+
 /// 3. Subsequent calls will attempt fresh connections
 /// 4. If node is still down, connection will fail fast (3s timeout)
 pub async fn node_service_time_out_client(
@@ -55,6 +57,8 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
     fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
         let headers = gen_signature_headers(TONIC_RPC_PREFIX, &Method::GET);
         req.metadata_mut().as_mut().extend(headers);
+        inject_trace_context_into_metadata(req.metadata_mut());
+        inject_request_id_into_metadata(req.metadata_mut());
         Ok(req)
     }
 }
@@ -84,3 +88,34 @@ impl tonic::service::Interceptor for TonicInterceptor {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tonic::service::Interceptor;
+
+    #[test]
+    fn test_signature_interceptor_keeps_auth_headers() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        assert!(req.metadata().contains_key("x-rustfs-signature"));
+        assert!(req.metadata().contains_key("x-rustfs-timestamp"));
+    }
+
+    #[test]
+    fn test_signature_interceptor_may_inject_request_id() {
+        let mut interceptor = TonicSignatureInterceptor;
+        let req = tonic::Request::new(());
+
+        let span = tracing::info_span!("grpc-rpc-test-span");
+        let _guard = span.enter();
+        let req = interceptor.call(req).expect("interceptor call should succeed");
+
+        if let Some(v) = req.metadata().get("x-request-id") {
+            assert!(!v.as_encoded_bytes().is_empty());
+        }
+    }
+}

crates/ecstore/src/rpc/context_propagation.rs

@@ -0,0 +1,223 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use http::{HeaderMap, HeaderValue};
+use opentelemetry::{global, propagation::Injector, trace::TraceContextExt};
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+pub(crate) const REQUEST_ID_HEADER: &str = "x-request-id";
+
+struct HttpHeaderInjector<'a> {
+    headers: &'a mut HeaderMap,
+}
+
+impl Injector for HttpHeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(name) = http::header::HeaderName::from_bytes(key.as_bytes()) else {
+            return;
+        };
+        let Ok(val) = HeaderValue::from_str(&value) else {
+            return;
+        };
+        self.headers.insert(name, val);
+    }
+}
+
+struct MetadataInjector<'a> {
+    metadata: &'a mut tonic::metadata::MetadataMap,
+}
+
+impl Injector for MetadataInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(meta_key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) else {
+            return;
+        };
+        let Ok(meta_value) = tonic::metadata::MetadataValue::try_from(value.as_str()) else {
+            return;
+        };
+        self.metadata.insert(meta_key, meta_value);
+    }
+}
+
+fn current_trace_id() -> Option<String> {
+    let current_context = Span::current().context();
+    let current_span = current_context.span();
+    let span_context = current_span.span_context();
+    if !span_context.is_valid() {
+        return None;
+    }
+    Some(span_context.trace_id().to_string())
+}
+
+fn fallback_request_id() -> String {
+    format!("req-{}", &uuid::Uuid::new_v4().to_string()[..8])
+}
+
+fn propagated_request_id() -> String {
+    current_trace_id()
+        .map(|trace_id| format!("trace-{trace_id}"))
+        .unwrap_or_else(fallback_request_id)
+}
+
+pub(crate) fn inject_trace_context_into_http_headers(headers: &mut HeaderMap) {
+    let current_context = Span::current().context();
+    global::get_text_map_propagator(|propagator| {
+        let mut injector = HttpHeaderInjector { headers };
+        propagator.inject_context(&current_context, &mut injector);
+    });
+}
+
+pub(crate) fn inject_request_id_into_http_headers(headers: &mut HeaderMap) {
+    if headers.contains_key(REQUEST_ID_HEADER) {
+        return;
+    }
+    let request_id = propagated_request_id();
+    if let Ok(value) = HeaderValue::from_str(&request_id) {
+        headers.insert(REQUEST_ID_HEADER, value);
+    }
+}
+
+pub(crate) fn inject_trace_context_into_metadata(metadata: &mut tonic::metadata::MetadataMap) {
+    let current_context = Span::current().context();
+    global::get_text_map_propagator(|propagator| {
+        let mut injector = MetadataInjector { metadata };
+        propagator.inject_context(&current_context, &mut injector);
+    });
+}
+
+pub(crate) fn inject_request_id_into_metadata(metadata: &mut tonic::metadata::MetadataMap) {
+    let request_id_key = tonic::metadata::MetadataKey::from_static(REQUEST_ID_HEADER);
+    if metadata.contains_key(&request_id_key) {
+        return;
+    }
+    let request_id = propagated_request_id();
+    let Ok(value) = tonic::metadata::MetadataValue::try_from(request_id.as_str()) else {
+        return;
+    };
+    metadata.insert(request_id_key, value);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _};
+    use opentelemetry_sdk::trace::SdkTracerProvider;
+    use tracing_opentelemetry::OpenTelemetrySpanExt;
+    use tracing_subscriber::{Registry, layer::SubscriberExt};
+
+    fn with_trace_parent<F>(trace_id_hex: &str, f: F)
+    where
+        F: FnOnce(),
+    {
+        let provider = SdkTracerProvider::builder().build();
+        let tracer = provider.tracer("context-propagation-tests");
+        let subscriber = Registry::default().with(tracing_opentelemetry::layer().with_tracer(tracer));
+
+        tracing::subscriber::with_default(subscriber, || {
+            let span = tracing::info_span!("context-propagation-test-span");
+
+            let trace_id = TraceId::from_hex(trace_id_hex).expect("trace id should be valid hex");
+            let span_id = opentelemetry::trace::SpanId::from_hex("0102030405060708").expect("span id should be valid hex");
+            let parent = SpanContext::new(trace_id, span_id, TraceFlags::SAMPLED, true, TraceState::default());
+            span.set_parent(opentelemetry::Context::new().with_remote_span_context(parent))
+                .expect("failed to set parent context");
+            let _guard = span.enter();
+
+            f();
+        });
+        let _ = provider.shutdown();
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_preserves_existing_value() {
+        let mut headers = HeaderMap::new();
+        headers.insert(REQUEST_ID_HEADER, HeaderValue::from_static("req-upstream-123"));
+
+        with_trace_parent("0123456789abcdef0123456789abcdef", || {
+            inject_request_id_into_http_headers(&mut headers);
+        });
+
+        assert_eq!(headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-123"));
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_uses_trace_id_when_missing() {
+        let trace_id = "abcdefabcdefabcdefabcdefabcdefab";
+        let mut headers = HeaderMap::new();
+
+        with_trace_parent(trace_id, || {
+            inject_request_id_into_http_headers(&mut headers);
+        });
+
+        assert_eq!(
+            headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()),
+            Some(format!("trace-{trace_id}").as_str())
+        );
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_preserves_existing_value() {
+        let mut metadata = tonic::metadata::MetadataMap::new();
+        metadata.insert(
+            tonic::metadata::MetadataKey::from_static(REQUEST_ID_HEADER),
+            tonic::metadata::MetadataValue::from_static("req-upstream-456"),
+        );
+
+        with_trace_parent("fedcba9876543210fedcba9876543210", || {
+            inject_request_id_into_metadata(&mut metadata);
+        });
+
+        assert_eq!(metadata.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-456"));
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_uses_trace_id_when_missing() {
+        let trace_id = "1234567890abcdef1234567890abcdef";
+        let mut metadata = tonic::metadata::MetadataMap::new();
+
+        with_trace_parent(trace_id, || {
+            inject_request_id_into_metadata(&mut metadata);
+        });
+
+        assert_eq!(
+            metadata.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()),
+            Some(format!("trace-{trace_id}").as_str())
+        );
+    }
+
+    #[test]
+    fn test_inject_request_id_into_http_headers_uses_req_fallback_when_trace_missing() {
+        let mut headers = HeaderMap::new();
+        inject_request_id_into_http_headers(&mut headers);
+
+        let request_id = headers
+            .get(REQUEST_ID_HEADER)
+            .and_then(|v| v.to_str().ok())
+            .expect("request id should be injected");
+        assert!(request_id.starts_with("req-"), "expected req- fallback, got: {request_id}");
+    }
+
+    #[test]
+    fn test_inject_request_id_into_metadata_uses_req_fallback_when_trace_missing() {
+        let mut metadata = tonic::metadata::MetadataMap::new();
+        inject_request_id_into_metadata(&mut metadata);
+
+        let request_id = metadata
+            .get(REQUEST_ID_HEADER)
+            .and_then(|v| v.to_str().ok())
+            .expect("request id should be injected");
+        assert!(request_id.starts_with("req-"), "expected req- fallback, got: {request_id}");
+    }
+}

crates/ecstore/src/rpc/http_auth.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers};
 use base64::Engine as _;
 use base64::engine::general_purpose;
 use hmac::{Hmac, KeyInit, Mac};
@@ -62,6 +63,8 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) {
     let auth_headers = gen_signature_headers(url, method);
 
     headers.extend(auth_headers);
+    inject_trace_context_into_http_headers(headers);
+    inject_request_id_into_http_headers(headers);
 }
 
 pub fn gen_signature_headers(url: &str, method: &Method) -> HeaderMap {
@@ -132,6 +135,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::rpc::context_propagation::REQUEST_ID_HEADER;
     use http::{HeaderMap, Method};
     use time::OffsetDateTime;
 
@@ -210,6 +214,33 @@ mod tests {
         assert!((current_time - timestamp).abs() <= 1, "Timestamp should be close to current time");
     }
 
+    #[test]
+    fn test_build_auth_headers_preserves_existing_request_id() {
+        let url = "http://example.com/api/test";
+        let method = Method::GET;
+        let mut headers = HeaderMap::new();
+        headers.insert(REQUEST_ID_HEADER, HeaderValue::from_static("req-upstream-123"));
+
+        build_auth_headers(url, &method, &mut headers);
+
+        assert_eq!(headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()), Some("req-upstream-123"));
+    }
+
+    #[test]
+    fn test_build_auth_headers_may_set_request_id_from_trace_id() {
+        let url = "http://example.com/api/test";
+        let method = Method::GET;
+        let mut headers = HeaderMap::new();
+
+        let span = tracing::info_span!("rpc-test-span");
+        let _guard = span.enter();
+        build_auth_headers(url, &method, &mut headers);
+
+        if let Some(value) = headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()) {
+            assert!(!value.is_empty(), "request id should not be empty");
+        }
+    }
+
     #[test]
     fn test_verify_rpc_signature_success() {
         let url = "http://example.com/api/test";

crates/ecstore/src/rpc/mod.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod client;
+mod context_propagation;
 mod http_auth;
 mod peer_rest_client;
 mod peer_s3_client;

crates/ecstore/src/rpc/remote_disk.rs

@@ -886,7 +886,7 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout_for_op(
             "write_metadata",
-            || async {
+            move || async move {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -896,8 +896,8 @@ impl DiskAPI for RemoteDisk {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
-                    file_info: file_info.clone(),
-                    file_info_bin: file_info_bin.clone(),
+                    file_info,
+                    file_info_bin: file_info_bin.into(),
                 });
 
                 let response = client.write_metadata(request).await?.into_inner();
@@ -951,7 +951,7 @@ impl DiskAPI for RemoteDisk {
 
         self.execute_with_timeout_for_op(
             "update_metadata",
-            || async {
+            move || async move {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -961,10 +961,10 @@ impl DiskAPI for RemoteDisk {
                     disk,
                     volume: volume.to_string(),
                     path: path.to_string(),
-                    file_info: file_info.clone(),
-                    opts: opts_str.clone(),
-                    file_info_bin: file_info_bin.clone(),
-                    opts_bin: opts_bin.clone(),
+                    file_info,
+                    opts: opts_str,
+                    file_info_bin: file_info_bin.into(),
+                    opts_bin: opts_bin.into(),
                 });
 
                 let response = client.update_metadata(request).await?.into_inner();
@@ -994,7 +994,7 @@ impl DiskAPI for RemoteDisk {
         let opts_bin = encode_msgpack(opts)?;
 
         self.execute_with_timeout(
-            || async {
+            move || async {
                 let disk = self.disk_ref().await;
                 let mut client = self
                     .get_client()
@@ -1005,8 +1005,8 @@ impl DiskAPI for RemoteDisk {
                     volume: volume.to_string(),
                     path: path.to_string(),
                     version_id: version_id.to_string(),
-                    opts: opts_str.clone(),
-                    opts_bin: opts_bin.clone(),
+                    opts: opts_str,
+                    opts_bin: opts_bin.into(),
                 });
 
                 let response = client.read_version(request).await?.into_inner();
@@ -1480,7 +1480,7 @@ impl DiskAPI for RemoteDisk {
                 let request = Request::new(ReadMultipleRequest {
                     disk,
                     read_multiple_req,
-                    read_multiple_req_bin,
+                    read_multiple_req_bin: read_multiple_req_bin.into(),
                 });
 
                 let response = client.read_multiple(request).await?.into_inner();

crates/ecstore/src/rpc/remote_locker.rs

@@ -190,11 +190,6 @@ impl LockClient for RemoteClient {
             Err(err) => return Ok(Self::rpc_failure_response(request, &err)),
         };
 
-        // Check for explicit error first
-        if let Some(error_info) = resp.error_info {
-            return Err(LockError::internal(error_info));
-        }
-
         // Check if the lock acquisition was successful
         if resp.success {
             Ok(LockResponse::success(
@@ -204,7 +199,8 @@ impl LockClient for RemoteClient {
         } else {
             // Lock acquisition failed
             Ok(LockResponse::failure(
-                "Lock acquisition failed on remote server".to_string(),
+                resp.error_info
+                    .unwrap_or_else(|| "Lock acquisition failed on remote server".to_string()),
                 std::time::Duration::ZERO,
             ))
         }

crates/ecstore/src/set_disk.rs

@@ -18,6 +18,7 @@
 use crate::batch_processor::{AsyncBatchProcessor, get_global_processors};
 use crate::bitrot::{create_bitrot_reader, create_bitrot_writer};
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
+use crate::bucket::object_lock::objectlock_sys::check_retention_for_modification;
 use crate::bucket::replication::check_replicate_delete;
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
@@ -707,6 +708,10 @@ impl ObjectIO for SetDisks {
         tokio::spawn(async move {
             let _guard = read_lock_guard; // keep guard alive until task ends (None if optimization enabled)
             let mut writer = wd;
+            // Do not wrap the entire read+write pipeline in `disk_read_timeout`.
+            // `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
+            // would incorrectly treat downstream backpressure as disk-read latency.
+            // Disk read timeouts must be enforced at the actual disk I/O operations.
             if let Err(e) = Self::get_object_with_fileinfo(
                 &bucket,
                 &object,
@@ -757,7 +762,6 @@ impl ObjectIO for SetDisks {
                 user_defined.insert(key.clone(), value.clone());
             }
         }
-
         let sc_parity_drives = {
             if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
                 sc.get_parity_for_sc(user_defined.get(AMZ_STORAGE_CLASS).cloned().unwrap_or_default().as_str())
@@ -1340,6 +1344,22 @@ impl BucketOperations for SetDisks {
     }
 }
 
+fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> {
+    if let Some(retention) = &opts.object_lock_retention
+        && check_retention_for_modification(
+            &obj_info.user_defined,
+            retention.mode.as_deref(),
+            retention.retain_until,
+            retention.bypass_governance,
+        )
+        .is_some()
+    {
+        return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
+    }
+
+    Ok(())
+}
+
 #[async_trait::async_trait]
 impl ObjectOperations for SetDisks {
     #[tracing::instrument(skip(self))]
@@ -1986,6 +2006,8 @@ impl ObjectOperations for SetDisks {
 
         let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
 
+        check_object_lock_retention_update(bucket, object, &obj_info, opts)?;
+
         for (k, v) in obj_info.user_defined {
             fi.metadata.insert(k, v);
         }
@@ -4126,6 +4148,7 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
                         total_space: res.total,
                         used_space: res.used,
                         available_space: res.free,
+                        physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
                         utilization: {
                             if res.total > 0 {
                                 res.used as f64 / res.total as f64 * 100_f64
@@ -5480,6 +5503,74 @@ mod tests {
         assert!(rendered.contains("object"), "{rendered}");
     }
 
+    #[test]
+    fn test_check_object_lock_retention_update_blocks_compliance_shorten() {
+        let now = OffsetDateTime::now_utc();
+        let existing_until = now + Duration::from_secs(60 * 60 * 24 * 60);
+        let requested_until = now + Duration::from_secs(60 * 60 * 24);
+
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
+        );
+
+        let obj_info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions {
+                mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()),
+                retain_until: Some(requested_until),
+                bypass_governance: true,
+            }),
+            ..Default::default()
+        };
+
+        let err = check_object_lock_retention_update("bucket", "object", &obj_info, &opts)
+            .expect_err("COMPLIANCE shortening must be blocked");
+
+        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
+    }
+
+    #[test]
+    fn test_check_object_lock_retention_update_allows_governance_shorten_with_bypass() {
+        let now = OffsetDateTime::now_utc();
+        let existing_until = now + Duration::from_secs(60 * 60 * 24 * 60);
+        let requested_until = now + Duration::from_secs(60 * 60 * 24);
+
+        let mut user_defined = HashMap::new();
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+            s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
+        );
+        user_defined.insert(
+            X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
+            existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
+        );
+
+        let obj_info = ObjectInfo {
+            user_defined,
+            ..Default::default()
+        };
+        let opts = ObjectOptions {
+            object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions {
+                mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()),
+                retain_until: Some(requested_until),
+                bypass_governance: true,
+            }),
+            ..Default::default()
+        };
+
+        check_object_lock_retention_update("bucket", "object", &obj_info, &opts)
+            .expect("GOVERNANCE shortening with bypass should remain allowed");
+    }
+
     #[test]
     fn test_should_prevent_write() {
         let oi = ObjectInfo {
@@ -5573,6 +5664,100 @@ mod tests {
         assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string())));
     }
 
+    #[tokio::test]
+    async fn range_reads_use_shard_span_length_for_non_zero_offsets() {
+        use tokio::io::AsyncReadExt;
+        use uuid::Uuid;
+
+        let tempdir = tempfile::tempdir().expect("tempdir should be created");
+        let endpoint =
+            Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("disk should be created");
+
+        let bucket = "bucket";
+        let object = "object";
+        let payload = vec![b'x'; 3 * 1024 * 1024 + 1234];
+        let range_offset = 2 * 1024 * 1024 + 17;
+        let range_length = 512 * 1024;
+
+        disk.make_volume(bucket).await.expect("bucket should be created");
+
+        let mut fi = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
+        let data_dir = Uuid::new_v4();
+        fi.data_dir = Some(data_dir);
+        fi.size = payload.len() as i64;
+        fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None);
+
+        let erasure = erasure_coding::Erasure::new_with_options(
+            fi.erasure.data_blocks,
+            fi.erasure.parity_blocks,
+            fi.erasure.block_size,
+            fi.uses_legacy_checksum,
+        );
+        let shard_path = format!("{object}/{data_dir}/part.1");
+        let checksum_info = fi.erasure.get_checksum_info(1);
+
+        let mut bitrot_writer = create_bitrot_writer(
+            true,
+            None,
+            bucket,
+            &shard_path,
+            payload.len() as i64,
+            erasure.shard_size(),
+            checksum_info.algorithm.clone(),
+        )
+        .await
+        .expect("bitrot writer should be created");
+
+        for chunk in payload.chunks(erasure.shard_size()) {
+            bitrot_writer.write(chunk).await.expect("payload chunk should be written");
+        }
+
+        let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
+        disk.write_all(bucket, &shard_path, Bytes::from(encoded))
+            .await
+            .expect("encoded shard should be stored");
+
+        let files = vec![fi.clone()];
+        let disks = vec![Some(disk.clone())];
+        let (mut reader, mut writer) = tokio::io::duplex(range_length * 2);
+
+        let read_task = tokio::spawn(async move {
+            SetDisks::get_object_with_fileinfo(
+                bucket,
+                object,
+                range_offset,
+                range_length as i64,
+                &mut writer,
+                fi,
+                files,
+                &disks,
+                0,
+                0,
+                true,
+            )
+            .await
+        });
+
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.expect("range bytes should be readable");
+
+        read_task
+            .await
+            .expect("read task should complete")
+            .expect("range read should succeed");
+
+        assert_eq!(out, payload[range_offset..range_offset + range_length]);
+    }
+
     #[test]
     fn parts_after_marker_uses_marker_position() {
         let part_numbers = (1..=1002).collect::<Vec<_>>();

crates/ecstore/src/store_api/types.rs

@@ -43,6 +43,13 @@ impl HTTPPreconditions {
     }
 }
 
+#[derive(Debug, Default, Clone)]
+pub struct ObjectLockRetentionOptions {
+    pub mode: Option<String>,
+    pub retain_until: Option<OffsetDateTime>,
+    pub bypass_governance: bool,
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -79,6 +86,7 @@ pub struct ObjectOptions {
     pub lifecycle_audit_event: LcAuditEvent,
 
     pub eval_metadata: Option<HashMap<String, String>>,
+    pub object_lock_retention: Option<ObjectLockRetentionOptions>,
 
     pub want_checksum: Option<Checksum>,
     pub skip_verify_bitrot: bool,

crates/ecstore/src/tier/tier.rs

@@ -808,14 +808,19 @@ impl TierConfigMgr {
             }
         }
         if !force {
-            let inuse = d.expect("err").in_use().await;
-            if let Err(err) = inuse {
-                let mut e = ERR_TIER_PERM_ERR.clone();
-                e.message.push('.');
-                e.message.push_str(&err.to_string());
-                return Err(e);
-            } else if inuse.expect("err") {
-                return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
+            if let Ok(driver) = d {
+                match driver.in_use().await {
+                    Err(err) => {
+                        let mut e = ERR_TIER_PERM_ERR.clone();
+                        e.message.push('.');
+                        e.message.push_str(&err.to_string());
+                        return Err(e);
+                    }
+                    Ok(in_use) if in_use => {
+                        return Err(ERR_TIER_BACKEND_NOT_EMPTY.clone());
+                    }
+                    _ => {}
+                }
             }
         }
         self.tiers.remove(tier_name);
@@ -842,11 +847,11 @@ impl TierConfigMgr {
     }
 
     pub fn tier_type(&self, tier_name: &str) -> String {
-        let cfg = self.tiers.get(tier_name);
-        if cfg.is_none() {
-            return "internal".to_string();
+        if let Some(cfg) = self.tiers.get(tier_name) {
+            cfg.tier_type.as_lowercase()
+        } else {
+            "internal".to_string()
         }
-        cfg.expect("err").tier_type.as_lowercase()
     }
 
     pub fn list_tiers(&self) -> Vec<TierConfig> {
@@ -876,81 +881,90 @@ impl TierConfigMgr {
         let mut tier_config = self.tiers[tier_name].clone();
         match tier_type {
             TierType::S3 => {
-                let mut s3 = tier_config.s3.as_mut().expect("err");
-                if creds.aws_role {
-                    s3.aws_role = true
-                }
-                if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
-                    s3.aws_role_arn = creds.aws_role_arn;
-                    s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
-                }
-                if creds.access_key != "" && creds.secret_key != "" {
-                    s3.access_key = creds.access_key;
-                    s3.secret_key = creds.secret_key;
+                if let Some(s3) = tier_config.s3.as_mut() {
+                    if creds.aws_role {
+                        s3.aws_role = true
+                    }
+                    if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" {
+                        s3.aws_role_arn = creds.aws_role_arn;
+                        s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file;
+                    }
+                    if creds.access_key != "" && creds.secret_key != "" {
+                        s3.access_key = creds.access_key;
+                        s3.secret_key = creds.secret_key;
+                    }
                 }
             }
             TierType::RustFS => {
-                let mut rustfs = tier_config.rustfs.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(rustfs) = tier_config.rustfs.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    rustfs.access_key = creds.access_key;
+                    rustfs.secret_key = creds.secret_key;
                 }
-                rustfs.access_key = creds.access_key;
-                rustfs.secret_key = creds.secret_key;
             }
             TierType::MinIO => {
-                let compatible_backend = tier_config.minio.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(compatible_backend) = tier_config.minio.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    compatible_backend.access_key = creds.access_key;
+                    compatible_backend.secret_key = creds.secret_key;
                 }
-                compatible_backend.access_key = creds.access_key;
-                compatible_backend.secret_key = creds.secret_key;
             }
             TierType::Aliyun => {
-                let mut aliyun = tier_config.aliyun.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(aliyun) = tier_config.aliyun.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    aliyun.access_key = creds.access_key;
+                    aliyun.secret_key = creds.secret_key;
                 }
-                aliyun.access_key = creds.access_key;
-                aliyun.secret_key = creds.secret_key;
             }
             TierType::Tencent => {
-                let mut tencent = tier_config.tencent.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(tencent) = tier_config.tencent.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    tencent.access_key = creds.access_key;
+                    tencent.secret_key = creds.secret_key;
                 }
-                tencent.access_key = creds.access_key;
-                tencent.secret_key = creds.secret_key;
             }
             TierType::Huaweicloud => {
-                let mut huaweicloud = tier_config.huaweicloud.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(huaweicloud) = tier_config.huaweicloud.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    huaweicloud.access_key = creds.access_key;
+                    huaweicloud.secret_key = creds.secret_key;
                 }
-                huaweicloud.access_key = creds.access_key;
-                huaweicloud.secret_key = creds.secret_key;
             }
             TierType::Azure => {
-                let mut azure = tier_config.azure.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(azure) = tier_config.azure.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    azure.access_key = creds.access_key;
+                    azure.secret_key = creds.secret_key;
                 }
-                azure.access_key = creds.access_key;
-                azure.secret_key = creds.secret_key;
             }
             TierType::GCS => {
-                let mut gcs = tier_config.gcs.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(gcs) = tier_config.gcs.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    gcs.creds = creds.access_key; //creds.creds_json
                 }
-                gcs.creds = creds.access_key; //creds.creds_json
             }
             TierType::R2 => {
-                let mut r2 = tier_config.r2.as_mut().expect("err");
-                if creds.access_key == "" || creds.secret_key == "" {
-                    return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                if let Some(r2) = tier_config.r2.as_mut() {
+                    if creds.access_key == "" || creds.secret_key == "" {
+                        return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
+                    }
+                    r2.access_key = creds.access_key;
+                    r2.secret_key = creds.secret_key;
                 }
-                r2.access_key = creds.access_key;
-                r2.secret_key = creds.secret_key;
             }
             _ => (),
         }
@@ -964,7 +978,7 @@ impl TierConfigMgr {
     pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> {
         // Return cached driver if present
         if self.driver_cache.contains_key(tier_name) {
-            return Ok(self.driver_cache.get(tier_name).unwrap());
+            return Ok(self.driver_cache.get(tier_name).expect("Driver not found in cache"));
         }
 
         // Get tier configuration and create new driver
@@ -974,7 +988,10 @@ impl TierConfigMgr {
 
         // Insert and return reference
         self.driver_cache.insert(tier_name.to_string(), driver);
-        Ok(self.driver_cache.get(tier_name).unwrap())
+        Ok(self
+            .driver_cache
+            .get(tier_name)
+            .expect("Driver not found in cache after insertion"))
     }
 
     pub async fn reload(&mut self, api: Arc<ECStore>) -> std::result::Result<(), std::io::Error> {
@@ -989,9 +1006,12 @@ impl TierConfigMgr {
         }
         self.driver_cache.clear();
         self.tiers.clear();
-        let new_config = new_config.expect("err");
-        for (tier, cfg) in new_config.tiers {
-            self.tiers.insert(tier, cfg);
+        if let Ok(config) = new_config {
+            for (tier, cfg) in config.tiers {
+                self.tiers.insert(tier, cfg);
+            }
+        } else {
+            return Err(std::io::Error::other("Failed to load tier configuration"));
         }
         self.last_refreshed_at = OffsetDateTime::now_utc();
         Ok(())

crates/ecstore/src/tier/tier_config.rs

@@ -155,49 +155,67 @@ impl Clone for TierConfig {
         let mut r2 = None;
         match self.tier_type {
             TierType::S3 => {
-                let mut s3_ = self.s3.as_ref().expect("err").clone();
-                s3_.secret_key = "REDACTED".to_string();
-                s3 = Some(s3_);
+                if let Some(s3_) = self.s3.as_ref() {
+                    let mut s3_clone = s3_.clone();
+                    s3_clone.secret_key = "REDACTED".to_string();
+                    s3 = Some(s3_clone);
+                }
             }
             TierType::RustFS => {
-                let mut r_ = self.rustfs.as_ref().expect("err").clone();
-                r_.secret_key = "REDACTED".to_string();
-                r = Some(r_);
+                if let Some(r_) = self.rustfs.as_ref() {
+                    let mut r_clone = r_.clone();
+                    r_clone.secret_key = "REDACTED".to_string();
+                    r = Some(r_clone);
+                }
             }
             TierType::MinIO => {
-                let mut compatible_backend_ = self.minio.as_ref().expect("err").clone();
-                compatible_backend_.secret_key = "REDACTED".to_string();
-                compatible_backend = Some(compatible_backend_);
+                if let Some(compatible_backend_) = self.minio.as_ref() {
+                    let mut compatible_backend_clone = compatible_backend_.clone();
+                    compatible_backend_clone.secret_key = "REDACTED".to_string();
+                    compatible_backend = Some(compatible_backend_clone);
+                }
             }
             TierType::Aliyun => {
-                let mut aliyun_ = self.aliyun.as_ref().expect("err").clone();
-                aliyun_.secret_key = "REDACTED".to_string();
-                aliyun = Some(aliyun_);
+                if let Some(aliyun_) = self.aliyun.as_ref() {
+                    let mut aliyun_clone = aliyun_.clone();
+                    aliyun_clone.secret_key = "REDACTED".to_string();
+                    aliyun = Some(aliyun_clone);
+                }
             }
             TierType::Tencent => {
-                let mut tencent_ = self.tencent.as_ref().expect("err").clone();
-                tencent_.secret_key = "REDACTED".to_string();
-                tencent = Some(tencent_);
+                if let Some(tencent_) = self.tencent.as_ref() {
+                    let mut tencent_clone = tencent_.clone();
+                    tencent_clone.secret_key = "REDACTED".to_string();
+                    tencent = Some(tencent_clone);
+                }
             }
             TierType::Huaweicloud => {
-                let mut huaweicloud_ = self.huaweicloud.as_ref().expect("err").clone();
-                huaweicloud_.secret_key = "REDACTED".to_string();
-                huaweicloud = Some(huaweicloud_);
+                if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
+                    let mut huaweicloud_clone = huaweicloud_.clone();
+                    huaweicloud_clone.secret_key = "REDACTED".to_string();
+                    huaweicloud = Some(huaweicloud_clone);
+                }
             }
             TierType::Azure => {
-                let mut azure_ = self.azure.as_ref().expect("err").clone();
-                azure_.secret_key = "REDACTED".to_string();
-                azure = Some(azure_);
+                if let Some(azure_) = self.azure.as_ref() {
+                    let mut azure_clone = azure_.clone();
+                    azure_clone.secret_key = "REDACTED".to_string();
+                    azure = Some(azure_clone);
+                }
             }
             TierType::GCS => {
-                let mut gcs_ = self.gcs.as_ref().expect("err").clone();
-                gcs_.creds = "REDACTED".to_string();
-                gcs = Some(gcs_);
+                if let Some(gcs_) = self.gcs.as_ref() {
+                    let mut gcs_clone = gcs_.clone();
+                    gcs_clone.creds = "REDACTED".to_string();
+                    gcs = Some(gcs_clone);
+                }
             }
             TierType::R2 => {
-                let mut r2_ = self.r2.as_ref().expect("err").clone();
-                r2_.secret_key = "REDACTED".to_string();
-                r2 = Some(r2_);
+                if let Some(r2_) = self.r2.as_ref() {
+                    let mut r2_clone = r2_.clone();
+                    r2_clone.secret_key = "REDACTED".to_string();
+                    r2 = Some(r2_clone);
+                }
             }
             _ => (),
         }
@@ -222,15 +240,15 @@ impl Clone for TierConfig {
 impl TierConfig {
     fn endpoint(&self) -> String {
         match self.tier_type {
-            TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(),
-            TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(),
-            TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(),
-            TierType::Aliyun => self.aliyun.as_ref().expect("err").endpoint.clone(),
-            TierType::Tencent => self.tencent.as_ref().expect("err").endpoint.clone(),
-            TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").endpoint.clone(),
-            TierType::Azure => self.azure.as_ref().expect("err").endpoint.clone(),
-            TierType::GCS => self.gcs.as_ref().expect("err").endpoint.clone(),
-            TierType::R2 => self.r2.as_ref().expect("err").endpoint.clone(),
+            TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.endpoint.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.endpoint.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.endpoint.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.endpoint.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.endpoint.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.endpoint.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.endpoint.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.endpoint.clone()).unwrap_or_default(),
             _ => {
                 info!("unexpected tier type {}", self.tier_type);
                 "".to_string()
@@ -240,15 +258,15 @@ impl TierConfig {
 
     fn bucket(&self) -> String {
         match self.tier_type {
-            TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(),
-            TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(),
-            TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(),
-            TierType::Aliyun => self.aliyun.as_ref().expect("err").bucket.clone(),
-            TierType::Tencent => self.tencent.as_ref().expect("err").bucket.clone(),
-            TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").bucket.clone(),
-            TierType::Azure => self.azure.as_ref().expect("err").bucket.clone(),
-            TierType::GCS => self.gcs.as_ref().expect("err").bucket.clone(),
-            TierType::R2 => self.r2.as_ref().expect("err").bucket.clone(),
+            TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.bucket.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.bucket.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.bucket.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.bucket.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.bucket.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.bucket.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.bucket.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.bucket.clone()).unwrap_or_default(),
             _ => {
                 info!("unexpected tier type {}", self.tier_type);
                 "".to_string()
@@ -258,15 +276,15 @@ impl TierConfig {
 
     fn prefix(&self) -> String {
         match self.tier_type {
-            TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(),
-            TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(),
-            TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(),
-            TierType::Aliyun => self.aliyun.as_ref().expect("err").prefix.clone(),
-            TierType::Tencent => self.tencent.as_ref().expect("err").prefix.clone(),
-            TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").prefix.clone(),
-            TierType::Azure => self.azure.as_ref().expect("err").prefix.clone(),
-            TierType::GCS => self.gcs.as_ref().expect("err").prefix.clone(),
-            TierType::R2 => self.r2.as_ref().expect("err").prefix.clone(),
+            TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.prefix.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.prefix.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.prefix.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.prefix.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.prefix.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.prefix.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.prefix.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.prefix.clone()).unwrap_or_default(),
             _ => {
                 info!("unexpected tier type {}", self.tier_type);
                 "".to_string()
@@ -276,15 +294,15 @@ impl TierConfig {
 
     fn region(&self) -> String {
         match self.tier_type {
-            TierType::S3 => self.s3.as_ref().expect("err").region.clone(),
-            TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(),
-            TierType::MinIO => self.minio.as_ref().expect("err").region.clone(),
-            TierType::Aliyun => self.aliyun.as_ref().expect("err").region.clone(),
-            TierType::Tencent => self.tencent.as_ref().expect("err").region.clone(),
-            TierType::Huaweicloud => self.huaweicloud.as_ref().expect("err").region.clone(),
-            TierType::Azure => self.azure.as_ref().expect("err").region.clone(),
-            TierType::GCS => self.gcs.as_ref().expect("err").region.clone(),
-            TierType::R2 => self.r2.as_ref().expect("err").region.clone(),
+            TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
+            TierType::RustFS => self.rustfs.as_ref().map(|r| r.region.clone()).unwrap_or_default(),
+            TierType::MinIO => self.minio.as_ref().map(|m| m.region.clone()).unwrap_or_default(),
+            TierType::Aliyun => self.aliyun.as_ref().map(|a| a.region.clone()).unwrap_or_default(),
+            TierType::Tencent => self.tencent.as_ref().map(|t| t.region.clone()).unwrap_or_default(),
+            TierType::Huaweicloud => self.huaweicloud.as_ref().map(|h| h.region.clone()).unwrap_or_default(),
+            TierType::Azure => self.azure.as_ref().map(|a| a.region.clone()).unwrap_or_default(),
+            TierType::GCS => self.gcs.as_ref().map(|g| g.region.clone()).unwrap_or_default(),
+            TierType::R2 => self.r2.as_ref().map(|r| r.region.clone()).unwrap_or_default(),
             _ => {
                 info!("unexpected tier type {}", self.tier_type);
                 "".to_string()

crates/ecstore/src/tier/warm_backend.rs

@@ -1,4 +1,3 @@
-#![allow(unused_imports)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
@@ -27,7 +27,7 @@ use crate::error::is_err_bucket_not_found;
 use crate::tier::{
     tier::ERR_TIER_TYPE_UNSUPPORTED,
     tier_config::{TierConfig, TierType},
-    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR},
+    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
     warm_backend_aliyun::WarmBackendAliyun,
     warm_backend_azure::WarmBackendAzure,
     warm_backend_gcs::WarmBackendGCS,
@@ -155,7 +155,7 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
 }
 
 pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
-    let w = w.expect("err");
+    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
     let remote_version_id = w
         .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
         .await;
@@ -176,9 +176,11 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
         return Err(ERR_TIER_PERM_ERR.clone());
         //}
     }
-    if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await {
-        return Err(ERR_TIER_PERM_ERR.clone());
-    };
+    if let Ok(version_id) = remote_version_id {
+        if let Err(err) = w.remove(PROBE_OBJECT, &version_id).await {
+            return Err(ERR_TIER_PERM_ERR.clone());
+        };
+    }
     Ok(())
 }
 
@@ -186,119 +188,195 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
     let mut d: Option<WarmBackendImpl> = None;
     match tier.tier_type {
         TierType::S3 => {
-            let dd = WarmBackendS3::new(tier.s3.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(s3_config) = tier.s3.as_ref() {
+                let dd = WarmBackendS3::new(s3_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create S3 backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "S3 tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::RustFS => {
-            let dd = WarmBackendRustFS::new(tier.rustfs.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(rustfs_config) = tier.rustfs.as_ref() {
+                let dd = WarmBackendRustFS::new(rustfs_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create RustFS backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "RustFS tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::MinIO => {
-            let dd = WarmBackendMinIO::new(tier.minio.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(minio_config) = tier.minio.as_ref() {
+                let dd = WarmBackendMinIO::new(minio_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create MinIO backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "MinIO tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::Aliyun => {
-            let dd = WarmBackendAliyun::new(tier.aliyun.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(aliyun_config) = tier.aliyun.as_ref() {
+                let dd = WarmBackendAliyun::new(aliyun_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Aliyun backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "Aliyun tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::Tencent => {
-            let dd = WarmBackendTencent::new(tier.tencent.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(tencent_config) = tier.tencent.as_ref() {
+                let dd = WarmBackendTencent::new(tencent_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Tencent backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "Tencent tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::Huaweicloud => {
-            let dd = WarmBackendHuaweicloud::new(tier.huaweicloud.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(huaweicloud_config) = tier.huaweicloud.as_ref() {
+                let dd = WarmBackendHuaweicloud::new(huaweicloud_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Huaweicloud backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "Huaweicloud tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::Azure => {
-            let dd = WarmBackendAzure::new(tier.azure.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(azure_config) = tier.azure.as_ref() {
+                let dd = WarmBackendAzure::new(azure_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create Azure backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "Azure tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::GCS => {
-            let dd = WarmBackendGCS::new(tier.gcs.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(gcs_config) = tier.gcs.as_ref() {
+                let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create GCS backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "GCS tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         TierType::R2 => {
-            let dd = WarmBackendR2::new(tier.r2.as_ref().expect("err"), &tier.name).await;
-            if let Err(err) = dd {
-                warn!("{}", err);
+            if let Some(r2_config) = tier.r2.as_ref() {
+                let dd = WarmBackendR2::new(r2_config, &tier.name).await;
+                if let Err(err) = dd {
+                    warn!("{}", err);
+                    return Err(AdminError {
+                        code: "XRustFSAdminTierInvalidConfig".to_string(),
+                        message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                        status_code: StatusCode::BAD_REQUEST,
+                    });
+                }
+                d = Some(Box::new(dd.expect("Failed to create R2 backend")));
+            } else {
                 return Err(AdminError {
                     code: "XRustFSAdminTierInvalidConfig".to_string(),
-                    message: format!("Unable to setup remote tier, check tier configuration: {}", err.to_string()),
+                    message: "R2 tier configuration not found".to_string(),
                     status_code: StatusCode::BAD_REQUEST,
                 });
             }
-            d = Some(Box::new(dd.expect("err")));
         }
         _ => {
             return Err(ERR_TIER_TYPE_UNSUPPORTED.clone());
         }
     }
 
-    Ok(d.expect("err"))
+    d.ok_or_else(|| AdminError {
+        code: "XRustFSAdminTierInvalidConfig".to_string(),
+        message: "Tier backend not initialized".to_string(),
+        status_code: StatusCode::BAD_REQUEST,
+    })
 }
 
 #[cfg(test)]

crates/ecstore/src/tier/warm_backend_aliyun.rs

@@ -76,12 +76,10 @@ impl WarmBackendAliyun {
         };
         let scheme = u.scheme();
         let default_port = if scheme == "https" { 443 } else { 80 };
-        let client = TransitionClient::new(
-            &format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
-            opts,
-            "aliyun",
-        )
-        .await?;
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "aliyun").await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));

crates/ecstore/src/tier/warm_backend_azure.rs

@@ -76,12 +76,10 @@ impl WarmBackendAzure {
         };
         let scheme = u.scheme();
         let default_port = if scheme == "https" { 443 } else { 80 };
-        let client = TransitionClient::new(
-            &format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
-            opts,
-            "azure",
-        )
-        .await?;
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "azure").await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));

crates/ecstore/src/tier/warm_backend_huaweicloud.rs

@@ -76,12 +76,11 @@ impl WarmBackendHuaweicloud {
         };
         let scheme = u.scheme();
         let default_port = if scheme == "https" { 443 } else { 80 };
-        let client = TransitionClient::new(
-            &format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
-            opts,
-            "huaweicloud",
-        )
-        .await?;
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client =
+            TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "huaweicloud").await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));

crates/ecstore/src/tier/warm_backend_minio.rs

@@ -75,12 +75,10 @@ impl WarmBackendMinIO {
         };
         let scheme = u.scheme();
         let default_port = if scheme == "https" { 443 } else { 80 };
-        let client = TransitionClient::new(
-            &format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
-            opts,
-            "minio",
-        )
-        .await?;
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "minio").await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));

crates/ecstore/src/tier/warm_backend_r2.rs

@@ -75,12 +75,10 @@ impl WarmBackendR2 {
         };
         let scheme = u.scheme();
         let default_port = if scheme == "https" { 443 } else { 80 };
-        let client = TransitionClient::new(
-            &format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
-            opts,
-            "r2",
-        )
-        .await?;
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "r2").await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));

crates/ecstore/src/tier/warm_backend_s3.rs

@@ -95,7 +95,10 @@ impl WarmBackendS3 {
             region: conf.region.clone(),
             ..Default::default()
         };
-        let client = TransitionClient::new(&u.host().expect("err").to_string(), opts, "s3").await?;
+        let host = u
+            .host()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&host.to_string(), opts, "s3").await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));
@@ -164,8 +167,10 @@ impl WarmBackend for WarmBackendS3 {
             ropts.version_id = rv.to_string();
         }
         let client = self.client.clone();
-        let err = client.remove_object(&self.bucket, &self.get_dest(object), ropts).await;
-        Err(std::io::Error::other(err.expect("err")))
+        match client.remove_object(&self.bucket, &self.get_dest(object), ropts).await {
+            None => Ok(()),
+            Some(err) => Err(std::io::Error::other(err)),
+        }
     }
 
     async fn in_use(&self) -> Result<bool, std::io::Error> {

crates/ecstore/src/tier/warm_backend_s3sdk.rs

@@ -190,6 +190,6 @@ impl WarmBackend for WarmBackendS3 {
             return Err(std::io::Error::other("list_objects_v2 error"));
         };
 
-        Ok(res.common_prefixes.unwrap().len() > 0 || res.contents.unwrap().len() > 0)
+        Ok(res.common_prefixes.unwrap_or_default().len() > 0 || res.contents.unwrap_or_default().len() > 0)
     }
 }

crates/ecstore/src/tier/warm_backend_tencent.rs

@@ -76,12 +76,10 @@ impl WarmBackendTencent {
         };
         let scheme = u.scheme();
         let default_port = if scheme == "https" { 443 } else { 80 };
-        let client = TransitionClient::new(
-            &format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)),
-            opts,
-            "tencent",
-        )
-        .await?;
+        let host = u
+            .host_str()
+            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+        let client = TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, "tencent").await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));

crates/ecstore/tests/protobuf_bytes_regression_test.rs

@@ -0,0 +1,36 @@
+// Copyright (c) RustFS contributors
+// SPDX-License-Identifier: Apache-2.0
+
+use bytes::Bytes;
+use rustfs_protos::proto_gen::node_service::{
+    ReadMultipleRequest, ReadMultipleResponse, ReadVersionResponse, ReadXlResponse, UpdateMetadataRequest, WriteMetadataRequest,
+};
+
+fn expect_bytes(_: &Bytes) {}
+
+#[test]
+fn protobuf_bytes_fields_use_bytes_consistently() {
+    let update = UpdateMetadataRequest::default();
+    expect_bytes(&update.file_info_bin);
+    expect_bytes(&update.opts_bin);
+
+    let write = WriteMetadataRequest::default();
+    expect_bytes(&write.file_info_bin);
+
+    let version = ReadVersionResponse::default();
+    expect_bytes(&version.file_info_bin);
+
+    let read_xl = ReadXlResponse::default();
+    expect_bytes(&read_xl.raw_file_info_bin);
+
+    let read_multiple = ReadMultipleRequest::default();
+    expect_bytes(&read_multiple.read_multiple_req_bin);
+
+    let read_multiple_response = ReadMultipleResponse::default();
+    let first = read_multiple_response
+        .read_multiple_resps_bin
+        .first()
+        .cloned()
+        .unwrap_or_default();
+    expect_bytes(&first);
+}

crates/iam/src/oidc.rs

@@ -18,11 +18,11 @@
 //! `openidconnect` crate for standards-compliant discovery, token exchange,
 //! and ID token verification.
 
-use crate::oidc_state::{OidcAuthSession, OidcStateStore};
-use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreProviderMetadata};
+use crate::oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore};
+use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken};
 use openidconnect::{
-    AsyncHttpClient, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce, PkceCodeChallenge,
-    PkceCodeVerifier, RedirectUrl, Scope,
+    AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, LogoutRequest, Nonce,
+    PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, ProviderMetadataWithLogout, RedirectUrl, Scope,
 };
 use reqwest::Client;
 use rustfs_config::oidc::*;
@@ -157,6 +157,7 @@ pub struct OidcProviderConfig {
     pub client_id: String,
     pub client_secret: Option<String>,
     pub scopes: Vec<String>,
+    pub other_audiences: Vec<String>,
     pub redirect_uri: Option<String>,
     pub redirect_uri_dynamic: bool,
     pub claim_name: String,
@@ -215,7 +216,7 @@ pub struct OidcClaims {
 /// on-the-fly from metadata when needed.
 #[derive(Clone)]
 struct ProviderState {
-    metadata: CoreProviderMetadata,
+    metadata: ProviderMetadataWithLogout,
     discovered_at: Instant,
 }
 
@@ -235,6 +236,15 @@ pub struct OidcSys {
     http_client: ReqwestHttpClient,
 }
 
+fn trusted_aud(other_audiences: &[String], audience: &Audience) -> bool {
+    for aud in other_audiences {
+        if audience.as_str() == aud.as_str() {
+            return true;
+        }
+    }
+    false
+}
+
 impl OidcSys {
     /// Parse environment variables and discover all configured OIDC providers.
     pub async fn new() -> Result<Self, String> {
@@ -354,7 +364,7 @@ impl OidcSys {
         state: &str,
         code: &str,
         redirect_uri: &str,
-    ) -> Result<(OidcClaims, String, OidcAuthSession), String> {
+    ) -> Result<(OidcClaims, String, OidcAuthSession, String), String> {
         // Retrieve and consume the state (single-use)
         let session = self
             .state_store
@@ -394,7 +404,9 @@ impl OidcSys {
             .id_token()
             .ok_or_else(|| "no id_token in token response".to_string())?;
 
-        let verifier = client.id_token_verifier();
+        let verifier = client
+            .id_token_verifier()
+            .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
         let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
         if let Err(e) = verified {
             let refreshed_state = self
@@ -416,7 +428,9 @@ impl OidcSys {
             )
             .set_auth_type(AuthType::RequestBody);
 
-            let verifier = client.id_token_verifier();
+            let verifier = client
+                .id_token_verifier()
+                .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
             id_token
                 .claims(&verifier, &Nonce::new(session.nonce.clone()))
                 .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
@@ -435,7 +449,63 @@ impl OidcSys {
             raw,
         };
 
-        Ok((claims, session.provider_id.clone(), session))
+        Ok((claims, session.provider_id.clone(), session, raw_jwt))
+    }
+
+    /// Store a one-time logout session keyed by an opaque token so the console can
+    /// trigger browser logout without persisting the raw ID token.
+    pub async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result<String, String> {
+        if !self.configs.contains_key(provider_id) {
+            return Err(format!("unknown OIDC provider: {provider_id}"));
+        }
+
+        let token = CsrfToken::new_random().secret().clone();
+        self.state_store
+            .insert_logout(
+                token.clone(),
+                OidcLogoutSession {
+                    provider_id: provider_id.to_string(),
+                    id_token: id_token.to_string(),
+                },
+            )
+            .await;
+
+        Ok(token)
+    }
+
+    /// Build the RP-initiated logout URL for a previously issued logout token.
+    /// Returns `Ok(None)` when the provider does not advertise an end-session endpoint.
+    pub async fn build_logout_url(&self, logout_token: &str, post_logout_redirect_uri: &str) -> Result<Option<String>, String> {
+        let session = self
+            .state_store
+            .take_logout(logout_token)
+            .await
+            .ok_or_else(|| "invalid or expired OIDC logout token".to_string())?;
+
+        let config = self
+            .configs
+            .get(&session.provider_id)
+            .ok_or_else(|| format!("unknown OIDC provider: {}", session.provider_id))?;
+        let state = self.ensure_provider_state(&session.provider_id, config).await?;
+        let Some(end_session_endpoint) = state.metadata.additional_metadata().end_session_endpoint.clone() else {
+            return Ok(None);
+        };
+
+        let id_token: CoreIdToken = session
+            .id_token
+            .parse()
+            .map_err(|e: serde_json::Error| format!("failed to parse ID token for logout: {e}"))?;
+        let post_logout_redirect_uri = PostLogoutRedirectUrl::new(post_logout_redirect_uri.to_string())
+            .map_err(|e| format!("invalid post logout redirect URI: {e}"))?;
+
+        let logout_url = LogoutRequest::from(end_session_endpoint)
+            .set_id_token_hint(&id_token)
+            .set_client_id(ClientId::new(config.client_id.clone()))
+            .set_post_logout_redirect_uri(post_logout_redirect_uri)
+            .http_get_url()
+            .to_string();
+
+        Ok(Some(logout_url))
     }
 
     /// Map OIDC claims to rustfs policy names.
@@ -529,7 +599,9 @@ impl OidcSys {
 
         // Verify the token (signature, issuer, audience, expiry) — skip nonce
         // (nonce is only required for the authorization code flow)
-        let verifier = client.id_token_verifier();
+        let verifier = client
+            .id_token_verifier()
+            .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
         if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
             state = self
                 .refresh_provider_state(&provider_id, &config)
@@ -544,7 +616,9 @@ impl OidcSys {
                 config.client_secret.as_ref().map(|s| ClientSecret::new(s.clone())),
             )
             .set_auth_type(AuthType::RequestBody);
-            let verifier = client.id_token_verifier();
+            let verifier = client
+                .id_token_verifier()
+                .set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
             id_token
                 .claims(&verifier, |_: Option<&Nonce>| Ok(()))
                 .map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
@@ -742,6 +816,14 @@ impl OidcSys {
             scopes_str.split(',').map(|s| s.trim().to_string()).collect()
         };
 
+        let other_audiences_str = get_env(ENV_IDENTITY_OPENID_OTHER_AUDIENCES);
+        let other_audiences = other_audiences_str
+            .split(',')
+            .map(|s| s.trim())
+            .filter(|s| !s.is_empty())
+            .map(|s| s.to_string())
+            .collect();
+
         let redirect_uri_dynamic_str = get_env(ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC);
         let redirect_uri_dynamic = redirect_uri_dynamic_str.is_empty()
             || redirect_uri_dynamic_str
@@ -802,6 +884,7 @@ impl OidcSys {
             client_id: get_env(ENV_IDENTITY_OPENID_CLIENT_ID),
             client_secret,
             scopes,
+            other_audiences,
             redirect_uri,
             redirect_uri_dynamic,
             claim_name,
@@ -835,6 +918,14 @@ impl OidcSys {
             scopes_str.split(',').map(|s| s.trim().to_string()).collect()
         };
 
+        let other_audiences_str = kvs.get(OIDC_OTHER_AUDIENCES);
+        let other_audiences = other_audiences_str
+            .split(',')
+            .map(|s| s.trim())
+            .filter(|s| !s.is_empty())
+            .map(|s| s.to_string())
+            .collect();
+
         let redirect_uri_dynamic = kvs
             .lookup(OIDC_REDIRECT_URI_DYNAMIC)
             .unwrap_or_else(|| EnableState::On.to_string())
@@ -868,6 +959,7 @@ impl OidcSys {
             client_id: kvs.get(OIDC_CLIENT_ID),
             client_secret,
             scopes,
+            other_audiences,
             redirect_uri,
             redirect_uri_dynamic,
             claim_name,
@@ -894,7 +986,7 @@ impl OidcSys {
             let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?;
 
             for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES {
-                match CoreProviderMetadata::discover_async(issuer_url.clone(), http_client)
+                match ProviderMetadataWithLogout::discover_async(issuer_url.clone(), http_client)
                     .await
                     .map_err(|e| format!("discovery failed: {e}"))
                 {
@@ -1352,6 +1444,7 @@ mod tests {
             client_id: "rustfs-oidc-test".to_string(),
             client_secret: None,
             scopes: vec!["openid".to_string()],
+            other_audiences: vec![],
             redirect_uri: None,
             redirect_uri_dynamic: false,
             claim_name: "sub".to_string(),
@@ -1717,6 +1810,7 @@ mod tests {
             client_id: "client-id".to_string(),
             client_secret: None,
             scopes: vec!["openid".to_string()],
+            other_audiences: vec![],
             redirect_uri: None,
             redirect_uri_dynamic: true,
             claim_name: "groups".to_string(),
@@ -1811,6 +1905,7 @@ mod tests {
             client_id: "my-client".to_string(),
             client_secret: Some("secret".to_string()),
             scopes: vec!["openid".to_string(), "profile".to_string(), "email".to_string()],
+            other_audiences: vec![],
             redirect_uri: None,
             redirect_uri_dynamic: true,
             claim_name: "groups".to_string(),

crates/iam/src/oidc_state.rs

@@ -32,11 +32,20 @@ pub struct OidcAuthSession {
     pub redirect_after: Option<String>,
 }
 
+/// Stores an ID token behind a one-time opaque handle so the console can trigger
+/// RP-initiated logout without persisting the raw token in browser storage.
+#[derive(Debug, Clone)]
+pub struct OidcLogoutSession {
+    pub provider_id: String,
+    pub id_token: String,
+}
+
 /// TTL cache for OIDC auth state (PKCE verifiers + nonces) during the authorization flow.
 /// Entries expire after 5 minutes and are single-use (removed on retrieval).
 #[derive(Clone)]
 pub struct OidcStateStore {
     cache: Cache<String, OidcAuthSession>,
+    logout_cache: Cache<String, OidcLogoutSession>,
     last_capacity_log_at: Arc<AtomicU64>,
 }
 
@@ -46,8 +55,13 @@ impl OidcStateStore {
             .max_capacity(OIDC_STATE_CAPACITY)
             .time_to_live(Duration::from_secs(300)) // 5 minute TTL
             .build();
+        let logout_cache = Cache::builder()
+            .max_capacity(OIDC_STATE_CAPACITY)
+            .time_to_live(Duration::from_secs(3600)) // 1 hour TTL to match console OIDC sessions
+            .build();
         Self {
             cache,
+            logout_cache,
             last_capacity_log_at: Arc::new(AtomicU64::new(0)),
         }
     }
@@ -98,6 +112,21 @@ impl OidcStateStore {
     pub async fn contains(&self, state: &str) -> bool {
         self.cache.get(state).await.is_some()
     }
+
+    /// Store a new one-time logout session keyed by an opaque logout token.
+    pub async fn insert_logout(&self, token: String, session: OidcLogoutSession) {
+        self.logout_cache.insert(token, session).await;
+    }
+
+    /// Retrieve and remove a logout session (single-use). Returns None if expired or not found.
+    pub async fn take_logout(&self, token: &str) -> Option<OidcLogoutSession> {
+        self.logout_cache.remove(token).await
+    }
+
+    /// Check if a logout token exists (without consuming it).
+    pub async fn contains_logout(&self, token: &str) -> bool {
+        self.logout_cache.get(token).await.is_some()
+    }
 }
 
 impl Default for OidcStateStore {
@@ -167,4 +196,24 @@ mod tests {
             assert!(store.take(&format!("state_{i}")).await.is_none());
         }
     }
+
+    #[tokio::test]
+    async fn test_logout_state_store_insert_and_take() {
+        let store = OidcStateStore::new();
+        let session = OidcLogoutSession {
+            provider_id: "okta".to_string(),
+            id_token: "jwt-token".to_string(),
+        };
+
+        store.insert_logout("logout_abc".to_string(), session.clone()).await;
+        assert!(store.contains_logout("logout_abc").await);
+
+        let retrieved = store.take_logout("logout_abc").await;
+        assert!(retrieved.is_some());
+        let retrieved = retrieved.unwrap();
+        assert_eq!(retrieved.provider_id, "okta");
+        assert_eq!(retrieved.id_token, "jwt-token");
+
+        assert!(store.take_logout("logout_abc").await.is_none());
+    }
 }

crates/io-core/src/pool.rs

@@ -84,6 +84,12 @@ struct PoolTier {
     available_buffers: Mutex<Vec<BytesMut>>,
     /// Metrics for tracking this tier
     metrics: Mutex<Option<Arc<BytesPoolMetrics>>>,
+    /// Total acquisitions for this tier
+    tier_total_acquires: AtomicU64,
+    /// Total hits for this tier
+    tier_pool_hits: AtomicU64,
+    /// Current allocated bytes for this tier
+    tier_current_allocated_bytes: AtomicU64,
 }
 
 /// Pool metrics for monitoring and optimization.
@@ -291,6 +297,9 @@ impl PoolTier {
             name,
             available_buffers: Mutex::new(Vec::new()),
             metrics: Mutex::new(None),
+            tier_total_acquires: AtomicU64::new(0),
+            tier_pool_hits: AtomicU64::new(0),
+            tier_current_allocated_bytes: AtomicU64::new(0),
         }
     }
 
@@ -298,6 +307,65 @@ impl PoolTier {
         *self.metrics.lock().unwrap() = Some(metrics);
     }
 
+    fn take_or_allocate_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> (BytesMut, bool) {
+        let buffer_opt = {
+            let mut available = self.available_buffers.lock().unwrap();
+            available.pop()
+        };
+        let was_reused = buffer_opt.is_some();
+
+        let buffer = if let Some(mut buf) = buffer_opt {
+            let previous_capacity = buf.capacity();
+            buf.clear();
+            if previous_capacity < size {
+                buf.reserve(size - previous_capacity);
+            }
+            let current_capacity = buf.capacity();
+            if current_capacity > previous_capacity {
+                let delta = (current_capacity - previous_capacity) as u64;
+                pool_metrics.total_bytes_allocated.fetch_add(delta, Ordering::Relaxed);
+                pool_metrics.current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+                self.tier_current_allocated_bytes.fetch_add(delta, Ordering::Relaxed);
+            }
+            buf
+        } else {
+            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
+            let allocated_bytes = buf.capacity() as u64;
+            pool_metrics
+                .total_bytes_allocated
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            pool_metrics
+                .current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            self.tier_current_allocated_bytes
+                .fetch_add(allocated_bytes, Ordering::Relaxed);
+            buf
+        };
+
+        (buffer, was_reused)
+    }
+
+    fn record_acquire_metrics(&self, pool_metrics: &BytesPoolMetrics, buffer_capacity: usize, was_reused: bool) {
+        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
+
+        if was_reused {
+            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
+            self.tier_pool_hits.fetch_add(1, Ordering::Relaxed);
+        } else {
+            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
+        }
+
+        let tier_total_acquires = self.tier_total_acquires.load(Ordering::Relaxed);
+        let tier_pool_hits = self.tier_pool_hits.load(Ordering::Relaxed);
+        let tier_hit_rate = if tier_total_acquires == 0 {
+            0.0
+        } else {
+            tier_pool_hits as f64 / tier_total_acquires as f64
+        };
+        rustfs_io_metrics::record_bytes_pool_hit_rate(self.name, tier_hit_rate);
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
+    }
+
     async fn acquire_buffer(&self, size: usize, pool_metrics: &BytesPoolMetrics) -> PooledBuffer {
         // Acquire semaphore permit (owned for storage in PooledBuffer)
         let permit = Arc::clone(&self.semaphore).acquire_owned().await.unwrap();
@@ -308,45 +376,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer - clear and ensure capacity
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -365,45 +399,11 @@ impl PoolTier {
 
         // Record acquisition
         pool_metrics.total_acquires.fetch_add(1, Ordering::Relaxed);
+        self.tier_total_acquires.fetch_add(1, Ordering::Relaxed);
 
-        // Try to get a buffer from the pool
-        let buffer_opt = {
-            let mut available = self.available_buffers.lock().unwrap();
-            available.pop()
-        };
-
-        let was_reused = buffer_opt.is_some();
-
-        let buffer = if let Some(mut buf) = buffer_opt {
-            // Reuse existing buffer
-            buf.clear();
-            if buf.capacity() < size {
-                buf.reserve(size - buf.capacity());
-            }
-            buf
-        } else {
-            // Allocate new buffer
-            let buf = BytesMut::with_capacity(size.max(self.buffer_size));
-            pool_metrics
-                .total_bytes_allocated
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            pool_metrics
-                .current_allocated_bytes
-                .fetch_add(buf.capacity() as u64, Ordering::Relaxed);
-            buf
-        };
-
+        let (buffer, was_reused) = self.take_or_allocate_buffer(size, pool_metrics);
         let buffer_capacity = buffer.capacity();
-
-        // Record metrics
-        rustfs_io_metrics::record_bytes_pool_acquire(self.name, buffer_capacity, was_reused);
-
-        // Record hit/miss (pool_metrics and metrics point to same Arc)
-        if was_reused {
-            pool_metrics.pool_hits.fetch_add(1, Ordering::Relaxed);
-        } else {
-            pool_metrics.pool_misses.fetch_add(1, Ordering::Relaxed);
-        }
+        self.record_acquire_metrics(pool_metrics, buffer_capacity, was_reused);
 
         Some(PooledBuffer {
             buffer: ManuallyDrop::new(buffer),
@@ -421,8 +421,24 @@ impl PoolTier {
             if let Some(ref metrics) = *self.metrics.lock().unwrap() {
                 metrics.available_buffers.fetch_add(1, Ordering::Relaxed);
             }
+        } else {
+            let released_bytes = buffer.capacity() as u64;
+            self.tier_current_allocated_bytes
+                .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                    Some(current.saturating_sub(released_bytes))
+                })
+                .ok();
+            if let Some(ref metrics) = *self.metrics.lock().unwrap() {
+                metrics
+                    .current_allocated_bytes
+                    .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                        Some(current.saturating_sub(released_bytes))
+                    })
+                    .ok();
+            }
         }
         // If pool is full, buffer is dropped and memory is freed
+        rustfs_io_metrics::record_bytes_pool_allocated(self.name, self.tier_current_allocated_bytes.load(Ordering::Relaxed));
     }
 }
 
@@ -617,4 +633,47 @@ mod tests {
         let delta_hits = pool.metrics().pool_hits.load(Ordering::Relaxed) - initial_hits;
         assert_eq!(delta_hits, 1);
     }
+
+    #[tokio::test]
+    async fn test_tier_allocated_bytes_tracks_real_allocations() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire allocates one small-tier buffer.
+        let buf1 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // Return and reuse should not increase allocated bytes.
+        drop(buf1);
+        let buf2 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 1024);
+
+        // A second in-flight buffer forces one more allocation.
+        let _buf3 = pool.acquire_buffer(512).await;
+        assert_eq!(pool.small_pool.tier_current_allocated_bytes.load(Ordering::Relaxed), 2048);
+
+        drop(buf2);
+    }
+
+    #[tokio::test]
+    async fn test_tier_hit_rate_counters_track_reuse() {
+        let pool = BytesPool::with_config(BytesPoolConfig {
+            small_size: 1024,
+            small_max: 2,
+            ..Default::default()
+        });
+
+        // First acquire is miss.
+        let buf1 = pool.acquire_buffer(512).await;
+        drop(buf1);
+
+        // Second acquire reuses previous buffer and counts as hit.
+        let _buf2 = pool.acquire_buffer(512).await;
+
+        assert_eq!(pool.small_pool.tier_total_acquires.load(Ordering::Relaxed), 2);
+        assert_eq!(pool.small_pool.tier_pool_hits.load(Ordering::Relaxed), 1);
+    }
 }

crates/lock/src/distributed_lock.rs

@@ -22,13 +22,15 @@ use futures::future::join_all;
 use rustfs_io_metrics::{
     record_read_lock_held_acquire, record_read_lock_held_release, record_write_lock_held_acquire, record_write_lock_held_release,
 };
-use std::sync::{Arc, LazyLock};
+use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::mpsc;
 use tokio::task::JoinSet;
-use tracing::warn;
+use tracing::{debug, warn};
 use uuid::Uuid;
 
+const UNLOCK_RETRY_ATTEMPTS: usize = 3;
+const UNLOCK_RETRY_BACKOFF: Duration = Duration::from_millis(100);
+
 /// Generate a new aggregate lock ID for multiple client locks
 fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
     LockId {
@@ -37,45 +39,6 @@ fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
     }
 }
 
-#[derive(Debug, Clone)]
-struct UnlockJob {
-    /// Entries to release: each (LockId, client) pair will be released independently.
-    entries: Vec<(LockId, Arc<dyn LockClient>)>,
-}
-
-#[derive(Debug)]
-struct UnlockRuntime {
-    tx: mpsc::Sender<UnlockJob>,
-}
-
-// Global unlock runtime with background worker
-static UNLOCK_RUNTIME: LazyLock<UnlockRuntime> = LazyLock::new(|| {
-    // Larger buffer to reduce contention during bursts
-    let (tx, mut rx) = mpsc::channel::<UnlockJob>(8192);
-
-    // Spawn background worker when first used; assumes a Tokio runtime is available
-    tokio::spawn(async move {
-        while let Some(job) = rx.recv().await {
-            // Best-effort release across all (LockId, client) entries.
-            let results = join_all(
-                job.entries
-                    .into_iter()
-                    .map(|(lock_id, client)| async move { client.release(&lock_id).await.unwrap_or(false) }),
-            )
-            .await;
-            let any_ok = results.into_iter().any(|released| released);
-
-            if !any_ok {
-                tracing::warn!("DistributedLockGuard background release failed for one or more entries");
-            } else {
-                tracing::debug!("DistributedLockGuard background released one or more entries");
-            }
-        }
-    });
-
-    UnlockRuntime { tx }
-});
-
 /// A RAII guard for distributed locks that releases the lock asynchronously when dropped.
 #[derive(Debug)]
 pub struct DistributedLockGuard {
@@ -126,47 +89,22 @@ impl DistributedLockGuard {
     }
 
     /// Manually release the lock early.
-    /// This sends a release job to the background worker and then disarms the guard
+    /// This spawns a background release task and then disarms the guard
     /// to prevent double-release on drop.
-    /// Returns true if the lock was released (or was already released), false otherwise.
+    /// Returns true if release was scheduled or the guard was already disarmed.
     pub fn release(&mut self) -> bool {
         if self.disarmed {
             // Lock was already released, return true to indicate lock is in released state
             return true;
         }
 
-        let job = UnlockJob {
-            entries: self.entries.clone(),
-        };
-
-        // Try a non-blocking send to avoid panics
-        let success = if let Err(err) = UNLOCK_RUNTIME.tx.try_send(job) {
-            // Channel full or closed; best-effort fallback: spawn a detached task
-            let entries = self.entries.clone();
-            tracing::warn!(
-                "DistributedLockGuard channel send failed ({}), spawning fallback unlock task for {} entries",
-                err,
-                entries.len()
-            );
-
-            // If runtime is not available, this will panic; but in RustFS we are inside Tokio contexts.
-            let handle = tokio::spawn(async move {
-                let futures_iter = entries
-                    .into_iter()
-                    .map(|(lock_id, client)| async move { client.release(&lock_id).await.unwrap_or(false) });
-                let _ = join_all(futures_iter).await;
-            });
-            // Explicitly drop the JoinHandle to acknowledge detaching the task.
-            drop(handle);
-            true // Consider it successful even if we had to use fallback
-        } else {
-            true
-        };
+        let entries = self.entries.clone();
+        DistributedLock::spawn_release_cleanup(entries, "distributed_lock_guard_release");
 
         // Disarm to prevent double-release on drop
         self.disarmed = true;
         record_lock_held_release(self.lock_type);
-        success
+        true
     }
 }
 
@@ -349,22 +287,64 @@ impl DistributedLock {
         pending
     }
 
-    async fn release_entries(entries: &[(LockId, Arc<dyn LockClient>)], context: &'static str) {
-        let release_results = join_all(
-            entries
-                .iter()
-                .map(|(lock_id, client)| async move { (lock_id, client.release(lock_id).await) }),
-        )
-        .await;
+    async fn release_entries(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
+        let mut pending = entries;
 
-        for (lock_id, result) in release_results {
-            match result {
-                Ok(true) | Ok(false) => {}
-                Err(err) => {
-                    tracing::warn!("{context}: failed to release lock {} on client: {}", lock_id, err);
+        for attempt in 1..=UNLOCK_RETRY_ATTEMPTS {
+            let release_results = join_all(pending.into_iter().map(|(lock_id, client)| async move {
+                match client.release(&lock_id).await {
+                    Ok(true) => None,
+                    Ok(false) => {
+                        warn!(%lock_id, attempt, context, "distributed unlock did not find lock on client");
+                        Some((lock_id, client))
+                    }
+                    Err(err) => {
+                        warn!(%lock_id, attempt, context, "distributed unlock failed on client: {}", err);
+                        Some((lock_id, client))
+                    }
                 }
+            }))
+            .await;
+
+            pending = release_results.into_iter().flatten().collect();
+            if pending.is_empty() {
+                debug!(attempt, context, "distributed unlock completed");
+                return;
+            }
+
+            if attempt < UNLOCK_RETRY_ATTEMPTS {
+                tokio::time::sleep(UNLOCK_RETRY_BACKOFF * attempt as u32).await;
             }
         }
+
+        warn!(
+            remaining = pending.len(),
+            attempts = UNLOCK_RETRY_ATTEMPTS,
+            context,
+            "distributed unlock left unreleased entries after retry"
+        );
+    }
+
+    fn spawn_release_cleanup(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
+        if entries.is_empty() {
+            return;
+        }
+
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            let join_handle = handle.spawn(async move {
+                Self::release_entries(entries, context).await;
+            });
+            drop(join_handle);
+            return;
+        }
+
+        let join_handle = std::thread::spawn(move || match tokio::runtime::Builder::new_current_thread().enable_all().build() {
+            Ok(runtime) => runtime.block_on(async move {
+                Self::release_entries(entries, context).await;
+            }),
+            Err(err) => warn!(context, "failed to create fallback unlock runtime: {}", err),
+        });
+        drop(join_handle);
     }
 
     fn spawn_pending_cleanup(
@@ -387,9 +367,7 @@ impl DistributedLock {
                             continue;
                         };
 
-                        if let Err(err) = client.release(&lock_id).await {
-                            tracing::warn!("{context}: failed to cleanup late lock {} on client {}: {}", lock_id, idx, err);
-                        }
+                        Self::release_entries(vec![(lock_id, client.clone())], context).await;
                     }
                     Ok((idx, Ok(resp))) => {
                         tracing::debug!(
@@ -506,7 +484,7 @@ impl DistributedLock {
 
             if individual_locks.len() + pending.len() < required_quorum {
                 let rollback_count = individual_locks.len();
-                Self::release_entries(&individual_locks, "distributed_lock_quorum_rollback").await;
+                Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
                 if !pending.is_empty() {
                     Self::spawn_pending_cleanup(
                         pending,
@@ -525,7 +503,7 @@ impl DistributedLock {
         }
 
         let rollback_count = individual_locks.len();
-        Self::release_entries(&individual_locks, "distributed_lock_quorum_rollback").await;
+        Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
         let resp = LockResponse::failure(
             format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
             Duration::ZERO,

crates/lock/src/namespace/tests.rs

@@ -16,7 +16,10 @@ use super::*;
 use crate::client::{ClientFactory, local::LocalClient};
 use crate::types::LockType;
 use crate::{GlobalLockManager, LockError, LockInfo, LockResponse, LockStats};
-use std::sync::Arc;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
 use std::time::Duration;
 
 #[derive(Debug, Default)]
@@ -107,6 +110,75 @@ impl crate::client::LockClient for DelayedClient {
     }
 }
 
+#[derive(Debug)]
+struct FlakyReleaseClient {
+    inner: LocalClient,
+    failed_releases_remaining: AtomicUsize,
+    release_attempts: AtomicUsize,
+}
+
+impl FlakyReleaseClient {
+    fn new(manager: Arc<GlobalLockManager>, failed_releases: usize) -> Self {
+        Self {
+            inner: LocalClient::with_manager(manager),
+            failed_releases_remaining: AtomicUsize::new(failed_releases),
+            release_attempts: AtomicUsize::new(0),
+        }
+    }
+
+    fn release_attempts(&self) -> usize {
+        self.release_attempts.load(Ordering::SeqCst)
+    }
+}
+
+#[async_trait::async_trait]
+impl crate::client::LockClient for FlakyReleaseClient {
+    async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
+        self.inner.acquire_lock(request).await
+    }
+
+    async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.release_attempts.fetch_add(1, Ordering::SeqCst);
+        if self
+            .failed_releases_remaining
+            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
+            .is_ok()
+        {
+            return Ok(false);
+        }
+
+        self.inner.release(lock_id).await
+    }
+
+    async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.inner.refresh(lock_id).await
+    }
+
+    async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
+        self.inner.force_release(lock_id).await
+    }
+
+    async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
+        self.inner.check_status(lock_id).await
+    }
+
+    async fn get_stats(&self) -> crate::Result<LockStats> {
+        self.inner.get_stats().await
+    }
+
+    async fn close(&self) -> crate::Result<()> {
+        self.inner.close().await
+    }
+
+    async fn is_online(&self) -> bool {
+        true
+    }
+
+    async fn is_local(&self) -> bool {
+        true
+    }
+}
+
 fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
     ObjectKey {
         bucket: Arc::from(bucket),
@@ -115,6 +187,41 @@ fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
     }
 }
 
+async fn wait_until_all_managers_can_write(managers: &[Arc<GlobalLockManager>], resource: ObjectKey) {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
+
+    loop {
+        let mut guards = Vec::with_capacity(managers.len());
+        let mut all_available = true;
+
+        for (idx, manager) in managers.iter().enumerate() {
+            let local_lock = NamespaceLock::with_local_manager(format!("probe-node-{idx}"), manager.clone());
+            match local_lock
+                .get_write_lock(resource.clone(), "probe-owner", Duration::from_millis(20))
+                .await
+            {
+                Ok(guard) => guards.push(guard),
+                Err(_) => {
+                    all_available = false;
+                    break;
+                }
+            }
+        }
+
+        drop(guards);
+
+        if all_available {
+            return;
+        }
+
+        assert!(
+            tokio::time::Instant::now() < deadline,
+            "distributed lock was not released on all simulated nodes"
+        );
+        tokio::time::sleep(Duration::from_millis(20)).await;
+    }
+}
+
 #[tokio::test]
 async fn test_namespace_lock_new() {
     let client = ClientFactory::create_local();
@@ -174,6 +281,24 @@ async fn test_lock_client_default_batch_acquire_and_release() {
     assert_eq!(released, vec![true, true]);
 }
 
+#[tokio::test]
+async fn test_local_client_uses_request_lock_id_for_release() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let client = LocalClient::with_manager(manager);
+    let resource = create_test_object_key("bucket", "object");
+    let request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner-a").with_acquire_timeout(Duration::from_secs(1));
+
+    let response = client.acquire_lock(&request).await.unwrap();
+    let lock_info = response.lock_info.expect("successful acquire should return lock info");
+    assert_eq!(lock_info.id, request.lock_id);
+
+    assert!(client.release(&request.lock_id).await.unwrap());
+
+    let second_request = LockRequest::new(resource, LockType::Exclusive, "owner-b").with_acquire_timeout(Duration::from_secs(1));
+    let second_response = client.acquire_lock(&second_request).await.unwrap();
+    assert!(second_response.success);
+}
+
 #[tokio::test]
 async fn test_namespace_lock_get_resource_key() {
     let client = ClientFactory::create_local();
@@ -488,6 +613,97 @@ async fn test_namespace_lock_distributed_with_clients_and_quorum() {
     drop(guard_b);
 }
 
+#[tokio::test]
+async fn test_namespace_lock_distributed_eight_node_write_releases_all_nodes() {
+    let managers = (0..8).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let clients = managers
+        .iter()
+        .map(|manager| Arc::new(LocalClient::with_manager(manager.clone())) as Arc<dyn LockClient>)
+        .collect::<Vec<_>>();
+
+    let lock = NamespaceLock::with_clients_and_quorum("eight-node".to_string(), clients, 5);
+    let resource = create_test_object_key("bucket", "object-eight-node");
+
+    let mut guard = lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
+        .await
+        .expect("owner-a should acquire write lock across eight simulated nodes");
+
+    let err = lock
+        .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
+        .await
+        .expect_err("owner-b should not acquire while owner-a holds all node locks");
+    let err_str = err.to_string();
+    assert!(
+        err_str.contains("required 5") && err_str.contains("achieved"),
+        "expected 8-node quorum failure below required write quorum, got: {err}"
+    );
+
+    assert!(guard.release(), "distributed guard should enqueue release");
+    wait_until_all_managers_can_write(&managers, resource).await;
+}
+
+#[tokio::test]
+async fn test_namespace_lock_distributed_unlock_retries_release_false() {
+    let managers = (0..3).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let flaky_clients = managers
+        .iter()
+        .map(|manager| Arc::new(FlakyReleaseClient::new(manager.clone(), 1)))
+        .collect::<Vec<_>>();
+    let clients = flaky_clients
+        .iter()
+        .map(|client| client.clone() as Arc<dyn LockClient>)
+        .collect::<Vec<_>>();
+
+    let lock = NamespaceLock::with_clients("flaky-release".to_string(), clients);
+    let resource = create_test_object_key("bucket", "object-flaky-release");
+
+    let mut guard = lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
+        .await
+        .expect("owner-a should acquire write lock before flaky release");
+
+    assert!(guard.release(), "distributed guard should enqueue release");
+    wait_until_all_managers_can_write(&managers, resource).await;
+
+    assert!(
+        flaky_clients.iter().all(|client| client.release_attempts() >= 2),
+        "each simulated node should be retried after an initial false release"
+    );
+}
+
+#[test]
+fn test_namespace_lock_distributed_drop_without_runtime_does_not_panic() {
+    let (manager, resource, guard) = {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should be created");
+        runtime.block_on(async {
+            let manager = Arc::new(GlobalLockManager::new());
+            let resource = create_test_object_key("bucket", "object-drop-no-runtime");
+            let lock = NamespaceLock::with_clients(
+                "drop-no-runtime".to_string(),
+                vec![Arc::new(LocalClient::with_manager(manager.clone()))],
+            );
+            let guard = lock
+                .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
+                .await
+                .expect("lock should be acquired");
+            (manager, resource, guard)
+        })
+    };
+
+    let drop_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(guard)));
+    assert!(drop_result.is_ok(), "dropping distributed guard without runtime should not panic");
+
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .expect("test runtime should be created");
+    runtime.block_on(wait_until_all_managers_can_write(&[manager], resource));
+}
+
 #[tokio::test]
 async fn test_namespace_lock_distributed_read_lock_succeeds_with_two_nodes_one_offline() {
     let manager = Arc::new(GlobalLockManager::new());
@@ -568,6 +784,39 @@ async fn test_namespace_lock_distributed_quorum_failure_rolls_back_successful_no
     drop(guard2);
 }
 
+#[tokio::test]
+async fn test_namespace_lock_distributed_quorum_rollback_retries_release_false() {
+    let managers = (0..2).map(|_| Arc::new(GlobalLockManager::new())).collect::<Vec<_>>();
+    let flaky_clients = managers
+        .iter()
+        .map(|manager| Arc::new(FlakyReleaseClient::new(manager.clone(), 1)))
+        .collect::<Vec<_>>();
+    let clients = vec![
+        flaky_clients[0].clone() as Arc<dyn LockClient>,
+        flaky_clients[1].clone() as Arc<dyn LockClient>,
+        Arc::new(FailingClient) as Arc<dyn LockClient>,
+    ];
+    let resource = create_test_object_key("bucket", "object-rollback-retry");
+    let lock = NamespaceLock::with_clients_and_quorum("rollback-retry".to_string(), clients, 3);
+
+    let err = lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_millis(100))
+        .await
+        .expect_err("write lock should fail when quorum requires the offline node");
+
+    let err_str = err.to_string().to_lowercase();
+    assert!(
+        err_str.contains("quorum") || err_str.contains("not reached"),
+        "expected quorum error, got: {err}"
+    );
+    wait_until_all_managers_can_write(&managers, resource).await;
+
+    assert!(
+        flaky_clients.iter().all(|client| client.release_attempts() >= 2),
+        "rollback should retry node releases that initially returned false"
+    );
+}
+
 #[tokio::test]
 async fn test_namespace_lock_distributed_even_node_read_write_quorum_split() {
     let manager1 = Arc::new(GlobalLockManager::new());

crates/madmin/Cargo.toml

@@ -38,3 +38,6 @@ time.workspace = true
 
 [lib]
 doctest = false
+
+[dev-dependencies]
+rmp-serde.workspace = true

crates/madmin/src/info_commands.rs

@@ -86,10 +86,6 @@ pub struct Disk {
     pub write_latency: f64,
     pub utilization: f64,
     pub metrics: Option<DiskMetrics>,
-    #[serde(rename = "runtimeState", default, skip_serializing_if = "Option::is_none")]
-    pub runtime_state: Option<String>,
-    #[serde(rename = "offlineDurationSeconds", default, skip_serializing_if = "Option::is_none")]
-    pub offline_duration_seconds: Option<u64>,
     pub heal_info: Option<HealingDisk>,
     pub used_inodes: u64,
     pub free_inodes: u64,
@@ -97,6 +93,13 @@ pub struct Disk {
     pub pool_index: i32,
     pub set_index: i32,
     pub disk_index: i32,
+    #[serde(rename = "runtimeState", default, skip_serializing_if = "Option::is_none")]
+    pub runtime_state: Option<String>,
+    #[serde(rename = "offlineDurationSeconds", default, skip_serializing_if = "Option::is_none")]
+    pub offline_duration_seconds: Option<u64>,
+    /// Leaf physical block devices backing this disk path when the platform can resolve them.
+    #[serde(rename = "physicalDeviceIds", default, skip_serializing_if = "Option::is_none")]
+    pub physical_device_ids: Option<Vec<String>>,
 }
 
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -353,10 +356,50 @@ pub struct InfoMessage {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rmp_serde::{Deserializer, Serializer};
     use serde_json;
-    use std::collections::HashMap;
+    use std::{collections::HashMap, io::Cursor};
     use time::OffsetDateTime;
 
+    #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
+    struct LegacyDiskCompat {
+        endpoint: String,
+        #[serde(rename = "rootDisk")]
+        root_disk: bool,
+        #[serde(rename = "path")]
+        drive_path: String,
+        healing: bool,
+        scanning: bool,
+        state: String,
+        uuid: String,
+        major: u32,
+        minor: u32,
+        model: Option<String>,
+        #[serde(rename = "totalspace")]
+        total_space: u64,
+        #[serde(rename = "usedspace")]
+        used_space: u64,
+        #[serde(rename = "availspace")]
+        available_space: u64,
+        #[serde(rename = "readthroughput")]
+        read_throughput: f64,
+        #[serde(rename = "writethroughput")]
+        write_throughput: f64,
+        #[serde(rename = "readlatency")]
+        read_latency: f64,
+        #[serde(rename = "writelatency")]
+        write_latency: f64,
+        utilization: f64,
+        metrics: Option<DiskMetrics>,
+        heal_info: Option<HealingDisk>,
+        used_inodes: u64,
+        free_inodes: u64,
+        local: bool,
+        pool_index: i32,
+        set_index: i32,
+        disk_index: i32,
+    }
+
     #[test]
     fn test_item_state_to_string() {
         assert_eq!(ItemState::Offline.to_string(), ITEM_OFFLINE);
@@ -441,6 +484,7 @@ mod tests {
         assert!(disk.heal_info.is_none());
         assert_eq!(disk.used_inodes, 0);
         assert_eq!(disk.free_inodes, 0);
+        assert!(disk.physical_device_ids.is_none());
         assert!(!disk.local);
         assert_eq!(disk.pool_index, 0);
         assert_eq!(disk.set_index, 0);
@@ -474,6 +518,7 @@ mod tests {
             heal_info: None,
             used_inodes: 1000000,
             free_inodes: 9000000,
+            physical_device_ids: Some(vec!["nvme0n1".to_string()]),
             local: true,
             pool_index: 0,
             set_index: 1,
@@ -493,9 +538,111 @@ mod tests {
         assert!(disk.metrics.is_some());
         assert_eq!(disk.runtime_state.as_deref(), Some("online"));
         assert_eq!(disk.offline_duration_seconds, Some(0));
+        assert_eq!(disk.physical_device_ids, Some(vec!["nvme0n1".to_string()]));
         assert!(disk.local);
     }
 
+    #[test]
+    fn test_disk_msgpack_backward_compat_from_legacy_layout() {
+        let legacy = LegacyDiskCompat {
+            endpoint: "http://legacy-node:9000".to_string(),
+            root_disk: false,
+            drive_path: "/data/legacy".to_string(),
+            healing: false,
+            scanning: false,
+            state: ITEM_ONLINE.to_string(),
+            uuid: "legacy-uuid".to_string(),
+            major: 8,
+            minor: 2,
+            model: Some("legacy".to_string()),
+            total_space: 42,
+            used_space: 12,
+            available_space: 30,
+            read_throughput: 1.0,
+            write_throughput: 2.0,
+            read_latency: 3.0,
+            write_latency: 4.0,
+            utilization: 5.0,
+            metrics: None,
+            heal_info: None,
+            used_inodes: 11_125,
+            free_inodes: 98_000,
+            local: true,
+            pool_index: 1,
+            set_index: 2,
+            disk_index: 3,
+        };
+
+        let mut encoded = Vec::new();
+        legacy.serialize(&mut Serializer::new(&mut encoded)).unwrap();
+
+        let mut decoder = Deserializer::new(Cursor::new(encoded));
+        let decoded: Disk = serde::Deserialize::deserialize(&mut decoder).unwrap();
+        assert_eq!(decoded.used_inodes, 11_125);
+        assert_eq!(decoded.runtime_state, None);
+        assert_eq!(decoded.offline_duration_seconds, None);
+        assert_eq!(decoded.physical_device_ids, None);
+    }
+
+    #[test]
+    fn test_disk_msgpack_forward_compat_to_legacy_layout() {
+        let current = Disk {
+            endpoint: "http://current-node:9000".to_string(),
+            root_disk: false,
+            drive_path: "/data/current".to_string(),
+            healing: false,
+            scanning: false,
+            state: ITEM_ONLINE.to_string(),
+            uuid: "current-uuid".to_string(),
+            major: 8,
+            minor: 3,
+            model: Some("current".to_string()),
+            total_space: 64,
+            used_space: 20,
+            available_space: 44,
+            read_throughput: 1.5,
+            write_throughput: 2.5,
+            read_latency: 3.5,
+            write_latency: 4.5,
+            utilization: 6.5,
+            metrics: None,
+            heal_info: None,
+            used_inodes: 22_250,
+            free_inodes: 97_000,
+            local: true,
+            pool_index: 1,
+            set_index: 2,
+            disk_index: 3,
+            physical_device_ids: Some(vec!["nvme0n1".to_string(), "nvme1n1".to_string()]),
+            runtime_state: Some("online".to_string()),
+            offline_duration_seconds: Some(0),
+        };
+
+        let mut encoded = Vec::new();
+        current
+            .serialize(&mut Serializer::new(&mut encoded).with_struct_map())
+            .unwrap();
+
+        let mut decoder = Deserializer::new(Cursor::new(encoded));
+        let decoded: LegacyDiskCompat = serde::Deserialize::deserialize(&mut decoder).unwrap();
+        assert_eq!(decoded.used_inodes, 22_250);
+        assert_eq!(decoded.disk_index, 3);
+        assert_eq!(decoded.endpoint, "http://current-node:9000");
+    }
+
+    #[test]
+    fn test_disk_serializes_physical_device_ids_when_present() {
+        let disk = Disk {
+            physical_device_ids: Some(vec!["nvme0n1".to_string(), "nvme1n1".to_string()]),
+            ..Default::default()
+        };
+
+        let json = serde_json::to_string(&disk).unwrap();
+        assert!(json.contains("physicalDeviceIds"));
+        assert!(json.contains("nvme0n1"));
+        assert!(json.contains("nvme1n1"));
+    }
+
     #[test]
     fn test_healing_disk_default() {
         let healing_disk = HealingDisk::default();

crates/notify/Cargo.toml

@@ -38,7 +38,6 @@ futures = { workspace = true }
 form_urlencoded = { workspace = true }
 hashbrown = { workspace = true }
 rayon = { workspace = true }
-rumqttc = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true }
 starshard = { workspace = true }

crates/notify/src/factory.rs

@@ -14,28 +14,19 @@
 
 use crate::Event;
 use async_trait::async_trait;
-use hashbrown::HashSet;
-use rumqttc::QoS;
-use rustfs_config::notify::{ENV_NOTIFY_MQTT_KEYS, ENV_NOTIFY_WEBHOOK_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_WEBHOOK_KEYS};
-use rustfs_config::{
-    DEFAULT_LIMIT, EVENT_DEFAULT_DIR, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
-    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
-    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
-};
+use rustfs_config::EVENT_DEFAULT_DIR;
+use rustfs_config::notify::{NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_NATS_KEYS, NOTIFY_PULSAR_KEYS, NOTIFY_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
-    error::TargetError,
-    target::{
-        mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
-        webhook::WebhookArgs,
+    config::{
+        build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+        validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
     },
+    error::TargetError,
+    target::TargetType,
 };
-use std::time::Duration;
-use tracing::{debug, warn};
-use url::Url;
+use std::collections::HashSet;
 
 /// Trait for creating targets from configuration
 #[async_trait]
@@ -49,10 +40,6 @@ pub trait TargetFactory: Send + Sync {
     /// Returns a set of valid configuration field names for this target type.
     /// This is used to filter environment variables.
     fn get_valid_fields(&self) -> HashSet<String>;
-
-    /// Returns a set of valid configuration env field names for this target type.
-    /// This is used to filter environment variables.
-    fn get_valid_env_fields(&self) -> HashSet<String>;
 }
 
 /// Factory for creating Webhook targets
@@ -61,75 +48,18 @@ pub struct WebhookTargetFactory;
 #[async_trait]
 impl TargetFactory for WebhookTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
-        // All config values are now read directly from the merged `config` KVS.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        let parsed_endpoint = endpoint.trim();
-        let endpoint_url = Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let args = WebhookArgs {
-            enable: true, // If we are here, it's already enabled.
-            endpoint: endpoint_url,
-            auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
-            queue_dir: config
-                .lookup(WEBHOOK_QUEUE_DIR)
-                .unwrap_or_else(|| EVENT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(WEBHOOK_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
-            client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
-            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
-            skip_tls_verify: config
-                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
-                .and_then(|v| v.parse::<bool>().ok())
-                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
-            target_type: rustfs_targets::target::TargetType::NotifyEvent,
-        };
-
+        let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
         let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        // Validation also uses the merged `config` KVS directly.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        debug!("endpoint: {}", endpoint);
-        let parsed_endpoint = endpoint.trim();
-        Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let client_cert = config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default();
-        let client_key = config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default();
-
-        if client_cert.is_empty() != client_key.is_empty() {
-            return Err(TargetError::Configuration(
-                "Both client_cert and client_key must be specified together".to_string(),
-            ));
-        }
-
-        let queue_dir = config
-            .lookup(WEBHOOK_QUEUE_DIR)
-            .unwrap_or_else(|| EVENT_DEFAULT_DIR.to_string());
-        if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
-            return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
-        }
-
-        Ok(())
+        validate_webhook_config(config, EVENT_DEFAULT_DIR)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         NOTIFY_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
     }
-
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_NOTIFY_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
-    }
 }
 
 /// Factory for creating MQTT targets
@@ -138,112 +68,73 @@ pub struct MQTTTargetFactory;
 #[async_trait]
 impl TargetFactory for MQTTTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let broker_url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let topic = config
-            .lookup(MQTT_TOPIC)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
-
-        let args = MQTTArgs {
-            enable: true, // Assumed enabled.
-            broker: broker_url,
-            topic,
-            qos: config
-                .lookup(MQTT_QOS)
-                .and_then(|v| v.parse::<u8>().ok())
-                .map(|q| match q {
-                    0 => QoS::AtMostOnce,
-                    1 => QoS::AtLeastOnce,
-                    2 => QoS::ExactlyOnce,
-                    _ => QoS::AtLeastOnce,
-                })
-                .unwrap_or(QoS::AtLeastOnce),
-            username: config.lookup(MQTT_USERNAME).unwrap_or_default(),
-            password: config.lookup(MQTT_PASSWORD).unwrap_or_default(),
-            max_reconnect_interval: config
-                .lookup(MQTT_RECONNECT_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(5)),
-            keep_alive: config
-                .lookup(MQTT_KEEP_ALIVE_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(30)),
-            tls: MQTTTlsConfig::from_values(
-                config.lookup(MQTT_TLS_POLICY).as_deref(),
-                config.lookup(MQTT_TLS_CA).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-                config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-                config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-            )?,
-            queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or_else(|| EVENT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(MQTT_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            target_type: rustfs_targets::target::TargetType::NotifyEvent,
-        };
-
+        let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
         let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let tls = MQTTTlsConfig::from_values(
-            config.lookup(MQTT_TLS_POLICY).as_deref(),
-            config.lookup(MQTT_TLS_CA).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-            config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-            config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-        )?;
-        validate_mqtt_broker_url(&url, &tls)?;
-
-        if config.lookup(MQTT_TOPIC).is_none() {
-            return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
-        }
-
-        if let Some(qos_str) = config.lookup(MQTT_QOS) {
-            let qos = qos_str
-                .parse::<u8>()
-                .map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
-            if qos > 2 {
-                return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
-            }
-        }
-
-        let queue_dir = config.lookup(MQTT_QUEUE_DIR).unwrap_or_default();
-        if !queue_dir.is_empty() {
-            if !std::path::Path::new(&queue_dir).is_absolute() {
-                return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
-            }
-            if let Some(qos_str) = config.lookup(MQTT_QOS)
-                && qos_str == "0"
-            {
-                warn!("Using queue_dir with QoS 0 may result in event loss");
-            }
-        }
-
-        Ok(())
+        validate_mqtt_config(config)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         NOTIFY_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
     }
+}
 
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_NOTIFY_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
+pub struct NATSTargetFactory;
+
+#[async_trait]
+impl TargetFactory for NATSTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
+        let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+        let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_nats_config(config, EVENT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        NOTIFY_NATS_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct PulsarTargetFactory;
+
+#[async_trait]
+impl TargetFactory for PulsarTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
+        let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+        let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_pulsar_config(config, EVENT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        NOTIFY_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct KafkaTargetFactory;
+
+#[async_trait]
+impl TargetFactory for KafkaTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
+        let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+        let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_kafka_config(config, EVENT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        NOTIFY_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }

crates/notify/src/integration.rs

@@ -15,11 +15,17 @@
 use crate::notification_system_subscriber::NotificationSystemSubscriberView;
 use crate::notifier::TargetList;
 use crate::{
-    Event, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream,
+    Event,
+    error::NotificationError,
+    notifier::EventNotifier,
+    registry::TargetRegistry,
+    rules::{BucketNotificationConfig, ParseConfigError},
+    stream,
 };
 use hashbrown::HashMap;
 use rustfs_config::notify::{
-    DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+    DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_s3_common::EventName;
@@ -32,14 +38,17 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
 use tokio::sync::{RwLock, Semaphore, broadcast, mpsc};
-use tracing::{debug, error, info, warn};
+use tracing::{debug, info, warn};
 
 const MAX_RECENT_LIVE_EVENTS: usize = 1024;
 
 fn subsystem_target_type(target_type: &str) -> &str {
     match target_type {
         NOTIFY_WEBHOOK_SUB_SYS => "webhook",
+        NOTIFY_KAFKA_SUB_SYS => "kafka",
         NOTIFY_MQTT_SUB_SYS => "mqtt",
+        NOTIFY_NATS_SUB_SYS => "nats",
+        NOTIFY_PULSAR_SUB_SYS => "pulsar",
         _ => target_type,
     }
 }
@@ -306,7 +315,10 @@ impl NotificationSystem {
 
         let config = {
             let guard = self.config.read().await;
-            debug!("Initializing notification system with config: {:?}", *guard);
+            debug!(
+                subsystem_count = guard.0.len(),
+                "Initializing notification system with configuration summary"
+            );
             guard.clone()
         };
 
@@ -514,7 +526,10 @@ impl NotificationSystem {
                 if !changed {
                     info!("Target {} of type {} not found, no changes made.", target_name, target_type);
                 }
-                debug!("Config after remove: {:?}", config);
+                debug!(
+                    subsystem_count = config.0.len(),
+                    "Target config removal processed and configuration summary updated"
+                );
                 changed
             })
             .await;
@@ -590,7 +605,6 @@ impl NotificationSystem {
         bucket: &str,
         cfg: &BucketNotificationConfig,
     ) -> Result<(), NotificationError> {
-        self.subscriber_view.apply_bucket_config(bucket, cfg);
         let arn_list = self.notifier.get_arn_list(&cfg.region).await;
         if arn_list.is_empty() {
             return Err(NotificationError::Configuration("No targets configured".to_string()));
@@ -599,13 +613,18 @@ impl NotificationSystem {
         // Validate the configuration against the available ARNs
         if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
             debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e);
-            if !e.to_string().contains("ARN not found") {
+            if !matches!(e, ParseConfigError::ArnNotFound(_)) {
                 return Err(NotificationError::BucketNotification(e.to_string()));
-            } else {
-                error!("config validate failed, err: {}", e);
             }
+            warn!(
+                bucket = %bucket,
+                region = %cfg.region,
+                error = %e,
+                "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
+            );
         }
 
+        self.subscriber_view.apply_bucket_config(bucket, cfg);
         let rules_map = cfg.get_rules_map();
         self.notifier.add_rules_map(bucket, rules_map.clone()).await;
         info!("Loaded notification config for bucket: {}", bucket);
@@ -746,4 +765,25 @@ mod tests {
         assert_eq!(target_id.id, "analytics");
         assert_eq!(target_id.name, "mqtt");
     }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
+        assert_eq!(target_id.id, "eventbus");
+        assert_eq!(target_id.name, "kafka");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
+        assert_eq!(target_id.id, "bus");
+        assert_eq!(target_id.name, "nats");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger");
+        assert_eq!(target_id.id, "ledger");
+        assert_eq!(target_id.name, "pulsar");
+    }
 }

crates/notify/src/registry.rs

@@ -13,15 +13,16 @@
 // limitations under the License.
 
 use crate::Event;
-use crate::factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory};
+use crate::factory::{
+    KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
+};
 use futures::stream::{FuturesUnordered, StreamExt};
-use hashbrown::{HashMap, HashSet};
-use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, notify::NOTIFY_ROUTE_PREFIX};
+use hashbrown::HashMap;
+use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
 use rustfs_ecstore::config::{Config, KVS};
-use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
-use std::str::FromStr;
+use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
 use std::sync::Arc;
-use tracing::{debug, error, info, warn};
+use tracing::{error, info};
 
 /// Registry for managing target factories
 pub struct TargetRegistry {
@@ -44,6 +45,9 @@ impl TargetRegistry {
         // Register built-in factories
         registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
+        registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
+        registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
+        registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
 
         registry
     }
@@ -85,143 +89,22 @@ impl TargetRegistry {
         &self,
         config: &Config,
     ) -> Result<Vec<Box<dyn Target<Event> + Send + Sync>>, TargetError> {
-        // Collect only environment variables with the relevant prefix to reduce memory usage
-        let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
-        // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
             info!("Start working on target types...");
-
-            // 2. Prepare the configuration source
-            // 2.1. Get the configuration segment in the file, e.g. 'notify_webhook'
-            let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
-            let file_configs = config.0.get(&section_name).cloned().unwrap_or_default();
-            // 2.2. Get the default configuration for that type
-            let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
-            debug!(?default_cfg, "Get the default configuration");
-
-            // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
-            debug!(?valid_fields, "Get the legitimate configuration fields");
-
-            // 3. Resolve instance IDs and configuration overrides from environment variables
-            let mut instance_ids_from_env = HashSet::new();
-            // 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
-            let enable_prefix =
-                format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
-                    .to_uppercase();
-            for (key, value) in &all_env {
-                if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
-                    && let Some(id) = key.strip_prefix(&enable_prefix)
-                    && !id.is_empty()
-                {
-                    instance_ids_from_env.insert(id.to_lowercase());
-                }
-            }
-
-            // 3.2. Parse all relevant environment variable configurations
-            // 3.2.1. Build environment variable prefixes such as 'RUSTFS_NOTIFY_WEBHOOK_'
-            let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
-            // 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
-            let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
-            for (key, value) in &all_env {
-                if let Some(rest) = key.strip_prefix(&env_prefix) {
-                    // Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
-                    // Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
-                    let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
-
-                    // The first part from the right is INSTANCE_ID
-                    let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
-                    // The remaining part is FIELD_NAME
-                    let field_name_part = parts.next();
-
-                    let (field_name, instance_id) = match field_name_part {
-                        // Case 1: The format is <FIELD_NAME>_<INSTANCE_ID>
-                        // e.g., rest = "ENDPOINT_PRIMARY" -> field_name="ENDPOINT", instance_id="PRIMARY"
-                        Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
-                        // Case 2: The format is <FIELD_NAME> (without INSTANCE_ID)
-                        // e.g., rest = "ENABLE" -> field_name="ENABLE", instance_id="" (Universal configuration `_ DEFAULT_DELIMITER`)
-                        None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
-                    };
-
-                    // *** Optimization point 2: Verify whether the parsed field_name is legal ***
-                    if !field_name.is_empty() && valid_fields.contains(&field_name) {
-                        debug!(
-                            instance_id = %if instance_id.is_empty() { DEFAULT_DELIMITER } else { &instance_id },
-                            %field_name,
-                            %value,
-                            "Parsing to environment variables"
-                        );
-                        env_overrides
-                            .entry(instance_id)
-                            .or_default()
-                            .insert(field_name, value.clone());
-                    } else {
-                        // Ignore illegal field names
-                        warn!(
-                            field_name = %field_name,
-                            "Ignore environment variable fields, not found in the list of valid fields for target type {}",
-                            target_type
-                        );
-                    }
-                }
-            }
-            debug!(?env_overrides, "Complete the environment variable analysis");
-
-            // 4. Determine all instance IDs that need to be processed
-            let mut all_instance_ids: HashSet<String> =
-                file_configs.keys().filter(|k| *k != DEFAULT_DELIMITER).cloned().collect();
-            all_instance_ids.extend(instance_ids_from_env);
-            debug!(?all_instance_ids, "Determine all instance IDs");
-
-            // 5. Merge configurations and create tasks for each instance
-            for id in all_instance_ids {
-                // 5.1. Merge configuration, priority: Environment variables > File instance configuration > File default configuration
-                let mut merged_config = default_cfg.clone();
-                // Instance-specific configuration in application files
-                if let Some(file_instance_cfg) = file_configs.get(&id) {
-                    merged_config.extend(file_instance_cfg.clone());
-                }
-                // Application instance-specific environment variable configuration
-                if let Some(env_instance_cfg) = env_overrides.get(&id) {
-                    // Convert HashMap<String, String> to KVS
-                    let mut kvs_from_env = KVS::new();
-                    for (k, v) in env_instance_cfg {
-                        kvs_from_env.insert(k.clone(), v.clone());
-                    }
-                    merged_config.extend(kvs_from_env);
-                }
-                debug!(instance_id = %id, ?merged_config, "Complete configuration merge");
-
-                // 5.2. Check if the instance is enabled
-                let enabled = merged_config
-                    .lookup(ENABLE_KEY)
-                    .map(|v| {
-                        EnableState::from_str(v.as_str())
-                            .ok()
-                            .map(|s| s.is_enabled())
-                            .unwrap_or(false)
-                    })
-                    .unwrap_or(false);
-
-                if enabled {
-                    info!(instance_id = %id, "Target is enabled, ready to create a task");
-                    // 5.3. Create asynchronous tasks for enabled instances
-                    let tid = id.clone();
-                    let merged_config_arc = Arc::new(merged_config);
-                    tasks.push(async move {
-                        let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (tid, result)
-                    });
-                } else {
-                    info!(instance_id = %id, "Skip disabled target");
-                }
+            for (id, merged_config) in collect_target_configs(config, NOTIFY_ROUTE_PREFIX, target_type, &valid_fields) {
+                info!(instance_id = %id, "Target is enabled, ready to create a task");
+                let tid = id.clone();
+                let merged_config_arc = Arc::new(merged_config);
+                tasks.push(async move {
+                    let result = factory.create_target(tid.clone(), &merged_config_arc).await;
+                    (tid, result)
+                });
             }
         }
 
-        // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
         while let Some((id, result)) = tasks.next().await {
             match result {

crates/object-capacity/src/capacity_manager.rs

@@ -20,13 +20,14 @@ use futures::FutureExt;
 use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope};
 use rustfs_config::{
     DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH,
-    DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, DEFAULT_CAPACITY_STALL_TIMEOUT_SECS,
-    DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE, DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS,
-    DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_TRIGGER_DELAY_SECS,
-    ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_FOLLOW_SYMLINKS,
-    ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT, ENV_CAPACITY_MIN_TIMEOUT,
-    ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL, ENV_CAPACITY_STALL_TIMEOUT, ENV_CAPACITY_STAT_TIMEOUT,
-    ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY,
+    DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS,
+    DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE,
+    DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD,
+    DEFAULT_WRITE_TRIGGER_DELAY_SECS, ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD,
+    ENV_CAPACITY_FOLLOW_SYMLINKS, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT,
+    ENV_CAPACITY_METRICS_INTERVAL, ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL,
+    ENV_CAPACITY_STALL_TIMEOUT, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
+    ENV_CAPACITY_WRITE_TRIGGER_DELAY,
 };
 use rustfs_io_metrics::capacity_metrics::{
     record_capacity_current_bytes, record_capacity_dirty_disk_count, record_capacity_refresh_inflight,
@@ -63,6 +64,8 @@ struct CachedCapacityConfig {
     stat_timeout: Duration,
     /// Sample rate
     sample_rate: usize,
+    /// Metrics logging interval
+    metrics_interval: Duration,
     /// Follow symlinks flag
     follow_symlinks: bool,
     /// Max symlink depth
@@ -97,6 +100,10 @@ impl CachedCapacityConfig {
             max_files_threshold: get_env_usize(ENV_CAPACITY_MAX_FILES_THRESHOLD, DEFAULT_MAX_FILES_THRESHOLD),
             stat_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_STAT_TIMEOUT, DEFAULT_STAT_TIMEOUT_SECS)),
             sample_rate: get_env_usize(ENV_CAPACITY_SAMPLE_RATE, DEFAULT_SAMPLE_RATE),
+            metrics_interval: Duration::from_secs(get_env_u64(
+                ENV_CAPACITY_METRICS_INTERVAL,
+                DEFAULT_CAPACITY_METRICS_INTERVAL_SECS,
+            )),
             follow_symlinks: get_env_bool(ENV_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_FOLLOW_SYMLINKS),
             max_symlink_depth: get_env_u64(ENV_CAPACITY_MAX_SYMLINK_DEPTH, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH as u64) as u8,
             enable_dynamic_timeout: get_env_bool(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT),
@@ -204,6 +211,18 @@ pub fn get_sample_rate() -> usize {
     get_cached_config().sample_rate
 }
 
+/// Get capacity metrics logging interval from environment or default
+#[cfg(not(test))]
+pub fn get_metrics_interval() -> Duration {
+    get_cached_config().metrics_interval
+}
+
+/// Get capacity metrics logging interval from environment or default (test mode)
+#[cfg(test)]
+pub fn get_metrics_interval() -> Duration {
+    get_cached_config().metrics_interval
+}
+
 /// Get follow symlinks flag from environment or default
 #[cfg(not(test))]
 pub fn get_follow_symlinks() -> bool {
@@ -462,6 +481,8 @@ pub struct HybridStrategyConfig {
     pub write_frequency_threshold: usize,
     /// Fast update threshold
     pub fast_update_threshold: Duration,
+    /// Metrics logging interval
+    pub metrics_interval: Duration,
     /// Enable smart update
     pub enable_smart_update: bool,
     /// Enable write trigger
@@ -475,6 +496,7 @@ impl Default for HybridStrategyConfig {
             write_trigger_delay: get_write_trigger_delay(),
             write_frequency_threshold: get_write_frequency_threshold(),
             fast_update_threshold: get_fast_update_threshold(),
+            metrics_interval: get_metrics_interval(),
             enable_smart_update: true,
             enable_write_trigger: true,
         }
@@ -877,6 +899,35 @@ impl HybridCapacityManager {
     pub async fn refresh_in_progress(&self) -> bool {
         self.refresh_state.lock().await.running
     }
+
+    /// Log capacity runtime summary for observability.
+    async fn log_runtime_summary(&self) {
+        let cached = self.get_capacity().await;
+        let recent_write_frequency = self.get_write_frequency().await;
+        let dirty_disks = self.get_dirty_disks().await;
+        let refresh_running = self.refresh_in_progress().await;
+
+        if let Some(cached) = cached {
+            info!(
+                total_used = cached.total_used,
+                file_count = cached.file_count,
+                estimated = cached.is_estimated,
+                source = ?cached.source,
+                cache_age_secs = cached.last_update.elapsed().as_secs(),
+                writes_per_minute = recent_write_frequency,
+                dirty_disk_count = dirty_disks.len(),
+                refresh_inflight = refresh_running,
+                "Capacity metrics summary"
+            );
+        } else {
+            info!(
+                writes_per_minute = recent_write_frequency,
+                dirty_disk_count = dirty_disks.len(),
+                refresh_inflight = refresh_running,
+                "Capacity metrics summary (cache empty)"
+            );
+        }
+    }
 }
 
 /// Global capacity manager instance
@@ -909,23 +960,30 @@ pub fn create_isolated_manager(config: HybridStrategyConfig) -> Arc<HybridCapaci
 /// Start background update task
 pub async fn start_background_task(disks: Vec<CapacityDiskRef>) {
     let manager = get_capacity_manager();
-    let mut interval = manager.get_config().scheduled_update_interval;
+    let manager_for_refresh = manager.clone();
+    let manager_for_metrics = manager.clone();
+    let mut refresh_interval = manager.get_config().scheduled_update_interval;
+    let mut metrics_interval = manager.get_config().metrics_interval;
 
     // Prevent panic in tokio::time::interval when misconfigured to 0
-    if interval.is_zero() {
+    if refresh_interval.is_zero() {
         warn!("RUSTFS_CAPACITY_SCHEDULED_INTERVAL is configured as 0; clamping to 1s to avoid panic");
-        interval = Duration::from_secs(1);
+        refresh_interval = Duration::from_secs(1);
+    }
+    if metrics_interval.is_zero() {
+        warn!("RUSTFS_CAPACITY_METRICS_INTERVAL is configured as 0; clamping to 1s to avoid panic");
+        metrics_interval = Duration::from_secs(1);
     }
 
     tokio::spawn(async move {
-        let mut timer = tokio::time::interval(interval);
+        let mut timer = tokio::time::interval(refresh_interval);
 
         loop {
             timer.tick().await;
 
             info!("Starting scheduled capacity update");
             let start = Instant::now();
-            let manager = manager.clone();
+            let manager = manager_for_refresh.clone();
             let disks = disks.clone();
             let started = manager
                 .clone()
@@ -942,6 +1000,14 @@ pub async fn start_background_task(disks: Vec<CapacityDiskRef>) {
             }
         }
     });
+
+    tokio::spawn(async move {
+        let mut timer = tokio::time::interval(metrics_interval);
+        loop {
+            timer.tick().await;
+            manager_for_metrics.log_runtime_summary().await;
+        }
+    });
 }
 
 // ============================================================================
@@ -953,8 +1019,9 @@ mod tests {
     use super::*;
     use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope};
     use rustfs_config::{
-        ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE,
-        ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY,
+        ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_METRICS_INTERVAL,
+        ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
+        ENV_CAPACITY_WRITE_TRIGGER_DELAY,
     };
     use serial_test::serial;
     use std::sync::Arc;
@@ -1009,6 +1076,13 @@ mod tests {
         assert_eq!(rate, 200);
     }
 
+    #[test]
+    #[serial]
+    fn test_get_metrics_interval() {
+        let interval = get_metrics_interval();
+        assert_eq!(interval, Duration::from_secs(600));
+    }
+
     #[test]
     #[serial]
     fn test_env_var_override_scheduled_interval() {
@@ -1045,6 +1119,15 @@ mod tests {
         });
     }
 
+    #[test]
+    #[serial]
+    fn test_env_var_override_metrics_interval() {
+        temp_env::with_var(ENV_CAPACITY_METRICS_INTERVAL, Some("90"), || {
+            let interval = get_metrics_interval();
+            assert_eq!(interval, Duration::from_secs(90));
+        });
+    }
+
     #[test]
     #[serial]
     fn test_env_var_override_max_files_threshold() {
@@ -1214,6 +1297,7 @@ mod tests {
             write_trigger_delay: Duration::from_millis(50),
             write_frequency_threshold: 1,
             fast_update_threshold: Duration::from_millis(10),
+            metrics_interval: Duration::from_secs(600),
             enable_smart_update: true,
             enable_write_trigger: true,
         });
@@ -1244,6 +1328,7 @@ mod tests {
             write_trigger_delay: Duration::from_secs(60),
             write_frequency_threshold: 1,
             fast_update_threshold: Duration::from_millis(10),
+            metrics_interval: Duration::from_secs(600),
             enable_smart_update: true,
             enable_write_trigger: false,
         });