fix: unify runtime readiness publication and graceful shutdown flow (#3087)

* fix(server): unify runtime readiness and shutdown flow

* refactor(server): decouple readiness shared types

* refactor(server): keep readiness types crate-private

* refactor(server): await protocol shutdown handles

* fix(server): bypass cached startup readiness

.agents/skills/rustfs-release-version-bump/SKILL.md

@@ -1,8 +1,7 @@
 ---
 name: rustfs-release-version-bump
-description: Publish a RustFS alpha/beta/stable release with an auditable flow: confirm target version and scope, update workspace and release assets (including strict rustfs.spec changelog identity/date/version format), run required verification, and finish with commit, push, and GitHub PR creation.
+description: "Publish a RustFS alpha/beta/stable release with an auditable flow: confirm target version and scope, update workspace and release assets (including strict rustfs.spec changelog identity/date/version format), run required verification, and finish with commit, push, and GitHub PR creation."
 ---
-
 # RustFS Release Version Bump
 
 Use this skill to publish a RustFS release (alpha, beta, or stable) with a minimal, auditable diff and a complete ship flow (`edit -> verify -> commit -> push -> PR`).

.docker/compose/docker-compose.cluster.local-build.yml

@@ -20,13 +20,16 @@ services:
       dockerfile: Dockerfile.source
     hostname: node1
     environment:
-      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
+      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
+      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
       - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -47,13 +50,16 @@ services:
       dockerfile: Dockerfile.source
     hostname: node2
     environment:
-      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
+      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
+      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
       - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -74,13 +80,16 @@ services:
       dockerfile: Dockerfile.source
     hostname: node3
     environment:
-      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
+      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
+      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
       - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -101,13 +110,16 @@ services:
       dockerfile: Dockerfile.source
     hostname: node4
     environment:
-      - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
+      - RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
       - RUSTFS_ADDRESS=:9000
       - RUSTFS_CONSOLE_ENABLE=true
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
       - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
       - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
+      - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
+      - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
+      - RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
       - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
     extra_hosts:
       - "host.docker.internal:host-gateway"

.github/actions/setup/action.yml

@@ -59,7 +59,7 @@ runs:
     - name: Install protoc
       uses: rustfs/setup-protoc@v3.0.1
       with:
-        version: "29.3"
+        version: "34.1"
         repo-token: ${{ github.token }}
 
     - name: Install flatc

.github/workflows/audit.yml

@@ -75,7 +75,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Dependency Review
-        uses: actions/dependency-review-action@v4
+        uses: actions/dependency-review-action@v5
         with:
           fail-on-severity: moderate
-          comment-summary-in-pr: true
+          comment-summary-in-pr: always

Cargo.toml

@@ -47,6 +47,7 @@ members = [
     "crates/signer", # client signer
     "crates/targets", # Target-specific configurations and utilities
     "crates/trusted-proxies", # Trusted proxies management
+    "crates/tls-runtime", # Project-wide TLS runtime foundation
     "crates/utils", # Utility functions and helpers
     "crates/io-metrics", # Zero-copy metrics collection for performance analysis
     "crates/io-core", # Zero-copy core reader and writer implementations
@@ -110,6 +111,7 @@ rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.4" }
 rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.4" }
 rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.4" }
 rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.4" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.4" }
 rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.4" }
 rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.4" }
 
@@ -126,7 +128,7 @@ futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
 pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
-lapin = { version = "4.7.4", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
+lapin = { version = "4.10.0", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
 hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
@@ -141,7 +143,7 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = ["logg
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
 tokio-util = { version = "0.7.18", features = ["io", "compat"] }
-tonic = { version = "0.14.6", features = ["gzip"] }
+tonic = { version = "0.14.6", features = ["gzip", "deflate"] }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
 tower = { version = "0.5.3", features = ["timeout"] }
@@ -158,7 +160,7 @@ quick-xml = "0.40.1"
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_json = { version = "1.0.149", features = ["raw_value"] }
+serde_json = { version = "1.0.150", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 
 # Cryptography and Security
@@ -189,7 +191,7 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros
 # Database
 deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
 tokio-postgres = { version = "0.7", default-features = false, features = ["runtime", "with-serde_json-1"] }
-tokio-postgres-rustls = "0.13"
+tokio-postgres-rustls = "0.14.0"
 
 # Utilities and Tools
 anyhow = "1.0.102"
@@ -197,9 +199,9 @@ arc-swap = "1.9.1"
 astral-tokio-tar = "0.6.2"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.16" }
+aws-config = { version = "1.8.17" }
 aws-credential-types = { version = "1.2.14" }
-aws-sdk-s3 = { version = "1.132.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-sdk-s3 = { version = "1.133.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.8" }
 base64 = "0.22.1"
@@ -253,7 +255,7 @@ rayon = "1.12.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
-rumqttc = { package = "rumqttc-next", version = "0.33.1", features = ["websocket"] }
+rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
 redis = { version = "1.2.1", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
@@ -307,7 +309,7 @@ unftp-core = "0.1.0"
 suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.8"
 russh = { version = "0.60.3", git = "https://github.com/Eugeny/russh", rev = "fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" }
-russh-sftp = "2.1.2"
+russh-sftp = "2.3.0"
 
 # WebDAV
 dav-server = "0.11.0"

INTERNODE_DATA_TRANSPORT_RFC.md

@@ -1,400 +0,0 @@
-# RFC: Pluggable Internode Data Transport
-
-> Status: draft
-> Last updated: 2026-05-19
-> Scope: internode data-path analysis, benchmark baseline, and transport boundary
-
-## Summary
-
-RustFS does not currently include RDMA, RoCE, InfiniBand, DPU, BlueField/DOCA,
-DPDK, SPDK, or SmartNIC offload support. The current distributed internode
-paths use TCP-based HTTP/gRPC transports:
-
-- `tonic` gRPC `NodeService` for most control, metadata, lock, health, and
-  peer operations.
-- HTTP streaming routes under `/rustfs/rpc/` for remote disk file streams.
-
-RDMA/RoCE is still a plausible future optimization for large internode disk
-data transfers, but it should not replace the whole internode RPC surface.
-The correct first step is to isolate the data plane, establish a TCP baseline,
-and introduce a pluggable transport boundary only around high-volume streams.
-
-## Goals
-
-- Document the current internode control plane and data plane.
-- Identify the existing transfer paths that could benefit from a future
-  high-throughput backend.
-- Define the minimum benchmark baseline required before transport changes.
-- Sketch a pluggable transport boundary that preserves the current TCP/HTTP
-  behavior as the default backend.
-- Reserve explicit boundaries for future RDMA/RoCE/InfiniBand work without
-  committing RustFS to a specific vendor stack.
-
-## Non-Goals
-
-- Implement RDMA, RoCE, InfiniBand, DPU, DOCA, DPDK, SPDK, or SmartNIC support.
-- Replace `tonic` gRPC for control-plane RPCs.
-- Redesign erasure coding, quorum handling, disk health tracking, or object
-  correctness semantics.
-- Require RDMA-capable hardware for default development, CI, or ordinary
-  RustFS deployments.
-
-## Current Internode Architecture
-
-### Server-side entry points
-
-The main HTTP server builds a hybrid service per connection:
-
-- `rustfs/src/server/http.rs` wires a `NodeServiceServer` for gRPC.
-- `rustfs/src/storage/rpc/InternodeRpcService` intercepts HTTP paths under
-  `/rustfs/rpc/`.
-- Other HTTP/S3 traffic continues through the normal S3 service.
-
-Compression logic already treats `/rustfs/rpc/` and `/rustfs/peer/` as internode
-RPC paths and skips normal response compression for them.
-
-### gRPC channel management
-
-`crates/protos/src/lib.rs` creates internode gRPC channels with `tonic`
-`Endpoint`:
-
-- connect timeout
-- TCP keepalive
-- HTTP/2 keepalive interval and timeout
-- request timeout
-- optional TLS configuration
-- global channel caching and failed-connection eviction
-
-This confirms the current gRPC transport is TCP/HTTP2-based.
-
-### NodeService layout
-
-`crates/protos/src/node.proto` defines one `NodeService` that mixes several
-classes of RPCs:
-
-- meta service: bucket and metadata operations
-- disk service: local/remote disk operations
-- lock service: distributed lock operations
-- peer rest service: node health, metrics, IAM/policy reload, rebalance,
-  profiling, events, and admin-style operations
-
-The service layout is practical today, but it is too broad to become an RDMA
-surface. A future high-throughput transport should target only disk data
-streams and keep this gRPC service as the control plane.
-
-## Control Plane vs Data Plane
-
-### Control plane
-
-These paths carry coordination, metadata, health, and administrative state.
-They should remain on gRPC/TCP:
-
-| Area | Client/server code | Examples | Notes |
-| --- | --- | --- | --- |
-| Bucket peer ops | `crates/ecstore/src/rpc/peer_s3_client.rs`, `rustfs/src/storage/rpc/bucket.rs` | `MakeBucket`, `ListBucket`, `DeleteBucket`, `GetBucketInfo`, `HealBucket` | Small metadata/control payloads. |
-| Locking | `crates/ecstore/src/rpc/remote_locker.rs`, `rustfs/src/storage/rpc/lock.rs` | `Lock`, `UnLock`, `Refresh`, batch lock/unlock | Latency-sensitive but not bulk data; correctness and timeout semantics matter more than transport bandwidth. |
-| Peer/admin state | `crates/ecstore/src/rpc/peer_rest_client.rs`, `rustfs/src/storage/rpc/health.rs`, `metrics.rs`, `event.rs` | `LocalStorageInfo`, `ServerInfo`, `GetMetrics`, `GetLiveEvents`, reload APIs, rebalance APIs | Operational control plane. |
-| Disk metadata/control | `crates/ecstore/src/rpc/remote_disk.rs`, `rustfs/src/storage/rpc/disk.rs` | `DiskInfo`, `ReadXL`, `ReadVersion`, `ReadMetadata`, `WriteMetadata`, `RenameFile`, `RenamePart`, `Delete*`, `VerifyFile`, `CheckParts` | Usually metadata, integrity checks, or namespace mutations. |
-| Connection health | `RemoteDisk`, `RemotePeerS3Client`, `PeerRestClient` | TCP connectivity probes and fault/recovery state | Must remain available even if an optional data backend is unavailable. |
-
-### Data plane candidates
-
-These paths move object shard bytes or stream potentially large disk data and
-are the only reasonable first candidates for a pluggable transport.
-
-| Priority | Path | Current client | Current server | Current transport | Why it matters |
-| --- | --- | --- | --- | --- | --- |
-| P0 | `read_file_stream` | `RemoteDisk::read_file_stream` | `handle_read_file` in `http_service.rs` | HTTP `GET /rustfs/rpc/read_file_stream` with a streaming response body | Main remote disk read stream used by bitrot readers and erasure reads. |
-| P0 | `put_file_stream` | `RemoteDisk::create_file` and `RemoteDisk::append_file` | `handle_put_file` in `http_service.rs` | HTTP `PUT /rustfs/rpc/put_file_stream` with a streaming request body | Main remote disk write stream used by bitrot writers and erasure writes. |
-| P1 | `walk_dir` | `RemoteDisk::walk_dir` | `handle_walk_dir` in `http_service.rs` | HTTP `GET /rustfs/rpc/walk_dir` with a streamed metadata listing | Can be high-volume during scans/healing, but it is metadata-oriented rather than object byte data. |
-| P1 | `ReadAll` / `WriteAll` | `RemoteDisk::read_all` / `write_all` | gRPC unary disk handlers | gRPC unary `bytes` payload | Moves bytes today, but should be measured before treating it as a high-throughput data path. |
-| P2 | proto `WriteStream` / `ReadAt` | currently not used | currently returns unimplemented | gRPC streaming definitions exist but are not implemented | Possible future API shape, not a current production path. |
-
-## Current Object Write Path
-
-For object PUTs in distributed erasure mode, the relevant flow is:
-
-1. Upper storage layers prepare object data and erasure metadata.
-2. `SetDisks` selects local and remote disks.
-3. `create_bitrot_writer` calls `disk.create_file(...)` for each shard writer.
-4. For a remote disk, `RemoteDisk::create_file` returns an `HttpWriter`.
-5. `HttpWriter` sends an HTTP `PUT` to `/rustfs/rpc/put_file_stream`.
-6. The remote node's `handle_put_file` opens the local file writer and copies
-   incoming body chunks into it.
-7. `Erasure::encode` writes shards through `MultiWriter` to all selected
-   writers while enforcing write quorum.
-
-This is the primary write data-plane candidate.
-
-## Current Object Read Path
-
-For object GETs and repair reads in distributed erasure mode, the relevant flow is:
-
-1. `SetDisks` prepares shard readers for the selected disks.
-2. `create_bitrot_reader` uses local zero-copy only when `disk.is_local()`.
-3. For a remote disk, it calls `disk.read_file_stream(...)`.
-4. `RemoteDisk::read_file_stream` returns an `HttpReader`.
-5. `HttpReader` sends an HTTP `GET` to `/rustfs/rpc/read_file_stream`.
-6. The remote node's `handle_read_file` opens the local disk stream and returns
-   it as an HTTP streaming body.
-7. The erasure decoder reads from the shard streams and reconstructs the object.
-
-This is the primary read data-plane candidate.
-
-## Existing Metrics and Benchmark Surface
-
-RustFS already has coarse internode metrics in `crates/common/src/internode_metrics.rs`:
-
-- sent bytes
-- received bytes
-- outgoing requests
-- incoming requests
-- errors
-- dial errors
-- average dial time
-
-These metrics are useful as a starting point, but they are not enough for a
-transport RFC. A transport benchmark needs route-level and operation-level
-measurements for at least:
-
-- `read_file_stream`
-- `put_file_stream`
-- `walk_dir`
-- gRPC `ReadAll` / `WriteAll`
-- gRPC control-plane request volume
-
-Existing benchmark assets:
-
-- `scripts/run_object_batch_bench.sh`
-- `scripts/run_object_batch_bench_enhanced.sh`
-- `scripts/run_object_batch_bench_abc.sh`
-- `scripts/run_four_node_cluster_failover_bench.sh`
-- `scripts/run_internode_transport_baseline.sh` (scenario matrix wrapper for local vs distributed TCP baseline artifacts)
-- Criterion benches under `crates/ecstore/benches/`
-
-These mostly cover S3/object workload or erasure coding performance. They do
-not yet isolate internode transport cost.
-
-## Required TCP Baseline
-
-Before adding a transport abstraction or any RDMA backend, collect a baseline
-for the current TCP/HTTP/gRPC implementation.
-
-### Topology
-
-Minimum:
-
-- 1-node local erasure deployment, to measure local disk and erasure overhead.
-- 4-node distributed erasure deployment, to measure internode overhead.
-
-Preferred:
-
-- Same host count and disk layout for every run.
-- Dedicated network interface or isolated VLAN.
-- Fixed CPU governor and no unrelated background load.
-- Recorded kernel version, NIC model, MTU, RustFS commit, Rust toolchain, and
-  benchmark tool versions.
-
-### Workloads
-
-| Workload | Sizes | Concurrency | Main signal |
-| --- | --- | --- | --- |
-| S3 PUT | 4 KiB, 1 MiB, 16 MiB, 128 MiB, 1 GiB | 1, 16, 64, 128 | End-to-end write throughput and tail latency. |
-| S3 GET | 4 KiB, 1 MiB, 16 MiB, 128 MiB, 1 GiB | 1, 16, 64, 128 | End-to-end read throughput and tail latency. |
-| Remote disk stream read | shard-sized ranges from `read_file_stream` | 1, 16, 64 | Isolated internode read path. |
-| Remote disk stream write | shard-sized writes through `put_file_stream` | 1, 16, 64 | Isolated internode write path. |
-| Healing / repair | missing disk or missing shard scenario | controlled | Rebuild throughput and read/write amplification. |
-| Scanner walk | large bucket/object namespace | controlled | Metadata streaming pressure, not primary RDMA target. |
-
-### Measurements
-
-Collect:
-
-- throughput in bytes/s and objects/s
-- p50, p95, p99, and max latency
-- CPU utilization per process and per core
-- memory RSS and allocation pressure where available
-- `rustfs_system_network_internode_*` metrics
-- TCP retransmits, socket errors, and NIC throughput
-- disk throughput and utilization
-- failure/retry/fallback counts
-
-The baseline should produce a machine-readable artifact, for example
-`target/bench/internode-transport/<timestamp>/summary.csv`, plus the exact
-commands and configuration used.
-
-### Baseline runner entry point
-
-Use `scripts/run_internode_transport_baseline.sh` to execute a reproducible
-S3 PUT/GET matrix against `local` and `distributed` scenarios and export:
-
-- `summary.csv` (throughput/latency summary per workload and object size)
-- `internode_metric_deltas.csv` (operation-level internode metric deltas when
-  `--metrics-url` is provided)
-
-## Transport Abstraction Proposal
-
-### Design principle
-
-Keep `NodeService` as the control plane. Introduce a separate data transport
-only below `RemoteDisk`, where remote disk byte streams are opened today.
-
-The first implementation should be a no-behavior-change TCP/HTTP backend that
-wraps the current `HttpReader`, `HttpWriter`, and `/rustfs/rpc/*` handlers.
-Only after that wrapper is benchmarked should an experimental RDMA/RoCE backend
-be considered.
-
-### Candidate boundary
-
-The narrowest useful boundary is remote disk stream transfer:
-
-```rust
-#[async_trait::async_trait]
-pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
-    async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
-    async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
-    async fn walk_dir(&self, request: WalkDirStreamRequest, writer: &mut dyn AsyncWrite) -> Result<()>;
-    fn capabilities(&self) -> InternodeTransportCapabilities;
-}
-```
-
-Initial request fields should mirror the current HTTP query parameters:
-
-- peer endpoint
-- disk reference
-- volume
-- path
-- offset
-- length
-- append/create mode
-- expected size
-- auth or transfer token material
-
-The initial TCP backend can keep the current signed HTTP URLs internally.
-
-### Integration point
-
-`RemoteDisk` should delegate only these methods to the data transport:
-
-- `read_file_stream`
-- `read_file_zero_copy` as a wrapper over `read_file_stream` unless the backend
-  supports a stronger zero-copy API
-- `append_file`
-- `create_file`
-- optionally `walk_dir`
-
-All other `RemoteDisk` methods should continue using the current gRPC client
-until measurements prove otherwise.
-
-### Capability model
-
-Avoid hard-coding RDMA assumptions into the generic interface. Use capabilities:
-
-- stream read
-- stream write
-- bounded range read
-- bidirectional streaming
-- registered memory support
-- scatter/gather support
-- zero-copy receive into caller-owned buffers
-- authenticated out-of-band transfer
-- transport fallback support
-
-The first TCP backend should report only capabilities that it actually provides.
-
-## TCP Fallback Requirements
-
-TCP/HTTP/gRPC must remain the default and required backend.
-
-Fallback rules:
-
-- If no explicit data transport is configured, use the current TCP/HTTP
-  implementation.
-- If an experimental backend fails initialization, either fail fast with a clear
-  error or fall back to TCP only when the configured policy allows fallback.
-- Runtime fallback must preserve object correctness and quorum semantics.
-- Fallback events must be logged and counted in metrics.
-- CI and local development must not require RDMA-capable hardware.
-
-Suggested future configuration shape:
-
-```text
-RUSTFS_INTERNODE_DATA_TRANSPORT=tcp
-RUSTFS_INTERNODE_DATA_TRANSPORT_FALLBACK=tcp
-```
-
-Do not add these settings until there is an implementation PR that uses them.
-
-## Future RDMA/RoCE/InfiniBand Boundary
-
-A future RDMA backend should be experimental and feature-gated. It should be
-designed as an optional data-plane backend, not as a replacement for the gRPC
-control plane.
-
-Required design areas:
-
-- peer capability discovery over the existing gRPC control plane
-- connection management and health mapping into existing disk fault handling
-- memory registration lifecycle and registration cache
-- buffer ownership, pinning, alignment, and lifetime rules
-- scatter/gather behavior for erasure shards
-- authentication and authorization for out-of-band data transfers
-- encryption/TLS-equivalent story or a documented deployment boundary
-- timeout, cancellation, retry, and fallback behavior
-- metrics for registration cost, transfer latency, bytes, queue depth, retries,
-  fallback, and errors
-- hardware and kernel compatibility matrix
-
-The first RDMA prototype should target `read_file_stream` and `put_file_stream`
-only. `walk_dir`, metadata RPCs, locks, admin RPCs, and bucket coordination
-should remain on gRPC unless a later benchmark identifies a specific bottleneck.
-
-## DPU, DOCA, DPDK, SPDK, and SmartNIC Notes
-
-These technologies should not drive the first abstraction:
-
-- DPU/BlueField/DOCA may become relevant for TLS, checksum, compression, or
-  storage/network offload, but they are vendor- and deployment-specific.
-- DPDK is a poor first fit because RustFS is currently an HTTP/S3 object store
-  and does not have a custom packet data plane.
-- SPDK may be relevant only if RustFS adds a raw block or NVMe-oriented local
-  storage backend. The current disk model is filesystem-based.
-- SmartNIC offload should be discussed only after the data-plane boundary and
-  baseline metrics show where CPU is spent.
-
-## Suggested PR Sequence
-
-1. Add this RFC and the current-path classification.
-2. Add route-level internode metrics for `/rustfs/rpc/read_file_stream`,
-   `/rustfs/rpc/put_file_stream`, `/rustfs/rpc/walk_dir`, and gRPC disk byte
-   calls.
-3. Add an internode transport benchmark harness that can run against a local
-   multi-node cluster and produce repeatable artifacts.
-4. Introduce an `InternodeDataTransport` wrapper with a TCP/HTTP backend that
-   preserves current behavior.
-5. Move `RemoteDisk` stream methods to the transport wrapper without changing
-   default behavior.
-6. Add an experimental feature-gated RDMA/RoCE backend only after the baseline
-   proves that internode byte transfer is a limiting factor.
-
-## Open Questions
-
-- Which production workload is the primary target: large-object throughput,
-  small-object tail latency, healing throughput, or rebalance throughput?
-- Should `ReadAll` and `WriteAll` stay as gRPC unary calls, or should large
-  payloads be redirected to the data transport?
-- Is `walk_dir` a metadata control stream or a secondary data-plane stream for
-  scanner/healing workloads?
-- What is the acceptable fallback policy for an explicitly configured
-  experimental backend?
-- How should an RDMA backend preserve authentication and encryption guarantees
-  currently provided by signed HTTP requests and TLS-capable gRPC/HTTP clients?
-- What hardware matrix is required before accepting a non-default RDMA backend?
-
-## Immediate Next Steps
-
-- Create a focused issue from this RFC.
-- Add route-level internode metrics before changing transport code.
-- Extend existing benchmark scripts or add a new script to isolate remote disk
-  stream read/write throughput.
-- Keep the first code PR behavior-preserving and TCP-only.

crates/common/src/globals.rs

@@ -17,6 +17,7 @@
 use chrono::{DateTime, Utc};
 use std::collections::HashMap;
 use std::sync::LazyLock;
+use std::sync::atomic::{AtomicU64, Ordering};
 use tokio::sync::RwLock;
 use tonic::transport::Channel;
 
@@ -27,6 +28,7 @@ pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLoc
 pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
 pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
 pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
+pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|| AtomicU64::new(0));
 /// Global initialization time of the RustFS node.
 pub static GLOBAL_INIT_TIME: LazyLock<RwLock<Option<DateTime<Utc>>>> = LazyLock::new(|| RwLock::new(None));
 
@@ -88,6 +90,16 @@ pub async fn set_global_mtls_identity(identity: Option<MtlsIdentityPem>) {
     *GLOBAL_MTLS_IDENTITY.write().await = identity;
 }
 
+/// Set the global outbound TLS generation.
+pub fn set_global_outbound_tls_generation(generation: u64) {
+    GLOBAL_OUTBOUND_TLS_GENERATION.store(generation, Ordering::Relaxed);
+}
+
+/// Get the global outbound TLS generation.
+pub fn get_global_outbound_tls_generation() -> u64 {
+    GLOBAL_OUTBOUND_TLS_GENERATION.load(Ordering::Relaxed)
+}
+
 /// Evict a stale/dead connection from the global connection cache.
 /// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off).
 /// By removing the cached connection, subsequent requests will establish a fresh connection.

crates/common/src/heal_channel.rs

@@ -260,9 +260,17 @@ pub enum HealChannelCommand {
         response_tx: oneshot::Sender<Result<HealAdmissionResult, String>>,
     },
     /// Query heal task status
-    Query { heal_path: String, client_token: String },
+    Query {
+        heal_path: String,
+        client_token: String,
+        response_tx: oneshot::Sender<Result<HealChannelResponse, String>>,
+    },
     /// Cancel heal task
-    Cancel { heal_path: String },
+    Cancel {
+        heal_path: String,
+        client_token: String,
+        response_tx: oneshot::Sender<Result<HealChannelResponse, String>>,
+    },
 }
 
 /// Heal request from admin to ahm
@@ -407,14 +415,36 @@ pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String
     }
 }
 
+async fn receive_heal_channel_response(
+    response_rx: oneshot::Receiver<Result<HealChannelResponse, String>>,
+) -> Result<HealChannelResponse, String> {
+    response_rx
+        .await
+        .map_err(|e| format!("Failed to receive heal channel response: {e}"))?
+}
+
 /// Send heal query request
-pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<(), String> {
-    send_heal_command(HealChannelCommand::Query { heal_path, client_token }).await
+pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<HealChannelResponse, String> {
+    let (response_tx, response_rx) = oneshot::channel();
+    send_heal_command(HealChannelCommand::Query {
+        heal_path,
+        client_token,
+        response_tx,
+    })
+    .await?;
+    receive_heal_channel_response(response_rx).await
 }
 
 /// Send heal cancel request
-pub async fn cancel_heal_task(heal_path: String) -> Result<(), String> {
-    send_heal_command(HealChannelCommand::Cancel { heal_path }).await
+pub async fn cancel_heal_task(heal_path: String, client_token: String) -> Result<HealChannelResponse, String> {
+    let (response_tx, response_rx) = oneshot::channel();
+    send_heal_command(HealChannelCommand::Cancel {
+        heal_path,
+        client_token,
+        response_tx,
+    })
+    .await?;
+    receive_heal_channel_response(response_rx).await
 }
 
 /// Create a new heal request

crates/config/README.md

@@ -83,6 +83,28 @@ Legacy compatibility fallback:
 - `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
   This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
 
+Drive timeout health-action policy:
+- `RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION`
+  - `mark_failure` (default): timeout marks failure and may transition drive runtime state.
+  - `ignore_scanner`: timeout does not mark failure for scanner-sensitive operations (`walk_dir`, `read_metadata`, `list_dir`, `disk_info`).
+
+Drive timeout profile preset:
+- `RUSTFS_DRIVE_TIMEOUT_PROFILE`
+  - `default` (default): keep current timeout defaults.
+  - `high_latency`: use 60s default timeout for scanner-sensitive operations when no per-operation timeout override is set (`read_metadata`, `disk_info`, `list_dir`, `walk_dir`, `walk_dir_stall`).
+- Precedence:
+  - Explicit per-operation timeout env (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) takes highest precedence.
+  - Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback.
+  - Then the profile-derived default (`default` or `high_latency`).
+
+## Startup filesystem boundary policy
+
+- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
+  - `warn` (default): log warning and continue startup.
+  - `fail`: abort startup with an error.
+
+RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard.
+
 ## 📄 License
 
 This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/config/src/constants/app.rs

@@ -161,6 +161,16 @@ pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
 /// provide non-default `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` values.
 pub const ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS: &str = "RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS";
 
+/// Environment variable controlling startup behavior when unsupported filesystem types are detected.
+///
+/// Accepted values:
+/// - "warn" (default): log a warning and continue startup
+/// - "fail": abort startup with an error
+pub const ENV_RUSTFS_UNSUPPORTED_FS_POLICY: &str = "RUSTFS_UNSUPPORTED_FS_POLICY";
+pub const RUSTFS_UNSUPPORTED_FS_POLICY_WARN: &str = "warn";
+pub const RUSTFS_UNSUPPORTED_FS_POLICY_FAIL: &str = "fail";
+pub const DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY: &str = RUSTFS_UNSUPPORTED_FS_POLICY_WARN;
+
 /// Environment variable for server OBS endpoint.
 pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
 

crates/config/src/constants/drive.rs

@@ -47,6 +47,16 @@ pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
 pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS";
 pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5;
 
+/// Timeout-to-health transition policy for drive operations.
+///
+/// Accepted values:
+/// - "mark_failure" (default): timeout marks failure and may transition runtime state.
+/// - "ignore_scanner": timeout does not mark failure for scanner-sensitive operations.
+pub const ENV_DRIVE_TIMEOUT_HEALTH_ACTION: &str = "RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION";
+pub const DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE: &str = "mark_failure";
+pub const DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER: &str = "ignore_scanner";
+pub const DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION: &str = DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE;
+
 /// Number of consecutive failures before a suspect drive is classified as offline.
 pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
 pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
@@ -66,3 +76,19 @@ pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
 /// Duration in seconds after which a recovered drive is classified as long offline.
 pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
 pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;
+
+/// Drive timeout profile preset.
+///
+/// Accepted values:
+/// - "default": keep current timeout defaults.
+/// - "high_latency": use a higher default timeout preset for scanner-sensitive and metadata operations.
+///
+/// Explicit per-operation overrides (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) still take precedence.
+pub const ENV_DRIVE_TIMEOUT_PROFILE: &str = "RUSTFS_DRIVE_TIMEOUT_PROFILE";
+pub const DRIVE_TIMEOUT_PROFILE_DEFAULT: &str = "default";
+pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY: &str = "high_latency";
+pub const DEFAULT_DRIVE_TIMEOUT_PROFILE: &str = DRIVE_TIMEOUT_PROFILE_DEFAULT;
+
+/// Timeout preset (seconds) used when `RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency`
+/// and no per-operation timeout override is provided.
+pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS: u64 = 60;

crates/config/src/constants/internode.rs

@@ -36,6 +36,12 @@ pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
 pub const ENV_RUSTFS_INTERNODE_DATA_TRANSPORT: &str = "RUSTFS_INTERNODE_DATA_TRANSPORT";
 pub const DEFAULT_INTERNODE_DATA_TRANSPORT: &str = "tcp-http";
 
+/// Legacy alias for "tcp-http". Both values select the TCP/HTTP transport backend.
+pub const INTERNODE_DATA_TRANSPORT_TCP: &str = "tcp";
+
+/// Known internode transport backend names accepted by the config parser.
+pub const KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS: &[&str] = &[DEFAULT_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP];
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -64,5 +70,7 @@ mod tests {
         assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
         assert_eq!(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, "RUSTFS_INTERNODE_DATA_TRANSPORT");
         assert_eq!(DEFAULT_INTERNODE_DATA_TRANSPORT, "tcp-http");
+        assert_eq!(INTERNODE_DATA_TRANSPORT_TCP, "tcp");
+        assert_eq!(KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS, &["tcp-http", "tcp"]);
     }
 }

crates/config/src/constants/scanner.rs

@@ -40,6 +40,37 @@ pub const ENV_SCANNER_SPEED: &str = "RUSTFS_SCANNER_SPEED";
 /// Default scanner speed preset.
 pub const DEFAULT_SCANNER_SPEED: &str = "default";
 
+/// Environment variable that specifies the periodic bitrot scan cycle in seconds.
+/// When set to `0`, `true`, `on`, or `yes`, every scanner cycle runs in deep mode.
+/// When set to `false`, `off`, `no`, or `disabled`, periodic deep scans are disabled.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_BITROT_CYCLE_SECS=2592000` (30 days)
+pub const ENV_SCANNER_BITROT_CYCLE_SECS: &str = "RUSTFS_SCANNER_BITROT_CYCLE_SECS";
+
+/// Default bitrot scan cycle used by the scanner.
+pub const DEFAULT_SCANNER_BITROT_CYCLE_SECS: u64 = 30 * 24 * 60 * 60;
+
+/// Environment variable that controls how many object versions trigger scanner alerts.
+/// - Example: `export RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS=100`
+pub const ENV_SCANNER_ALERT_EXCESS_VERSIONS: &str = "RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS";
+
+/// Default object version count that triggers scanner alerts.
+pub const DEFAULT_SCANNER_ALERT_EXCESS_VERSIONS: u64 = 100;
+
+/// Environment variable that controls how many cumulative bytes across object versions trigger scanner alerts.
+/// - Example: `export RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE=1099511627776`
+pub const ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE: &str = "RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE";
+
+/// Default cumulative object version bytes that trigger scanner alerts.
+pub const DEFAULT_SCANNER_ALERT_EXCESS_VERSION_SIZE: u64 = 1024 * 1024 * 1024 * 1024;
+
+/// Environment variable that controls how many subfolders trigger scanner alerts.
+/// - Example: `export RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS=50000`
+pub const ENV_SCANNER_ALERT_EXCESS_FOLDERS: &str = "RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS";
+
+/// Default subfolder count that triggers scanner alerts.
+pub const DEFAULT_SCANNER_ALERT_EXCESS_FOLDERS: u64 = 50_000;
+
 /// Environment variable that controls whether the scanner sleeps between operations.
 /// When `true` (default), the scanner throttles itself. When `false`, it runs at full speed.
 /// - Example: `export RUSTFS_SCANNER_IDLE_MODE=false`

crates/credentials/Cargo.toml

@@ -1,3 +1,17 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 [package]
 name = "rustfs-credentials"
 edition.workspace = true
@@ -12,9 +26,11 @@ categories = ["web-programming", "development-tools", "data-structures", "securi
 
 [dependencies]
 base64-simd = { workspace = true }
+hmac = { workspace = true }
 rand = { workspace = true }
 serde = { workspace = true }
 serde_json.workspace = true
+sha2 = { workspace = true }
 time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }
 
 [lints]

crates/credentials/src/credentials.rs

@@ -12,10 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{DEFAULT_SECRET_KEY, ENV_RPC_SECRET, IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
+use crate::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ENV_RPC_SECRET, IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
+use base64_simd::URL_SAFE_NO_PAD;
+use hmac::{Hmac, KeyInit, Mac};
 use rand::{Rng, RngExt};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
+use sha2::Sha256;
 use std::collections::HashMap;
 use std::env;
 use std::fmt;
@@ -33,7 +36,12 @@ pub static GLOBAL_RUSTFS_RPC_SECRET: OnceLock<String> = OnceLock::new();
 pub const RPC_SECRET_REQUIRED_MESSAGE: &str = "RPC authentication secret is not configured";
 
 /// Operator-facing guidance for configuring RPC authentication safely.
-pub const RPC_SECRET_REQUIRED_OPERATOR_MESSAGE: &str = "RUSTFS_RPC_SECRET must be set to a non-default value or RUSTFS_SECRET_KEY must be changed from the default for RPC authentication";
+pub const RPC_SECRET_REQUIRED_OPERATOR_MESSAGE: &str =
+    "RUSTFS_RPC_SECRET can be set explicitly; otherwise the RPC secret is derived from the active access/secret key pair";
+
+type HmacSha256 = Hmac<Sha256>;
+
+const RPC_SECRET_DERIVATION_CONTEXT: &[u8] = b"rustfs-rpc-secret:v1";
 
 /// Error type for credentials operations
 #[derive(Debug)]
@@ -217,37 +225,59 @@ pub fn gen_secret_key(length: usize) -> std::io::Result<String> {
     Ok(encoded)
 }
 
-/// Get the RPC authentication token from environment variable
+/// Get the RPC authentication token from the environment or derive it from the active credentials.
 ///
 /// # Returns
 /// * `String` - The RPC authentication token
 ///
-fn resolve_rpc_secret(env_secret: Option<&str>, global_secret: Option<&str>) -> Option<String> {
-    if let Some(secret) = env_secret.map(str::trim).filter(|secret| !secret.is_empty()) {
-        return (secret != DEFAULT_SECRET_KEY).then(|| secret.to_string());
+fn normalize_rpc_secret(secret: &str) -> Option<String> {
+    let secret = secret.trim();
+    (!secret.is_empty() && secret != DEFAULT_SECRET_KEY && secret != DEFAULT_ACCESS_KEY).then(|| secret.to_string())
+}
+
+fn derive_rpc_secret(access_key: &str, secret_key: &str) -> Option<String> {
+    let access_key = access_key.trim();
+    let secret_key = secret_key.trim();
+
+    if access_key.is_empty() || secret_key.is_empty() {
+        return None;
     }
 
-    global_secret
-        .map(str::trim)
-        .filter(|secret| !secret.is_empty() && *secret != DEFAULT_SECRET_KEY)
-        .map(ToOwned::to_owned)
+    let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret_key.as_bytes()).expect("HMAC can take key of any size");
+    mac.update(RPC_SECRET_DERIVATION_CONTEXT);
+    mac.update(&[0]);
+    mac.update(access_key.as_bytes());
+
+    Some(URL_SAFE_NO_PAD.encode_to_string(mac.finalize().into_bytes()))
+}
+
+fn resolve_rpc_secret(env_secret: Option<&str>, global_access: Option<&str>, global_secret: Option<&str>) -> Option<String> {
+    if let Some(secret) = env_secret.map(str::trim).filter(|secret| !secret.is_empty()) {
+        return normalize_rpc_secret(secret);
+    }
+
+    match (global_access, global_secret) {
+        (Some(access_key), Some(secret_key)) => derive_rpc_secret(access_key, secret_key),
+        _ => None,
+    }
 }
 
 pub fn try_get_rpc_token() -> std::io::Result<String> {
     if let Some(secret) = GLOBAL_RUSTFS_RPC_SECRET.get() {
-        return resolve_rpc_secret(None, Some(secret)).ok_or_else(|| Error::other(RPC_SECRET_REQUIRED_MESSAGE));
+        return normalize_rpc_secret(secret).ok_or_else(|| Error::other(RPC_SECRET_REQUIRED_MESSAGE));
     }
 
     let env_secret = env::var(ENV_RPC_SECRET).ok();
+    let global_access = get_global_access_key_opt();
     let global_secret = get_global_secret_key_opt();
-    let secret = resolve_rpc_secret(env_secret.as_deref(), global_secret.as_deref())
+    let secret = resolve_rpc_secret(env_secret.as_deref(), global_access.as_deref(), global_secret.as_deref())
         .ok_or_else(|| Error::other(RPC_SECRET_REQUIRED_MESSAGE))?;
 
     match GLOBAL_RUSTFS_RPC_SECRET.set(secret.clone()) {
         Ok(()) => Ok(secret),
         Err(_) => GLOBAL_RUSTFS_RPC_SECRET
             .get()
-            .and_then(|stored| resolve_rpc_secret(None, Some(stored)))
+            .and_then(|stored| normalize_rpc_secret(stored))
             .ok_or_else(|| Error::other(RPC_SECRET_REQUIRED_MESSAGE)),
     }
 }
@@ -407,7 +437,7 @@ impl Credentials {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
+    use crate::{DEFAULT_ACCESS_KEY, IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
     use time::Duration;
 
     #[test]
@@ -510,11 +540,12 @@ mod tests {
         // If it hasn't already been initialized, the test automatically generates logic
         if get_global_action_cred().is_none() {
             init_global_action_credentials(None, None).ok();
-            let ak = get_global_access_key();
-            let sk = get_global_secret_key();
-            assert_eq!(ak.len(), 20);
-            assert_eq!(sk.len(), 32);
         }
+
+        let ak = get_global_access_key();
+        let sk = get_global_secret_key();
+        assert!(ak.len() >= 3);
+        assert!(sk.len() >= 8);
     }
 
     #[test]
@@ -528,10 +559,26 @@ mod tests {
     }
 
     #[test]
-    fn test_resolve_rpc_secret_rejects_default_fallback() {
-        assert!(resolve_rpc_secret(None, None).is_none());
-        assert!(resolve_rpc_secret(None, Some(DEFAULT_SECRET_KEY)).is_none());
-        assert!(resolve_rpc_secret(Some(DEFAULT_SECRET_KEY), Some("custom-global-secret")).is_none());
+    fn test_gen_access_key_length_and_charset() {
+        let err = gen_access_key(2).expect_err("length below 3 should fail");
+        assert_eq!(err.to_string(), "access key length is too short");
+
+        let key = gen_access_key(20).expect("access key should generate");
+        assert_eq!(key.len(), 20);
+        assert!(key.chars().all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit()));
+    }
+
+    #[test]
+    fn test_resolve_rpc_secret_allows_default_credentials_for_derivation() {
+        assert!(resolve_rpc_secret(None, None, None).is_none());
+
+        let expected = derive_rpc_secret(DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY).expect("secret should derive");
+        assert_eq!(
+            resolve_rpc_secret(None, Some(DEFAULT_ACCESS_KEY), Some(DEFAULT_SECRET_KEY)).as_deref(),
+            Some(expected.as_str())
+        );
+
+        assert!(resolve_rpc_secret(Some(DEFAULT_SECRET_KEY), Some("custom-access"), Some("custom-global-secret")).is_none());
     }
 
     #[test]
@@ -551,37 +598,64 @@ mod tests {
 
     #[test]
     fn test_resolve_rpc_secret_accepts_non_default_secret() {
-        assert_eq!(resolve_rpc_secret(Some("custom-rpc-secret"), None).as_deref(), Some("custom-rpc-secret"));
         assert_eq!(
-            resolve_rpc_secret(None, Some("custom-global-secret")).as_deref(),
-            Some("custom-global-secret")
+            resolve_rpc_secret(Some("custom-rpc-secret"), None, None).as_deref(),
+            Some("custom-rpc-secret")
+        );
+        let expected = derive_rpc_secret("custom-access", "custom-global-secret").expect("secret should derive");
+        assert_eq!(
+            resolve_rpc_secret(None, Some("custom-access"), Some("custom-global-secret")).as_deref(),
+            Some(expected.as_str())
         );
     }
 
     #[test]
     fn test_resolve_rpc_secret_trims_and_falls_back_from_blank_env() {
+        let expected = derive_rpc_secret("custom-access", "custom-global-secret").expect("secret should derive");
         assert_eq!(
-            resolve_rpc_secret(Some("  custom-rpc-secret  "), None).as_deref(),
+            resolve_rpc_secret(Some("  custom-rpc-secret  "), None, None).as_deref(),
             Some("custom-rpc-secret")
         );
         assert_eq!(
-            resolve_rpc_secret(Some(""), Some("custom-global-secret")).as_deref(),
-            Some("custom-global-secret")
+            resolve_rpc_secret(Some(""), Some("custom-access"), Some("custom-global-secret")).as_deref(),
+            Some(expected.as_str())
         );
         assert_eq!(
-            resolve_rpc_secret(Some("  "), Some("  custom-global-secret  ")).as_deref(),
-            Some("custom-global-secret")
+            resolve_rpc_secret(Some("  "), Some("  custom-access  "), Some("  custom-global-secret  ")).as_deref(),
+            Some(expected.as_str())
         );
         assert_eq!(
-            resolve_rpc_secret(Some("  "), Some("custom-global-secret")).as_deref(),
-            Some("custom-global-secret")
+            resolve_rpc_secret(Some("  "), Some("custom-access"), Some("custom-global-secret")).as_deref(),
+            Some(expected.as_str())
         );
     }
 
     #[test]
     fn test_resolve_rpc_secret_returns_none_for_trimmed_default_secret() {
         let padded_default_secret = format!("  {}  ", DEFAULT_SECRET_KEY);
-        assert!(resolve_rpc_secret(Some(padded_default_secret.as_str()), Some("custom-global-secret")).is_none());
+        assert!(
+            resolve_rpc_secret(Some(padded_default_secret.as_str()), Some("custom-access"), Some("custom-global-secret"))
+                .is_none()
+        );
+
+        let padded_default_access = format!("  {}  ", DEFAULT_ACCESS_KEY);
+        assert!(
+            resolve_rpc_secret(Some(padded_default_access.as_str()), Some("custom-access"), Some("custom-global-secret"))
+                .is_none()
+        );
+    }
+
+    #[test]
+    fn test_derive_rpc_secret_is_stable_and_not_plaintext() {
+        let first = derive_rpc_secret("custom-access", "custom-secret").expect("secret should derive");
+        let second = derive_rpc_secret("custom-access", "custom-secret").expect("secret should derive");
+
+        assert_eq!(first, second);
+        assert_ne!(first, "custom-access");
+        assert_ne!(first, "custom-secret");
+        assert_ne!(first, format!("{}{}", "custom-access", "custom-secret"));
+        assert!(!first.contains("custom-access"));
+        assert!(!first.contains("custom-secret"));
     }
 
     #[test]

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -0,0 +1,212 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Erasure-set healing regression tests.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, execute_awscurl, init_logging};
+    use aws_sdk_s3::primitives::ByteStream;
+    use serial_test::serial;
+    use std::collections::HashSet;
+    use std::path::{Path, PathBuf};
+    use tokio::time::{Duration, sleep};
+    use tracing::info;
+
+    fn has_file_under(path: &Path) -> bool {
+        let Ok(entries) = std::fs::read_dir(path) else {
+            return false;
+        };
+
+        for entry in entries.filter_map(Result::ok) {
+            let path = entry.path();
+            if path.is_dir() {
+                if has_file_under(&path) {
+                    return true;
+                }
+            } else {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn object_metadata_exists_on_disk(disk: &Path, bucket: &str, key: &str) -> bool {
+        disk.join(bucket).join(key).join("xl.meta").is_file()
+    }
+
+    async fn assert_object_body(env: &RustFSTestEnvironment, bucket: &str, key: &str, expected: &[u8]) {
+        let client = env.create_s3_client();
+        let response = client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("GET should succeed during/after heal");
+        let body = response.body.collect().await.expect("GET body should collect").into_bytes();
+        assert_eq!(body.as_ref(), expected, "object body changed for {key}");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_admin_deep_heal_rebuilds_cleared_disk_in_single_node_erasure_set() {
+        init_logging();
+        info!("Discussion #2964: admin deep heal should rebuild a wiped disk in a 4-disk single-node erasure set");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        let root = PathBuf::from(env.temp_dir.clone());
+        let disk0 = root.join("disk0");
+        let disk1 = root.join("disk1");
+        let disk2 = root.join("disk2");
+        let disk3 = root.join("disk3");
+        for disk in [&disk0, &disk1, &disk2, &disk3] {
+            std::fs::create_dir_all(disk).expect("disk directory should be created");
+        }
+
+        // The test helper always appends env.temp_dir as the final storage path.
+        // Point it at disk3 and pass the other three disks explicitly.
+        env.temp_dir = disk3.to_string_lossy().to_string();
+        let disk0_arg = disk0.to_string_lossy().to_string();
+        let disk1_arg = disk1.to_string_lossy().to_string();
+        let disk2_arg = disk2.to_string_lossy().to_string();
+        env.start_rustfs_server_with_env(
+            vec![disk0_arg.as_str(), disk1_arg.as_str(), disk2_arg.as_str()],
+            &[("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true")],
+        )
+        .await
+        .expect("Failed to start 4-disk RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "heal-cleared-disk-regression";
+        let target_object_count = std::env::var("RUSTFS_HEAL_REBUILD_OBJECT_COUNT")
+            .ok()
+            .and_then(|value| value.parse::<usize>().ok())
+            .unwrap_or(4)
+            .max(4);
+        let heal_timeout_secs = std::env::var("RUSTFS_HEAL_REBUILD_TIMEOUT_SECS")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .unwrap_or(60);
+
+        let mut objects: Vec<(String, Vec<u8>, &'static str)> = vec![
+            (
+                "中文/报告-0001.json".to_string(),
+                "{\"message\":\"hello 中文\"}".as_bytes().to_vec(),
+                "application/json",
+            ),
+            (
+                "english/images/photo-0002.jpg".to_string(),
+                vec![0xff, 0xd8, 0xff, 0x00, 0x42, 0x24],
+                "image/jpeg",
+            ),
+            (
+                "mixed/空 格 + symbols @#%.txt".to_string(),
+                b"text object with spaces and symbols".to_vec(),
+                "text/plain; charset=utf-8",
+            ),
+            (
+                "bin/archive-0004.bin".to_string(),
+                (0..=255).collect::<Vec<u8>>(),
+                "application/octet-stream",
+            ),
+        ];
+        for index in objects.len()..target_object_count {
+            objects.push((
+                format!("bulk/prefix-{}/object-{index:04}.txt", index % 17),
+                format!("bulk object {index}: heal regression payload").into_bytes(),
+                "text/plain; charset=utf-8",
+            ));
+        }
+
+        let object_keys = objects.iter().map(|(key, _, _)| key.clone()).collect::<Vec<_>>();
+        let mut remaining_rebuild_keys: HashSet<String> = object_keys.iter().cloned().collect();
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("bucket create should succeed");
+        for (key, body, content_type) in &objects {
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(key)
+                .content_type(*content_type)
+                .body(ByteStream::from(body.clone()))
+                .send()
+                .await
+                .expect("PUT should succeed");
+        }
+
+        assert!(has_file_under(&disk0), "disk0 should contain object shards before wipe");
+        env.stop_server();
+
+        std::fs::remove_dir_all(&disk0).expect("disk0 wipe should succeed");
+        std::fs::create_dir_all(&disk0).expect("disk0 should be recreated empty");
+        assert!(!has_file_under(&disk0), "disk0 must be empty before restart");
+
+        env.start_rustfs_server_with_env(
+            vec![disk0_arg.as_str(), disk1_arg.as_str(), disk2_arg.as_str()],
+            &[("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true")],
+        )
+        .await
+        .expect("Failed to restart 4-disk RustFS after disk wipe");
+        // The helper's Drop cleanup removes env.temp_dir. Reset it to the parent
+        // directory after server startup so all four disk directories are cleaned
+        // without manually deleting a path Drop will also try to remove.
+        env.temp_dir = root.to_string_lossy().to_string();
+
+        let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+        let heal_url = format!("{}/rustfs/admin/v3/heal/{}?forceStart=true", env.url, bucket);
+        execute_awscurl(&heal_url, "POST", Some(heal_body), &env.access_key, &env.secret_key)
+            .await
+            .expect("admin deep heal should be accepted");
+
+        for _ in 0..heal_timeout_secs {
+            if !remaining_rebuild_keys.is_empty() {
+                let mut rebuilt = Vec::new();
+                for key in &remaining_rebuild_keys {
+                    if object_metadata_exists_on_disk(&disk0, bucket, key) {
+                        rebuilt.push(key.clone());
+                    }
+                }
+                for key in rebuilt {
+                    let _ = remaining_rebuild_keys.remove(&key);
+                }
+            }
+
+            if remaining_rebuild_keys.is_empty() {
+                for (key, body, _) in &objects {
+                    assert_object_body(&env, bucket, key, body).await;
+                }
+
+                env.stop_server();
+                for key in &object_keys {
+                    assert!(
+                        object_metadata_exists_on_disk(&disk0, bucket, key),
+                        "wiped disk should contain rebuilt xl.meta for {key}"
+                    );
+                }
+                return;
+            }
+
+            sleep(Duration::from_secs(1)).await;
+        }
+
+        panic!("admin deep heal did not rebuild all files on the wiped disk within timeout");
+    }
+}

crates/e2e_test/src/lib.rs

@@ -114,6 +114,9 @@ mod head_object_range_test;
 #[cfg(test)]
 mod head_object_consistency_test;
 
+#[cfg(test)]
+mod heal_erasure_disk_rebuild_test;
+
 #[cfg(test)]
 mod copy_object_metadata_test;
 

crates/e2e_test/src/list_objects_v2_pagination_test.rs

@@ -612,4 +612,421 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Regression test: delimiter listing must not produce false-positive truncation
+    /// when many raw keys collapse into a small number of CommonPrefixes.
+    ///
+    /// Scenario: 30 objects across 3 directories + 2 direct files under prefix.
+    /// With max_keys=1000, all 5 visible results (3 prefixes + 2 objects) fit in one
+    /// page, so IsTruncated must be false even though raw entry count is much larger.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_delimiter_collapsed_prefix_no_false_truncation() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 delimiter collapsed-prefix no false truncation");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-collapsed-prefix";
+        let prefix = "data/";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        // Create objects under 3 subdirectories (10 each = 30 raw keys)
+        let dirs = ["alpha/", "beta/", "gamma/"];
+        let mut expected_keys = Vec::new();
+        for dir in &dirs {
+            for i in 0..10 {
+                let key = format!("{prefix}{dir}file{i:02}.txt");
+                client
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .body(ByteStream::from_static(b"x"))
+                    .send()
+                    .await
+                    .expect("Failed to put object");
+                expected_keys.push(key);
+            }
+        }
+        // Add 2 direct objects under prefix
+        for name in ["readme.txt", "config.json"] {
+            let key = format!("{prefix}{name}");
+            client
+                .put_object()
+                .bucket(bucket)
+                .key(&key)
+                .body(ByteStream::from_static(b"x"))
+                .send()
+                .await
+                .expect("Failed to put object");
+            expected_keys.push(key);
+        }
+
+        // List with delimiter="/", max_keys large enough to cover all visible results
+        // Visible: 3 common prefixes + 2 direct objects = 5
+        let output = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix(prefix)
+            .delimiter("/")
+            .max_keys(1000)
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        let listed_keys: Vec<String> = output
+            .contents()
+            .iter()
+            .filter_map(|obj| obj.key())
+            .map(|k| k.to_string())
+            .collect();
+        let listed_prefixes: Vec<String> = output
+            .common_prefixes()
+            .iter()
+            .filter_map(|cp| cp.prefix())
+            .map(|p| p.to_string())
+            .collect();
+
+        // KEY ASSERTION: IsTruncated must be false because all visible results fit within max_keys
+        let is_truncated = output.is_truncated().unwrap_or(false);
+        assert!(
+            !is_truncated,
+            "BUG: IsTruncated should be false when all visible results ({} objects + {} prefixes = {}) fit within max_keys (1000)",
+            listed_keys.len(),
+            listed_prefixes.len(),
+            listed_keys.len() + listed_prefixes.len()
+        );
+
+        assert!(
+            output.next_continuation_token().is_none(),
+            "NextContinuationToken should be None when IsTruncated is false"
+        );
+
+        // All 32 objects must be covered (listed_keys + keys under listed_prefixes)
+        let total_raw_count = expected_keys.len();
+        let keys_under_prefixes: usize = listed_prefixes
+            .iter()
+            .map(|p| {
+                let dir = p.trim_end_matches('/');
+                expected_keys.iter().filter(|k| k.starts_with(&format!("{dir}/"))).count()
+            })
+            .sum();
+        let covered = listed_keys.len() + keys_under_prefixes;
+
+        assert_eq!(
+            covered,
+            total_raw_count,
+            "Collapsed-prefix listing must cover all {} objects, got {} (keys={}, under_prefixes={})",
+            total_raw_count,
+            covered,
+            listed_keys.len(),
+            keys_under_prefixes
+        );
+
+        info!(
+            "Collapsed-prefix test passed: {} objects covered ({} keys + {} prefixes), IsTruncated=false",
+            covered,
+            listed_keys.len(),
+            listed_prefixes.len()
+        );
+
+        env.stop_server();
+    }
+
+    /// Regression test: delimiter listing pagination must traverse all keys
+    /// even when the visible result count per page is much smaller than the
+    /// raw entry count (due to CommonPrefix collapse).
+    ///
+    /// Scenario: 1000 objects across 100 directories, paginated with max_keys=50.
+    /// Each page returns up to 50 CommonPrefixes. The server must correctly set
+    /// IsTruncated and provide a valid continuation token across all pages.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_delimiter_small_page_traverses_all() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 delimiter small page traverses all keys");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-delimiter-small-page";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        // Create 1000 objects: 100 directories with 10 files each
+        let mut all_keys = Vec::new();
+        let dirs: Vec<String> = (0..100).map(|i| format!("dir-{:03}/", i)).collect();
+        for dir in &dirs {
+            for i in 0..10 {
+                let key = format!("{dir}file{:02}.txt", i);
+                client
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .body(ByteStream::from_static(b"x"))
+                    .send()
+                    .await
+                    .expect("Failed to put object");
+                all_keys.push(key);
+            }
+        }
+
+        // Paginate with delimiter="/" and max_keys=50
+        // Visible per page: up to 50 CommonPrefixes
+        let mut listed_keys = Vec::new();
+        let mut listed_prefixes = Vec::new();
+        let mut continuation_token: Option<String> = None;
+        let mut page_count = 0;
+        let mut last_page_is_truncated: bool;
+
+        loop {
+            let mut request = client.list_objects_v2().bucket(bucket).delimiter("/").max_keys(50);
+
+            if let Some(token) = continuation_token.take() {
+                request = request.continuation_token(token);
+            }
+
+            let output = request.send().await.expect("Failed to list objects");
+            last_page_is_truncated = output.is_truncated().unwrap_or(false);
+
+            for obj in output.contents() {
+                if let Some(key) = obj.key() {
+                    listed_keys.push(key.to_string());
+                }
+            }
+            for cp in output.common_prefixes() {
+                if let Some(p) = cp.prefix() {
+                    listed_prefixes.push(p.to_string());
+                }
+            }
+
+            page_count += 1;
+
+            if last_page_is_truncated {
+                continuation_token = output.next_continuation_token().map(|s| s.to_string());
+                assert!(
+                    continuation_token.is_some(),
+                    "BUG: NextContinuationToken must be present when IsTruncated is true"
+                );
+            } else {
+                break;
+            }
+
+            if page_count > 20 {
+                panic!("Too many pages, possible infinite loop in delimiter pagination");
+            }
+        }
+
+        // Last page must have IsTruncated=false
+        assert!(
+            !last_page_is_truncated,
+            "BUG: Last page must have IsTruncated=false after all results returned"
+        );
+
+        // Verify all objects are covered via listed prefixes
+        let keys_under_prefixes: HashSet<String> = all_keys
+            .iter()
+            .filter(|k| {
+                listed_prefixes
+                    .iter()
+                    .any(|p| k.starts_with(&format!("{}/", p.trim_end_matches('/'))))
+            })
+            .cloned()
+            .collect();
+        let listed_set: HashSet<String> = listed_keys.iter().cloned().collect();
+        let covered: HashSet<String> = listed_set.union(&keys_under_prefixes).cloned().collect();
+        let expected_set: HashSet<String> = all_keys.iter().cloned().collect();
+
+        assert_eq!(
+            covered,
+            expected_set,
+            "Delimiter pagination must cover all {} objects, missing: {:?}",
+            expected_set.difference(&covered).count(),
+            expected_set.difference(&covered)
+        );
+
+        info!(
+            "Delimiter small-page test passed: {} objects covered in {} pages",
+            covered.len(),
+            page_count
+        );
+
+        env.stop_server();
+    }
+
+    /// Regression test: raw entries exceed MaxKeys but all collapse into fewer
+    /// visible CommonPrefixes — IsTruncated must be false because there are no
+    /// more visible results beyond the collapsed prefixes.
+    ///
+    /// Scenario: 10 directories with 200 files each = 2000 raw keys.
+    /// With MaxKeys=1000, the raw entry count exceeds MaxKeys (disk_has_more=true),
+    /// but after delimiter collapse only 10 CommonPrefixes are visible (10 < 1000).
+    /// IsTruncated must be false since there are no additional visible results.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 raw > MaxKeys but visible < MaxKeys after collapse");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-raw-exceeds-visible-below";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        // Create 10 directories × 200 files = 2000 raw keys
+        let dir_count = 10;
+        let files_per_dir = 200;
+        let mut all_keys = Vec::new();
+        for d in 0..dir_count {
+            let dir = format!("dir-{:02}/", d);
+            for f in 0..files_per_dir {
+                let key = format!("{}file{:03}.txt", dir, f);
+                client
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .body(ByteStream::from_static(b"x"))
+                    .send()
+                    .await
+                    .expect("Failed to put object");
+                all_keys.push(key);
+            }
+        }
+
+        // List with delimiter="/", max_keys=1000
+        // Visible: 10 CommonPrefixes (dir-00/ .. dir-09/) = 10 visible << 1000 MaxKeys
+        // But raw entry count (2000+1001 requested) exceeds MaxKeys
+        let output = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .delimiter("/")
+            .max_keys(1000)
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        let listed_keys: Vec<String> = output
+            .contents()
+            .iter()
+            .filter_map(|obj| obj.key())
+            .map(|k| k.to_string())
+            .collect();
+        let listed_prefixes: Vec<String> = output
+            .common_prefixes()
+            .iter()
+            .filter_map(|cp| cp.prefix())
+            .map(|p| p.to_string())
+            .collect();
+
+        // KEY ASSERTION: IsTruncated must be false because visible results (10)
+        // are far below MaxKeys (1000), even though raw entries exceeded MaxKeys.
+        let is_truncated = output.is_truncated().unwrap_or(false);
+        assert!(
+            !is_truncated,
+            "BUG: IsTruncated should be false when visible results ({} objects + {} prefixes = {}) < MaxKeys (1000), even if raw entries exceeded MaxKeys",
+            listed_keys.len(),
+            listed_prefixes.len(),
+            listed_keys.len() + listed_prefixes.len()
+        );
+
+        assert!(
+            output.next_continuation_token().is_none(),
+            "NextContinuationToken should be None when IsTruncated is false"
+        );
+
+        // All objects must be covered by listed prefixes
+        assert_eq!(
+            listed_prefixes.len(),
+            dir_count,
+            "Expected {} prefixes, got {}",
+            dir_count,
+            listed_prefixes.len()
+        );
+        for d in 0..dir_count {
+            let prefix = format!("dir-{:02}/", d);
+            assert!(listed_prefixes.contains(&prefix), "Missing prefix: {}", prefix);
+        }
+
+        info!(
+            "Raw-exceeds-visible test passed: {} raw keys → {} prefixes, IsTruncated=false",
+            all_keys.len(),
+            listed_prefixes.len()
+        );
+
+        env.stop_server();
+    }
+
+    /// Regression test: pagination with MaxKeys exceeding S3 limit (1000) and delimiter.
+    /// The server caps MaxKeys to 1000. With delimiter="/", many raw keys collapse into
+    /// few CommonPrefixes. Visible results (prefixes) < capped MaxKeys → IsTruncated=false.
+    ///
+    /// This complements test_list_objects_v2_max_keys_above_limit_returns_token which
+    /// tests the non-delimiter case.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_objects_v2_maxkeys_above_limit_with_delimiter() {
+        init_logging();
+        info!("Starting test: ListObjectsV2 MaxKeys above limit with delimiter");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-maxkeys-limit-delimiter";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+
+        // 12 dirs × 100 files = 1200 raw keys
+        let dir_count = 12;
+        let files_per_dir = 100;
+        for d in 0..dir_count {
+            for f in 0..files_per_dir {
+                let key = format!("dir-{:02}/file{:03}.txt", d, f);
+                client
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .body(ByteStream::from_static(b"x"))
+                    .send()
+                    .await
+                    .expect("Failed to put object");
+            }
+        }
+
+        // With delimiter: 12 CommonPrefixes visible, all fit within capped 1000
+        let output = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .delimiter("/")
+            .max_keys(2000)
+            .send()
+            .await
+            .expect("Failed to list objects");
+
+        assert_eq!(
+            output.common_prefixes().len(),
+            dir_count,
+            "Expected {} prefixes, got {}",
+            dir_count,
+            output.common_prefixes().len()
+        );
+        assert_eq!(output.max_keys(), Some(1000));
+        // 12 visible < 1000 capped MaxKeys → not truncated
+        assert!(
+            !output.is_truncated().unwrap_or(false),
+            "BUG: IsTruncated should be false when visible ({}) < capped MaxKeys (1000)",
+            output.common_prefixes().len()
+        );
+
+        info!("MaxKeys above limit with delimiter test passed");
+
+        env.stop_server();
+    }
 }

crates/ecstore/Cargo.toml

@@ -38,6 +38,7 @@ rustfs-filemeta.workspace = true
 rustfs-utils = { workspace = true, features = ["full"] }
 rustfs-rio.workspace = true
 rustfs-signer.workspace = true
+rustfs-tls-runtime.workspace = true
 rustfs-checksums.workspace = true
 rustfs-config = { workspace = true, features = ["constants", "notify", "audit"] }
 rustfs-credentials = { workspace = true }
@@ -91,6 +92,7 @@ hyper.workspace = true
 hyper-util.workspace = true
 hyper-rustls.workspace = true
 rustls.workspace = true
+rustls-pki-types.workspace = true
 tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }

crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md

@@ -0,0 +1,484 @@
+# RFC: Internode Transport Adapter Boundary
+
+> Status: draft
+> Last updated: 2026-05-22
+> Scope: OSS internode data-plane adapter analysis, benchmark baseline, and
+> transport boundary
+
+## Summary
+
+The current distributed internode paths use TCP-based HTTP/gRPC transports:
+
+- `tonic` gRPC `NodeService` for most control, metadata, lock, health, and
+  peer operations.
+- HTTP streaming routes under `/rustfs/rpc/` for remote disk file streams.
+
+This document frames the existing work as an OSS `InternodeDataTransport`
+adapter boundary. The adapter keeps RustFS data-plane logic separate from the
+concrete transport backend while preserving the current TCP/HTTP behavior as
+the default implementation.
+
+Current implementation status:
+
+- `InternodeDataTransport` exists in
+  `crates/ecstore/src/rpc/internode_data_transport.rs`.
+- The default and only production backend is `tcp-http`; `tcp` is accepted as
+  an alias.
+- `RUSTFS_INTERNODE_DATA_TRANSPORT` selects the backend. Blank or unset values
+  use `tcp-http`; invalid values fail closed.
+- `RemoteDisk::read_file_stream`, `RemoteDisk::create_file`,
+  `RemoteDisk::append_file`, and `RemoteDisk::walk_dir` delegate to the
+  transport.
+- `NodeService` gRPC remains the internode control plane and continues to carry
+  metadata/control operations.
+
+Related design notes in this directory:
+
+- `transport-capabilities.md`
+- `transport-buffer-lifecycle.md`
+- `transport-buffer-contract.md`
+- `transport-fallback-and-selection.md`
+- `transport-metrics-and-baseline.md`
+
+## Open-source Scope
+
+The OSS scope is:
+
+- define a clear `InternodeDataTransport` adapter boundary;
+- keep `tcp-http` as the default backend;
+- keep existing TCP/HTTP behavior unchanged;
+- keep internode data-plane behavior observable through metrics and baseline
+  tooling;
+- document buffer ownership, fallback, and capability expectations for
+  maintainable transport code;
+- avoid adding dependencies or backend implementations.
+
+The OSS scope is not:
+
+- adding another transport backend;
+- replacing the gRPC control plane;
+- adding benchmark plans for another transport;
+- adding runtime plugin loading;
+- changing object correctness semantics.
+
+## Goals
+
+- Document the current internode control plane and data plane.
+- Identify the existing transfer paths covered by the
+  `InternodeDataTransport` adapter and the paths that remain on gRPC.
+- Define the minimum benchmark baseline required before transport changes.
+- Sketch a pluggable transport boundary that preserves the current TCP/HTTP
+  behavior as the default backend.
+- Document backend-neutral capability, fallback, buffer ownership, and
+  observability expectations.
+
+## Non-Goals
+
+- Implement another transport backend.
+- Replace `tonic` gRPC for control-plane RPCs.
+- Redesign erasure coding, quorum handling, disk health tracking, or object
+  correctness semantics.
+- Change default development, CI, or ordinary RustFS deployment requirements.
+
+## Current Internode Architecture
+
+### Server-side entry points
+
+The main HTTP server builds a hybrid service per connection:
+
+- `rustfs/src/server/http.rs` wires a `NodeServiceServer` for gRPC.
+- `rustfs/src/storage/rpc/InternodeRpcService` intercepts HTTP paths under
+  `/rustfs/rpc/`.
+- Other HTTP/S3 traffic continues through the normal S3 service.
+
+Compression logic already treats `/rustfs/rpc/` and `/rustfs/peer/` as internode
+RPC paths and skips normal response compression for them.
+
+### gRPC channel management
+
+`crates/protos/src/lib.rs` creates internode gRPC channels with `tonic`
+`Endpoint`:
+
+- connect timeout
+- TCP keepalive
+- HTTP/2 keepalive interval and timeout
+- request timeout
+- optional TLS configuration
+- global channel caching and failed-connection eviction
+
+This confirms the current gRPC transport is TCP/HTTP2-based.
+
+### NodeService layout
+
+`crates/protos/src/node.proto` defines one `NodeService` that mixes several
+classes of RPCs:
+
+- meta service: bucket and metadata operations
+- disk service: local/remote disk operations
+- lock service: distributed lock operations
+- peer rest service: node health, metrics, IAM/policy reload, rebalance,
+  profiling, events, and admin-style operations
+
+The service layout is practical today, but it is too broad to become the
+transport adapter surface. A pluggable data transport should target only disk
+data streams and keep this gRPC service as the control plane.
+
+## Control Plane vs Data Plane
+
+### Control plane
+
+These paths carry coordination, metadata, health, and administrative state.
+They should remain on gRPC/TCP:
+
+| Area | Client/server code | Examples | Notes |
+| --- | --- | --- | --- |
+| Bucket peer ops | `crates/ecstore/src/rpc/peer_s3_client.rs`, `rustfs/src/storage/rpc/bucket.rs` | `MakeBucket`, `ListBucket`, `DeleteBucket`, `GetBucketInfo`, `HealBucket` | Small metadata/control payloads. |
+| Locking | `crates/ecstore/src/rpc/remote_locker.rs`, `rustfs/src/storage/rpc/lock.rs` | `Lock`, `UnLock`, `Refresh`, batch lock/unlock | Latency-sensitive but not bulk data; correctness and timeout semantics matter more than transport bandwidth. |
+| Peer/admin state | `crates/ecstore/src/rpc/peer_rest_client.rs`, `rustfs/src/storage/rpc/health.rs`, `metrics.rs`, `event.rs` | `LocalStorageInfo`, `ServerInfo`, `GetMetrics`, `GetLiveEvents`, reload APIs, rebalance APIs | Operational control plane. |
+| Disk metadata/control | `crates/ecstore/src/rpc/remote_disk.rs`, `rustfs/src/storage/rpc/disk.rs` | `DiskInfo`, `ReadXL`, `ReadVersion`, `ReadMetadata`, `WriteMetadata`, `RenameFile`, `RenamePart`, `Delete*`, `VerifyFile`, `CheckParts` | Usually metadata, integrity checks, or namespace mutations. |
+| Connection health | `RemoteDisk`, `RemotePeerS3Client`, `PeerRestClient` | TCP connectivity probes and fault/recovery state | Must remain available even if an optional data backend is unavailable. |
+
+### Data plane candidates
+
+These paths move object shard bytes or stream potentially large disk data and
+are the only reasonable first candidates for a pluggable transport.
+
+| Priority | Path | Current client | Current server | Current transport | Why it matters |
+| --- | --- | --- | --- | --- | --- |
+| P0 | `read_file_stream` | `RemoteDisk::read_file_stream` | `handle_read_file` in `http_service.rs` | HTTP `GET /rustfs/rpc/read_file_stream` with a streaming response body | Main remote disk read stream used by bitrot readers and erasure reads. |
+| P0 | `put_file_stream` | `RemoteDisk::create_file` and `RemoteDisk::append_file` | `handle_put_file` in `http_service.rs` | HTTP `PUT /rustfs/rpc/put_file_stream` with a streaming request body | Main remote disk write stream used by bitrot writers and erasure writes. |
+| P1 | `walk_dir` | `RemoteDisk::walk_dir` | `handle_walk_dir` in `http_service.rs` | HTTP `GET /rustfs/rpc/walk_dir` with a streamed metadata listing | Can be high-volume during scans/healing, but it is metadata-oriented rather than object byte data. |
+| P1 | `ReadAll` / `WriteAll` | `RemoteDisk::read_all` / `write_all` | gRPC unary disk handlers | gRPC unary `bytes` payload | Moves bytes today, but should be measured before treating it as a high-throughput data path. |
+| P2 | proto `WriteStream` / `ReadAt` | currently not used | currently returns unimplemented | gRPC streaming definitions exist but are not implemented | Declared proto shape, not a current production path. |
+
+## P1 Data Path Inventory
+
+Classification:
+
+- Covered by `InternodeDataTransport`: `RemoteDisk` opens the transfer through
+  the transport abstraction.
+- Still direct TCP/HTTP/gRPC: bytes move over a fixed internode protocol
+  outside the transport abstraction.
+- Metadata/control-plane only: payloads are expected to be small metadata,
+  namespace, lock, health, or admin messages.
+- Not relevant: declared or test-only paths that are not current production
+  data paths.
+
+### Covered by `InternodeDataTransport`
+
+| Path | Owner references | Server references | Classification | Notes |
+| --- | --- | --- | --- | --- |
+| Remote shard read stream | `crates/ecstore/src/rpc/remote_disk.rs::RemoteDisk::read_file_stream`; `crates/ecstore/src/rpc/internode_data_transport.rs::InternodeDataTransport::open_read`; `crates/ecstore/src/bitrot.rs::create_bitrot_reader` | `rustfs/src/storage/rpc/http_service.rs::handle_read_file` | Covered by `InternodeDataTransport` | Object GET, repair reads, and erasure decode use this path for remote shard bytes. |
+| Remote shard write stream | `RemoteDisk::create_file`; `RemoteDisk::append_file`; `InternodeDataTransport::open_write`; `crates/ecstore/src/bitrot.rs::create_bitrot_writer` | `rustfs/src/storage/rpc/http_service.rs::handle_put_file` | Covered by `InternodeDataTransport` | Object PUT and multipart part upload use this path for remote shard bytes. |
+| Remote namespace walk stream | `RemoteDisk::walk_dir`; `InternodeDataTransport::open_walk_dir`; `crates/ecstore/src/cache_value/metacache_set.rs` walk producers | `rustfs/src/storage/rpc/http_service.rs::handle_walk_dir` | Covered by `InternodeDataTransport` | High-volume listing/scanner/heal metadata stream. It is not object byte data, but it is a large internode stream. |
+| Remote zero-copy read fallback | `RemoteDisk::read_file_zero_copy` | same as remote shard read stream | Covered by `InternodeDataTransport` through `read_file_stream` | The remote path buffers the stream into `Bytes`; true zero-copy is not guaranteed for remote disks. |
+
+### Still Direct TCP/HTTP/gRPC
+
+| Path | Owner references | Server references | Classification | Notes |
+| --- | --- | --- | --- | --- |
+| `ReadAll` | `RemoteDisk::read_all`; `crates/ecstore/src/store_init.rs`; heal resume metadata readers | `rustfs/src/storage/rpc/disk.rs::handle_read_all` | Still direct gRPC | Unary `bytes` response. Currently used mostly for metadata/config files; measure before moving. |
+| `WriteAll` | `RemoteDisk::write_all`; `crates/ecstore/src/store_init.rs`; heal resume metadata writers | `rustfs/src/storage/rpc/disk.rs::handle_write_all` | Still direct gRPC | Unary `bytes` request. Currently used mostly for metadata/config/checkpoint writes. |
+| `ReadMultiple` | `RemoteDisk::read_multiple`; `crates/ecstore/src/set_disk/read.rs::read_multiple_files` | `rustfs/src/storage/rpc/disk.rs::handle_read_multiple` | Still direct gRPC | Returns multiple small file payloads, usually metadata/listing support. Could become large with many entries. |
+| `ReadParts` | `RemoteDisk::read_parts`; `crates/ecstore/src/set_disk/read.rs::read_parts`; multipart list/complete paths | `rustfs/src/storage/rpc/disk.rs::handle_read_parts` | Still direct gRPC | Encoded `ObjectPartInfo` metadata, not object data. |
+| `RenamePart` | `RemoteDisk::rename_part`; `crates/ecstore/src/set_disk/write.rs::rename_part` | `rustfs/src/storage/rpc/disk.rs::handle_rename_part` | Still direct gRPC | Carries part metadata while committing multipart data already written through stream writers. |
+| `ListDir` | `RemoteDisk::list_dir`; multipart/lifecycle metadata listing callers | `rustfs/src/storage/rpc/disk.rs::handle_list_dir` | Still direct gRPC | Directory name listing, metadata/control-plane unless measured otherwise. |
+| Legacy gRPC `WalkDir` | `rustfs/src/storage/rpc/node_service.rs::NodeService::walk_dir` | same file | Still direct gRPC | Server implementation remains, but current `RemoteDisk::walk_dir` uses HTTP through the transport. Keep until callers are audited or compatibility policy is set. |
+
+### Metadata/control-plane only
+
+| Area | Owner references | Classification | Notes |
+| --- | --- | --- | --- |
+| Disk metadata and namespace mutations | `RemoteDisk::{read_metadata,write_metadata,update_metadata,read_version,read_xl,rename_data,rename_file,delete*,verify_file,check_parts,disk_info}` | Metadata/control-plane only | These remain on gRPC by design. |
+| Peer/bucket/admin operations | `crates/ecstore/src/rpc/{peer_s3_client.rs,peer_rest_client.rs,remote_locker.rs}` and matching `rustfs/src/storage/rpc/*` handlers | Metadata/control-plane only | Not candidates for a data-plane backend without separate measurements. |
+| Store init and format operations | `crates/ecstore/src/store_init.rs` | Metadata/control-plane only | Uses `ReadAll`/`WriteAll` for small format/config objects. |
+| Heal orchestration | `crates/heal/src/heal/storage.rs` and `crates/ecstore/src/set_disk.rs::heal_object` | Metadata/control-plane plus covered data reads | Heal object data reads go through `get_object_reader` and then covered shard streams; resume/checkpoint metadata uses direct gRPC disk metadata calls. |
+
+### Not Relevant Current Paths
+
+| Path | Owner references | Classification | Notes |
+| --- | --- | --- | --- |
+| Proto `Write` | `crates/protos/src/node.proto`; `rustfs/src/storage/rpc/disk.rs::handle_write` | Not relevant | Handler is unimplemented. |
+| Proto `WriteStream` | `crates/protos/src/node.proto`; `rustfs/src/storage/rpc/node_service.rs::write_stream` | Not relevant | Returns `unimplemented`. |
+| Proto `ReadAt` | `crates/protos/src/node.proto`; `rustfs/src/storage/rpc/node_service.rs::read_at` | Not relevant | Returns `unimplemented`. |
+| E2E reliant gRPC helpers | `crates/e2e_test/src/reliant/*` | Not relevant | Test harnesses, not production internode data-path callers. |
+
+### Current Limitations
+
+| Risk | Limitation |
+| --- | --- |
+| Medium | `ReadAll` and `WriteAll` still carry unary `bytes` over gRPC. They appear metadata-oriented today, but there is no size threshold or routing policy. |
+| Medium | `ReadMultiple` can aggregate many metadata files into one gRPC response. |
+| Low | Legacy gRPC `WalkDir` remains implemented while `RemoteDisk::walk_dir` uses HTTP through the transport. |
+| Medium | Remote `read_file_zero_copy` is a buffered read over the transport, not a remote zero-copy contract. |
+| Medium | Server-side TCP HTTP route handling is outside the client-side trait. |
+
+## Current Object Write Path
+
+For object PUTs in distributed erasure mode, the relevant flow is:
+
+1. Upper storage layers prepare object data and erasure metadata.
+2. `SetDisks` selects local and remote disks.
+3. `create_bitrot_writer` calls `disk.create_file(...)` for each shard writer.
+4. For a remote disk, `RemoteDisk::create_file` delegates to
+   `InternodeDataTransport::open_write`.
+5. `HttpWriter` sends an HTTP `PUT` to `/rustfs/rpc/put_file_stream`.
+6. The remote node's `handle_put_file` opens the local file writer and copies
+   incoming body chunks into it.
+7. `Erasure::encode` writes shards through `MultiWriter` to all selected
+   writers while enforcing write quorum.
+
+This is the primary write data-plane candidate.
+
+## Current Object Read Path
+
+For object GETs and repair reads in distributed erasure mode, the relevant flow is:
+
+1. `SetDisks` prepares shard readers for the selected disks.
+2. `create_bitrot_reader` uses local zero-copy only when `disk.is_local()`.
+3. For a remote disk, it calls `disk.read_file_stream(...)`.
+4. `RemoteDisk::read_file_stream` delegates to
+   `InternodeDataTransport::open_read`.
+5. `HttpReader` sends an HTTP `GET` to `/rustfs/rpc/read_file_stream`.
+6. The remote node's `handle_read_file` opens the local disk stream and returns
+   it as an HTTP streaming body.
+7. The erasure decoder reads from the shard streams and reconstructs the object.
+
+This is the primary read data-plane candidate.
+
+## Existing Metrics and Benchmark Surface
+
+RustFS already has coarse internode metrics in `crates/io-metrics/src/internode_metrics.rs`:
+
+- sent bytes
+- received bytes
+- outgoing requests
+- incoming requests
+- errors
+- dial errors
+- average dial time
+
+These metrics are useful as a starting point. For backend comparisons, the
+relevant route-level and operation-level dimensions are:
+
+- `read_file_stream`
+- `put_file_stream`
+- `walk_dir`
+- gRPC `ReadAll` / `WriteAll`
+- gRPC control-plane request volume
+
+Existing benchmark assets:
+
+- `scripts/run_object_batch_bench.sh`
+- `scripts/run_object_batch_bench_enhanced.sh`
+- `scripts/run_object_batch_bench_abc.sh`
+- `scripts/run_four_node_cluster_failover_bench.sh`
+- `scripts/run_internode_transport_baseline.sh` (scenario matrix wrapper for local vs distributed TCP baseline artifacts)
+- Criterion benches under `crates/ecstore/benches/`
+
+These mostly cover S3/object workload or erasure coding performance. They do
+not yet isolate internode transport cost.
+
+## Required TCP Baseline
+
+Before changing internode data transport behavior or comparing a non-default
+backend, collect a baseline for the current TCP/HTTP/gRPC implementation.
+
+### Topology
+
+Minimum:
+
+- 1-node local erasure deployment, to measure local disk and erasure overhead.
+- 4-node distributed erasure deployment, to measure internode overhead.
+
+Preferred:
+
+- Same host count and disk layout for every run.
+- Dedicated network interface or isolated VLAN.
+- Fixed CPU governor and no unrelated background load.
+- Recorded kernel version, NIC model, MTU, RustFS commit, Rust toolchain, and
+  benchmark tool versions.
+
+### Workloads
+
+| Workload | Sizes | Concurrency | Main signal |
+| --- | --- | --- | --- |
+| S3 PUT | 4 KiB, 1 MiB, 16 MiB, 128 MiB, 1 GiB | 1, 16, 64, 128 | End-to-end write throughput and tail latency. |
+| S3 GET | 4 KiB, 1 MiB, 16 MiB, 128 MiB, 1 GiB | 1, 16, 64, 128 | End-to-end read throughput and tail latency. |
+| Remote disk stream read | shard-sized ranges from `read_file_stream` | 1, 16, 64 | Isolated internode read path. |
+| Remote disk stream write | shard-sized writes through `put_file_stream` | 1, 16, 64 | Isolated internode write path. |
+| Healing / repair | missing disk or missing shard scenario | controlled | Rebuild throughput and read/write amplification. |
+| Scanner walk | large bucket/object namespace | controlled | Metadata streaming pressure, not the primary object-byte transport path. |
+
+### Measurements
+
+Collect:
+
+- throughput in bytes/s and objects/s
+- p50, p95, p99, and max latency
+- CPU utilization per process and per core
+- memory RSS and allocation pressure where available
+- `rustfs_system_network_internode_*` metrics
+- TCP retransmits, socket errors, and NIC throughput
+- disk throughput and utilization
+- failure/retry/fallback counts
+
+The baseline should produce a machine-readable artifact, for example
+`target/bench/internode-transport/<timestamp>/summary.csv`, plus the exact
+commands and configuration used.
+
+### Baseline runner entry point
+
+Use `scripts/run_internode_transport_baseline.sh` to execute a reproducible
+S3 PUT/GET matrix against `local` and `distributed` scenarios and export:
+
+- `summary.csv` (throughput/latency summary per workload and object size)
+- `internode_metric_deltas.csv` (operation-level internode metric deltas when
+  `--metrics-url` is provided)
+
+See `transport-metrics-and-baseline.md` for current metric names, labels,
+operation values, baseline inputs, and baseline artifact fields.
+
+## Transport Abstraction Proposal
+
+### Design principle
+
+Keep `NodeService` as the control plane. Introduce a separate data transport
+only below `RemoteDisk`, where remote disk byte streams are opened today.
+
+The first implementation should be a no-behavior-change TCP/HTTP backend that
+wraps the current `HttpReader`, `HttpWriter`, and `/rustfs/rpc/*` handlers.
+Non-default backend work should not proceed until the default wrapper is
+measured and adapter gaps are documented.
+
+### Candidate boundary
+
+The current boundary is remote disk stream transfer:
+
+```rust
+#[async_trait::async_trait]
+pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
+    async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
+    async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
+    async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
+    fn name(&self) -> &'static str;
+    fn capabilities(&self) -> InternodeDataTransportCapabilities;
+}
+```
+
+Initial request fields should mirror the current HTTP query parameters:
+
+- peer endpoint
+- disk reference
+- volume
+- path
+- offset
+- length
+- append/create mode
+- expected size
+- optional stall timeout for long-running listing streams
+
+The initial TCP backend can keep the current signed HTTP URLs internally.
+
+### Integration point
+
+`RemoteDisk` delegates only these methods to the data transport:
+
+- `read_file_stream`
+- `read_file_zero_copy` as the current wrapper over `read_file_stream`
+- `append_file`
+- `create_file`
+- `walk_dir`
+
+All other `RemoteDisk` methods continue using the current gRPC client
+in this adapter scope.
+
+### Capability model
+
+Avoid hard-coding transport-specific assumptions into the generic interface.
+The current conservative capability fields are:
+
+- streaming read
+- streaming write
+- streaming walk-dir
+- ordered delivery
+- maximum transfer size
+- fallback support
+
+The TCP/HTTP backend should report only capabilities that it actually provides.
+
+## TCP Fallback Requirements
+
+TCP/HTTP/gRPC must remain the default and required backend.
+
+Fallback rules:
+
+- If no explicit data transport is configured, use the current TCP/HTTP
+  implementation.
+- The current accepted values for `RUSTFS_INTERNODE_DATA_TRANSPORT` are
+  `tcp-http` and the `tcp` alias. Empty and unset values use `tcp-http`.
+- Invalid configured values fail closed with an error that includes the env var
+  name and invalid value.
+- Unsupported configured backends fail closed during transport construction.
+- Runtime fallback must preserve object correctness and quorum semantics.
+- Fallback events must be logged and counted in metrics.
+
+Do not add fallback settings until there is an implementation PR that uses them.
+
+## Baseline Validation Commands
+
+Dry-run command:
+
+```bash
+scripts/run_internode_transport_baseline.sh \
+  --access-key minioadmin \
+  --secret-key minioadmin \
+  --scenarios local=http://127.0.0.1:9000,distributed=http://127.0.0.1:9001 \
+  --sizes 4KiB,1MiB \
+  --concurrencies 1 \
+  --duration 10s \
+  --dry-run
+```
+
+Real TCP baseline command with metrics:
+
+```bash
+RUSTFS_INTERNODE_DATA_TRANSPORT=tcp-http \
+scripts/run_internode_transport_baseline.sh \
+  --access-key "$RUSTFS_ACCESS_KEY" \
+  --secret-key "$RUSTFS_SECRET_KEY" \
+  --scenarios local=http://127.0.0.1:9000,distributed=http://127.0.0.1:9001 \
+  --metrics-url http://127.0.0.1:9000/metrics \
+  --out-dir target/bench/internode-transport/manual-run
+```
+
+Expected artifacts:
+
+- `run_manifest.txt`
+- `summary.csv`
+- `internode_metric_deltas.csv` when `--metrics-url` is provided
+
+The baseline validates the default TCP/HTTP path only. It must not be used to
+claim support or performance for any other transport.
+
+## Adapter Constraints
+
+The current adapter boundary has these constraints:
+
+- `tcp-http` is the default and only OSS backend.
+- Backend selection is explicit and fail-closed.
+- The gRPC control plane remains responsible for metadata, health, locks, and
+  coordination.
+- Transport errors must preserve existing disk health, quorum, timeout, and
+  integrity semantics.
+- Metrics must identify the selected backend and operation without high-cardinality
+  labels.
+
+The current `RemoteDisk::walk_dir` stream is routed through the adapter.
+Metadata RPCs, locks, admin RPCs, bucket coordination, and the legacy gRPC
+`WalkDir` handler remain outside the current data-plane boundary.
+
+## Out of Scope
+
+This RFC does not add a plugin system, split the adapter into a separate crate,
+add accepted backend values, or implement a new transport backend.

crates/ecstore/docs/internode-transport/transport-buffer-contract.md

@@ -0,0 +1,91 @@
+# Internode Transport Buffer Contract
+
+Status: design note only. This document defines a backend-neutral buffer
+ownership and lifecycle contract for the `InternodeDataTransport` adapter. It
+does not implement a new backend and does not change production behavior.
+
+## Open-source Scope
+
+The open-source RustFS path keeps `tcp-http` as the default internode data
+transport. This document defines adapter contracts only:
+
+- no additional production backend is introduced;
+- no dependency is added;
+- no new accepted production backend value is added;
+- RustFS core data-plane logic remains independent of the concrete transport
+  implementation.
+
+## Current Adapter Surface
+
+The current data-plane surface is byte-stream based:
+
+| Current path | Current API shape | Current ownership |
+| --- | --- | --- |
+| Remote read stream | `InternodeDataTransport::open_read(...) -> FileReader` | Backend returns boxed `AsyncRead`; callers provide temporary `ReadBuf` storage per poll. |
+| Remote write stream | `InternodeDataTransport::open_write(...) -> FileWriter` | Callers pass borrowed `&[u8]` slices into boxed `AsyncWrite`; the backend owns any async body staging. |
+| Walk-dir stream | `InternodeDataTransport::open_walk_dir(...) -> FileReader` | Same boxed stream model as read, with a small serialized request body. |
+
+This API is correct for the current TCP/HTTP backend. The adapter contract
+describes current ownership boundaries without assuming implementation details
+outside `TcpHttpInternodeDataTransport`.
+
+## Buffer Ownership Model
+
+| Buffer role | Allocator | Lifetime owner | Transport state | TCP/HTTP behavior |
+| --- | --- | --- | --- | --- |
+| Send buffer | Caller or RustFS-owned pool | Caller until the writer copies or accepts bytes for the HTTP body | Existing HTTP body staging | Copy into the existing `AsyncWrite` path when the writer cannot use the borrowed slice directly. |
+| Receive buffer | Caller-provided storage | Reader while filling; caller after `poll_read` returns | Existing HTTP response body chunks | Copy from `AsyncRead` into caller storage as today. |
+| Control metadata | RustFS caller | Caller/request object | Not buffer-managed by the data-plane backend | Serialize into HTTP/gRPC/control-plane messages. |
+| Fallback staging | TCP/HTTP backend | TCP/HTTP backend | Existing `HttpReader`/`HttpWriter` buffers | Existing buffering semantics. |
+
+The current writer must not retain borrowed caller slices beyond the write call.
+When bytes must outlive the call, they are copied into owned HTTP body chunks.
+
+This contract does not claim zero-copy behavior. The current TCP/HTTP path
+documents where copies occur.
+
+## Compatibility Contract
+
+The current stream API remains the OSS compatibility contract:
+
+```rust
+#[async_trait::async_trait]
+pub trait InternodeDataTransport {
+    async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
+    async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter>;
+    async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader>;
+}
+```
+
+## Current Adapter Contract
+
+| Area | Required contract |
+| --- | --- |
+| Ownership | Define when caller-owned bytes are copied or accepted by the transport. |
+| Completion | Return from stream operations only when bytes are accepted or an error is reported. |
+| Staging | Keep staging behavior inside the TCP/HTTP implementation. |
+| Size limits | Report any RustFS-visible `max_transfer_size`; TCP/HTTP currently reports none. |
+| Ordering | Preserve ordered byte-stream semantics. |
+| Copy accounting | Document known copy boundaries and avoid unmeasured zero-copy claims. |
+
+## Current API Limitations
+
+| Current API | Current limitation |
+| --- | --- |
+| `FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>` | `AsyncRead` exposes temporary caller `ReadBuf` storage. |
+| `FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>` | `AsyncWrite::poll_write` receives borrowed `&[u8]` that cannot outlive the poll. |
+| `HttpWriter` | The async HTTP body must own `Bytes`, so borrowed write buffers are copied into `BytesMut` or `Bytes`. |
+| `write_body_chunks_to_writer` | Server-side HTTP body chunks are copied into `BytesMut` before local disk write. |
+| Erasure encode output | Encoded shards are represented as `Vec<Bytes>` and written through `AsyncWrite`. |
+| Erasure decode input | Shard reads allocate `Vec<u8>` buffers before decode. |
+
+These limitations do not block the current `tcp-http` backend.
+
+## Adapter Stability
+
+`InternodeDataTransport` should keep RustFS core data-plane logic separate from
+the concrete transport implementation. The trait and `tcp-http` backend remain
+inside `ecstore`.
+
+This PR does not perform a crate split, add runtime loading, introduce a plugin
+system, add a backend value, or implement a new transport backend.

crates/ecstore/docs/internode-transport/transport-buffer-lifecycle.md

@@ -0,0 +1,122 @@
+# Internode Buffer Lifecycle and Copy Count
+
+Status: P1-D analysis only. This document records the current TCP/HTTP
+internode data path and the ownership boundaries that matter for the
+backend-neutral `InternodeDataTransport` adapter. It does not implement a new
+backend or change production behavior.
+
+## Open-source Scope
+
+The OSS scope is:
+
+- define buffer ownership and copy-count behavior for the current
+  `InternodeDataTransport` adapter;
+- keep `tcp-http` as the default backend;
+- keep existing TCP/HTTP behavior unchanged;
+- document copy hotspots and ownership gaps for maintainable transport code;
+- avoid adding dependencies or backend implementations.
+
+The OSS scope is not:
+
+- adding another transport backend;
+- replacing the current TCP/HTTP path;
+- adding benchmark plans for another transport;
+- changing object correctness semantics.
+
+## Scope
+
+The covered paths are the large internode data-plane calls currently routed
+through `InternodeDataTransport`:
+
+| Path | Entry | Transport owner | Server owner |
+| --- | --- | --- | --- |
+| Read stream | `RemoteDisk::read_file_stream` in `crates/ecstore/src/rpc/remote_disk.rs` | `TcpHttpInternodeDataTransport::open_read` in `crates/ecstore/src/rpc/internode_data_transport.rs`, `HttpReader` in `crates/rio/src/http_reader.rs` | `handle_read_file` in `rustfs/src/storage/rpc/http_service.rs` |
+| Write stream | `RemoteDisk::create_file`, `RemoteDisk::append_file` in `crates/ecstore/src/rpc/remote_disk.rs` | `TcpHttpInternodeDataTransport::open_write` in `crates/ecstore/src/rpc/internode_data_transport.rs`, `HttpWriter` in `crates/rio/src/http_reader.rs` | `handle_put_file` in `rustfs/src/storage/rpc/http_service.rs` |
+| Walk dir stream | `RemoteDisk::walk_dir` in `crates/ecstore/src/rpc/remote_disk.rs` | `TcpHttpInternodeDataTransport::open_walk_dir`, `HttpReader` | `handle_walk_dir` in `rustfs/src/storage/rpc/http_service.rs` |
+
+Object read/write/heal callers enter these streams through
+`create_bitrot_reader` and `create_bitrot_writer` in
+`crates/ecstore/src/bitrot.rs`. Erasure decode and encode then move data
+through `ParallelReader` in `crates/ecstore/src/erasure_coding/decode.rs` and
+`MultiWriter` in `crates/ecstore/src/erasure_coding/encode.rs`.
+
+## Read Stream
+
+| Step | Owner | Buffer type | Copy? | Reason |
+| --- | --- | --- | --- | --- |
+| Build request | `RemoteDisk::read_file_stream` | `String` fields in `ReadStreamRequest` | Yes | Volume, path, endpoint, and disk references are copied into an owned request before async transport dispatch. This is metadata, not payload. |
+| Select transport | `TcpHttpInternodeDataTransport::open_read` | URL `String`, `HeaderMap` | Yes | URL and auth headers are HTTP control data. No object bytes are copied here. |
+| Open local file on server | `handle_read_file`, `LocalDisk::read_file_stream` | `FileCacheReclaimReader` boxed as `FileReader` | No payload copy | The server owns an async file reader positioned at the requested offset. |
+| File to HTTP body | `read_file_body_stream` | `ReaderStream<AsyncRead>` yielding `Bytes` | Yes | `ReaderStream::with_capacity` reads from the file into chunk buffers. This is the file-to-network buffer materialization point. |
+| Length limiting | `rustfs_utils::net::bytes_stream` | `Bytes` | Usually no | `Bytes::truncate` adjusts the chunk view when the last chunk exceeds the requested length. It does not copy the retained prefix. |
+| HTTP receive | `HttpReader::with_capacity_and_stall_timeout` | `reqwest::Response::bytes_stream()` yielding `Bytes` | Network stack dependent | The user-level object is `Bytes`; any kernel/TLS/hyper copy is below the current RustFS abstraction. |
+| Stream to caller buffer | `HttpReader::poll_read` | `StreamReader<Stream<Item = Bytes>>`, caller `ReadBuf` | Yes | `StreamReader` exposes `AsyncRead`, so it copies bytes from each `Bytes` chunk into the caller-provided `ReadBuf`. |
+| Bitrot verification | `BitrotReader::read` | caller `&mut [u8]`, `hash_buf: Vec<u8>` | No additional payload copy | The bitrot reader reads hash bytes into `hash_buf` and payload bytes directly into the supplied output slice. Hash calculation reads the slice. |
+| Erasure shard read | `ParallelReader::read` | `Vec<u8>` per shard | Yes | Each shard read allocates `vec![0u8; shard_size]`; data is filled there before decode/reconstruction. |
+| Object response write | `write_data_blocks` | slices of shard `Vec<u8>` | No extra staging copy | Decoded data block slices are written to the target writer with `write_all`; the target may copy internally. |
+| Remote zero-copy helper | `RemoteDisk::read_file_zero_copy` | `Vec<u8>` then `Bytes` | Yes | The remote implementation reads the full stream into a `Vec` and converts it into `Bytes`. It is a convenience fallback, not network zero-copy. |
+
+## Write Stream
+
+| Step | Owner | Buffer type | Copy? | Reason |
+| --- | --- | --- | --- | --- |
+| Build writer request | `RemoteDisk::create_file`, `RemoteDisk::append_file` | `String` fields in `WriteStreamRequest` | Yes | Volume, path, endpoint, and disk references are copied into an owned request. This is metadata. |
+| Select transport | `TcpHttpInternodeDataTransport::open_write` | URL `String`, `HeaderMap` | Yes | URL and auth headers are HTTP control data. No object bytes are copied here. |
+| Erasure encode input | `Erasure::encode` in `encode.rs` | reusable `Vec<u8>` sized to `block_size` | Yes | `rustfs_utils::read_full` fills a block buffer from the source reader before encoding. |
+| Erasure encode output | `Erasure::encode_data` caller in `encode.rs` | `Vec<Bytes>` per encoded block | Yes | Encoding creates shard `Bytes` for data and parity blocks before queueing them to writers. |
+| Multi-writer fanout | `MultiWriter::write` | borrowed `Bytes` shards | No additional fanout copy | The writer fanout passes borrowed `Bytes` references to each `BitrotWriterWrapper`. |
+| Bitrot write | `BitrotWriter::write` | shard `&[u8]`, checksum bytes | Yes for checksum bytes | The payload slice is passed to the inner writer, while checksum bytes are generated and written before payload when enabled. |
+| Client HTTP writer buffer | `HttpWriter::poll_write` and `poll_write_vectored` | `BytesMut` pending chunk or `Bytes::copy_from_slice` | Yes | Small writes are coalesced with `BytesMut::extend_from_slice`; large single writes still copy into an owned `Bytes` because the async body must outlive the caller's borrowed buffer. |
+| Client channel to reqwest | `HttpWriter::poll_send_pending_chunk`, `ReceiverStream` | `Bytes` | No | `BytesMut::split().freeze()` transfers owned chunk storage to `Bytes`; the mpsc channel and stream move the `Bytes` handle. |
+| HTTP receive body on server | `handle_put_file` | `Incoming::into_data_stream()` yielding `Bytes` | Network stack dependent | The server receives owned `Bytes` chunks from hyper. |
+| Server body coalescing | `write_body_chunks_to_writer` | `BytesMut` sized to `DEFAULT_READ_BUFFER_SIZE` | Yes | Each incoming `Bytes` chunk is copied into `pending` before writing to the local file writer. This normalizes chunk size but adds a full payload copy. |
+| Local file write | `LocalDisk::create_file`, `LocalDisk::append_file`, `FileCacheReclaimWriter` | `&[u8]` into `tokio::fs::File` | Kernel dependent | RustFS passes slices to Tokio file writes. Kernel page-cache copies are below the RustFS abstraction. |
+
+## Request and Serialization Boundaries
+
+| Boundary | Owner | Buffer type | Copy? | Notes |
+| --- | --- | --- | --- | --- |
+| Read/write query parameters | `build_read_file_stream_url`, `build_put_file_stream_url` | URL-encoded `String` | Yes | Metadata only. It includes disk, volume, path, offset, length, append, and size. |
+| Auth headers | `build_auth_headers` callers | `HeaderMap` | Yes | Metadata only. This is currently tied to HTTP request construction. |
+| Walk dir request | `RemoteDisk::walk_dir`, `open_walk_dir`, `handle_walk_dir` | JSON `Vec<u8>` body, collected `Bytes` on server | Yes | Walk dir is a streamed response but its request body is serialized JSON control data. |
+| gRPC read/write-all | `RemoteDisk::read_all`, `RemoteDisk::write_all`, `NodeService::{handle_read_all,handle_write_all}` | Prost `Bytes`/message bodies | Yes | These paths are still gRPC byte paths, not `InternodeDataTransport`; they matter for metrics and inventory but are outside this P1-D stream-copy count. |
+
+## Hotspots
+
+| Rank | Hotspot | Impact | Reason |
+| --- | --- | --- | --- |
+| 1 | `HttpWriter::poll_write` and `poll_write_vectored` | High on write path | Every borrowed caller buffer is copied into owned `BytesMut` or `Bytes` before it can be sent by an async HTTP body. |
+| 2 | `write_body_chunks_to_writer` | High on write path | The server copies every received `Bytes` chunk into a coalescing `BytesMut` before local disk write. |
+| 3 | `ParallelReader::read` shard buffers | High on read path | Each shard read allocates and fills a `Vec<u8>` before decode can proceed. This is also where degraded reads wait on quorum. |
+| 4 | `ReaderStream::with_capacity` plus `StreamReader` | Medium on read path | Server file reads create `Bytes` chunks, then client `AsyncRead` copies those chunks into the caller's `ReadBuf`. |
+| 5 | `Erasure::encode` block and shard materialization | Medium on write path | Source data is first read into a block `Vec<u8>`, then encoded into per-shard `Bytes`. This is necessary for the current erasure API. |
+| 6 | `RemoteDisk::read_file_zero_copy` | Medium when used | Remote zero-copy reads buffer the whole stream into memory. The name does not mean zero-copy over the network. |
+| 7 | URL/query/header/JSON serialization | Low | Metadata copies are small and not on the large payload hot path. |
+
+## Adapter Ownership Gaps
+
+1. `FileReader` and `FileWriter` are boxed `AsyncRead`/`AsyncWrite` trait
+   objects. They expose borrowed buffers per poll, not stable backend-owned
+   regions, transfer handles, or explicit completion ownership.
+2. `InternodeDataTransport` currently returns stream traits only. Its
+   capabilities advertise that TCP/HTTP does not require backend-specific
+   buffer registration and is not a zero-copy candidate, but there is no
+   backend API to pass stable backend-managed buffers.
+3. `HttpWriter` must own outgoing chunks because the async request body outlives
+   the caller's borrowed `&[u8]`. A lower-copy backend would need a different
+   lifetime contract or an owned buffer pool.
+4. Server write handling normalizes all incoming body chunks into a new
+   `BytesMut`. Avoiding that copy would require passing incoming `Bytes` or
+   backend-owned receive buffers directly into the disk/bitrot writer contract.
+5. Erasure decode owns shard `Vec<u8>` buffers and write-back happens through
+   `AsyncWrite`. A lower-copy backend would need explicit ownership of shard
+   buffers across decode, reconstruction, and network completion.
+6. Erasure encode materializes `Vec<Bytes>` blocks before fanout. A
+   backend that can send multiple stable slices would need an encode output
+   representation that can be transferred without repacking.
+7. The HTTP auth and URL construction boundary is part of the current TCP/HTTP
+   backend. A non-HTTP backend would need equivalent peer authentication and
+   disk addressing without assuming URL query parameters.
+8. Local disk zero-copy exists only for local reads via `read_file_zero_copy`.
+   Remote disks deliberately fall back to network streaming and full-buffer
+   collection for the zero-copy helper.

crates/ecstore/docs/internode-transport/transport-capabilities.md

@@ -0,0 +1,67 @@
+# Internode Transport Capabilities
+
+Status: design note for backend-neutral capability reporting. This document
+does not add a backend or transport crate.
+
+## Open-source Scope
+
+The OSS scope is:
+
+- define honest capability reporting for the `InternodeDataTransport` adapter;
+- keep `tcp-http` as the default backend;
+- keep existing TCP/HTTP behavior unchanged;
+- document the capability fields needed for maintainable transport code;
+- avoid adding dependencies or backend implementations.
+
+The OSS scope is not:
+
+- adding another transport backend;
+- replacing the current TCP/HTTP path;
+- adding benchmark plans for another transport;
+- changing object correctness semantics.
+
+## Purpose
+
+`InternodeDataTransportCapabilities` describes what a backend can honestly do
+for RustFS internode data-plane transfers. The fields describe observable
+adapter behavior without naming a specific transport implementation.
+
+The capability report is descriptive. It does not select a backend, negotiate
+with peers, or weaken object correctness semantics.
+
+## Capability Fields
+
+| Field | Meaning |
+| --- | --- |
+| `streaming_read` | The backend can open a remote disk reader for `read_file_stream`. |
+| `streaming_write` | The backend can open a remote disk writer for `create_file` or `append_file`. |
+| `streaming_walk_dir` | The backend can stream `walk_dir` responses. |
+| `ordered_delivery` | Bytes for each opened transfer are delivered in order. |
+| `max_transfer_size` | Optional RustFS-level cap for a single transfer. `None` means no additional cap beyond the backend/protocol/runtime limits. |
+| `fallback_supported` | The backend can participate in the behavior-preserving TCP fallback path. |
+
+## TCP/HTTP Backend
+
+The default TCP/HTTP backend reports only capabilities it actually provides:
+
+| Field | TCP/HTTP value | Reason |
+| --- | --- | --- |
+| `streaming_read` | `true` | `HttpReader` streams `/rustfs/rpc/read_file_stream` responses. |
+| `streaming_write` | `true` | `HttpWriter` streams `/rustfs/rpc/put_file_stream` request bodies. |
+| `streaming_walk_dir` | `true` | `HttpReader` streams `/rustfs/rpc/walk_dir` responses. |
+| `ordered_delivery` | `true` | Each HTTP request body or response body is consumed as an ordered byte stream. |
+| `max_transfer_size` | `None` | RustFS does not impose an extra per-transfer cap at the capability layer. |
+| `fallback_supported` | `true` | TCP/HTTP is the behavior-preserving default and fallback path. |
+
+## Capability Compatibility
+
+Any new capability field should describe observable RustFS behavior without
+assuming a specific transport implementation:
+
+| Capability shape | Interpretation |
+| --- | --- |
+| `max_transfer_size=Some(n)` | The backend has a RustFS-visible transfer size ceiling and callers must split larger transfers or use fallback behavior. |
+| `ordered_delivery=false` | The backend cannot be used behind the current stream API without an ordering or reassembly layer. |
+
+Unsupported or mismatched capabilities must not silently change quorum,
+integrity verification, retry, or timeout semantics.

crates/ecstore/docs/internode-transport/transport-fallback-and-selection.md

@@ -0,0 +1,135 @@
+# Internode Transport Fallback and Backend Selection Model
+
+Status: design note only. This document defines backend-neutral selection,
+fallback, failure handling, negotiation, security, and observability rules for
+the `InternodeDataTransport` adapter. It does not implement a new backend and
+does not change production behavior.
+
+## Open-source Scope
+
+The open-source RustFS path keeps `tcp-http` as the default internode data
+transport. This document defines adapter contracts only:
+
+- no additional production backend is introduced;
+- no dependency is added;
+- no new accepted production backend value is added;
+- RustFS core data-plane logic remains independent of the concrete transport
+  implementation.
+
+## Static Backend Selection
+
+Static config is the first selection model. Existing accepted values remain:
+
+| Config value | Meaning |
+| --- | --- |
+| unset | Use default TCP/HTTP backend. |
+| `tcp-http` | Use default TCP/HTTP backend. |
+| `tcp` | Alias for `tcp-http`. |
+| any unsupported value | Fail closed with a diagnostic naming `RUSTFS_INTERNODE_DATA_TRANSPORT` and the invalid value. |
+
+Unknown backend values must fail closed. Unsupported backend values must fail
+closed. Any additional backend value must be explicitly added and must not
+silently replace `tcp-http`.
+
+Backend selection must expose an observable backend identity for metrics, logs,
+and benchmark interpretation. The default and fallback path remains `tcp-http`.
+
+## Fallback Contract
+
+Fallback must be explicit and observable. Silent fallback is not allowed for
+benchmark or production interpretation because it hides which backend moved the
+payload.
+
+| Condition | Default behavior | Explicit fallback behavior | Observability |
+| --- | --- | --- | --- |
+| Unsupported configured backend | Fail closed during transport construction. | Fall back only when a separately configured policy explicitly allows unsupported-backend fallback. | Error includes config key and invalid value; fallback event is counted when fallback is enabled. |
+| Peer does not support selected backend | Fail before payload transfer. | Use TCP/HTTP only when both local policy and peer policy allow it. | Count peer mismatch and selected fallback backend. |
+| Capability mismatch | Fail before payload transfer. | Use TCP/HTTP only if it satisfies the operation and policy allows fallback. | Record missing capability names or a low-cardinality reason. |
+| Connection setup failure | Fail the operation. | Retry on TCP/HTTP only when fallback is allowed and no payload bytes have transferred. | Count setup failure, retry, fallback backend, and fallback result. |
+| Partial transfer failure | Fail the operation and let existing object/quorum logic decide retry behavior. | Do not silently resume on another backend unless the transfer protocol can prove byte range, checksum, and idempotency boundaries. | Count partial failure with bytes completed. |
+| Max transfer size exceeded | Fail before payload transfer or split at a higher layer. | Use TCP/HTTP if policy allows and TCP has no RustFS-level cap. | Record rejected size and selected backend. |
+| Auth or encryption mismatch | Fail closed. | No fallback unless the fallback path satisfies the same or stronger security requirements. | Security failure metric and audit log entry. |
+
+Fallback settings should not be added until there is an implementation that
+uses them. A backend must define failure behavior before production use.
+
+## Dynamic Negotiation Boundary
+
+Dynamic negotiation is not implemented by this PR. If it is added later, it
+belongs on the existing gRPC control plane. Data transfer must start only after
+both peers agree on:
+
+| Negotiated item | Required property |
+| --- | --- |
+| Backend name | Both peers know the backend and have it enabled. |
+| Capability set | Required capabilities match the operation. |
+| Max transfer size | The selected operation fits or is split before transfer starts. |
+| Buffer rules | Both peers agree on staging and ownership rules. |
+| Completion semantics | Both peers agree when a transfer is considered complete and when buffers may be reused. |
+| Security mode | Authentication and encryption requirements are satisfied before any out-of-band transfer. |
+| Fallback policy | Both peers agree whether TCP/HTTP fallback is allowed for this operation. |
+
+Negotiation must not silently downgrade security or bypass existing disk
+health, quorum, timeout, and integrity semantics.
+
+## Failure Handling Requirements
+
+| Failure mode | Requirement |
+| --- | --- |
+| Invalid config | Fail closed with `RUSTFS_INTERNODE_DATA_TRANSPORT` and the invalid value. |
+| Backend disabled | Fail closed with the selected backend name and the missing enablement condition. |
+| Backend unavailable | Fail closed with an actionable diagnostic; do not silently use TCP/HTTP. |
+| Peer mismatch | Fail before payload transfer unless explicit fallback is configured. |
+| Connection failure | Fail the operation and record setup failure; fallback only if policy allows and no payload bytes moved. |
+| Completion failure | Return an operation error and release backend-owned resources. |
+| Timeout | Return an operation error and preserve existing disk health and quorum semantics. |
+| Partial transfer | Do not silently resume on another backend without a safe byte-range/checksum/idempotency proof. |
+| Unsupported operation | Return a clear unsupported-operation error. |
+
+## Security Requirements
+
+- Backend selection must preserve peer authentication.
+- Fallback must not weaken encryption or authorization.
+- Partial transfers must not bypass bitrot verification or erasure quorum
+  handling.
+- Any adapter implementation must preserve the same request authority, disk,
+  volume, path, and operation binding as the current TCP/HTTP path.
+
+## Metrics and Observability Requirements
+
+Metrics and logs must use low-cardinality labels or metadata:
+
+- selected backend;
+- requested backend;
+- fallback backend, when used;
+- operation name;
+- success/failure;
+- transferred bytes;
+- setup failure count;
+- partial transfer failure count;
+- capability mismatch count;
+- fallback decision count.
+
+Adapter implementations must not add high-cardinality labels such as object
+names, full paths, full URLs, peer-specific dynamic strings, memory addresses,
+or buffer identifiers.
+
+## TCP/HTTP Compatibility
+
+The `tcp-http` backend remains the default and behavior-preserving path. It
+uses ordinary byte streams, does not require backend-specific buffer
+registration, and remains suitable as the fallback path when an explicit
+fallback policy exists.
+
+Any adapter implementation must not change the correctness semantics of
+object writes, object reads, healing, bitrot verification, erasure quorum,
+timeouts, or disk health handling.
+
+## Adapter Stability
+
+`InternodeDataTransport` should keep RustFS core data-plane logic separate from
+the concrete transport implementation. The trait and `tcp-http` backend remain
+inside `ecstore`.
+
+This PR does not perform a crate split, add runtime loading, introduce a plugin
+system, add a backend value, or implement a new transport backend.

crates/ecstore/docs/internode-transport/transport-metrics-and-baseline.md

@@ -0,0 +1,143 @@
+# Internode Transport Metrics and Baseline
+
+Status: current OSS behavior. This document describes the metrics and baseline
+runner used to observe the existing `tcp-http` internode transport adapter. It
+does not add a backend, change runtime behavior, or make performance claims.
+
+## Scope
+
+The current OSS adapter scope is:
+
+- `tcp-http` is the default and only supported internode data transport backend;
+- `tcp` is a legacy alias for `tcp-http`;
+- unsupported backend values fail closed during transport construction;
+- metrics and baseline artifacts are used to understand the current data-plane
+  behavior.
+
+## Operation Metrics
+
+Operation-level internode metrics are defined in
+`crates/io-metrics/src/internode_metrics.rs`.
+
+| Metric | Labels | Meaning |
+| --- | --- | --- |
+| `rustfs_system_network_internode_operation_sent_bytes_total` | `operation`, `backend` | Bytes sent for a known internode operation. |
+| `rustfs_system_network_internode_operation_recv_bytes_total` | `operation`, `backend` | Bytes received for a known internode operation. |
+| `rustfs_system_network_internode_operation_requests_outgoing_total` | `operation`, `backend` | Outgoing requests opened for a known internode operation. |
+| `rustfs_system_network_internode_operation_requests_incoming_total` | `operation`, `backend` | Incoming requests handled for a known internode operation. |
+| `rustfs_system_network_internode_operation_errors_total` | `operation`, `backend` | Errors observed for a known internode operation. |
+
+Current operation label values are:
+
+| Operation | Transport path | Notes |
+| --- | --- | --- |
+| `read_file_stream` | HTTP `/rustfs/rpc/read_file_stream` | Remote disk read stream opened by `InternodeDataTransport::open_read`. |
+| `put_file_stream` | HTTP `/rustfs/rpc/put_file_stream` | Remote disk write stream opened by `InternodeDataTransport::open_write`. |
+| `walk_dir` | HTTP `/rustfs/rpc/walk_dir` | Remote walk-dir stream opened by `InternodeDataTransport::open_walk_dir`. |
+| `grpc_read_all` | gRPC `ReadAll` | Unary bytes response outside the adapter. |
+| `grpc_write_all` | gRPC `WriteAll` | Unary bytes request outside the adapter. |
+
+Current backend label values are:
+
+| Backend | Meaning |
+| --- | --- |
+| `tcp-http` | Current HTTP stream backend for adapter-routed operations. |
+| `grpc` | Current gRPC path for `ReadAll` and `WriteAll`. |
+| `unknown` | Aggregate internode metric path when an operation-specific backend is not available. |
+
+The `backend` label reflects the current instrumentation source, for example
+the `tcp-http` stream path, gRPC path, or aggregate unknown path. It does not
+imply additional supported transport backends.
+
+Metric labels must stay low cardinality. Do not add object names, full paths,
+full URLs, peer-specific dynamic strings, request IDs, or other per-request
+values as labels.
+
+## Current Instrumentation Points
+
+| Area | File | Current signal |
+| --- | --- | --- |
+| Outgoing HTTP stream requests | `crates/rio/src/http_reader.rs` | Outgoing request count, sent bytes for writer bodies, received bytes for reader bodies, and errors for known `/rustfs/rpc/` operations. |
+| Incoming HTTP stream requests | `rustfs/src/storage/rpc/http_service.rs` | Incoming request count, sent bytes for read and walk streams, received bytes for put streams, and route errors. |
+| gRPC `ReadAll` / `WriteAll` | `rustfs/src/storage/rpc/disk.rs` | Incoming request count, sent/received bytes, and errors with the `grpc` backend label. |
+
+Known gaps:
+
+- Operation-level metrics do not currently expose latency histograms.
+- The baseline runner reads Prometheus text output only when `--metrics-url` is
+  provided.
+- Metrics are useful for attribution, but they do not replace end-to-end
+  correctness tests or object benchmark output.
+
+## Baseline Runner
+
+Use `scripts/run_internode_transport_baseline.sh` to run the current S3 PUT/GET
+matrix against one or more configured endpoints.
+
+Required inputs:
+
+- `--access-key`
+- `--secret-key`
+
+Common optional inputs:
+
+- `--tool warp|s3bench`
+- `--scenarios name=url,...`
+- `--sizes 4KiB,1MiB,...`
+- `--concurrencies 1,16,...`
+- `--duration 90s`
+- `--metrics-url http://host:port/metrics`
+- `--out-dir target/bench/internode-transport/manual-run`
+- `--dry-run`
+
+Dry-run example:
+
+```bash
+scripts/run_internode_transport_baseline.sh \
+  --access-key minioadmin \
+  --secret-key minioadmin \
+  --scenarios local=http://127.0.0.1:9000,distributed=http://127.0.0.1:9001 \
+  --sizes 4KiB,1MiB \
+  --concurrencies 1 \
+  --duration 10s \
+  --dry-run
+```
+
+Real baseline example with metrics:
+
+```bash
+RUSTFS_INTERNODE_DATA_TRANSPORT=tcp-http \
+scripts/run_internode_transport_baseline.sh \
+  --access-key "$RUSTFS_ACCESS_KEY" \
+  --secret-key "$RUSTFS_SECRET_KEY" \
+  --scenarios local=http://127.0.0.1:9000,distributed=http://127.0.0.1:9001 \
+  --metrics-url http://127.0.0.1:9000/metrics \
+  --out-dir target/bench/internode-transport/manual-run
+```
+
+The real baseline requires a running RustFS deployment and working benchmark
+tool. Do not invent or copy synthetic benchmark numbers into PR descriptions.
+
+## Output Artifacts
+
+The baseline runner writes:
+
+| File | Created when | Contents |
+| --- | --- | --- |
+| `run_manifest.txt` | Always | Timestamp, git commit, dirty flag, Rust compiler version, kernel string, scenario matrix, tool, workload settings, metrics URL, and redacted credentials. |
+| `summary.csv` | Always | `scenario`, `endpoint`, `workload`, `concurrency`, `size`, `status`, `throughput`, `requests_per_sec`, `avg_latency`, `error_count`, `log_file`, and `run_dir`. |
+| `internode_metric_deltas.csv` | When `--metrics-url` is set | `scenario`, `workload`, `concurrency`, `size`, metric name, operation, backend, before value, after value, and delta. |
+
+In dry-run mode, the runner still creates the manifest and CSV headers, and the
+underlying object benchmark command is printed with credentials redacted.
+
+## Interpretation
+
+Use baseline artifacts to record and inspect the current `tcp-http` adapter
+behavior across commits, environments, and scenario matrices. A baseline run is
+valid only for the exact environment and command recorded in
+`run_manifest.txt`.
+
+Baseline artifacts should state whether metrics were collected. When
+`internode_metric_deltas.csv` is absent, benchmark output cannot attribute
+internode operation deltas from Prometheus metrics.

crates/ecstore/src/admin_server_info.rs

@@ -17,7 +17,7 @@ use crate::error::{Error, Result};
 use crate::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
 use crate::{
     disk::endpoint::Endpoint,
-    global::{GLOBAL_BOOT_TIME, GLOBAL_Endpoints},
+    global::{GLOBAL_BOOT_TIME, GLOBAL_Endpoints, get_global_deployment_id},
     new_object_layer_fn,
     notification_sys::get_global_notification_sys,
     store_api::StorageAPI,
@@ -291,7 +291,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
         domain: None,
         region: None,
         sqs_arn: None,
-        deployment_id: None,
+        deployment_id: get_global_deployment_id(),
         buckets: Some(buckets),
         objects: Some(objects),
         versions: Some(versions),
@@ -393,3 +393,21 @@ pub fn get_commit_id() -> String {
 
     format!("{}@{}", build::COMMIT_DATE_3339, ver)
 }
+
+#[cfg(test)]
+mod tests {
+    use serial_test::serial;
+
+    use crate::global::get_global_deployment_id;
+
+    use super::get_server_info;
+
+    #[serial]
+    #[tokio::test]
+    async fn server_info_includes_global_deployment_id() {
+        let expected_deployment_id = get_global_deployment_id();
+        let info = get_server_info(false).await;
+
+        assert_eq!(info.deployment_id, expected_deployment_id);
+    }
+}

crates/ecstore/src/bucket/lifecycle/core.rs

@@ -34,7 +34,8 @@ const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at leas
 const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule";
 const _ERR_XML_NOT_WELL_FORMED: &str =
     "The XML you provided was not well-formed or did not validate against our published schema";
-const ERR_LIFECYCLE_BUCKET_LOCKED: &str = "ExpiredObjectDeleteMarker is not allowed on a bucket with Object Lock enabled";
+const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
+    "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
 const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
 const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must not be negative";
 const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str = "Lifecycle noncurrent expiration days must not be negative";
@@ -320,14 +321,15 @@ impl Lifecycle for BucketLifecycleConfiguration {
             r.validate()?;
             if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref()
                 && object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED
-                && let Some(expiration) = r.expiration.as_ref()
             {
-                // Object Lock + ExpiredObjectDeleteMarker conflict
-                if expiration.expired_object_delete_marker.is_some_and(|v| v) {
-                    return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
+                if let Some(expiration) = r.expiration.as_ref() {
+                    // Object Lock + ExpiredObjectAllVersions conflict (MinIO extension)
+                    if expiration.expired_object_all_versions.is_some_and(|v| v) {
+                        return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
+                    }
                 }
-                // Object Lock + ExpiredObjectAllVersions conflict (MinIO extension)
-                if expiration.expired_object_all_versions.is_some_and(|v| v) {
+                // Object Lock + DelMarkerExpiration conflict
+                if r.del_marker_expiration.is_some() {
                     return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
                 }
             }
@@ -2060,11 +2062,11 @@ mod tests {
         assert_eq!(event_after.due, Some(future_date));
     }
 
-    // --- TASK-002 tests: Object Lock + ExpiredObjectDeleteMarker conflict ---
+    // --- TASK-002 tests: Object Lock + ExpiredObjectDeleteMarker compatibility ---
 
     #[tokio::test]
     #[serial]
-    async fn validate_rejects_expired_object_delete_marker_on_locked_bucket() {
+    async fn validate_allows_expired_object_delete_marker_on_locked_bucket() {
         let lc = BucketLifecycleConfiguration {
             expiry_updated_at: None,
             rules: vec![LifecycleRule {
@@ -2089,8 +2091,9 @@ mod tests {
             ..Default::default()
         };
 
-        let err = lc.validate(&locked_config).await.unwrap_err();
-        assert_eq!(err.to_string(), ERR_LIFECYCLE_BUCKET_LOCKED);
+        lc.validate(&locked_config)
+            .await
+            .expect("expected validation to pass for ExpiredObjectDeleteMarker on locked bucket");
     }
 
     #[tokio::test]
@@ -2154,6 +2157,68 @@ mod tests {
             .expect("expected days-based expiration to pass on locked bucket");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn validate_rejects_del_marker_expiration_on_locked_bucket() {
+        let lc = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: Some(s3s::dto::DelMarkerExpiration { days: Some(1) }),
+                filter: None,
+                id: Some("test-rule".to_string()),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let locked_config = ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            ..Default::default()
+        };
+
+        let err = lc.validate(&locked_config).await.unwrap_err();
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_BUCKET_LOCKED);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn validate_rejects_zero_day_del_marker_expiration_on_locked_bucket() {
+        let lc = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    days: Some(30),
+                    ..Default::default()
+                }),
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: Some(s3s::dto::DelMarkerExpiration { days: Some(0) }),
+                filter: None,
+                id: Some("test-rule".to_string()),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        let locked_config = ObjectLockConfiguration {
+            object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
+            ..Default::default()
+        };
+
+        let err = lc.validate(&locked_config).await.unwrap_err();
+        assert_eq!(err.to_string(), ERR_LIFECYCLE_BUCKET_LOCKED);
+    }
+
     // --- TASK-003 tests: Round up to next UTC processing boundary ---
 
     #[test]

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -929,7 +929,7 @@ impl ReplicationResyncer {
 }
 
 fn heal_should_use_check_replicate_delete(oi: &ObjectInfo) -> bool {
-    oi.delete_marker || (!oi.replication_status.is_empty() && oi.replication_status != ReplicationStatusType::Failed)
+    oi.delete_marker || !oi.version_purge_status.is_empty()
 }
 
 pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationConfig) -> ReplicateObjectInfo {
@@ -939,8 +939,8 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
     if let Some(rc) = rcfg.config.as_ref()
         && !rc.role.is_empty()
     {
-        if !oi.replication_status.is_empty() {
-            oi.replication_status_internal = Some(format!("{}={};", rc.role, oi.replication_status.as_str()));
+        if !oi.version_purge_status.is_empty() {
+            oi.version_purge_status_internal = Some(format!("{}={};", rc.role, oi.version_purge_status.as_str()));
         }
 
         if !oi.replication_status.is_empty() {
@@ -3742,7 +3742,7 @@ mod tests {
     }
 
     #[test]
-    fn test_heal_should_use_check_replicate_delete_pending_uses_delete_path() {
+    fn test_heal_should_use_check_replicate_delete_pending_non_delete_marker_uses_must_replicate_path() {
         let oi = ObjectInfo {
             bucket: "b".to_string(),
             name: "obj".to_string(),
@@ -3751,8 +3751,8 @@ mod tests {
             ..Default::default()
         };
         assert!(
-            heal_should_use_check_replicate_delete(&oi),
-            "Pending (non-Failed) status with non-empty replication uses check_replicate_delete path"
+            !heal_should_use_check_replicate_delete(&oi),
+            "Pending non-delete-marker object must use must_replicate path to evaluate current target set"
         );
     }
 
@@ -3771,6 +3771,36 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_heal_should_use_check_replicate_delete_version_purge_status_uses_delete_path() {
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            delete_marker: false,
+            version_purge_status: VersionPurgeStatusType::Pending,
+            ..Default::default()
+        };
+        assert!(
+            heal_should_use_check_replicate_delete(&oi),
+            "Version purge entries must use check_replicate_delete path"
+        );
+    }
+
+    #[test]
+    fn test_heal_should_use_check_replicate_delete_completed_non_delete_marker_uses_must_replicate_path() {
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            delete_marker: false,
+            replication_status: ReplicationStatusType::Completed,
+            ..Default::default()
+        };
+        assert!(
+            !heal_should_use_check_replicate_delete(&oi),
+            "Completed non-delete-marker object must use must_replicate path so new targets can be evaluated"
+        );
+    }
+
     #[test]
     fn test_delete_replication_object_opts_marks_replica_deletes() {
         let dobj = ObjectToDelete {
@@ -4051,6 +4081,32 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_get_heal_replicate_object_info_maps_version_purge_status_for_role() {
+        let role = "arn:rustfs:replication::target:bucket";
+        let oi = ObjectInfo {
+            bucket: "test-bucket".to_string(),
+            name: "key".to_string(),
+            delete_marker: false,
+            version_purge_status: VersionPurgeStatusType::Pending,
+            version_id: Some(Uuid::nil()),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        };
+        let rcfg = ReplicationConfig::new(
+            Some(ReplicationConfiguration {
+                role: role.to_string(),
+                rules: vec![],
+            }),
+            None,
+        );
+        let roi = get_heal_replicate_object_info(&oi, &rcfg).await;
+
+        assert_eq!(roi.replication_status_internal, None);
+        assert_eq!(roi.version_purge_status_internal.as_deref(), Some(format!("{role}=PENDING;").as_str()));
+        assert_eq!(roi.target_purge_statuses.get(role), Some(&VersionPurgeStatusType::Pending));
+    }
+
     #[tokio::test]
     async fn test_cancel_marks_only_matching_bucket_target_token() {
         let resyncer = ReplicationResyncer::new().await;

crates/ecstore/src/client/transition_api.rs

@@ -51,6 +51,7 @@ use md5::Md5;
 use rand::{Rng, RngExt};
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
 use rustfs_rio::HashReader;
+use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_generation};
 use rustfs_utils::HashAlgorithm;
 use rustfs_utils::{
     net::get_endpoint_url,
@@ -58,6 +59,8 @@ use rustfs_utils::{
         DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer, is_http_status_retryable, is_s3code_retryable,
     },
 };
+use rustls_pki_types::PrivateKeyDer;
+use rustls_pki_types::pem::PemObject;
 use s3s::S3ErrorCode;
 use s3s::dto::Owner;
 use s3s::dto::ReplicationStatus;
@@ -152,24 +155,11 @@ pub enum BucketLookupType {
     BucketLookupPath,
 }
 
-fn load_root_store_from_tls_path() -> Option<rustls::RootCertStore> {
-    // Load the root certificate bundle from the path specified by the
-    // RUSTFS_TLS_PATH environment variable.
-    let tp = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
-    // If no TLS path is configured, do not fall back to a CA bundle in the current directory.
-    if tp.is_empty() {
-        return None;
-    }
-    let ca = std::path::Path::new(&tp).join(rustfs_config::RUSTFS_CA_CERT);
-    if !ca.exists() {
-        return None;
-    }
-
-    let der_list = rustfs_utils::load_cert_bundle_der_bytes(ca.to_str().unwrap_or_default()).ok()?;
+fn build_root_store_from_der_list(der_list: Vec<Vec<u8>>) -> Option<rustls::RootCertStore> {
     let mut store = rustls::RootCertStore::empty();
     for der in der_list {
         if let Err(e) = store.add(der.into()) {
-            warn!("Warning: failed to add certificate from '{}' to root store: {e}", ca.display());
+            warn!("Warning: failed to add certificate to root store: {e}");
         }
     }
     Some(store)
@@ -199,18 +189,38 @@ where
     })
 }
 
-fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
-    with_rustls_init_guard(|| {
-        let config = if let Some(store) = load_root_store_from_tls_path() {
-            rustls::ClientConfig::builder()
-                .with_root_certificates(store)
-                .with_no_client_auth()
-        } else {
-            rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth()
-        };
+async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
+    with_rustls_init_guard(|| Ok(()))?;
 
-        Ok(config)
-    })
+    let outbound_tls = load_global_outbound_tls_state().await;
+    record_tls_generation("ecstore_transition_client", outbound_tls.generation.0);
+    let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
+        let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
+        let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|e| std::io::Error::other(format!("failed to parse published root CA PEM: {e}")))?;
+
+        let root_store = build_root_store_from_der_list(certs_der.into_iter().map(|cert| cert.to_vec()).collect::<Vec<_>>())
+            .ok_or_else(|| std::io::Error::other("published outbound root CA material could not build root store"))?;
+        rustls::ClientConfig::builder().with_root_certificates(root_store)
+    } else {
+        rustls::ClientConfig::builder().with_native_roots()?
+    };
+
+    let config = if let Some(identity) = outbound_tls.mtls_identity.as_ref() {
+        let certs = rustls_pki_types::CertificateDer::pem_reader_iter(&mut std::io::BufReader::new(identity.cert_pem.as_slice()))
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(|e| std::io::Error::other(format!("failed to parse published client cert PEM: {e}")))?;
+        let key = PrivateKeyDer::from_pem_reader(&mut std::io::BufReader::new(identity.key_pem.as_slice()))
+            .map_err(|e| std::io::Error::other(format!("failed to parse published client key PEM: {e}")))?;
+        builder
+            .with_client_auth_cert(certs, key)
+            .map_err(|e| std::io::Error::other(format!("failed to build client mTLS identity: {e}")))?
+    } else {
+        builder.with_no_client_auth()
+    };
+
+    Ok(config)
 }
 
 impl TransitionClient {
@@ -234,7 +244,7 @@ impl TransitionClient {
 
         let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
 
-        let tls = build_tls_config()?;
+        let tls = build_tls_config().await?;
 
         let https = hyper_rustls::HttpsConnectorBuilder::new()
             .with_tls_config(tls)
@@ -1374,9 +1384,7 @@ pub struct CreateBucketConfiguration {
 
 #[cfg(test)]
 mod tests {
-    use super::{
-        build_tls_config, load_root_store_from_tls_path, signer_error_to_io_error, validate_header_values, with_rustls_init_guard,
-    };
+    use super::{build_tls_config, signer_error_to_io_error, validate_header_values, with_rustls_init_guard};
     use http::{HeaderMap, HeaderValue};
 
     #[test]
@@ -1395,22 +1403,6 @@ mod tests {
         assert!(outcome.is_ok(), "TLS config creation should not panic");
     }
 
-    /// When RUSTFS_TLS_PATH is not set, `load_root_store_from_tls_path` must return `None`
-    /// (i.e. it must not silently look for a CA bundle in the current working directory).
-    #[test]
-    fn tls_path_unset_returns_none() {
-        let result = temp_env::with_var_unset(rustfs_config::ENV_RUSTFS_TLS_PATH, || load_root_store_from_tls_path());
-        assert!(result.is_none(), "expected None when RUSTFS_TLS_PATH is unset, but got a root store");
-    }
-
-    /// When RUSTFS_TLS_PATH is set to an empty string, `load_root_store_from_tls_path` must
-    /// return `None` to avoid accidentally trusting a CA bundle in the current directory.
-    #[test]
-    fn tls_path_empty_returns_none() {
-        let result = temp_env::with_var(rustfs_config::ENV_RUSTFS_TLS_PATH, Some(""), || load_root_store_from_tls_path());
-        assert!(result.is_none(), "expected None when RUSTFS_TLS_PATH is empty, but got a root store");
-    }
-
     /// Installing the rustls crypto provider when one is already set must not panic or return
     /// an error that surfaces to callers (the race-safe `get_default` check guards the install).
     #[test]

crates/ecstore/src/data_usage.rs

@@ -48,7 +48,14 @@ const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
 pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
 const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
 
-type UsageMemoryCache = Arc<RwLock<HashMap<String, (u64, SystemTime)>>>;
+#[derive(Debug, Clone)]
+struct CachedBucketUsage {
+    usage: BucketUsageInfo,
+    refreshed_at: SystemTime,
+    usage_updated_at: SystemTime,
+}
+
+type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
 type CacheUpdating = Arc<RwLock<bool>>;
 
 static USAGE_MEMORY_CACHE: OnceLock<UsageMemoryCache> = OnceLock::new();
@@ -403,21 +410,90 @@ pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Res
     Ok(usage)
 }
 
-/// Fast in-memory increment for immediate quota consistency
-pub async fn increment_bucket_usage_memory(bucket: &str, size_increment: u64) {
+async fn ensure_bucket_usage_cached(bucket: &str) {
+    let cache = memory_cache().read().await;
+    if cache.contains_key(bucket) {
+        return;
+    }
+    drop(cache);
+
+    update_usage_cache_if_needed().await;
+}
+
+fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTime) -> CachedBucketUsage {
+    CachedBucketUsage {
+        usage,
+        refreshed_at: SystemTime::now(),
+        usage_updated_at: updated_at,
+    }
+}
+
+fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
+    let now = SystemTime::now();
+    CachedBucketUsage {
+        usage,
+        refreshed_at: now,
+        usage_updated_at: now,
+    }
+}
+
+fn data_usage_info_updated_at(data_usage_info: &DataUsageInfo) -> SystemTime {
+    data_usage_info.last_update.unwrap_or(SystemTime::UNIX_EPOCH)
+}
+
+/// Fast in-memory update for immediate quota and admin usage consistency.
+pub async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option<u64>, new_size: u64) {
+    ensure_bucket_usage_cached(bucket).await;
+
     let mut cache = memory_cache().write().await;
-    let current = cache.entry(bucket.to_string()).or_insert_with(|| (0, SystemTime::now()));
-    current.0 += size_increment;
-    current.1 = SystemTime::now();
+    let entry = cache
+        .entry(bucket.to_string())
+        .or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
+
+    match previous_current_size {
+        Some(previous_size) => {
+            entry.usage.size = entry.usage.size.saturating_sub(previous_size).saturating_add(new_size);
+        }
+        None => {
+            entry.usage.size = entry.usage.size.saturating_add(new_size);
+            entry.usage.objects_count = entry.usage.objects_count.saturating_add(1);
+            entry.usage.versions_count = entry.usage.versions_count.saturating_add(1);
+        }
+    }
+
+    let now = SystemTime::now();
+    entry.refreshed_at = now;
+    entry.usage_updated_at = now;
+}
+
+/// Fast in-memory increment for immediate quota consistency.
+pub async fn increment_bucket_usage_memory(bucket: &str, size_increment: u64) {
+    record_bucket_object_write_memory(bucket, None, size_increment).await;
+}
+
+/// Fast in-memory update for successful object deletes.
+pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) {
+    ensure_bucket_usage_cached(bucket).await;
+
+    let mut cache = memory_cache().write().await;
+    let entry = cache
+        .entry(bucket.to_string())
+        .or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
+
+    entry.usage.size = entry.usage.size.saturating_sub(deleted_size);
+    if removed_current_object {
+        entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1);
+        entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1);
+    }
+
+    let now = SystemTime::now();
+    entry.refreshed_at = now;
+    entry.usage_updated_at = now;
 }
 
 /// Fast in-memory decrement for immediate quota consistency
 pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) {
-    let mut cache = memory_cache().write().await;
-    if let Some(current) = cache.get_mut(bucket) {
-        current.0 = current.0.saturating_sub(size_decrement);
-        current.1 = SystemTime::now();
-    }
+    record_bucket_object_delete_memory(bucket, size_decrement, size_decrement > 0).await;
 }
 
 /// Get bucket usage from in-memory cache
@@ -425,7 +501,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
     update_usage_cache_if_needed().await;
 
     let cache = memory_cache().read().await;
-    cache.get(bucket).map(|(usage, _)| *usage)
+    cache.get(bucket).map(|cached| cached.usage.size)
 }
 
 async fn update_usage_cache_if_needed() {
@@ -434,7 +510,7 @@ async fn update_usage_cache_if_needed() {
     let now = SystemTime::now();
 
     let cache = memory_cache().read().await;
-    let earliest_timestamp = cache.values().map(|(_, ts)| *ts).min();
+    let earliest_timestamp = cache.values().map(|cached| cached.refreshed_at).min();
     drop(cache);
 
     let age = match earliest_timestamp {
@@ -461,8 +537,12 @@ async fn update_usage_cache_if_needed() {
                 && let Ok(data_usage_info) = load_data_usage_from_backend(store.clone()).await
             {
                 let mut cache = cache_clone.write().await;
+                let usage_updated_at = data_usage_info_updated_at(&data_usage_info);
                 for (bucket_name, bucket_usage) in data_usage_info.buckets_usage.iter() {
-                    cache.insert(bucket_name.clone(), (bucket_usage.size, SystemTime::now()));
+                    cache.insert(
+                        bucket_name.clone(),
+                        cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at),
+                    );
                 }
             }
             let mut updating = updating_clone.write().await;
@@ -488,8 +568,12 @@ async fn update_usage_cache_if_needed() {
         && let Ok(data_usage_info) = load_data_usage_from_backend(store.clone()).await
     {
         let mut cache = memory_cache().write().await;
+        let usage_updated_at = data_usage_info_updated_at(&data_usage_info);
         for (bucket_name, bucket_usage) in data_usage_info.buckets_usage.iter() {
-            cache.insert(bucket_name.clone(), (bucket_usage.size, SystemTime::now()));
+            cache.insert(
+                bucket_name.clone(),
+                cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at),
+            );
         }
     }
 
@@ -497,15 +581,62 @@ async fn update_usage_cache_if_needed() {
     *updating = false;
 }
 
+pub async fn replace_bucket_usage_memory_from_info(data_usage_info: &DataUsageInfo) {
+    let usage_updated_at = data_usage_info_updated_at(data_usage_info);
+    let mut cache = memory_cache().write().await;
+    let mut next_cache = HashMap::new();
+
+    for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
+        if let Some(existing) = cache.get(bucket)
+            && existing.usage_updated_at > usage_updated_at
+        {
+            next_cache.insert(bucket.clone(), existing.clone());
+            continue;
+        }
+
+        next_cache.insert(bucket.clone(), cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at));
+    }
+
+    for (bucket, existing) in cache.iter() {
+        if !data_usage_info.buckets_usage.contains_key(bucket) && existing.usage_updated_at > usage_updated_at {
+            next_cache.insert(bucket.clone(), existing.clone());
+        }
+    }
+
+    *cache = next_cache;
+}
+
+pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageInfo) {
+    let cache = memory_cache().read().await;
+    if cache.is_empty() {
+        return;
+    }
+
+    let persisted_update = data_usage_info.last_update;
+    let mut changed = false;
+
+    for (bucket, cached) in cache.iter() {
+        if persisted_update.is_some_and(|persisted| cached.usage_updated_at <= persisted) {
+            continue;
+        }
+
+        data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
+        data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
+        changed = true;
+    }
+
+    if changed {
+        data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
+        data_usage_info.calculate_totals();
+    }
+}
+
 /// Sync memory cache with backend data (called by scanner)
 pub async fn sync_memory_cache_with_backend() -> Result<(), Error> {
     if let Some(store) = crate::global::GLOBAL_OBJECT_API.get() {
         match load_data_usage_from_backend(store.clone()).await {
             Ok(data_usage_info) => {
-                let mut cache = memory_cache().write().await;
-                for (bucket, bucket_usage) in data_usage_info.buckets_usage.iter() {
-                    cache.insert(bucket.clone(), (bucket_usage.size, SystemTime::now()));
-                }
+                replace_bucket_usage_memory_from_info(&data_usage_info).await;
             }
             Err(e) => {
                 debug!("Failed to sync memory cache with backend: {}", e);
@@ -687,6 +818,32 @@ pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate:
 mod tests {
     use super::*;
     use rustfs_data_usage::BucketUsageInfo;
+    use serial_test::serial;
+
+    async fn clear_usage_memory_cache_for_test() {
+        memory_cache().write().await.clear();
+        *cache_updating().write().await = false;
+    }
+
+    fn data_usage_info_for_test(bucket: &str, objects_count: u64, size: u64, last_update: SystemTime) -> DataUsageInfo {
+        let mut info = DataUsageInfo {
+            last_update: Some(last_update),
+            ..Default::default()
+        };
+        info.buckets_usage.insert(
+            bucket.to_string(),
+            BucketUsageInfo {
+                objects_count,
+                versions_count: objects_count,
+                size,
+                ..Default::default()
+            },
+        );
+        info.bucket_sizes.insert(bucket.to_string(), size);
+        info.buckets_count = info.buckets_usage.len() as u64;
+        info.calculate_totals();
+        info
+    }
 
     fn aggregate_for_test(
         inputs: Vec<(DiskUsageStatus, Result<Option<LocalUsageSnapshot>, Error>)>,
@@ -772,4 +929,101 @@ mod tests {
         assert_eq!(aggregated.buckets_count, 1);
         assert_eq!(aggregated.buckets_usage.get("bucket-a").map(|b| (b.objects_count, b.size)), Some((3, 42)));
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn memory_overlay_reflects_recent_delete_before_scanner_persists() {
+        clear_usage_memory_cache_for_test().await;
+
+        let persisted = data_usage_info_for_test("bucket-a", 1, 42, SystemTime::now() - Duration::from_secs(10));
+        replace_bucket_usage_memory_from_info(&persisted).await;
+        record_bucket_object_delete_memory("bucket-a", 42, true).await;
+
+        let mut response = persisted.clone();
+        apply_bucket_usage_memory_overlay(&mut response).await;
+
+        assert_eq!(response.objects_total_count, 0);
+        assert_eq!(response.objects_total_size, 0);
+        assert_eq!(
+            response
+                .buckets_usage
+                .get("bucket-a")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((0, 0))
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn memory_overlay_preserves_object_count_for_overwrite() {
+        clear_usage_memory_cache_for_test().await;
+
+        let persisted = data_usage_info_for_test("bucket-a", 1, 10, SystemTime::now() - Duration::from_secs(10));
+        replace_bucket_usage_memory_from_info(&persisted).await;
+        record_bucket_object_write_memory("bucket-a", Some(10), 20).await;
+
+        let mut response = persisted.clone();
+        apply_bucket_usage_memory_overlay(&mut response).await;
+
+        assert_eq!(response.objects_total_count, 1);
+        assert_eq!(response.objects_total_size, 20);
+        assert_eq!(
+            response
+                .buckets_usage
+                .get("bucket-a")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((1, 20))
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn memory_overlay_does_not_replace_newer_persisted_usage() {
+        clear_usage_memory_cache_for_test().await;
+
+        let now = SystemTime::now();
+        let old_persisted = data_usage_info_for_test("bucket-a", 1, 42, now - Duration::from_secs(10));
+        replace_bucket_usage_memory_from_info(&old_persisted).await;
+        record_bucket_object_delete_memory("bucket-a", 42, true).await;
+
+        let mut newer_persisted = data_usage_info_for_test("bucket-a", 2, 84, now + Duration::from_secs(10));
+        apply_bucket_usage_memory_overlay(&mut newer_persisted).await;
+
+        assert_eq!(newer_persisted.objects_total_count, 2);
+        assert_eq!(newer_persisted.objects_total_size, 84);
+        assert_eq!(
+            newer_persisted
+                .buckets_usage
+                .get("bucket-a")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((2, 84))
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn scanner_sync_preserves_newer_memory_update() {
+        clear_usage_memory_cache_for_test().await;
+
+        let now = SystemTime::now();
+        let old_persisted = data_usage_info_for_test("bucket-a", 1, 42, now - Duration::from_secs(10));
+        replace_bucket_usage_memory_from_info(&old_persisted).await;
+        record_bucket_object_delete_memory("bucket-a", 42, true).await;
+
+        let scanner_snapshot = data_usage_info_for_test("bucket-a", 1, 42, now - Duration::from_secs(5));
+        replace_bucket_usage_memory_from_info(&scanner_snapshot).await;
+
+        let mut response = scanner_snapshot.clone();
+        apply_bucket_usage_memory_overlay(&mut response).await;
+
+        assert_eq!(response.objects_total_count, 0);
+        assert_eq!(response.objects_total_size, 0);
+        assert_eq!(
+            response
+                .buckets_usage
+                .get("bucket-a")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((0, 0))
+        );
+    }
 }

crates/ecstore/src/disk/disk_store.rs

@@ -27,6 +27,8 @@ use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
 use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
+#[cfg(not(test))]
+use std::sync::OnceLock;
 use std::{
     path::PathBuf,
     sync::{
@@ -50,10 +52,54 @@ enum TimeoutHealthAction {
     IgnoreFailure,
 }
 
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum TimeoutHealthPolicy {
+    MarkFailure,
+    IgnoreScanner,
+}
+
+impl TimeoutHealthPolicy {
+    fn parse(raw: &str) -> Option<Self> {
+        match raw.trim().to_ascii_lowercase().as_str() {
+            rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE => Some(Self::MarkFailure),
+            rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER => Some(Self::IgnoreScanner),
+            _ => None,
+        }
+    }
+
+    fn scanner_timeout_health_action(self) -> TimeoutHealthAction {
+        match self {
+            Self::MarkFailure => TimeoutHealthAction::MarkFailure,
+            Self::IgnoreScanner => TimeoutHealthAction::IgnoreFailure,
+        }
+    }
+}
+
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
 
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum DriveTimeoutProfile {
+    Default,
+    HighLatency,
+}
+
+impl DriveTimeoutProfile {
+    fn parse(raw: &str) -> Option<Self> {
+        match raw.trim().to_ascii_lowercase().as_str() {
+            rustfs_config::DRIVE_TIMEOUT_PROFILE_DEFAULT => Some(Self::Default),
+            rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY => Some(Self::HighLatency),
+            _ => None,
+        }
+    }
+}
+
+#[cfg(not(test))]
+static DRIVE_TIMEOUT_PROFILE_CACHE: OnceLock<DriveTimeoutProfile> = OnceLock::new();
+#[cfg(not(test))]
+static DRIVE_TIMEOUT_HEALTH_POLICY_CACHE: OnceLock<TimeoutHealthPolicy> = OnceLock::new();
+
 lazy_static::lazy_static! {
     static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]);
     static ref TEST_BUCKET: String = ".rustfs.sys/tmp".to_string();
@@ -66,10 +112,39 @@ pub fn get_max_timeout_duration() -> Duration {
     ))
 }
 
-fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
+fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile {
+    let raw = rustfs_utils::get_env_str(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE);
+    if let Some(profile) = DriveTimeoutProfile::parse(&raw) {
+        return profile;
+    }
+    warn!(
+        env = rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
+        value = %raw,
+        default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE,
+        "Invalid drive timeout profile; falling back to default"
+    );
+    DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default)
+}
+
+fn get_drive_timeout_profile() -> DriveTimeoutProfile {
+    #[cfg(test)]
+    {
+        resolve_drive_timeout_profile_from_env()
+    }
+    #[cfg(not(test))]
+    {
+        *DRIVE_TIMEOUT_PROFILE_CACHE.get_or_init(resolve_drive_timeout_profile_from_env)
+    }
+}
+
+fn get_drive_timeout_duration(env_key: &str, default_secs: u64, high_latency_secs: Option<u64>) -> Duration {
+    let fallback_default = match (get_drive_timeout_profile(), high_latency_secs) {
+        (DriveTimeoutProfile::HighLatency, Some(secs)) => secs,
+        _ => default_secs,
+    };
     Duration::from_secs(
         rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
-            .unwrap_or(default_secs),
+            .unwrap_or(fallback_default),
     )
 }
 
@@ -77,6 +152,7 @@ pub fn get_drive_metadata_timeout() -> Duration {
     get_drive_timeout_duration(
         rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
         rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
+        Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
     )
 }
 
@@ -84,6 +160,7 @@ pub fn get_drive_disk_info_timeout() -> Duration {
     get_drive_timeout_duration(
         rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
         rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
+        Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
     )
 }
 
@@ -91,6 +168,7 @@ pub fn get_drive_list_dir_timeout() -> Duration {
     get_drive_timeout_duration(
         rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
         rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
+        Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
     )
 }
 
@@ -98,6 +176,7 @@ pub fn get_drive_walkdir_timeout() -> Duration {
     get_drive_timeout_duration(
         rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
         rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
+        Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
     )
 }
 
@@ -105,6 +184,7 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
     get_drive_timeout_duration(
         rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
         rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+        Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
     )
 }
 
@@ -122,6 +202,34 @@ pub fn get_drive_active_check_timeout() -> Duration {
     ))
 }
 
+fn resolve_drive_timeout_health_policy_from_env() -> TimeoutHealthPolicy {
+    let raw = rustfs_utils::get_env_str(
+        rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
+        rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION,
+    );
+    if let Some(policy) = TimeoutHealthPolicy::parse(&raw) {
+        return policy;
+    }
+    warn!(
+        env = rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
+        value = %raw,
+        default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION,
+        "Invalid drive timeout health action policy; falling back to default"
+    );
+    TimeoutHealthPolicy::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION).unwrap_or(TimeoutHealthPolicy::MarkFailure)
+}
+
+fn get_drive_timeout_health_policy() -> TimeoutHealthPolicy {
+    #[cfg(test)]
+    {
+        resolve_drive_timeout_health_policy_from_env()
+    }
+    #[cfg(not(test))]
+    {
+        *DRIVE_TIMEOUT_HEALTH_POLICY_CACHE.get_or_init(resolve_drive_timeout_health_policy_from_env)
+    }
+}
+
 /// DiskHealthTracker tracks the health status of a disk.
 /// Similar to Go's diskHealthTracker.
 #[derive(Debug)]
@@ -470,6 +578,8 @@ pub struct LocalDiskWrapper {
     cancel_token: CancellationToken,
     /// Disk ID for stale checking
     disk_id: Arc<RwLock<Option<Uuid>>>,
+    /// Timeout policy for scanner-sensitive operations, loaded once on wrapper initialization.
+    timeout_health_policy: TimeoutHealthPolicy,
 }
 
 impl LocalDiskWrapper {
@@ -486,6 +596,7 @@ impl LocalDiskWrapper {
             health_check: health_check && env_health_check,
             cancel_token: CancellationToken::new(),
             disk_id: Arc::new(RwLock::new(None)),
+            timeout_health_policy: get_drive_timeout_health_policy(),
         };
         record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
         wrapper
@@ -511,6 +622,10 @@ impl LocalDiskWrapper {
         self.health.record_capacity_probe(total, used, free);
     }
 
+    fn scanner_timeout_health_action(&self) -> TimeoutHealthAction {
+        self.timeout_health_policy.scanner_timeout_health_action()
+    }
+
     #[cfg(test)]
     pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
         self.health.force_runtime_state_for_test(state);
@@ -558,6 +673,7 @@ impl LocalDiskWrapper {
     /// Monitor disk writability periodically
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         let mut interval = time::interval(get_drive_active_check_interval());
+        let active_check_timeout = get_drive_active_check_timeout();
 
         loop {
             tokio::select! {
@@ -596,7 +712,7 @@ impl LocalDiskWrapper {
                         &test_obj,
                         &TEST_DATA,
                         true,
-                        get_drive_active_check_timeout(),
+                        active_check_timeout,
                     )
                     .await
                     .is_err()
@@ -691,6 +807,7 @@ impl LocalDiskWrapper {
     /// Monitor disk status and try to bring it back online
     async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         let check_every = get_drive_returning_probe_interval();
+        let active_check_timeout = get_drive_active_check_timeout();
 
         let mut interval = time::interval(check_every);
 
@@ -711,7 +828,7 @@ impl LocalDiskWrapper {
                         &test_obj,
                         &TEST_DATA,
                         false,
-                        get_drive_active_check_timeout(),
+                        active_check_timeout,
                     )
                     .await
                     {
@@ -893,10 +1010,11 @@ impl LocalDiskWrapper {
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        self.track_disk_health_with_op(
+        self.track_disk_health_with_op_and_timeout_action(
             "read_metadata",
             || async { self.disk.read_metadata(volume, path).await },
             get_drive_metadata_timeout(),
+            self.scanner_timeout_health_action(),
         )
         .await
     }
@@ -973,7 +1091,7 @@ impl DiskAPI for LocalDiskWrapper {
             return Err(DiskError::FaultyDisk);
         }
 
-        self.track_disk_health_with_op(
+        self.track_disk_health_with_op_and_timeout_action(
             "disk_info",
             || async {
                 let result = self.disk.disk_info(opts).await?;
@@ -987,6 +1105,7 @@ impl DiskAPI for LocalDiskWrapper {
                 Ok(result)
             },
             get_drive_disk_info_timeout(),
+            self.scanner_timeout_health_action(),
         )
         .await
     }
@@ -1021,7 +1140,7 @@ impl DiskAPI for LocalDiskWrapper {
             "walk_dir",
             || async { self.disk.walk_dir(opts, wr).await },
             get_drive_walkdir_timeout(),
-            TimeoutHealthAction::IgnoreFailure,
+            self.scanner_timeout_health_action(),
         )
         .await
     }
@@ -1130,10 +1249,11 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
-        self.track_disk_health_with_op(
+        self.track_disk_health_with_op_and_timeout_action(
             "list_dir",
             || async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
             get_drive_list_dir_timeout(),
+            self.scanner_timeout_health_action(),
         )
         .await
     }
@@ -1256,14 +1376,48 @@ mod tests {
     fn drive_metadata_timeout_uses_default_when_unset() {
         temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
             temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
-                assert_eq!(
-                    get_drive_metadata_timeout(),
-                    Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, || {
+                    assert_eq!(
+                        get_drive_metadata_timeout(),
+                        Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                    );
+                });
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_uses_high_latency_profile_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
+                temp_env::with_var(
+                    rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
+                    Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY),
+                    || {
+                        assert_eq!(
+                            get_drive_metadata_timeout(),
+                            Duration::from_secs(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS)
+                        );
+                    },
                 );
             });
         });
     }
 
+    #[test]
+    fn drive_metadata_timeout_invalid_profile_falls_back_to_default() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
+                temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, Some("invalid"), || {
+                    assert_eq!(
+                        get_drive_metadata_timeout(),
+                        Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                    );
+                });
+            });
+        });
+    }
+
     #[test]
     fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
         temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
@@ -1398,6 +1552,58 @@ mod tests {
 
     #[tokio::test]
     async fn walk_dir_writer_backpressure_timeout_does_not_mark_drive_failure() {
+        temp_env::async_with_vars(
+            [
+                (rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1")),
+                (
+                    rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
+                    Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER),
+                ),
+            ],
+            async {
+                let dir = tempfile::tempdir().expect("temp dir should be created");
+                let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
+                    .expect("endpoint should parse");
+                let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
+                let wrapper = LocalDiskWrapper::new(disk, false);
+                let bucket = "test-bucket";
+                let object = "test-object";
+
+                wrapper.make_volume(bucket).await.expect("bucket should be created");
+
+                let mut file_info = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
+                file_info.volume = bucket.to_string();
+                file_info.name = object.to_string();
+                file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
+                file_info.erasure.index = 1;
+
+                wrapper
+                    .write_metadata("", bucket, object, file_info)
+                    .await
+                    .expect("object metadata should be written");
+
+                let mut writer = PendingWriter;
+                let result = wrapper
+                    .walk_dir(
+                        WalkDirOptions {
+                            bucket: bucket.to_string(),
+                            recursive: true,
+                            ..Default::default()
+                        },
+                        &mut writer,
+                    )
+                    .await;
+
+                assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout);
+                assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
+                assert!(!wrapper.health.is_faulty());
+            },
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn walk_dir_writer_backpressure_timeout_marks_drive_failure_by_default() {
         temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
             let dir = tempfile::tempdir().expect("temp dir should be created");
             let endpoint =
@@ -1433,8 +1639,7 @@ mod tests {
                 .await;
 
             assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout);
-            assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
-            assert!(!wrapper.health.is_faulty());
+            assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect);
         })
         .await;
     }
@@ -1462,6 +1667,40 @@ mod tests {
         assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect);
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn drive_timeout_health_policy_defaults_to_mark_failure() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, || {
+            let policy = get_drive_timeout_health_policy();
+            assert_eq!(policy, TimeoutHealthPolicy::MarkFailure);
+            assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure);
+        });
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn drive_timeout_health_policy_respects_ignore_scanner() {
+        temp_env::with_var(
+            rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
+            Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER),
+            || {
+                let policy = get_drive_timeout_health_policy();
+                assert_eq!(policy, TimeoutHealthPolicy::IgnoreScanner);
+                assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::IgnoreFailure);
+            },
+        );
+    }
+
+    #[test]
+    #[serial_test::serial]
+    fn drive_timeout_health_policy_invalid_value_falls_back_to_default() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, Some("invalid"), || {
+            let policy = get_drive_timeout_health_policy();
+            assert_eq!(policy, TimeoutHealthPolicy::MarkFailure);
+            assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure);
+        });
+    }
+
     #[test]
     fn reset_for_store_init_retry_clears_faulty_and_back_online() {
         let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry").expect("endpoint should parse");

crates/ecstore/src/disk/local.rs

@@ -1163,26 +1163,45 @@ impl LocalDisk {
     }
     // write_all_internal do write file
     async fn write_all_internal(&self, file_path: &Path, data: InternalBuf<'_>, sync: bool, skip_parent: &Path) -> Result<()> {
-        let flags = O_CREATE | O_WRONLY | O_TRUNC;
-
-        let mut f = {
-            if sync {
-                // TODO: support sync
-                self.open_file(file_path, flags, skip_parent).await?
-            } else {
-                self.open_file(file_path, flags, skip_parent).await?
-            }
+        let skip_parent = if skip_parent.as_os_str().is_empty() {
+            self.root.as_path()
+        } else {
+            skip_parent
         };
 
         match data {
             InternalBuf::Ref(buf) => {
+                let mut f = self.open_file(file_path, O_CREATE | O_WRONLY | O_TRUNC, skip_parent).await?;
                 f.write_all(buf).await.map_err(to_file_error)?;
             }
             InternalBuf::Owned(buf) => {
-                f.write_all(buf.as_ref()).await.map_err(to_file_error)?;
+                let path = file_path.to_path_buf();
+                if let Some(parent) = path.parent()
+                    && parent != skip_parent
+                {
+                    os::make_dir_all(parent, skip_parent).await?;
+                }
+
+                tokio::task::spawn_blocking(move || {
+                    let mut f = std::fs::OpenOptions::new()
+                        .create(true)
+                        .write(true)
+                        .truncate(true)
+                        .open(path)
+                        .map_err(to_file_error)?;
+
+                    std::io::Write::write_all(&mut f, buf.as_ref()).map_err(to_file_error)?;
+
+                    Ok::<(), std::io::Error>(())
+                })
+                .await
+                .map_err(DiskError::from)??;
             }
         }
 
+        // Keep existing durability contract: this path intentionally ignores `sync`.
+        let _ = sync;
+
         Ok(())
     }
 
@@ -1993,11 +2012,7 @@ impl DiskAPI for LocalDisk {
             let meta_op = match lstat_std(&src_file_path).map_err(|e| to_file_error(e).into()) {
                 Ok(meta) => Some(meta),
                 Err(e) => {
-                    if e != DiskError::FileNotFound {
-                        return Err(e);
-                    }
-
-                    None
+                    return Err(e);
                 }
             };
 
@@ -2009,10 +2024,22 @@ impl DiskAPI for LocalDisk {
             }
 
             remove_std(&dst_file_path).map_err(to_file_error)?;
+        } else {
+            let meta = lstat_std(&src_file_path).map_err(|e| -> DiskError { to_file_error(e).into() })?;
+            if meta.is_dir() {
+                warn!("rename_part src is dir {:?}", &src_file_path);
+                return Err(DiskError::FileAccessDenied);
+            }
         }
 
         rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await?;
 
+        let dst_meta = lstat_std(&dst_file_path).map_err(|e| -> DiskError { to_file_error(e).into() })?;
+        if src_is_dir != dst_meta.is_dir() {
+            warn!("rename_part dst type changed after rename {:?}", &dst_file_path);
+            return Err(DiskError::FileAccessDenied);
+        }
+
         self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
 
         if let Some(parent) = src_file_path.parent() {

crates/ecstore/src/erasure_coding/erasure.rs

@@ -106,7 +106,7 @@ impl LegacyReedSolomonEncoder {
         Ok(())
     }
 
-    fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    fn reconstruct_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         let shard_len = shards
             .iter()
             .find_map(|s| s.as_ref().map(|v| v.len()))
@@ -172,6 +172,15 @@ impl LegacyReedSolomonEncoder {
 
         Ok(())
     }
+
+    fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+        self.reconstruct_data(shards)?;
+        self.encode_parity(shards)
+    }
+
+    fn encode_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+        encode_parity_shards(shards, self.data_shards, self.parity_shards, |shards| self.encode(shards))
+    }
 }
 
 /// Reed-Solomon encoder using reed-solomon-erasure
@@ -224,8 +233,8 @@ impl ReedSolomonEncoder {
         }
     }
 
-    /// Reconstruct missing shards.
-    pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+    /// Reconstruct missing data shards.
+    pub fn reconstruct_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         if let Some(ref rs) = self.encoder {
             rs.reconstruct_data(shards)
                 .map_err(|e| io::Error::other(format!("Reed-Solomon reconstruct failed: {e:?}")))
@@ -233,6 +242,58 @@ impl ReedSolomonEncoder {
             Ok(())
         }
     }
+
+    /// Reconstruct missing data shards and regenerate parity shards.
+    pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+        self.reconstruct_data(shards)?;
+        self.encode_parity(shards)
+    }
+
+    fn encode_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+        encode_parity_shards(shards, self.data_shards, self.parity_shards, |shards| self.encode(shards))
+    }
+}
+
+fn encode_parity_shards<F>(shards: &mut [Option<Vec<u8>>], data_shards: usize, parity_shards: usize, encode: F) -> io::Result<()>
+where
+    F: FnOnce(SmallVec<[&mut [u8]; 16]>) -> io::Result<()>,
+{
+    let expected_shards = data_shards + parity_shards;
+    if shards.len() != expected_shards {
+        return Err(io::Error::other(format!(
+            "invalid shard count: got {}, expected {}",
+            shards.len(),
+            expected_shards
+        )));
+    }
+
+    let shard_len = shards
+        .iter()
+        .find_map(|s| s.as_ref().map(Vec::len))
+        .ok_or_else(|| io::Error::other("No valid shards found for parity encoding"))?;
+
+    for shard in shards.iter_mut().skip(data_shards) {
+        if shard.is_none() {
+            *shard = Some(vec![0; shard_len]);
+        }
+    }
+
+    let mut shard_refs: SmallVec<[&mut [u8]; 16]> = SmallVec::new();
+    for (index, shard) in shards.iter_mut().enumerate() {
+        let shard = shard
+            .as_mut()
+            .ok_or_else(|| io::Error::other(format!("missing shard {index} after data reconstruction")))?;
+        if shard.len() != shard_len {
+            return Err(io::Error::other(format!(
+                "inconsistent shard length at index {index}: got {}, expected {}",
+                shard.len(),
+                shard_len
+            )));
+        }
+        shard_refs.push(shard.as_mut_slice());
+    }
+
+    encode(shard_refs)
 }
 
 /// Erasure coding utility for data reliability using Reed-Solomon codes.
@@ -392,7 +453,7 @@ impl Erasure {
         Ok(shards)
     }
 
-    /// Decode and reconstruct missing shards in-place.
+    /// Decode and reconstruct missing data shards in-place.
     ///
     /// # Arguments
     /// * `shards` - Mutable slice of optional shard data. Missing shards should be `None`.
@@ -400,6 +461,25 @@ impl Erasure {
     /// # Returns
     /// Ok if reconstruction succeeds, error otherwise.
     pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
+        if self.parity_shards > 0 {
+            if self.uses_legacy {
+                if let Some(encoder) = self.legacy_encoder.as_ref() {
+                    encoder.reconstruct_data(shards)?;
+                } else {
+                    warn!("parity_shards > 0, uses_legacy but legacy_encoder is None");
+                }
+            } else if let Some(encoder) = self.encoder.as_ref() {
+                encoder.reconstruct_data(shards)?;
+            } else {
+                warn!("parity_shards > 0, but encoder is None");
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Decode and reconstruct missing data shards, then regenerate parity shards.
+    pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         if self.parity_shards > 0 {
             if self.uses_legacy {
                 if let Some(encoder) = self.legacy_encoder.as_ref() {
@@ -534,6 +614,131 @@ mod tests {
 
     use super::*;
 
+    fn optional_shards(shards: &[Bytes]) -> Vec<Option<Vec<u8>>> {
+        shards.iter().map(|shard| Some(shard.to_vec())).collect()
+    }
+
+    #[test]
+    fn decode_data_keeps_missing_parity_shard_unreconstructed() {
+        let erasure = Erasure::new(2, 2, 64);
+        let data = b"read decode should not rebuild parity";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let mut shards = optional_shards(&encoded);
+        let missing_parity = erasure.data_shards;
+        shards[missing_parity] = None;
+
+        erasure.decode_data(&mut shards).expect("decode should succeed");
+
+        assert!(shards[missing_parity].is_none(), "read decode should leave parity missing");
+        for index in 0..erasure.data_shards {
+            assert_eq!(
+                shards[index].as_deref(),
+                Some(encoded[index].as_ref()),
+                "data shard {index} should remain unchanged"
+            );
+        }
+    }
+
+    #[test]
+    fn legacy_decode_data_keeps_missing_parity_shard_unreconstructed() {
+        let erasure = Erasure::new_with_options(2, 2, 64, true);
+        let data = b"legacy read decode should not rebuild parity";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let mut shards = optional_shards(&encoded);
+        let missing_parity = erasure.data_shards + 1;
+        shards[missing_parity] = None;
+
+        erasure.decode_data(&mut shards).expect("decode should succeed");
+
+        assert!(shards[missing_parity].is_none(), "legacy read decode should leave parity missing");
+        for index in 0..erasure.data_shards {
+            assert_eq!(
+                shards[index].as_deref(),
+                Some(encoded[index].as_ref()),
+                "legacy data shard {index} should remain unchanged"
+            );
+        }
+    }
+
+    #[test]
+    fn decode_data_and_parity_leaves_complete_shards_unchanged() {
+        let erasure = Erasure::new(4, 2, 128);
+        let data = b"complete shards should not be changed";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let original = optional_shards(&encoded);
+        let mut shards = original.clone();
+
+        erasure
+            .decode_data_and_parity(&mut shards)
+            .expect("decode should succeed without missing shards");
+
+        assert_eq!(shards, original);
+    }
+
+    #[test]
+    fn decode_data_and_parity_reconstructs_missing_parity_shard() {
+        let erasure = Erasure::new(2, 2, 64);
+        let data = b"parity shard must be rebuilt";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let mut shards = optional_shards(&encoded);
+        shards[2] = None;
+
+        erasure
+            .decode_data_and_parity(&mut shards)
+            .expect("decode should rebuild parity");
+
+        for (index, shard) in shards.iter().enumerate() {
+            assert_eq!(
+                shard.as_deref(),
+                Some(encoded[index].as_ref()),
+                "shard {index} should match encoded source"
+            );
+        }
+    }
+
+    #[test]
+    fn decode_data_and_parity_reconstructs_missing_data_and_parity_shards() {
+        let erasure = Erasure::new(4, 2, 128);
+        let data = b"data and parity shards should both be reconstructed";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let mut shards = optional_shards(&encoded);
+        shards[1] = None;
+        shards[4] = None;
+
+        erasure
+            .decode_data_and_parity(&mut shards)
+            .expect("decode should rebuild all missing shards");
+
+        for (index, shard) in shards.iter().enumerate() {
+            assert_eq!(
+                shard.as_deref(),
+                Some(encoded[index].as_ref()),
+                "shard {index} should match encoded source"
+            );
+        }
+    }
+
+    #[test]
+    fn legacy_decode_data_and_parity_reconstructs_missing_parity_shard() {
+        let erasure = Erasure::new_with_options(2, 2, 64, true);
+        let data = b"legacy parity shard must be rebuilt";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let mut shards = optional_shards(&encoded);
+        shards[3] = None;
+
+        erasure
+            .decode_data_and_parity(&mut shards)
+            .expect("decode should rebuild parity");
+
+        for (index, shard) in shards.iter().enumerate() {
+            assert_eq!(
+                shard.as_deref(),
+                Some(encoded[index].as_ref()),
+                "shard {index} should match encoded source"
+            );
+        }
+    }
+
     #[test]
     fn test_shard_file_size_cases2() {
         let erasure = Erasure::new(12, 4, 1024 * 1024);

crates/ecstore/src/erasure_coding/heal.rs

@@ -49,6 +49,10 @@ impl super::Erasure {
             end_block += 1;
         }
 
+        let available_writers = writers.iter().filter(|w| w.is_some()).count();
+        let write_quorum = available_writers.max(1);
+        let mut writers = MultiWriter::new(writers, write_quorum);
+
         for _ in start_block..end_block {
             let (mut shards, errs) = reader.read().await;
 
@@ -63,7 +67,7 @@ impl super::Erasure {
             }
 
             if self.parity_shards > 0 {
-                self.decode_data(&mut shards)?;
+                self.decode_data_and_parity(&mut shards)?;
             }
 
             let shards = shards
@@ -71,15 +75,122 @@ impl super::Erasure {
                 .map(|s| Bytes::from(s.unwrap_or_default()))
                 .collect::<Vec<_>>();
 
-            // Calculate proper write quorum for heal operation
-            // For heal, we only write to disks that need healing, so write quorum should be
-            // the number of available writers (disks that need healing)
-            let available_writers = writers.iter().filter(|w| w.is_some()).count();
-            let write_quorum = available_writers.max(1); // At least 1 writer must succeed
-            let mut writers = MultiWriter::new(writers, write_quorum);
             writers.write(shards).await?;
         }
 
+        writers.shutdown().await?;
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::erasure_coding::{CustomWriter, Erasure};
+    use rustfs_utils::HashAlgorithm;
+    use std::io::Cursor;
+
+    #[tokio::test]
+    async fn heal_reconstructs_missing_parity_shard() {
+        let erasure = Erasure::new(2, 2, 64);
+        let data = b"heal should write a rebuilt parity shard";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+        let missing_parity = erasure.data_shards;
+
+        let readers = encoded
+            .iter()
+            .enumerate()
+            .map(|(index, shard)| {
+                if index == missing_parity {
+                    None
+                } else {
+                    Some(BitrotReader::new(
+                        Cursor::new(shard.to_vec()),
+                        erasure.shard_size(),
+                        HashAlgorithm::None,
+                        false,
+                    ))
+                }
+            })
+            .collect::<Vec<_>>();
+
+        let mut writers = (0..erasure.total_shard_count())
+            .map(|index| {
+                if index == missing_parity {
+                    Some(BitrotWriterWrapper::new(
+                        CustomWriter::new_inline_buffer(),
+                        erasure.shard_size(),
+                        HashAlgorithm::None,
+                    ))
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        erasure
+            .heal(&mut writers, readers, data.len(), &[])
+            .await
+            .expect("heal should rebuild parity");
+
+        let healed = writers[missing_parity]
+            .take()
+            .expect("parity writer should remain")
+            .into_inline_data()
+            .expect("inline writer should retain data");
+        assert_eq!(healed, encoded[missing_parity].to_vec());
+    }
+
+    #[tokio::test]
+    async fn heal_reconstructs_missing_data_shard_across_multiple_blocks() {
+        let erasure = Erasure::new(3, 2, 96);
+        let data = (0..erasure.block_size * 2 + 17)
+            .map(|index| (index % 251) as u8)
+            .collect::<Vec<_>>();
+        let encoded = erasure.encode_data(&data).expect("encode should succeed");
+        let missing_data = 1;
+
+        let readers = encoded
+            .iter()
+            .enumerate()
+            .map(|(index, shard)| {
+                if index == missing_data {
+                    None
+                } else {
+                    Some(BitrotReader::new(
+                        Cursor::new(shard.to_vec()),
+                        erasure.shard_size(),
+                        HashAlgorithm::None,
+                        false,
+                    ))
+                }
+            })
+            .collect::<Vec<_>>();
+
+        let mut writers = (0..erasure.total_shard_count())
+            .map(|index| {
+                if index == missing_data {
+                    Some(BitrotWriterWrapper::new(
+                        CustomWriter::new_inline_buffer(),
+                        erasure.shard_size(),
+                        HashAlgorithm::None,
+                    ))
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        erasure
+            .heal(&mut writers, readers, data.len(), &[])
+            .await
+            .expect("heal should rebuild data");
+
+        let healed = writers[missing_data]
+            .take()
+            .expect("data writer should remain")
+            .into_inline_data()
+            .expect("inline writer should retain data");
+        assert_eq!(healed, encoded[missing_data].to_vec());
+    }
+}

crates/ecstore/src/rpc/http_auth.rs

@@ -72,7 +72,7 @@ fn signature_payload(url: &str, method: &Method, timestamp: i64) -> String {
 /// Generate HMAC-SHA256 signature for the given data
 fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) -> String {
     let data = signature_payload(url, method, timestamp);
-    let mut mac = HmacSha256::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
+    let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
     mac.update(data.as_bytes());
     let result = mac.finalize();
     general_purpose::STANDARD.encode(result.into_bytes())
@@ -84,7 +84,7 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
     };
 
     let data = signature_payload(url, method, timestamp);
-    let mut mac = HmacSha256::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
+    let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
     mac.update(data.as_bytes());
     mac.verify_slice(&signature).is_ok()
 }

crates/ecstore/src/rpc/internode_data_transport.rs

@@ -17,12 +17,14 @@ use crate::disk::{FileReader, FileWriter};
 use crate::rpc::build_auth_headers;
 use async_trait::async_trait;
 use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
-use rustfs_config::{DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT};
+use rustfs_config::{
+    DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
+    KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
+};
 use rustfs_rio::{HttpReader, HttpWriter};
 use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 
-pub const INTERNODE_DATA_TRANSPORT_TCP: &str = "tcp";
 static INTERNODE_DATA_TRANSPORT: OnceLock<std::result::Result<Arc<dyn InternodeDataTransport>, String>> = OnceLock::new();
 
 const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
@@ -32,29 +34,36 @@ const CONTENT_TYPE_JSON: &str = "application/json";
 
 fn unsupported_transport_message(transport: &str) -> String {
     format!(
-        "invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {DEFAULT_INTERNODE_DATA_TRANSPORT}, {INTERNODE_DATA_TRANSPORT_TCP}"
+        "invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {}",
+        KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS.join(", ")
     )
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 pub struct InternodeDataTransportCapabilities {
-    pub stream_read: bool,
-    pub stream_write: bool,
-    pub walk_dir: bool,
-    pub registered_memory: bool,
-    pub scatter_gather: bool,
-    pub zero_copy_receive: bool,
+    /// Backend can open a streaming remote disk reader.
+    pub streaming_read: bool,
+    /// Backend can open a streaming remote disk writer.
+    pub streaming_write: bool,
+    /// Backend can stream walk-dir responses.
+    pub streaming_walk_dir: bool,
+    /// Backend preserves in-order delivery for each opened transfer.
+    pub ordered_delivery: bool,
+    /// Largest payload the backend accepts for one transfer, or no RustFS-level cap.
+    pub max_transfer_size: Option<usize>,
+    /// Backend can participate in the behavior-preserving TCP fallback path.
+    pub fallback_supported: bool,
 }
 
 impl InternodeDataTransportCapabilities {
     pub const fn tcp_http() -> Self {
         Self {
-            stream_read: true,
-            stream_write: true,
-            walk_dir: true,
-            registered_memory: false,
-            scatter_gather: false,
-            zero_copy_receive: false,
+            streaming_read: true,
+            streaming_write: true,
+            streaming_walk_dir: true,
+            ordered_delivery: true,
+            max_transfer_size: None,
+            fallback_supported: true,
         }
     }
 }
@@ -87,6 +96,14 @@ pub struct WalkDirStreamRequest {
     pub stall_timeout: Option<Duration>,
 }
 
+/// Data-plane stream opener used by `RemoteDisk`.
+///
+/// This boundary is limited to remote disk streams that can move large payloads.
+/// Internode metadata, lock, health, and administrative calls remain on the
+/// existing gRPC control plane.
+///
+/// Buffer ownership, backend selection, and fallback expectations are documented
+/// in `crates/ecstore/docs/internode-transport/`.
 #[async_trait]
 pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
     async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader>;
@@ -215,16 +232,25 @@ mod tests {
         assert_eq!(
             transport.capabilities(),
             InternodeDataTransportCapabilities {
-                stream_read: true,
-                stream_write: true,
-                walk_dir: true,
-                registered_memory: false,
-                scatter_gather: false,
-                zero_copy_receive: false,
+                streaming_read: true,
+                streaming_write: true,
+                streaming_walk_dir: true,
+                ordered_delivery: true,
+                max_transfer_size: None,
+                fallback_supported: true,
             }
         );
     }
 
+    #[test]
+    fn tcp_http_capabilities_are_conservative() {
+        let capabilities = TcpHttpInternodeDataTransport.capabilities();
+
+        assert!(capabilities.ordered_delivery);
+        assert_eq!(capabilities.max_transfer_size, None);
+        assert!(capabilities.fallback_supported);
+    }
+
     #[test]
     fn read_file_stream_url_encodes_query_values() {
         let url = build_read_file_stream_url(&ReadStreamRequest {
@@ -302,20 +328,36 @@ mod tests {
         }
     }
 
+    #[test]
+    fn transport_config_known_backends_are_current_oss_values() {
+        assert_eq!(
+            KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
+            &[DEFAULT_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP]
+        );
+
+        for configured in KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS {
+            let transport = build_internode_data_transport(Some(configured)).unwrap();
+
+            assert_eq!(transport.name(), DEFAULT_INTERNODE_DATA_TRANSPORT);
+        }
+    }
+
     #[test]
     fn transport_config_rejects_unknown_backend() {
-        let err = build_internode_data_transport(Some("rdma")).expect_err("unknown backend should fail closed");
+        let err = build_internode_data_transport(Some("unsupported-backend")).expect_err("unknown backend should fail closed");
 
         assert!(err.to_string().contains(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT));
-        assert!(err.to_string().contains("rdma"));
+        assert!(err.to_string().contains("unsupported-backend"));
+        assert!(err.to_string().contains("supported values: tcp-http, tcp"));
     }
 
     #[test]
     fn cached_transport_config_error_uses_raw_message() {
-        let err = build_internode_data_transport_result(Some("rdma")).expect_err("unknown backend should fail closed");
+        let err =
+            build_internode_data_transport_result(Some("unsupported-backend")).expect_err("unknown backend should fail closed");
 
         assert!(!err.starts_with("io error "));
         assert!(err.contains(ENV_RUSTFS_INTERNODE_DATA_TRANSPORT));
-        assert!(err.contains("rdma"));
+        assert!(err.contains("unsupported-backend"));
     }
 }

crates/ecstore/src/rpc/remote_disk.rs

@@ -35,7 +35,8 @@ use futures::lock::Mutex;
 use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_io_metrics::internode_metrics::{
-    INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, global_internode_metrics,
+    INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC,
+    global_internode_metrics,
 };
 use rustfs_protos::evict_failed_connection;
 use rustfs_protos::proto_gen::node_service::RenamePartRequest;
@@ -1558,7 +1559,10 @@ impl DiskAPI for RemoteDisk {
                 let data_len = data.len();
                 let disk = self.disk_ref().await;
                 let mut client = self.get_client().await.map_err(|err| {
-                    global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
+                    global_internode_metrics().record_error_for_operation_and_backend(
+                        INTERNODE_OPERATION_GRPC_WRITE_ALL,
+                        INTERNODE_TRANSPORT_BACKEND_GRPC,
+                    );
                     Error::other(format!("can not get client, err: {err}"))
                 })?;
                 let request = Request::new(WriteAllRequest {
@@ -1568,19 +1572,32 @@ impl DiskAPI for RemoteDisk {
                     data,
                 });
 
-                global_internode_metrics().record_outgoing_request_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
+                global_internode_metrics().record_outgoing_request_for_operation_and_backend(
+                    INTERNODE_OPERATION_GRPC_WRITE_ALL,
+                    INTERNODE_TRANSPORT_BACKEND_GRPC,
+                );
                 let response = match client.write_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
-                        global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
+                        global_internode_metrics().record_error_for_operation_and_backend(
+                            INTERNODE_OPERATION_GRPC_WRITE_ALL,
+                            INTERNODE_TRANSPORT_BACKEND_GRPC,
+                        );
                         return Err(err.into());
                     }
                 };
 
-                global_internode_metrics().record_sent_bytes_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL, data_len);
+                global_internode_metrics().record_sent_bytes_for_operation_and_backend(
+                    INTERNODE_OPERATION_GRPC_WRITE_ALL,
+                    INTERNODE_TRANSPORT_BACKEND_GRPC,
+                    data_len,
+                );
 
                 if !response.success {
-                    global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
+                    global_internode_metrics().record_error_for_operation_and_backend(
+                        INTERNODE_OPERATION_GRPC_WRITE_ALL,
+                        INTERNODE_TRANSPORT_BACKEND_GRPC,
+                    );
                     return Err(response.error.unwrap_or_default().into());
                 }
 
@@ -1599,7 +1616,10 @@ impl DiskAPI for RemoteDisk {
             || async {
                 let disk = self.disk_ref().await;
                 let mut client = self.get_client().await.map_err(|err| {
-                    global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
+                    global_internode_metrics().record_error_for_operation_and_backend(
+                        INTERNODE_OPERATION_GRPC_READ_ALL,
+                        INTERNODE_TRANSPORT_BACKEND_GRPC,
+                    );
                     Error::other(format!("can not get client, err: {err}"))
                 })?;
                 let request = Request::new(ReadAllRequest {
@@ -1608,22 +1628,34 @@ impl DiskAPI for RemoteDisk {
                     path: path.to_string(),
                 });
 
-                global_internode_metrics().record_outgoing_request_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
+                global_internode_metrics().record_outgoing_request_for_operation_and_backend(
+                    INTERNODE_OPERATION_GRPC_READ_ALL,
+                    INTERNODE_TRANSPORT_BACKEND_GRPC,
+                );
                 let response = match client.read_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
-                        global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
+                        global_internode_metrics().record_error_for_operation_and_backend(
+                            INTERNODE_OPERATION_GRPC_READ_ALL,
+                            INTERNODE_TRANSPORT_BACKEND_GRPC,
+                        );
                         return Err(err.into());
                     }
                 };
 
                 if !response.success {
-                    global_internode_metrics().record_error_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
+                    global_internode_metrics().record_error_for_operation_and_backend(
+                        INTERNODE_OPERATION_GRPC_READ_ALL,
+                        INTERNODE_TRANSPORT_BACKEND_GRPC,
+                    );
                     return Err(response.error.unwrap_or_default().into());
                 }
 
-                global_internode_metrics()
-                    .record_recv_bytes_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL, response.data.len());
+                global_internode_metrics().record_recv_bytes_for_operation_and_backend(
+                    INTERNODE_OPERATION_GRPC_READ_ALL,
+                    INTERNODE_TRANSPORT_BACKEND_GRPC,
+                    response.data.len(),
+                );
                 Ok(response.data)
             },
             get_max_timeout_duration(),
@@ -1671,10 +1703,12 @@ impl DiskAPI for RemoteDisk {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::rpc::TcpHttpInternodeDataTransport;
+    use crate::rpc::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
     use rustfs_common::GLOBAL_CONN_MAP;
-    use std::sync::Once;
-    use tokio::io::duplex;
+    use std::pin::Pin;
+    use std::sync::{Mutex as StdMutex, Once};
+    use std::task::{Context, Poll};
+    use tokio::io::{ReadBuf, duplex};
     use tokio::net::TcpListener;
     use tonic::transport::Endpoint as TonicEndpoint;
     use tracing::Level;
@@ -1682,6 +1716,96 @@ mod tests {
 
     static INIT: Once = Once::new();
 
+    #[derive(Debug, Clone)]
+    enum RecordedTransportCall {
+        Read(ReadStreamRequest),
+        Write(WriteStreamRequest),
+        WalkDir(WalkDirStreamRequest),
+    }
+
+    #[derive(Debug, Clone, Default)]
+    struct RecordingInternodeDataTransport {
+        calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
+    }
+
+    impl RecordingInternodeDataTransport {
+        fn calls(&self) -> Vec<RecordedTransportCall> {
+            self.calls.lock().expect("recorded transport calls lock poisoned").clone()
+        }
+
+        fn record(&self, call: RecordedTransportCall) {
+            self.calls.lock().expect("recorded transport calls lock poisoned").push(call);
+        }
+    }
+
+    #[derive(Debug, Default)]
+    struct EmptyTestReader;
+
+    impl AsyncRead for EmptyTestReader {
+        fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[derive(Debug, Default)]
+    struct SinkTestWriter;
+
+    impl AsyncWrite for SinkTestWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
+            Poll::Ready(Ok(buf.len()))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl InternodeDataTransport for RecordingInternodeDataTransport {
+        async fn open_read(&self, request: ReadStreamRequest) -> Result<FileReader> {
+            self.record(RecordedTransportCall::Read(request));
+            Ok(Box::new(EmptyTestReader))
+        }
+
+        async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
+            self.record(RecordedTransportCall::Write(request));
+            Ok(Box::new(SinkTestWriter))
+        }
+
+        async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader> {
+            self.record(RecordedTransportCall::WalkDir(request));
+            Ok(Box::new(EmptyTestReader))
+        }
+
+        fn name(&self) -> &'static str {
+            "recording"
+        }
+
+        fn capabilities(&self) -> InternodeDataTransportCapabilities {
+            InternodeDataTransportCapabilities::tcp_http()
+        }
+    }
+
+    async fn new_remote_disk_with_transport(data_transport: Arc<dyn InternodeDataTransport>) -> RemoteDisk {
+        let endpoint = Endpoint {
+            url: url::Url::parse("http://remote-node:9000/data/rustfs0").unwrap(),
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+        let disk_option = DiskOption {
+            cleanup: false,
+            health_check: false,
+        };
+
+        RemoteDisk::new(&endpoint, &disk_option, data_transport).await.unwrap()
+    }
+
     fn init_tracing(filter_level: Level) {
         INIT.call_once(|| {
             let _ = tracing_subscriber::fmt()
@@ -1925,6 +2049,102 @@ mod tests {
         assert_eq!(remote_disk.disk_ref().await, disk_id.to_string());
     }
 
+    #[tokio::test]
+    async fn test_remote_disk_read_file_stream_uses_configured_data_transport() {
+        let transport = RecordingInternodeDataTransport::default();
+        let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
+        let expected_disk = remote_disk.disk_ref().await;
+
+        let _reader = remote_disk.read_file_stream("bucket", "object/part.1", 7, 11).await.unwrap();
+
+        let calls = transport.calls();
+        assert_eq!(calls.len(), 1);
+        match &calls[0] {
+            RecordedTransportCall::Read(request) => {
+                assert_eq!(request.endpoint, "http://remote-node:9000");
+                assert_eq!(request.disk, expected_disk);
+                assert_eq!(request.volume, "bucket");
+                assert_eq!(request.path, "object/part.1");
+                assert_eq!(request.offset, 7);
+                assert_eq!(request.length, 11);
+            }
+            other => panic!("expected read transport call, got {other:?}"),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_remote_disk_create_and_append_file_use_configured_data_transport() {
+        let transport = RecordingInternodeDataTransport::default();
+        let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
+        let expected_disk = remote_disk.disk_ref().await;
+
+        let _created = remote_disk
+            .create_file("orig-bucket", "bucket", "object/part.1", 4096)
+            .await
+            .unwrap();
+        let _appended = remote_disk.append_file("bucket", "object/part.2").await.unwrap();
+
+        let calls = transport.calls();
+        assert_eq!(calls.len(), 2);
+
+        match &calls[0] {
+            RecordedTransportCall::Write(request) => {
+                assert_eq!(request.endpoint, "http://remote-node:9000");
+                assert_eq!(request.disk, expected_disk);
+                assert_eq!(request.volume, "bucket");
+                assert_eq!(request.path, "object/part.1");
+                assert!(!request.append);
+                assert_eq!(request.size, 4096);
+            }
+            other => panic!("expected create write transport call, got {other:?}"),
+        }
+
+        match &calls[1] {
+            RecordedTransportCall::Write(request) => {
+                assert_eq!(request.endpoint, "http://remote-node:9000");
+                assert_eq!(request.disk, expected_disk);
+                assert_eq!(request.volume, "bucket");
+                assert_eq!(request.path, "object/part.2");
+                assert!(request.append);
+                assert_eq!(request.size, 0);
+            }
+            other => panic!("expected append write transport call, got {other:?}"),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_remote_disk_walk_dir_uses_configured_data_transport() {
+        let transport = RecordingInternodeDataTransport::default();
+        let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
+        let expected_disk = remote_disk.disk_ref().await;
+        let opts = WalkDirOptions {
+            bucket: "bucket".to_string(),
+            base_dir: "prefix".to_string(),
+            recursive: true,
+            report_notfound: false,
+            filter_prefix: Some("part".to_string()),
+            forward_to: None,
+            limit: 10,
+            disk_id: String::new(),
+        };
+        let expected_body = serde_json::to_vec(&opts).unwrap();
+        let mut writer = Vec::new();
+
+        remote_disk.walk_dir(opts, &mut writer).await.unwrap();
+
+        let calls = transport.calls();
+        assert_eq!(calls.len(), 1);
+        match &calls[0] {
+            RecordedTransportCall::WalkDir(request) => {
+                assert_eq!(request.endpoint, "http://remote-node:9000");
+                assert_eq!(request.disk, expected_disk);
+                assert_eq!(request.body, expected_body);
+                assert_eq!(request.stall_timeout, Some(get_drive_walkdir_stall_timeout()));
+            }
+            other => panic!("expected walk-dir transport call, got {other:?}"),
+        }
+    }
+
     #[tokio::test]
     async fn test_remote_disk_endpoints_with_different_schemes() {
         let test_cases = vec![

crates/ecstore/src/set_disk.rs

@@ -30,7 +30,6 @@ use crate::disk::{
 };
 use crate::disk::{STORAGE_FORMAT_FILE, count_part_not_success};
 use crate::erasure_coding;
-use crate::erasure_coding::bitrot_verify;
 use crate::error::{Error, Result, is_err_version_not_found};
 use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
 use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
@@ -130,6 +129,7 @@ pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
 pub const MAX_PARTS_COUNT: usize = 10000;
 pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
 pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
+const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
 
 pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, String>) {
     metadata.remove(RUSTFS_MULTIPART_BUCKET_KEY);
@@ -270,6 +270,10 @@ fn record_lock_release(bucket: &str, object: &str, lock_id: &str, lock_type: &st
     );
 }
 
+fn issue3031_diag_enabled() -> bool {
+    rustfs_utils::get_env_bool(ENV_ISSUE3031_DIAG_ENABLE, false)
+}
+
 fn build_tiered_decommission_file_info(
     bucket: &str,
     object: &str,
@@ -3262,6 +3266,7 @@ impl MultipartOperations for SetDisks {
         let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
 
         let write_quorum = fi.write_quorum(self.default_write_quorum());
+        let read_quorum = fi.read_quorum(self.default_read_quorum());
 
         let disks = self.disks.read().await;
 
@@ -3278,7 +3283,7 @@ impl MultipartOperations for SetDisks {
         let part_numbers = uploaded_parts.iter().map(|v| v.part_num).collect::<Vec<usize>>();
 
         let object_parts =
-            Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, write_quorum).await?;
+            Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum).await?;
 
         if object_parts.len() != uploaded_parts.len() {
             return Err(Error::other("part result number err"));
@@ -3302,6 +3307,21 @@ impl MultipartOperations for SetDisks {
         for (i, part) in object_parts.iter().enumerate() {
             if let Some(err) = &part.error {
                 error!("complete_multipart_upload part error: {:?}", &err);
+                if issue3031_diag_enabled() {
+                    warn!(
+                        target: "rustfs_ecstore::set_disk",
+                        op = "complete_multipart_upload",
+                        bucket = %bucket,
+                        object = %object,
+                        upload_id = %upload_id,
+                        uploaded_part_num = uploaded_parts[i].part_num,
+                        observed_part_num = part.number,
+                        read_quorum = read_quorum,
+                        write_quorum = write_quorum,
+                        error = %err,
+                        "issue3031_complete_part_error"
+                    );
+                }
             }
 
             if uploaded_parts[i].part_num != part.number {
@@ -4028,32 +4048,16 @@ async fn disks_with_all_parts(
             continue;
         }
 
-        // Always check data, if we got it.
+        // Inline data is stored inside xl.meta, so there is no separate part file to
+        // verify here. Treat the shard as present once metadata was read successfully;
+        // object reads/heal will validate the inline shard through the normal bitrot
+        // reader path. Running bitrot_verify directly here can falsely mark small
+        // inline shards corrupt when older metadata has no per-part checksum entries.
         if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() {
-            if let Some(data) = &meta.data {
-                let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
-                let checksum_algo = if meta.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
-                    HashAlgorithm::HighwayHash256SLegacy
-                } else {
-                    checksum_info.algorithm
-                };
-                let data_len = data.len();
-                let verify_err = bitrot_verify(
-                    Box::new(Cursor::new(data.clone())),
-                    data_len,
-                    meta.erasure.shard_file_size(meta.size) as usize,
-                    checksum_algo,
-                    checksum_info.hash,
-                    meta.erasure.shard_size(),
-                )
-                .await
-                .err();
-
-                if let Some(vec) = data_errs_by_part.get_mut(&0)
-                    && index < vec.len()
-                {
-                    vec[index] = conv_part_err_to_int(&verify_err.map(|e| e.into()));
-                }
+            if let Some(vec) = data_errs_by_part.get_mut(&0)
+                && index < vec.len()
+            {
+                vec[index] = CHECK_PART_SUCCESS;
             }
             continue;
         }
@@ -4108,18 +4112,24 @@ async fn disks_with_all_parts(
         }
     }
 
-    // Build dataErrsByDisk from dataErrsByPart
-    for (part, disks) in data_errs_by_part.iter() {
-        for disk_idx in disks.iter() {
-            if let Some(parts) = data_errs_by_disk.get_mut(disk_idx)
-                && *part < parts.len()
+    populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
+
+    Ok((data_errs_by_disk, data_errs_by_part))
+}
+
+fn populate_data_errs_by_disk(
+    data_errs_by_disk: &mut HashMap<usize, Vec<usize>>,
+    data_errs_by_part: &HashMap<usize, Vec<usize>>,
+) {
+    for (part_index, part_errs) in data_errs_by_part {
+        for (disk_index, part_err) in part_errs.iter().enumerate() {
+            if let Some(disk_errs) = data_errs_by_disk.get_mut(&disk_index)
+                && *part_index < disk_errs.len()
             {
-                parts[*part] = disks[*disk_idx];
+                disk_errs[*part_index] = *part_err;
             }
         }
     }
-
-    Ok((data_errs_by_disk, data_errs_by_part))
 }
 
 pub fn should_heal_object_on_disk(
@@ -5366,6 +5376,62 @@ mod tests {
         assert!(has_part_err(&unknown_errors));
     }
 
+    #[test]
+    fn test_populate_data_errs_by_disk_uses_disk_index_not_error_code() {
+        let mut data_errs_by_disk = HashMap::from([
+            (0, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
+            (1, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
+            (2, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
+        ]);
+        let data_errs_by_part = HashMap::from([
+            (0, vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]),
+            (1, vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS]),
+        ]);
+
+        populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
+
+        assert_eq!(data_errs_by_disk.get(&0).unwrap(), &vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS]);
+        assert_eq!(data_errs_by_disk.get(&1).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_CORRUPT]);
+        assert_eq!(data_errs_by_disk.get(&2).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
+
+        let mut data_errs_by_disk = HashMap::from([
+            (0, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
+            (1, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
+            (2, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
+            (3, vec![CHECK_PART_UNKNOWN, CHECK_PART_UNKNOWN]),
+        ]);
+        let data_errs_by_part = HashMap::from([
+            (
+                0,
+                vec![
+                    CHECK_PART_FILE_NOT_FOUND,
+                    CHECK_PART_SUCCESS,
+                    CHECK_PART_SUCCESS,
+                    CHECK_PART_SUCCESS,
+                ],
+            ),
+            (
+                1,
+                vec![
+                    CHECK_PART_FILE_CORRUPT,
+                    CHECK_PART_SUCCESS,
+                    CHECK_PART_SUCCESS,
+                    CHECK_PART_SUCCESS,
+                ],
+            ),
+        ]);
+
+        populate_data_errs_by_disk(&mut data_errs_by_disk, &data_errs_by_part);
+
+        assert_eq!(
+            data_errs_by_disk.get(&0).unwrap(),
+            &vec![CHECK_PART_FILE_NOT_FOUND, CHECK_PART_FILE_CORRUPT]
+        );
+        assert_eq!(data_errs_by_disk.get(&1).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
+        assert_eq!(data_errs_by_disk.get(&2).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
+        assert_eq!(data_errs_by_disk.get(&3).unwrap(), &vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS]);
+    }
+
     #[test]
     fn test_should_heal_object_on_disk() {
         // Test healing decision logic

crates/ecstore/src/set_disk/heal.rs

@@ -350,15 +350,6 @@ impl SetDisks {
 
                             for (part_index, part) in latest_meta.parts.iter().enumerate() {
                                 let till_offset = erasure.shard_file_offset(0, part.size, part.size);
-                                let checksum_info = erasure_info.get_checksum_info(part.number);
-                                let checksum_algo = if latest_meta.uses_legacy_checksum
-                                    && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S
-                                {
-                                    rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
-                                } else {
-                                    checksum_info.algorithm
-                                };
-
                                 // Read zero-copy configuration from environment variable
                                 // Default: enabled (true) for performance
                                 let use_zero_copy =
@@ -382,6 +373,15 @@ impl SetDisks {
                                     }
 
                                     if let (Some(disk), Some(metadata)) = (disk, &copy_parts_metadata[index]) {
+                                        let checksum_info = metadata.erasure.get_checksum_info(part.number);
+                                        let checksum_algo = if metadata.uses_legacy_checksum
+                                            && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S
+                                        {
+                                            rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
+                                        } else {
+                                            checksum_info.algorithm
+                                        };
+
                                         match create_bitrot_reader(
                                             metadata.data.as_deref(),
                                             Some(disk),

crates/ecstore/src/set_disk/read.rs

@@ -144,18 +144,24 @@ impl SetDisks {
         for (part_idx, part_info) in part_meta_paths.iter().enumerate() {
             let mut part_meta_quorum = HashMap::new();
             let mut part_infos = Vec::new();
-            for (j, parts) in object_parts.iter().enumerate() {
+            let mut present_count = 0usize;
+            let mut missing_or_empty_count = 0usize;
+            let mut mismatched_response_count = 0usize;
+            for parts in object_parts.iter() {
                 if parts.len() != part_meta_paths.len() {
+                    mismatched_response_count += 1;
                     *part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1;
                     continue;
                 }
 
                 if !parts[part_idx].etag.is_empty() {
+                    present_count += 1;
                     *part_meta_quorum.entry(parts[part_idx].etag.clone()).or_insert(0) += 1;
                     part_infos.push(parts[part_idx].clone());
                     continue;
                 }
 
+                missing_or_empty_count += 1;
                 *part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1;
             }
 
@@ -194,6 +200,22 @@ impl SetDisks {
             {
                 ret[part_idx] = found.clone();
             } else {
+                if issue3031_diag_enabled() {
+                    warn!(
+                        target: "rustfs_ecstore::set_disk",
+                        bucket = %bucket,
+                        part_meta_path = %part_info,
+                        part_id = part_numbers[part_idx],
+                        read_quorum = read_quorum,
+                        max_quorum = max_quorum,
+                        disk_response_count = object_parts.len(),
+                        present_count = present_count,
+                        missing_or_empty_count = missing_or_empty_count,
+                        mismatched_response_count = mismatched_response_count,
+                        max_vote_is_missing_marker = max_etag.map(|etag| etag == part_info).unwrap_or(false),
+                        "issue3031_read_parts_part_quorum"
+                    );
+                }
                 ret[part_idx] = ObjectPartInfo {
                     number: part_numbers[part_idx],
                     error: Some(format!("part.{} not found", part_numbers[part_idx])),

crates/ecstore/src/set_disk/write.rs

@@ -202,6 +202,9 @@ impl SetDisks {
 
     #[tracing::instrument(skip(self))]
     pub(super) async fn cleanup_multipart_path(&self, paths: &[String]) {
+        if paths.is_empty() {
+            return;
+        }
         let disks = self.get_disks_internal().await;
 
         let mut errs = Vec::with_capacity(disks.len());
@@ -290,6 +293,27 @@ impl SetDisks {
             }
         }
 
+        if issue3031_diag_enabled() {
+            let success_count = errs.iter().filter(|err| err.is_none()).count();
+            let error_count = errs.len().saturating_sub(success_count);
+            let disk_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::DiskNotFound))).count();
+            let file_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::FileNotFound))).count();
+            warn!(
+                target: "rustfs_ecstore::set_disk",
+                src_bucket = %src_bucket,
+                src_object = %src_object,
+                dst_bucket = %dst_bucket,
+                dst_object = %dst_object,
+                write_quorum = write_quorum,
+                disk_count = errs.len(),
+                success_count = success_count,
+                error_count = error_count,
+                disk_not_found_count = disk_not_found_count,
+                file_not_found_count = file_not_found_count,
+                "issue3031_rename_part_context"
+            );
+        }
+
         if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
             warn!("rename_part errs {:?}", &errs);
             self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])

crates/ecstore/src/store_list_objects.rs

@@ -340,6 +340,9 @@ impl ECStore {
                 ..Default::default()
             });
 
+        // err=None means gather_results filled its limit → disk has more data
+        let disk_has_more = list_result.err.is_none();
+
         if let Some(err) = list_result.err.take()
             && err != rustfs_filemeta::Error::Unexpected
         {
@@ -371,7 +374,7 @@ impl ECStore {
             get_objects.truncate(max_keys as usize);
         }
 
-        let next_marker = {
+        let mut next_marker = {
             if is_truncated {
                 get_objects.last().map(|last| last.name.clone())
             } else {
@@ -398,6 +401,27 @@ impl ECStore {
             }
         }
 
+        // After delimiter collapse, re-evaluate is_truncated based on visible results.
+        // No delimiter: reduction is from skipped entries → disk_has_more && non-empty.
+        // With delimiter: reduction may be from collapse → only when visible >= max_keys.
+        if !is_truncated && disk_has_more {
+            let visible_count = objects.len() + prefixes.len();
+            let should_truncate = if delimiter.is_none() {
+                visible_count > 0
+            } else {
+                visible_count >= max_keys as usize
+            };
+            if should_truncate {
+                is_truncated = true;
+                // Compute next_marker from visible results since get_objects was consumed.
+                // Prefer last object name; fall back to last prefix for marker.
+                next_marker = objects
+                    .last()
+                    .map(|last| last.name.clone())
+                    .or_else(|| prefixes.last().cloned());
+            }
+        }
+
         Ok(ListObjectsInfo {
             is_truncated,
             next_marker,
@@ -453,6 +477,9 @@ impl ECStore {
                 ..Default::default()
             });
 
+        // err=None means gather_results filled its limit → disk has more data
+        let disk_has_more = list_result.err.is_none();
+
         if let Some(err) = list_result.err.take()
             && err != rustfs_filemeta::Error::Unexpected
         {
@@ -483,22 +510,13 @@ impl ECStore {
             get_objects.truncate(max_keys as usize);
         }
 
-        let (next_marker, next_version_idmarker) = {
-            if is_truncated {
-                get_objects
-                    .last()
-                    .map(|last| {
-                        (
-                            Some(last.name.clone()),
-                            // AWS S3 API returns "null" for non-versioned objects
-                            Some(last.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())),
-                        )
-                    })
-                    .unwrap_or_default()
-            } else {
-                (None, None)
-            }
-        };
+        let mut next_marker: Option<String> = None;
+        let mut next_version_idmarker: Option<String> = None;
+        if is_truncated && let Some(last) = get_objects.last() {
+            next_marker = Some(last.name.clone());
+            // AWS S3 API returns "null" for non-versioned objects
+            next_version_idmarker = Some(last.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string()));
+        }
 
         let mut prefixes: Vec<String> = Vec::new();
         let mut prefix_set: HashSet<String> = HashSet::new();
@@ -519,6 +537,30 @@ impl ECStore {
             }
         }
 
+        // After delimiter collapse, re-evaluate is_truncated based on visible results.
+        // Two distinct scenarios (see list_objects_generic for detailed rationale):
+        // 1. No delimiter: reduction from skipped entries → disk_has_more && non-empty
+        // 2. With delimiter: reduction from collapse → only when visible >= max_keys
+        if !is_truncated && disk_has_more {
+            let visible_count = objects.len() + prefixes.len();
+            let should_truncate = if delimiter.is_none() {
+                visible_count > 0
+            } else {
+                visible_count >= max_keys as usize
+            };
+            if should_truncate {
+                is_truncated = true;
+                // Compute markers from visible results since get_objects was consumed.
+                if let Some(last) = objects.last() {
+                    next_marker = Some(last.name.clone());
+                    next_version_idmarker = Some(last.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string()));
+                } else if let Some(last_prefix) = prefixes.last().cloned() {
+                    next_marker = Some(last_prefix);
+                    next_version_idmarker = None;
+                }
+            }
+        }
+
         Ok(ListObjectVersionsInfo {
             is_truncated,
             next_marker,

crates/heal/src/error.rs

@@ -59,6 +59,9 @@ pub enum Error {
     #[error("Heal task already exists: {task_id}")]
     TaskAlreadyExists { task_id: String },
 
+    #[error("Invalid heal client token")]
+    InvalidClientToken,
+
     #[error("Heal manager is not running")]
     ManagerNotRunning,
 

crates/heal/src/heal/channel.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use crate::heal::{
-    manager::HealManager,
-    task::{HealOptions, HealPriority, HealRequest, HealType},
+    manager::{HealManager, HealTaskReport},
+    task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
     utils,
 };
 use crate::{Error, Result};
@@ -22,6 +22,8 @@ use rustfs_common::heal_channel::{
     HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse,
     HealScanMode, publish_heal_response,
 };
+use rustfs_madmin::heal_commands::HealResultItem;
+use serde::Serialize;
 use std::sync::Arc;
 use tokio::sync::{mpsc, oneshot};
 use tracing::{debug, error, info};
@@ -36,6 +38,12 @@ pub struct HealChannelProcessor {
     response_receiver: mpsc::UnboundedReceiver<HealChannelResponse>,
 }
 
+#[derive(Serialize)]
+struct HealTaskStatusPayload {
+    summary: String,
+    items: Vec<HealResultItem>,
+}
+
 impl HealChannelProcessor {
     /// Create new HealChannelProcessor
     pub fn new(heal_manager: Arc<HealManager>) -> Self {
@@ -83,8 +91,16 @@ impl HealChannelProcessor {
     async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
         match command {
             HealChannelCommand::Start { request, response_tx } => self.process_start_request(request, response_tx).await,
-            HealChannelCommand::Query { heal_path, client_token } => self.process_query_request(heal_path, client_token).await,
-            HealChannelCommand::Cancel { heal_path } => self.process_cancel_request(heal_path).await,
+            HealChannelCommand::Query {
+                heal_path,
+                client_token,
+                response_tx,
+            } => self.process_query_request(heal_path, client_token, response_tx).await,
+            HealChannelCommand::Cancel {
+                heal_path,
+                client_token,
+                response_tx,
+            } => self.process_cancel_request(heal_path, client_token, response_tx).await,
         }
     }
 
@@ -166,36 +182,114 @@ impl HealChannelProcessor {
     }
 
     /// Process query request
-    async fn process_query_request(&self, heal_path: String, client_token: String) -> Result<()> {
+    async fn process_query_request(
+        &self,
+        heal_path: String,
+        client_token: String,
+        response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
+    ) -> Result<()> {
         info!("Processing heal query request for path: {}", heal_path);
 
-        // TODO: Implement query logic based on heal_path and client_token
-        // For now, return a placeholder response
+        let (summary, detail, items) = match self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await {
+            Ok(HealTaskReport {
+                status: HealTaskStatus::Pending | HealTaskStatus::Running,
+                result_items,
+            }) => ("running".to_string(), None, result_items),
+            Ok(HealTaskReport {
+                status: HealTaskStatus::Completed,
+                result_items,
+            }) => ("finished".to_string(), None, result_items),
+            Ok(HealTaskReport {
+                status: HealTaskStatus::Cancelled,
+                result_items,
+            }) => ("stopped".to_string(), Some("heal task cancelled".to_string()), result_items),
+            Ok(HealTaskReport {
+                status: HealTaskStatus::Timeout,
+                result_items,
+            }) => ("stopped".to_string(), Some("heal task timed out".to_string()), result_items),
+            Ok(HealTaskReport {
+                status: HealTaskStatus::Failed { error },
+                result_items,
+            }) => ("stopped".to_string(), Some(error), result_items),
+            Err(crate::Error::TaskNotFound { .. }) => ("finished".to_string(), None, Vec::new()),
+            Err(crate::Error::InvalidClientToken) => {
+                let response = HealChannelResponse {
+                    request_id: client_token,
+                    success: false,
+                    data: None,
+                    error: Some("invalid heal client token".to_string()),
+                };
+                let _ = response_tx.send(Ok(response.clone()));
+                self.publish_response(response);
+                return Ok(());
+            }
+            Err(err) => {
+                let error_text = err.to_string();
+                let response = HealChannelResponse {
+                    request_id: client_token,
+                    success: false,
+                    data: None,
+                    error: Some(error_text.clone()),
+                };
+                let _ = response_tx.send(Ok(response.clone()));
+                self.publish_response(response);
+                return Ok(());
+            }
+        };
+
+        let data = serde_json::to_vec(&HealTaskStatusPayload { summary, items })
+            .map_err(|e| crate::Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
+
         let response = HealChannelResponse {
             request_id: client_token,
             success: true,
-            data: Some(format!("Query result for path: {heal_path}").into_bytes()),
-            error: None,
+            data: Some(data),
+            error: detail,
         };
 
+        let _ = response_tx.send(Ok(response.clone()));
         self.publish_response(response);
 
         Ok(())
     }
 
     /// Process cancel request
-    async fn process_cancel_request(&self, heal_path: String) -> Result<()> {
+    async fn process_cancel_request(
+        &self,
+        heal_path: String,
+        client_token: String,
+        response_tx: oneshot::Sender<std::result::Result<HealChannelResponse, String>>,
+    ) -> Result<()> {
         info!("Processing heal cancel request for path: {}", heal_path);
 
-        // TODO: Implement cancel logic based on heal_path
-        // For now, return a placeholder response
-        let response = HealChannelResponse {
-            request_id: heal_path.clone(),
-            success: true,
-            data: Some(format!("Cancel request for path: {heal_path}").into_bytes()),
-            error: None,
+        let request_id = if client_token.is_empty() {
+            heal_path.clone()
+        } else {
+            client_token.clone()
         };
 
+        let cancel_result = if client_token.is_empty() {
+            self.heal_manager.cancel_tasks_for_path(&heal_path).await.map(|_| ())
+        } else {
+            self.heal_manager.cancel_task(&client_token).await
+        };
+
+        let response = match cancel_result {
+            Ok(()) => HealChannelResponse {
+                request_id,
+                success: true,
+                data: Some("stopped".as_bytes().to_vec()),
+                error: None,
+            },
+            Err(err) => HealChannelResponse {
+                request_id,
+                success: false,
+                data: None,
+                error: Some(err.to_string()),
+            },
+        };
+
+        let _ = response_tx.send(Ok(response.clone()));
         self.publish_response(response);
 
         Ok(())
@@ -237,7 +331,7 @@ impl HealChannelProcessor {
         };
 
         // Build HealOptions with all available fields
-        let mut options = HealOptions {
+        let options = HealOptions {
             scan_mode: request.scan_mode.unwrap_or(HealScanMode::Normal),
             remove_corrupted: request.remove_corrupted.unwrap_or(false),
             recreate_missing: request.recreate_missing.unwrap_or(true),
@@ -249,14 +343,14 @@ impl HealChannelProcessor {
             set_index: request.set_index,
         };
 
-        // Apply force_start overrides
-        if request.force_start {
-            options.remove_corrupted = true;
-            options.recreate_missing = true;
-            options.update_parity = true;
-        }
-
-        Ok(HealRequest::new(heal_type, options, priority))
+        let mut heal_request = HealRequest::new(heal_type, options, priority);
+        heal_request.id = request.id;
+        // force_start controls admission/queue semantics only. Do not reinterpret it as
+        // destructive heal options: admin clients commonly pass forceStart=true together
+        // with remove=false, and turning that into remove_corrupted=true can delete the
+        // remaining healthy bucket volumes before object shards are rebuilt.
+        heal_request.force_start = request.force_start;
+        Ok(heal_request)
     }
 
     fn publish_response(&self, response: HealChannelResponse) {
@@ -416,6 +510,7 @@ mod tests {
         };
 
         let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
+        assert_eq!(heal_request.id, "test-id");
         assert!(matches!(heal_request.heal_type, HealType::Bucket { .. }));
         assert_eq!(heal_request.priority, HealPriority::Normal);
     }
@@ -567,13 +662,14 @@ mod tests {
             timeout_seconds: None,
             pool_index: None,
             set_index: None,
-            force_start: true, // Should override the above false values
+            force_start: true, // Admission force only; must not override explicit heal options.
         };
 
         let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
-        assert!(heal_request.options.remove_corrupted);
-        assert!(heal_request.options.recreate_missing);
-        assert!(heal_request.options.update_parity);
+        assert!(heal_request.force_start);
+        assert!(!heal_request.options.remove_corrupted);
+        assert!(!heal_request.options.recreate_missing);
+        assert!(!heal_request.options.update_parity);
     }
 
     #[tokio::test]
@@ -691,4 +787,201 @@ mod tests {
             .expect("processor should surface invalid request through response channel");
         assert!(rx.await.expect("oneshot should resolve").is_err());
     }
+
+    #[tokio::test]
+    async fn test_process_query_request_reports_finished_when_task_is_not_active() {
+        let heal_manager = create_test_heal_manager();
+        let processor = HealChannelProcessor::new(heal_manager);
+        let (tx, rx) = oneshot::channel();
+
+        processor
+            .process_query_request("bucket".to_string(), "completed-token".to_string(), tx)
+            .await
+            .expect("query should process");
+
+        let response = rx
+            .await
+            .expect("oneshot should resolve")
+            .expect("query response should be returned");
+        assert!(response.success);
+        assert_eq!(response.request_id, "completed-token");
+        let payload: serde_json::Value =
+            serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
+                .expect("status payload should be json");
+        assert_eq!(payload["summary"], "finished");
+        assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_process_query_request_reports_running_for_queued_task() {
+        let heal_manager = create_test_heal_manager();
+        let request = HealRequest::bucket("bucket".to_string());
+        let task_id = request.id.clone();
+        assert_eq!(
+            heal_manager
+                .submit_heal_request(request)
+                .await
+                .expect("request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+
+        let processor = HealChannelProcessor::new(heal_manager);
+        let (tx, rx) = oneshot::channel();
+
+        processor
+            .process_query_request("bucket".to_string(), task_id.clone(), tx)
+            .await
+            .expect("query should process");
+
+        let response = rx
+            .await
+            .expect("oneshot should resolve")
+            .expect("query response should be returned");
+        assert!(response.success);
+        assert_eq!(response.request_id, task_id);
+        let payload: serde_json::Value =
+            serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
+                .expect("status payload should be json");
+        assert_eq!(payload["summary"], "running");
+        assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_process_query_request_rejects_wrong_token_for_active_path() {
+        let heal_manager = create_test_heal_manager();
+        let request = HealRequest::bucket("bucket".to_string());
+        assert_eq!(
+            heal_manager
+                .submit_heal_request(request)
+                .await
+                .expect("request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+
+        let processor = HealChannelProcessor::new(heal_manager);
+        let (tx, rx) = oneshot::channel();
+
+        processor
+            .process_query_request("bucket".to_string(), "wrong-token".to_string(), tx)
+            .await
+            .expect("query should process");
+
+        let response = rx
+            .await
+            .expect("oneshot should resolve")
+            .expect("query response should be returned");
+        assert!(!response.success);
+        assert_eq!(response.request_id, "wrong-token");
+        assert_eq!(response.error.as_deref(), Some("invalid heal client token"));
+    }
+
+    #[tokio::test]
+    async fn test_process_query_request_empty_path_ignores_unrelated_tasks() {
+        let heal_manager = create_test_heal_manager();
+        heal_manager
+            .submit_heal_request(HealRequest::bucket("bucket".to_string()))
+            .await
+            .expect("request should be accepted");
+
+        let processor = HealChannelProcessor::new(heal_manager);
+        let (tx, rx) = oneshot::channel();
+
+        processor
+            .process_query_request(String::new(), "wrong-token".to_string(), tx)
+            .await
+            .expect("query should process");
+
+        let response = rx
+            .await
+            .expect("oneshot should resolve")
+            .expect("query response should be returned");
+        assert!(response.success);
+        let payload: serde_json::Value =
+            serde_json::from_slice(response.data.as_deref().expect("status payload should be present"))
+                .expect("status payload should be json");
+        assert_eq!(payload["summary"], "finished");
+        assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_process_cancel_request_cancels_queued_task_by_token() {
+        let heal_manager = create_test_heal_manager();
+        let request = HealRequest::bucket("bucket".to_string());
+        let task_id = request.id.clone();
+        heal_manager
+            .submit_heal_request(request)
+            .await
+            .expect("request should be accepted");
+
+        let processor = HealChannelProcessor::new(heal_manager.clone());
+        let (tx, rx) = oneshot::channel();
+
+        processor
+            .process_cancel_request("bucket".to_string(), task_id.clone(), tx)
+            .await
+            .expect("cancel should process");
+
+        let response = rx
+            .await
+            .expect("oneshot should resolve")
+            .expect("cancel response should be returned");
+        assert!(response.success);
+        assert_eq!(response.request_id, task_id);
+        assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
+        assert!(matches!(
+            heal_manager.get_task_status(&response.request_id).await,
+            Err(crate::Error::TaskNotFound { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_process_cancel_request_cancels_queued_task_by_path() {
+        let heal_manager = create_test_heal_manager();
+        let request = HealRequest::bucket("bucket".to_string());
+        let task_id = request.id.clone();
+        heal_manager
+            .submit_heal_request(request)
+            .await
+            .expect("request should be accepted");
+
+        let processor = HealChannelProcessor::new(heal_manager.clone());
+        let (tx, rx) = oneshot::channel();
+
+        processor
+            .process_cancel_request("bucket".to_string(), String::new(), tx)
+            .await
+            .expect("cancel should process");
+
+        let response = rx
+            .await
+            .expect("oneshot should resolve")
+            .expect("cancel response should be returned");
+        assert!(response.success);
+        assert_eq!(response.request_id, "bucket");
+        assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
+        assert!(matches!(
+            heal_manager.get_task_status(&task_id).await,
+            Err(crate::Error::TaskNotFound { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_process_cancel_request_reports_unknown_task() {
+        let heal_manager = create_test_heal_manager();
+        let processor = HealChannelProcessor::new(heal_manager);
+        let (tx, rx) = oneshot::channel();
+
+        processor
+            .process_cancel_request("missing".to_string(), "missing-token".to_string(), tx)
+            .await
+            .expect("cancel should process");
+
+        let response = rx
+            .await
+            .expect("oneshot should resolve")
+            .expect("cancel response should be returned");
+        assert!(!response.success);
+        assert_eq!(response.request_id, "missing-token");
+        assert!(response.error.unwrap_or_default().contains("Heal task not found"));
+    }
 }

crates/heal/src/heal/erasure_healer.rs

@@ -35,6 +35,7 @@ pub struct ErasureSetHealer {
     progress: Arc<RwLock<HealProgress>>,
     cancel_token: tokio_util::sync::CancellationToken,
     disk: DiskStore,
+    heal_opts: HealOpts,
 }
 
 impl ErasureSetHealer {
@@ -61,17 +62,31 @@ impl ErasureSetHealer {
         }
     }
 
+    fn effective_heal_page_object_concurrency_for_scan_mode(scan_mode: HealScanMode) -> usize {
+        if matches!(scan_mode, HealScanMode::Deep) {
+            1
+        } else {
+            Self::effective_heal_page_object_concurrency()
+        }
+    }
+
+    fn is_object_not_found_message(message: &str) -> bool {
+        message.contains("File not found") || message.contains("not found")
+    }
+
     pub fn new(
         storage: Arc<dyn HealStorageAPI>,
         progress: Arc<RwLock<HealProgress>>,
         cancel_token: tokio_util::sync::CancellationToken,
         disk: DiskStore,
+        heal_opts: HealOpts,
     ) -> Self {
         Self {
             storage,
             progress,
             cancel_token,
             disk,
+            heal_opts,
         }
     }
 
@@ -289,7 +304,7 @@ impl ErasureSetHealer {
         // 2. process objects with pagination to avoid loading all objects into memory
         let mut continuation_token: Option<String> = None;
         let mut global_obj_idx = 0usize;
-        let page_concurrency_limit = Self::effective_heal_page_object_concurrency();
+        let page_concurrency_limit = Self::effective_heal_page_object_concurrency_for_scan_mode(self.heal_opts.scan_mode);
         let in_flight = Arc::new(AtomicUsize::new(0));
 
         loop {
@@ -325,23 +340,51 @@ impl ErasureSetHealer {
                 let cancel_token = self.cancel_token.clone();
                 let in_flight = in_flight.clone();
                 let set_label = set_disk_id.to_string();
-                let permit = semaphore
-                    .clone()
-                    .acquire_owned()
-                    .await
-                    .map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}")))?;
-
-                let current_in_flight = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
-                gauge!(
-                    "rustfs_heal_page_concurrency_current",
-                    "set" => set_label.clone()
-                )
-                .set(current_in_flight as f64);
+                let heal_opts = self.heal_opts;
+                let deep_scan = matches!(heal_opts.scan_mode, HealScanMode::Deep);
+                let semaphore = semaphore.clone();
 
                 page_tasks.push(async move {
-                    let _permit = permit;
+                    let permit = semaphore
+                        .clone()
+                        .acquire_owned()
+                        .await
+                        .map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}")));
+
+                    let _permit = match permit {
+                        Ok(permit) => permit,
+                        Err(err) => return (object_name, Err(err)),
+                    };
+
+                    let current_in_flight = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
+                    gauge!(
+                        "rustfs_heal_page_concurrency_current",
+                        "set" => set_label.clone()
+                    )
+                    .set(current_in_flight as f64);
+
                     let result = if cancel_token.is_cancelled() {
                         Err(Error::TaskCancelled)
+                    } else if deep_scan {
+                        match storage.heal_object(&bucket_name, &object_name, None, &heal_opts).await {
+                            Ok((_result, None)) => Ok(true),
+                            Ok((_, Some(err))) => {
+                                let err_msg = err.to_string();
+                                if Self::is_object_not_found_message(&err_msg) {
+                                    Ok(false)
+                                } else {
+                                    Err(Error::other(err))
+                                }
+                            }
+                            Err(err) => {
+                                let err_msg = err.to_string();
+                                if Self::is_object_not_found_message(&err_msg) {
+                                    Ok(false)
+                                } else {
+                                    Err(err)
+                                }
+                            }
+                        }
                     } else {
                         let object_exists = match storage.object_exists(&bucket_name, &object_name).await {
                             Ok(exists) => exists,
@@ -375,12 +418,6 @@ impl ErasureSetHealer {
                         if !object_exists {
                             Ok(false)
                         } else {
-                            let heal_opts = HealOpts {
-                                scan_mode: HealScanMode::Normal,
-                                remove: true,
-                                recreate: true,
-                                ..Default::default()
-                            };
                             match storage.heal_object(&bucket_name, &object_name, None, &heal_opts).await {
                                 Ok((_result, None)) => Ok(true),
                                 Ok((_, Some(err))) => Err(Error::other(err)),
@@ -678,6 +715,7 @@ impl ErasureSetHealer {
 #[cfg(test)]
 mod tests {
     use super::ErasureSetHealer;
+    use rustfs_common::heal_channel::HealScanMode;
 
     #[test]
     fn heal_page_object_concurrency_uses_default_when_env_is_unset() {
@@ -704,4 +742,24 @@ mod tests {
             });
         });
     }
+
+    #[test]
+    fn deep_scan_heal_page_object_concurrency_is_serial() {
+        temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("11"), || {
+            assert_eq!(
+                ErasureSetHealer::effective_heal_page_object_concurrency_for_scan_mode(HealScanMode::Deep),
+                1
+            );
+        });
+    }
+
+    #[test]
+    fn normal_scan_heal_page_object_concurrency_uses_effective_limit() {
+        temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("11"), || {
+            assert_eq!(
+                ErasureSetHealer::effective_heal_page_object_concurrency_for_scan_mode(HealScanMode::Normal),
+                11
+            );
+        });
+    }
 }

crates/heal/src/heal/manager.rs

@@ -23,8 +23,9 @@ use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
 use rustfs_ecstore::disk::DiskAPI;
 use rustfs_ecstore::disk::error::DiskError;
 use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
+use rustfs_madmin::heal_commands::HealResultItem;
 use std::{
-    collections::{BinaryHeap, HashMap, HashSet},
+    collections::{BinaryHeap, HashMap},
     sync::Arc,
     time::{Duration, SystemTime},
 };
@@ -35,6 +36,8 @@ use tokio::{
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
 
+const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
+
 /// Priority queue wrapper for heal requests
 /// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items
 #[derive(Debug)]
@@ -43,8 +46,8 @@ struct PriorityHealQueue {
     heap: BinaryHeap<PriorityQueueItem>,
     /// Sequence counter for FIFO ordering within same priority
     sequence: u64,
-    /// Set of request keys to prevent duplicates
-    dedup_keys: HashSet<String>,
+    /// Deduplication key reference counts for queued requests
+    dedup_keys: HashMap<String, usize>,
 }
 
 /// Wrapper for heap items to implement proper ordering
@@ -88,12 +91,26 @@ enum QueuePushOutcome {
     Merged,
 }
 
+#[derive(Debug, Clone)]
+struct CompletedHealStatus {
+    heal_type: HealType,
+    status: HealTaskStatus,
+    result_items: Vec<HealResultItem>,
+    completed_at: SystemTime,
+}
+
+#[derive(Debug, Clone)]
+pub struct HealTaskReport {
+    pub status: HealTaskStatus,
+    pub result_items: Vec<HealResultItem>,
+}
+
 impl PriorityHealQueue {
     fn new() -> Self {
         Self {
             heap: BinaryHeap::new(),
             sequence: 0,
-            dedup_keys: HashSet::new(),
+            dedup_keys: HashMap::new(),
         }
     }
 
@@ -104,7 +121,7 @@ impl PriorityHealQueue {
     fn pop_next(&mut self) -> Option<HealRequest> {
         self.heap.pop().map(|item| {
             let key = Self::make_dedup_key(&item.request);
-            self.dedup_keys.remove(&key);
+            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
             item.request
         })
     }
@@ -116,12 +133,13 @@ impl PriorityHealQueue {
     fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
         let key = Self::make_dedup_key(&request);
 
-        // Check for duplicates
-        if self.dedup_keys.contains(&key) {
+        // Check for duplicates unless the caller explicitly forces admission.
+        if self.dedup_keys.contains_key(&key) && !request.force_start {
             return QueuePushOutcome::Merged;
         }
-
-        self.dedup_keys.insert(key);
+        // Track dedup keys for both normal and forced requests so queued forced work
+        // also reserves the dedup key for later non-forced duplicates.
+        *self.dedup_keys.entry(key).or_insert(0) += 1;
         self.sequence += 1;
         self.heap.push(PriorityQueueItem {
             priority: request.priority,
@@ -144,7 +162,7 @@ impl PriorityHealQueue {
     fn pop(&mut self) -> Option<HealRequest> {
         self.heap.pop().map(|item| {
             let key = Self::make_dedup_key(&item.request);
-            self.dedup_keys.remove(&key);
+            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
             item.request
         })
     }
@@ -184,7 +202,7 @@ impl PriorityHealQueue {
         (
             selected.map(|item| {
                 let key = Self::make_dedup_key(&item.request);
-                self.dedup_keys.remove(&key);
+                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
                 item.request
             }),
             skipped,
@@ -223,17 +241,99 @@ impl PriorityHealQueue {
         }
     }
 
+    fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap<String, usize>, key: &str) {
+        if let Some(count) = dedup_keys.get_mut(key) {
+            if *count <= 1 {
+                dedup_keys.remove(key);
+            } else {
+                *count -= 1;
+            }
+        }
+    }
+
     /// Check if a request with the same key already exists in the queue
     #[allow(dead_code)]
     fn contains_key(&self, request: &HealRequest) -> bool {
         let key = Self::make_dedup_key(request);
-        self.dedup_keys.contains(&key)
+        self.dedup_keys.contains_key(&key)
     }
 
     /// Check if an erasure set heal request for a specific set_disk_id exists
     fn contains_erasure_set(&self, set_disk_id: &str) -> bool {
         let key = format!("erasure_set:{set_disk_id}");
-        self.dedup_keys.contains(&key)
+        self.dedup_keys.contains_key(&key)
+    }
+
+    fn contains_request_id(&self, request_id: &str) -> bool {
+        self.heap.iter().any(|item| item.request.id == request_id)
+    }
+
+    fn contains_request_id_matching_path(&self, request_id: &str, heal_path: &str) -> bool {
+        self.heap
+            .iter()
+            .any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path))
+    }
+
+    fn contains_matching<F>(&self, mut matches: F) -> bool
+    where
+        F: FnMut(&HealRequest) -> bool,
+    {
+        self.heap.iter().any(|item| matches(&item.request))
+    }
+
+    fn remove_request_id(&mut self, request_id: &str) -> Option<HealRequest> {
+        let mut retained = BinaryHeap::new();
+        let mut removed = None;
+
+        while let Some(item) = self.heap.pop() {
+            if removed.is_none() && item.request.id == request_id {
+                let key = Self::make_dedup_key(&item.request);
+                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
+                removed = Some(item.request);
+            } else {
+                retained.push(item);
+            }
+        }
+
+        self.heap = retained;
+        removed
+    }
+
+    fn remove_matching<F>(&mut self, mut should_remove: F) -> usize
+    where
+        F: FnMut(&HealRequest) -> bool,
+    {
+        let mut retained = BinaryHeap::new();
+        let mut removed_count = 0;
+
+        while let Some(item) = self.heap.pop() {
+            if should_remove(&item.request) {
+                let key = Self::make_dedup_key(&item.request);
+                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
+                removed_count += 1;
+            } else {
+                retained.push(item);
+            }
+        }
+
+        self.heap = retained;
+        removed_count
+    }
+}
+
+fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool {
+    let heal_path = heal_path.trim_matches('/');
+    if heal_path.is_empty() {
+        return false;
+    }
+
+    match heal_type {
+        HealType::Object { bucket, object, .. }
+        | HealType::Metadata { bucket, object }
+        | HealType::ECDecode { bucket, object, .. } => heal_path == bucket || heal_path == format!("{bucket}/{object}"),
+        HealType::Bucket { bucket } => heal_path == bucket,
+        HealType::ErasureSet { set_disk_id, .. } => heal_path == set_disk_id,
+        HealType::MRF { meta_path } => heal_path == meta_path.trim_matches('/'),
     }
 }
 
@@ -357,6 +457,8 @@ pub struct HealManager {
     active_heals: Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
     /// Heal queue (priority-based)
     heal_queue: Arc<Mutex<PriorityHealQueue>>,
+    /// Recently completed heal statuses retained for status queries.
+    completed_heals: Arc<Mutex<HashMap<String, CompletedHealStatus>>>,
     /// Storage layer interface
     storage: Arc<dyn HealStorageAPI>,
     /// Cancel token
@@ -384,6 +486,7 @@ impl HealManager {
             state: Arc::new(RwLock::new(HealState::default())),
             active_heals: Arc::new(Mutex::new(HashMap::new())),
             heal_queue: Arc::new(Mutex::new(PriorityHealQueue::new())),
+            completed_heals: Arc::new(Mutex::new(HashMap::new())),
             storage,
             cancel_token: CancellationToken::new(),
             statistics: Arc::new(RwLock::new(HealStatistics::new())),
@@ -429,6 +532,7 @@ impl HealManager {
         }
         active_heals.clear();
         publish_active_heal_count(&active_heals);
+        self.completed_heals.lock().await.clear();
         crate::set_heal_queue_length(0);
 
         // update state
@@ -448,7 +552,7 @@ impl HealManager {
         publish_heal_queue_length(&queue);
         let queue_capacity = config.queue_size;
 
-        if queue.contains_key(&request) {
+        if !request.force_start && queue.contains_key(&request) {
             let admission = if request.priority == HealPriority::Low && !config.low_priority_merge_enable {
                 HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
             } else {
@@ -473,7 +577,7 @@ impl HealManager {
             return Ok(admission);
         }
 
-        if queue_len >= queue_capacity {
+        if queue_len >= queue_capacity && !request.force_start {
             let admission = Self::classify_full_admission(&request, &config);
             match admission {
                 HealAdmissionResult::Dropped(reason) => {
@@ -544,14 +648,139 @@ impl HealManager {
 
     /// Get task status
     pub async fn get_task_status(&self, task_id: &str) -> Result<HealTaskStatus> {
-        let active_heals = self.active_heals.lock().await;
-        if let Some(task) = active_heals.get(task_id) {
-            Ok(task.get_status().await)
-        } else {
-            Err(Error::TaskNotFound {
-                task_id: task_id.to_string(),
-            })
+        {
+            let active_heals = self.active_heals.lock().await;
+            if let Some(task) = active_heals.get(task_id) {
+                return Ok(task.get_status().await);
+            }
         }
+
+        let queue = self.heal_queue.lock().await;
+        if queue.contains_request_id(task_id) {
+            return Ok(HealTaskStatus::Pending);
+        }
+        drop(queue);
+
+        let mut completed_heals = self.completed_heals.lock().await;
+        prune_completed_heal_statuses(&mut completed_heals);
+        if let Some(completed) = completed_heals.get(task_id) {
+            return Ok(completed.status.clone());
+        }
+
+        Err(Error::TaskNotFound {
+            task_id: task_id.to_string(),
+        })
+    }
+
+    pub async fn get_task_report_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskReport> {
+        {
+            let active_heals = self.active_heals.lock().await;
+            if let Some(task) = active_heals.get(task_id)
+                && heal_type_matches_path(&task.heal_type, heal_path)
+            {
+                return Ok(HealTaskReport {
+                    status: task.get_status().await,
+                    result_items: task.get_result_items().await,
+                });
+            }
+        }
+
+        {
+            let queue = self.heal_queue.lock().await;
+            if queue.contains_request_id_matching_path(task_id, heal_path) {
+                return Ok(HealTaskReport {
+                    status: HealTaskStatus::Pending,
+                    result_items: Vec::new(),
+                });
+            }
+        }
+
+        {
+            let mut completed_heals = self.completed_heals.lock().await;
+            prune_completed_heal_statuses(&mut completed_heals);
+            if let Some(completed) = completed_heals.get(task_id)
+                && heal_type_matches_path(&completed.heal_type, heal_path)
+            {
+                return Ok(HealTaskReport {
+                    status: completed.status.clone(),
+                    result_items: completed.result_items.clone(),
+                });
+            }
+        }
+
+        if self.path_has_task(heal_path).await {
+            return Err(Error::InvalidClientToken);
+        }
+
+        Err(Error::TaskNotFound {
+            task_id: task_id.to_string(),
+        })
+    }
+
+    /// Get task status for a path-bound client token.
+    ///
+    /// If the token is unknown but no task remains for the path, the caller can
+    /// treat it as an already-finished sequence. If the path still has a live or
+    /// recently completed task, a different token is invalid for that path.
+    pub async fn get_task_status_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskStatus> {
+        {
+            let active_heals = self.active_heals.lock().await;
+            if let Some(task) = active_heals.get(task_id)
+                && heal_type_matches_path(&task.heal_type, heal_path)
+            {
+                return Ok(task.get_status().await);
+            }
+        }
+
+        {
+            let queue = self.heal_queue.lock().await;
+            if queue.contains_request_id_matching_path(task_id, heal_path) {
+                return Ok(HealTaskStatus::Pending);
+            }
+        }
+
+        {
+            let mut completed_heals = self.completed_heals.lock().await;
+            prune_completed_heal_statuses(&mut completed_heals);
+            if let Some(completed) = completed_heals.get(task_id)
+                && heal_type_matches_path(&completed.heal_type, heal_path)
+            {
+                return Ok(completed.status.clone());
+            }
+        }
+
+        if self.path_has_task(heal_path).await {
+            return Err(Error::InvalidClientToken);
+        }
+
+        Err(Error::TaskNotFound {
+            task_id: task_id.to_string(),
+        })
+    }
+
+    async fn path_has_task(&self, heal_path: &str) -> bool {
+        {
+            let active_heals = self.active_heals.lock().await;
+            if active_heals
+                .values()
+                .any(|task| heal_type_matches_path(&task.heal_type, heal_path))
+            {
+                return true;
+            }
+        }
+
+        {
+            let queue = self.heal_queue.lock().await;
+            if queue.contains_matching(|request| heal_type_matches_path(&request.heal_type, heal_path)) {
+                return true;
+            }
+        }
+
+        let mut completed_heals = self.completed_heals.lock().await;
+        prune_completed_heal_statuses(&mut completed_heals);
+        completed_heals
+            .values()
+            .any(|completed| heal_type_matches_path(&completed.heal_type, heal_path))
     }
 
     /// Get task progress
@@ -574,18 +803,68 @@ impl HealManager {
 
     /// Cancel task
     pub async fn cancel_task(&self, task_id: &str) -> Result<()> {
-        let mut active_heals = self.active_heals.lock().await;
-        if let Some(task) = active_heals.get(task_id) {
-            task.cancel().await?;
-            active_heals.remove(task_id);
-            publish_active_heal_count(&active_heals);
-            info!("Cancelled heal task: {}", task_id);
-            Ok(())
-        } else {
-            Err(Error::TaskNotFound {
-                task_id: task_id.to_string(),
-            })
+        {
+            let mut active_heals = self.active_heals.lock().await;
+            if let Some(task) = active_heals.get(task_id) {
+                task.cancel().await?;
+                active_heals.remove(task_id);
+                publish_active_heal_count(&active_heals);
+                info!("Cancelled active heal task: {}", task_id);
+                return Ok(());
+            }
         }
+
+        let mut queue = self.heal_queue.lock().await;
+        if queue.remove_request_id(task_id).is_some() {
+            publish_heal_queue_length(&queue);
+            info!("Cancelled queued heal task: {}", task_id);
+            return Ok(());
+        }
+
+        Err(Error::TaskNotFound {
+            task_id: task_id.to_string(),
+        })
+    }
+
+    /// Cancel all queued or active tasks matching a heal path.
+    pub async fn cancel_tasks_for_path(&self, heal_path: &str) -> Result<usize> {
+        let mut cancelled = 0usize;
+
+        {
+            let mut active_heals = self.active_heals.lock().await;
+            let task_ids = active_heals
+                .iter()
+                .filter_map(|(task_id, task)| heal_type_matches_path(&task.heal_type, heal_path).then_some(task_id.clone()))
+                .collect::<Vec<_>>();
+
+            for task_id in task_ids {
+                if let Some(task) = active_heals.get(&task_id) {
+                    task.cancel().await?;
+                }
+                active_heals.remove(&task_id);
+                cancelled += 1;
+            }
+
+            if cancelled > 0 {
+                publish_active_heal_count(&active_heals);
+            }
+        }
+
+        let mut queue = self.heal_queue.lock().await;
+        let queued_cancelled = queue.remove_matching(|request| heal_type_matches_path(&request.heal_type, heal_path));
+        if queued_cancelled > 0 {
+            publish_heal_queue_length(&queue);
+            cancelled += queued_cancelled;
+        }
+
+        if cancelled == 0 {
+            return Err(Error::TaskNotFound {
+                task_id: heal_path.to_string(),
+            });
+        }
+
+        info!("Cancelled {} heal task(s) for path: {}", cancelled, heal_path);
+        Ok(cancelled)
     }
 
     /// Get statistics
@@ -612,6 +891,7 @@ impl HealManager {
         let config = self.config.clone();
         let heal_queue = self.heal_queue.clone();
         let active_heals = self.active_heals.clone();
+        let completed_heals = self.completed_heals.clone();
         let cancel_token = self.cancel_token.clone();
         let statistics = self.statistics.clone();
         let storage = self.storage.clone();
@@ -628,10 +908,10 @@ impl HealManager {
                         break;
                     }
                     _ = notify.notified(), if event_driven_scheduler_enable => {
-                        Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage, &notify).await;
+                        Self::process_heal_queue(&heal_queue, &active_heals, &completed_heals, &config, &statistics, &storage, &notify).await;
                     }
                     _ = interval.tick() => {
-                        Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage, &notify).await;
+                        Self::process_heal_queue(&heal_queue, &active_heals, &completed_heals, &config, &statistics, &storage, &notify).await;
                     }
                 }
             }
@@ -768,6 +1048,7 @@ impl HealManager {
     async fn process_heal_queue(
         heal_queue: &Arc<Mutex<PriorityHealQueue>>,
         active_heals: &Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
+        completed_heals: &Arc<Mutex<HashMap<String, CompletedHealStatus>>>,
         config: &Arc<RwLock<HealConfig>>,
         statistics: &Arc<RwLock<HealStatistics>>,
         storage: &Arc<dyn HealStorageAPI>,
@@ -827,6 +1108,7 @@ impl HealManager {
                 publish_active_heal_count(&active_heals_guard);
                 update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
                 let active_heals_clone = active_heals.clone();
+                let completed_heals_clone = completed_heals.clone();
                 let statistics_clone = statistics.clone();
                 let notify_clone = notify.clone();
                 let task_type_label_for_spawn = task_type_label.clone();
@@ -851,9 +1133,19 @@ impl HealManager {
                     if let Some(completed_task) = active_heals_guard.remove(&task_id) {
                         publish_active_heal_count(&active_heals_guard);
                         update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
+                        let completed_status = completed_task.get_status().await;
+                        let completed_status_entry = CompletedHealStatus {
+                            heal_type: completed_task.heal_type.clone(),
+                            status: completed_status.clone(),
+                            result_items: completed_task.get_result_items().await,
+                            completed_at: SystemTime::now(),
+                        };
+                        let mut completed_heals_guard = completed_heals_clone.lock().await;
+                        prune_completed_heal_statuses(&mut completed_heals_guard);
+                        completed_heals_guard.insert(task_id.clone(), completed_status_entry);
                         // update statistics
                         let mut stats = statistics_clone.write().await;
-                        match completed_task.get_status().await {
+                        match completed_status {
                             HealTaskStatus::Completed => {
                                 stats.update_task_completion(true);
                             }
@@ -959,6 +1251,20 @@ fn running_erasure_set_counts(active_heals: &HashMap<String, Arc<HealTask>>) ->
     running
 }
 
+fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, CompletedHealStatus>) {
+    let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
+        return;
+    };
+
+    completed_heals.retain(|_, completed| {
+        completed
+            .completed_at
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
+            .unwrap_or(false)
+    });
+}
+
 fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap<String, usize>, max_concurrent_per_set: usize) -> bool {
     match heal_request_set_key(request) {
         Some(set_key) => running_per_set.get(&set_key).copied().unwrap_or(0) < max_concurrent_per_set,
@@ -1497,6 +1803,258 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_get_task_status_reports_pending_for_queued_request() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        let request = HealRequest::bucket("bucket".to_string());
+        let request_id = request.id.clone();
+
+        assert_eq!(
+            manager
+                .submit_heal_request(request)
+                .await
+                .expect("request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .get_task_status(&request_id)
+                .await
+                .expect("queued request should have status"),
+            HealTaskStatus::Pending
+        );
+    }
+
+    #[tokio::test]
+    async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        manager
+            .submit_heal_request(HealRequest::bucket("bucket".to_string()))
+            .await
+            .expect("request should be accepted");
+
+        assert!(matches!(
+            manager.get_task_status_for_path("bucket", "wrong-token").await,
+            Err(Error::InvalidClientToken)
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_get_task_status_for_path_rejects_token_from_other_active_path() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        let bucket_request = HealRequest::bucket("bucket".to_string());
+        let other_request = HealRequest::bucket("other".to_string());
+        let other_request_id = other_request.id.clone();
+
+        manager
+            .submit_heal_request(bucket_request)
+            .await
+            .expect("bucket request should be accepted");
+        manager
+            .submit_heal_request(other_request)
+            .await
+            .expect("other request should be accepted");
+
+        assert!(matches!(
+            manager.get_task_status_for_path("bucket", &other_request_id).await,
+            Err(Error::InvalidClientToken)
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        let request = HealRequest::bucket("bucket".to_string());
+        let request_id = request.id.clone();
+
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("request should be accepted");
+
+        assert!(matches!(
+            manager.get_task_status_for_path("other", &request_id).await,
+            Err(Error::TaskNotFound { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        assert!(matches!(
+            manager.get_task_status_for_path("bucket", "old-token").await,
+            Err(Error::TaskNotFound { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        let request = HealRequest::bucket("bucket".to_string());
+        let request_id = request.id.clone();
+
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("request should be accepted");
+
+        assert!(matches!(
+            manager.get_task_status_for_path("", &request_id).await,
+            Err(Error::TaskNotFound { .. })
+        ));
+        assert!(matches!(
+            manager.get_task_status_for_path("", "wrong-token").await,
+            Err(Error::TaskNotFound { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_get_task_status_reads_recent_completed_status() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        manager.completed_heals.lock().await.insert(
+            "completed-token".to_string(),
+            CompletedHealStatus {
+                heal_type: HealType::Bucket {
+                    bucket: "bucket".to_string(),
+                },
+                status: HealTaskStatus::Completed,
+                result_items: Vec::new(),
+                completed_at: SystemTime::now(),
+            },
+        );
+
+        assert_eq!(
+            manager
+                .get_task_status_for_path("bucket", "completed-token")
+                .await
+                .expect("recent completed task should be queryable"),
+            HealTaskStatus::Completed
+        );
+    }
+
+    #[tokio::test]
+    async fn test_get_task_report_for_path_reads_completed_items() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        manager.completed_heals.lock().await.insert(
+            "completed-token".to_string(),
+            CompletedHealStatus {
+                heal_type: HealType::Object {
+                    bucket: "bucket".to_string(),
+                    object: "object".to_string(),
+                    version_id: None,
+                },
+                status: HealTaskStatus::Completed,
+                result_items: vec![HealResultItem {
+                    bucket: "bucket".to_string(),
+                    object: "object".to_string(),
+                    object_size: 1024,
+                    ..Default::default()
+                }],
+                completed_at: SystemTime::now(),
+            },
+        );
+
+        let report = manager
+            .get_task_report_for_path("bucket/object", "completed-token")
+            .await
+            .expect("recent completed task report should be queryable");
+
+        assert_eq!(report.status, HealTaskStatus::Completed);
+        assert_eq!(report.result_items.len(), 1);
+        assert_eq!(report.result_items[0].object_size, 1024);
+    }
+
+    #[tokio::test]
+    async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        manager
+            .submit_heal_request(HealRequest::bucket("bucket".to_string()))
+            .await
+            .expect("request should be accepted");
+
+        assert!(matches!(
+            manager.get_task_report_for_path("", "wrong-token").await,
+            Err(Error::TaskNotFound { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_cancel_task_removes_queued_request() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        let request = HealRequest::bucket("bucket".to_string());
+        let request_id = request.id.clone();
+
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("request should be accepted");
+        manager
+            .cancel_task(&request_id)
+            .await
+            .expect("queued request should be cancelled");
+
+        assert!(matches!(manager.get_task_status(&request_id).await, Err(Error::TaskNotFound { .. })));
+    }
+
+    #[tokio::test]
+    async fn test_cancel_tasks_for_path_removes_matching_queued_requests() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        let bucket_request = HealRequest::bucket("bucket".to_string());
+        let bucket_request_id = bucket_request.id.clone();
+        let other_request = HealRequest::bucket("other".to_string());
+        let other_request_id = other_request.id.clone();
+
+        manager
+            .submit_heal_request(bucket_request)
+            .await
+            .expect("bucket request should be accepted");
+        manager
+            .submit_heal_request(other_request)
+            .await
+            .expect("other request should be accepted");
+
+        assert_eq!(
+            manager
+                .cancel_tasks_for_path("bucket")
+                .await
+                .expect("matching request should be cancelled"),
+            1
+        );
+        assert!(matches!(
+            manager.get_task_status(&bucket_request_id).await,
+            Err(Error::TaskNotFound { .. })
+        ));
+        assert_eq!(
+            manager
+                .get_task_status(&other_request_id)
+                .await
+                .expect("unmatched request should remain queued"),
+            HealTaskStatus::Pending
+        );
+    }
+
     #[tokio::test]
     async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
         let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
@@ -1577,6 +2135,122 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_force_start_bypasses_duplicate_and_full_admission() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(
+            storage,
+            Some(HealConfig {
+                queue_size: 1,
+                low_priority_drop_when_full: true,
+                ..HealConfig::default()
+            }),
+        );
+
+        let normal = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+        let mut forced_duplicate = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+        forced_duplicate.force_start = true;
+
+        let subsequent_duplicate = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+
+        assert_eq!(
+            manager
+                .submit_heal_request(normal)
+                .await
+                .expect("first request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .submit_heal_request(forced_duplicate)
+                .await
+                .expect("force start should bypass duplicate/full policy"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .submit_heal_request(subsequent_duplicate)
+                .await
+                .expect("subsequent non-force duplicate should be merged"),
+            HealAdmissionResult::Merged
+        );
+    }
+
+    #[tokio::test]
+    async fn test_force_start_marks_dedup_key_for_future_duplicates() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(
+            storage,
+            Some(HealConfig {
+                queue_size: 1,
+                ..HealConfig::default()
+            }),
+        );
+
+        let normal = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+        let mut forced = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+        forced.force_start = true;
+        let duplicate = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+
+        assert_eq!(
+            manager
+                .submit_heal_request(normal)
+                .await
+                .expect("first request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .submit_heal_request(forced)
+                .await
+                .expect("forced request should bypass duplicate/full admission"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .submit_heal_request(duplicate)
+                .await
+                .expect("non-forced duplicate should merge while forced request is queued"),
+            HealAdmissionResult::Merged
+        );
+    }
+
     #[test]
     fn test_running_erasure_set_counts_groups_only_erasure_tasks() {
         let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);

crates/heal/src/heal/task.rs

@@ -16,6 +16,7 @@ use crate::heal::{ErasureSetHealer, progress::HealProgress, storage::HealStorage
 use crate::{Error, Result};
 use metrics::{counter, histogram};
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
+use rustfs_madmin::heal_commands::HealResultItem;
 use serde::{Deserialize, Serialize};
 use std::{
     future::Future,
@@ -132,6 +133,8 @@ pub struct HealRequest {
     pub options: HealOptions,
     /// Priority
     pub priority: HealPriority,
+    /// Whether this request should bypass queue admission dedup/full policies.
+    pub force_start: bool,
     /// Created time
     pub created_at: SystemTime,
     /// Queue admission time used for scheduler delay metrics
@@ -146,6 +149,7 @@ impl HealRequest {
             heal_type,
             options,
             priority,
+            force_start: false,
             created_at: now,
             enqueued_at: now,
         }
@@ -196,6 +200,8 @@ pub struct HealTask {
     pub status: Arc<RwLock<HealTaskStatus>>,
     /// Progress tracking
     pub progress: Arc<RwLock<HealProgress>>,
+    /// Result items collected from storage heal calls.
+    pub result_items: Arc<RwLock<Vec<HealResultItem>>>,
     /// Created time
     pub created_at: SystemTime,
     /// Queue admission time
@@ -220,6 +226,7 @@ impl HealTask {
             options: request.options,
             status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
             progress: Arc::new(RwLock::new(HealProgress::new())),
+            result_items: Arc::new(RwLock::new(Vec::new())),
             created_at: request.created_at,
             enqueued_at: request.enqueued_at,
             started_at: Arc::new(RwLock::new(None)),
@@ -411,6 +418,14 @@ impl HealTask {
         self.progress.read().await.clone()
     }
 
+    pub async fn get_result_items(&self) -> Vec<HealResultItem> {
+        self.result_items.read().await.clone()
+    }
+
+    async fn record_result_item(&self, result: HealResultItem) {
+        self.result_items.write().await.push(result);
+    }
+
     // specific heal implementation method
     #[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
     async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
@@ -521,6 +536,7 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_progress(3, 3, object_size, object_size);
                 }
+                self.record_result_item(result).await;
                 Ok(())
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
@@ -601,6 +617,7 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_progress(4, 4, object_size, object_size);
                 }
+                self.record_result_item(result).await;
                 Ok(())
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
@@ -645,7 +662,11 @@ impl HealTask {
         let heal_opts = HealOpts {
             recursive: self.options.recursive,
             dry_run: self.options.dry_run,
-            remove: self.options.remove_corrupted,
+            remove: if self.options.recursive {
+                false
+            } else {
+                self.options.remove_corrupted
+            },
             recreate: self.options.recreate_missing,
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
@@ -659,8 +680,13 @@ impl HealTask {
         match heal_result {
             Ok(result) => {
                 info!("Bucket heal completed successfully: {} ({} drives)", bucket, result.after.drives.len());
+                self.record_result_item(result).await;
 
-                {
+                if self.options.recursive {
+                    self.heal_bucket_objects(bucket).await?;
+                }
+
+                if !self.options.recursive {
                     let mut progress = self.progress.write().await;
                     progress.update_progress(3, 3, 0, 0);
                 }
@@ -681,6 +707,98 @@ impl HealTask {
         }
     }
 
+    async fn heal_bucket_objects(&self, bucket: &str) -> Result<()> {
+        let mut continuation_token: Option<String> = None;
+        let mut scanned = 0u64;
+        let mut healed = 0u64;
+        let mut failed = 0u64;
+        let mut bytes = 0u64;
+
+        let heal_opts = HealOpts {
+            recursive: false,
+            dry_run: self.options.dry_run,
+            remove: self.options.remove_corrupted,
+            recreate: self.options.recreate_missing,
+            scan_mode: self.options.scan_mode,
+            update_parity: self.options.update_parity,
+            no_lock: false,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
+        };
+
+        loop {
+            self.check_control_flags().await?;
+            let (objects, next_token, is_truncated) = self
+                .await_with_control(
+                    self.storage
+                        .list_objects_for_heal_page(bucket, "", continuation_token.as_deref()),
+                )
+                .await?;
+
+            for object in objects {
+                self.check_control_flags().await?;
+                scanned += 1;
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.set_current_object(Some(format!("{bucket}/{object}")));
+                    progress.update_progress(scanned, healed, failed, bytes);
+                }
+
+                match self.await_with_control(self.storage.object_exists(bucket, &object)).await {
+                    Ok(false) => {
+                        healed += 1;
+                    }
+                    Ok(true) => match self
+                        .await_with_control(self.storage.heal_object(bucket, &object, None, &heal_opts))
+                        .await
+                    {
+                        Ok((result, None)) => {
+                            healed += 1;
+                            bytes = bytes.saturating_add(result.object_size as u64);
+                            self.record_result_item(result).await;
+                        }
+                        Ok((_, Some(err))) => {
+                            failed += 1;
+                            warn!("Failed to heal object {}/{}: {}", bucket, object, err);
+                        }
+                        Err(err) => {
+                            failed += 1;
+                            warn!("Failed to heal object {}/{}: {}", bucket, object, err);
+                        }
+                    },
+                    Err(err) => {
+                        failed += 1;
+                        warn!("Failed to check object {}/{} before heal: {}", bucket, object, err);
+                    }
+                }
+
+                {
+                    let mut progress = self.progress.write().await;
+                    progress.update_progress(scanned, healed, failed, bytes);
+                }
+            }
+
+            if !is_truncated {
+                break;
+            }
+
+            continuation_token = next_token;
+            if continuation_token.is_none() {
+                warn!("List is truncated but no continuation token was returned for bucket {}", bucket);
+                break;
+            }
+        }
+
+        if failed > 0 {
+            return Err(Error::TaskExecutionFailed {
+                message: format!("Failed to heal {failed} object(s) in bucket {bucket}"),
+            });
+        }
+
+        info!("Recursive bucket heal completed for {}: {} scanned, {} healed", bucket, scanned, healed);
+        Ok(())
+    }
+
     async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
         info!("Healing metadata: {}/{}", bucket, object);
 
@@ -755,6 +873,7 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_progress(3, 3, 0, 0);
                 }
+                self.record_result_item(result).await;
                 Ok(())
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
@@ -830,6 +949,7 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_progress(2, 2, 0, 0);
                 }
+                self.record_result_item(result).await;
                 Ok(())
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
@@ -923,6 +1043,7 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_progress(3, 3, object_size, object_size);
                 }
+                self.record_result_item(result).await;
                 Ok(())
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
@@ -1013,23 +1134,53 @@ impl HealTask {
 
         // Step 3: Heal bucket structure
         // Check control flags before each iteration to ensure timely cancellation.
-        // Each heal_bucket call may handle timeout/cancellation internally, see its implementation for details.
+        let bucket_heal_opts = HealOpts {
+            recursive: false,
+            dry_run: self.options.dry_run,
+            remove: false,
+            recreate: self.options.recreate_missing,
+            scan_mode: self.options.scan_mode,
+            update_parity: self.options.update_parity,
+            no_lock: false,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
+        };
+
         for bucket in buckets.iter() {
             // Check control flags before starting each bucket heal
             self.check_control_flags().await?;
-            // heal_bucket internally uses await_with_control for timeout/cancellation handling
-            if let Err(err) = self.heal_bucket(bucket).await {
-                // Check if error is due to cancellation or timeout
-                if matches!(err, Error::TaskCancelled | Error::TaskTimeout) {
-                    return Err(err);
+            let heal_result = self
+                .await_with_control(self.storage.heal_bucket(bucket, &bucket_heal_opts))
+                .await;
+            match heal_result {
+                Ok(result) => {
+                    self.record_result_item(result).await;
+                }
+                Err(err) => {
+                    // Check if error is due to cancellation or timeout
+                    if matches!(err, Error::TaskCancelled | Error::TaskTimeout) {
+                        return Err(err);
+                    }
+                    info!("Bucket heal failed: {}", err.to_string());
                 }
-                info!("Bucket heal failed: {}", err.to_string());
             }
         }
 
-        // Step 3: Create erasure set healer with resume support
-        info!("Step 3: Creating erasure set healer with resume support");
-        let erasure_healer = ErasureSetHealer::new(self.storage.clone(), self.progress.clone(), self.cancel_token.clone(), disk);
+        // Create erasure set healer with resume support
+        info!("Creating erasure set healer with resume support");
+        let heal_opts = HealOpts {
+            recursive: self.options.recursive,
+            dry_run: self.options.dry_run,
+            remove: self.options.remove_corrupted,
+            recreate: self.options.recreate_missing,
+            scan_mode: self.options.scan_mode,
+            update_parity: self.options.update_parity,
+            no_lock: false,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
+        };
+        let erasure_healer =
+            ErasureSetHealer::new(self.storage.clone(), self.progress.clone(), self.cancel_token.clone(), disk, heal_opts);
 
         {
             let mut progress = self.progress.write().await;
@@ -1072,3 +1223,203 @@ impl std::fmt::Debug for HealTask {
             .finish()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::heal::storage::DiskStatus;
+    use rustfs_ecstore::{
+        disk::{DiskStore, endpoint::Endpoint},
+        store_api::{BucketInfo, ObjectInfo},
+    };
+    use rustfs_madmin::heal_commands::HealResultItem;
+    use std::sync::Mutex;
+
+    #[derive(Default)]
+    struct MockStorage {
+        listed: Mutex<bool>,
+        healed_objects: Mutex<Vec<String>>,
+        bucket_heal_opts: Mutex<Vec<HealOpts>>,
+        object_heal_opts: Mutex<Vec<HealOpts>>,
+    }
+
+    #[async_trait::async_trait]
+    impl HealStorageAPI for MockStorage {
+        async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<ObjectInfo>> {
+            Ok(None)
+        }
+
+        async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result<Option<Vec<u8>>> {
+            Ok(None)
+        }
+
+        async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> {
+            Ok(())
+        }
+
+        async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<()> {
+            Ok(())
+        }
+
+        async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result<bool> {
+            Ok(true)
+        }
+
+        async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result<Vec<u8>> {
+            Ok(Vec::new())
+        }
+
+        async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result<DiskStatus> {
+            Ok(DiskStatus::Ok)
+        }
+
+        async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> {
+            Ok(())
+        }
+
+        async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
+            Ok(Some(BucketInfo {
+                name: bucket.to_string(),
+                ..Default::default()
+            }))
+        }
+
+        async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> {
+            Ok(())
+        }
+
+        async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
+            Ok(Vec::new())
+        }
+
+        async fn object_exists(&self, _bucket: &str, _object: &str) -> Result<bool> {
+            Ok(true)
+        }
+
+        async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result<Option<u64>> {
+            Ok(None)
+        }
+
+        async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result<Option<String>> {
+            Ok(None)
+        }
+
+        async fn heal_object(
+            &self,
+            _bucket: &str,
+            object: &str,
+            _version_id: Option<&str>,
+            opts: &HealOpts,
+        ) -> Result<(HealResultItem, Option<Error>)> {
+            self.healed_objects.lock().unwrap().push(object.to_string());
+            self.object_heal_opts.lock().unwrap().push(*opts);
+            Ok((
+                HealResultItem {
+                    object_size: 1,
+                    ..Default::default()
+                },
+                None,
+            ))
+        }
+
+        async fn heal_bucket(&self, _bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+            self.bucket_heal_opts.lock().unwrap().push(*opts);
+            Ok(HealResultItem::default())
+        }
+
+        async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+            Ok((HealResultItem::default(), None))
+        }
+
+        async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result<Vec<String>> {
+            Ok(vec!["object-a".to_string(), "object-b".to_string()])
+        }
+
+        async fn list_objects_for_heal_page(
+            &self,
+            _bucket: &str,
+            _prefix: &str,
+            continuation_token: Option<&str>,
+        ) -> Result<(Vec<String>, Option<String>, bool)> {
+            let mut listed = self.listed.lock().unwrap();
+            if continuation_token.is_none() && !*listed {
+                *listed = true;
+                Ok((vec!["object-a".to_string(), "object-b".to_string()], None, false))
+            } else {
+                Ok((Vec::new(), None, false))
+            }
+        }
+
+        async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result<DiskStore> {
+            Err(Error::other("not implemented in tests"))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_recursive_bucket_heal_visits_objects() {
+        let storage = Arc::new(MockStorage::default());
+        let request = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-a".to_string(),
+            },
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        let task = HealTask::from_request(request, storage.clone());
+
+        task.heal_bucket("bucket-a")
+            .await
+            .expect("recursive bucket heal should succeed");
+
+        assert_eq!(
+            storage.healed_objects.lock().unwrap().as_slice(),
+            ["object-a".to_string(), "object-b".to_string()]
+        );
+        let progress = task.get_progress().await;
+        assert_eq!(progress.objects_scanned, 2);
+        assert_eq!(progress.objects_healed, 2);
+        let result_items = task.get_result_items().await;
+        assert_eq!(result_items.len(), 3);
+        assert_eq!(result_items.iter().filter(|item| item.object_size == 1).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_recursive_bucket_heal_does_not_remove_bucket_metadata() {
+        let storage = Arc::new(MockStorage::default());
+        let request = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-a".to_string(),
+            },
+            HealOptions {
+                recursive: true,
+                remove_corrupted: true,
+                recreate_missing: true,
+                scan_mode: HealScanMode::Deep,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        let task = HealTask::from_request(request, storage.clone());
+
+        task.heal_bucket("bucket-a")
+            .await
+            .expect("recursive bucket heal should succeed");
+
+        let bucket_opts = storage.bucket_heal_opts.lock().unwrap();
+        assert_eq!(bucket_opts.len(), 1);
+        assert!(!bucket_opts[0].remove);
+        assert!(bucket_opts[0].recreate);
+        assert_eq!(bucket_opts[0].scan_mode, HealScanMode::Deep);
+
+        let object_opts = storage.object_heal_opts.lock().unwrap();
+        assert_eq!(object_opts.len(), 2);
+        assert!(object_opts.iter().all(|opts| opts.remove));
+        assert!(object_opts.iter().all(|opts| opts.recreate));
+        assert!(object_opts.iter().all(|opts| opts.scan_mode == HealScanMode::Deep));
+    }
+}

crates/iam/Cargo.toml

@@ -42,7 +42,6 @@ thiserror.workspace = true
 arc-swap = { workspace = true }
 rustfs-crypto = { workspace = true }
 futures.workspace = true
-rand.workspace = true
 base64-simd = { workspace = true }
 jsonwebtoken = { workspace = true }
 tracing.workspace = true

crates/iam/src/utils.rs

@@ -13,75 +13,8 @@
 // limitations under the License.
 
 use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header};
-use rand::{Rng, RngExt};
 use serde::{Serialize, de::DeserializeOwned};
 use std::collections::HashSet;
-use std::io::{Error, Result};
-
-/// Generates a random access key of the specified length.
-///
-/// # Arguments
-///
-/// * `length` - The length of the access key to be generated.
-///
-/// # Returns
-///
-/// * `Result<String>` - A result containing the generated access key or an error if the length is invalid.
-///
-/// # Errors
-///
-/// * Returns an error if the length is less than 3.
-///
-pub fn gen_access_key(length: usize) -> Result<String> {
-    const ALPHA_NUMERIC_TABLE: [char; 36] = [
-        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N',
-        'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
-    ];
-
-    if length < 3 {
-        return Err(Error::other("access key length is too short"));
-    }
-
-    let mut result = String::with_capacity(length);
-    let mut rng = rand::rng();
-
-    for _ in 0..length {
-        result.push(ALPHA_NUMERIC_TABLE[rng.random_range(0..ALPHA_NUMERIC_TABLE.len())]);
-    }
-
-    Ok(result)
-}
-
-/// Generates a random secret key of the specified length.
-///
-/// # Arguments
-///
-/// * `length` - The length of the secret key to be generated.
-///
-/// # Returns
-///
-/// * `Result<String>` - A result containing the generated secret key or an error if the length is invalid.
-///
-/// # Errors
-///
-/// * Returns an error if the length is less than 8.
-///
-pub fn gen_secret_key(length: usize) -> Result<String> {
-    use base64_simd::URL_SAFE_NO_PAD;
-
-    if length < 8 {
-        return Err(Error::other("secret key length is too short"));
-    }
-    let mut rng = rand::rng();
-
-    let mut key = vec![0u8; URL_SAFE_NO_PAD.estimated_decoded_length(length)];
-    rng.fill_bytes(&mut key);
-
-    let encoded = URL_SAFE_NO_PAD.encode_to_string(&key);
-    let key_str = encoded.replace("/", "+");
-
-    Ok(key_str)
-}
 
 pub fn generate_jwt<T: Serialize>(claims: &T, secret: &str) -> std::result::Result<String, jsonwebtoken::errors::Error> {
     let header = Header::new(Algorithm::HS512);
@@ -111,99 +44,9 @@ pub fn extract_claims_allow_missing_exp<T: DeserializeOwned + Clone>(
 
 #[cfg(test)]
 mod tests {
-    use super::{extract_claims, gen_access_key, gen_secret_key, generate_jwt};
+    use super::{extract_claims, generate_jwt};
     use serde::{Deserialize, Serialize};
 
-    #[test]
-    fn test_gen_access_key_valid_length() {
-        // Test valid access key generation
-        let key = gen_access_key(10).unwrap();
-        assert_eq!(key.len(), 10);
-
-        // Test different lengths
-        let key_20 = gen_access_key(20).unwrap();
-        assert_eq!(key_20.len(), 20);
-
-        let key_3 = gen_access_key(3).unwrap();
-        assert_eq!(key_3.len(), 3);
-    }
-
-    #[test]
-    fn test_gen_access_key_uniqueness() {
-        // Test that generated keys are unique
-        let key1 = gen_access_key(16).unwrap();
-        let key2 = gen_access_key(16).unwrap();
-        assert_ne!(key1, key2, "Generated access keys should be unique");
-    }
-
-    #[test]
-    fn test_gen_access_key_character_set() {
-        // Test that generated keys only contain valid characters
-        let key = gen_access_key(100).unwrap();
-        for ch in key.chars() {
-            assert!(ch.is_ascii_alphanumeric(), "Access key should only contain alphanumeric characters");
-            assert!(
-                ch.is_ascii_uppercase() || ch.is_ascii_digit(),
-                "Access key should only contain uppercase letters and digits"
-            );
-        }
-    }
-
-    #[test]
-    fn test_gen_access_key_invalid_length() {
-        // Test error cases for invalid lengths
-        assert!(gen_access_key(0).is_err(), "Should fail for length 0");
-        assert!(gen_access_key(1).is_err(), "Should fail for length 1");
-        assert!(gen_access_key(2).is_err(), "Should fail for length 2");
-
-        // Verify error message
-        let error = gen_access_key(2).unwrap_err();
-        assert_eq!(error.to_string(), "access key length is too short");
-    }
-
-    #[test]
-    fn test_gen_secret_key_valid_length() {
-        // Test valid secret key generation
-        let key = gen_secret_key(10).unwrap();
-        assert!(!key.is_empty(), "Secret key should not be empty");
-
-        let key_20 = gen_secret_key(20).unwrap();
-        assert!(!key_20.is_empty(), "Secret key should not be empty");
-    }
-
-    #[test]
-    fn test_gen_secret_key_uniqueness() {
-        // Test that generated secret keys are unique
-        let key1 = gen_secret_key(16).unwrap();
-        let key2 = gen_secret_key(16).unwrap();
-        assert_ne!(key1, key2, "Generated secret keys should be unique");
-    }
-
-    #[test]
-    fn test_gen_secret_key_base64_format() {
-        // Test that secret key is valid base64-like format
-        let key = gen_secret_key(32).unwrap();
-
-        // Should not contain invalid characters for URL-safe base64
-        for ch in key.chars() {
-            assert!(
-                ch.is_ascii_alphanumeric() || ch == '+' || ch == '-' || ch == '_',
-                "Secret key should be URL-safe base64 compatible"
-            );
-        }
-    }
-
-    #[test]
-    fn test_gen_secret_key_invalid_length() {
-        // Test error cases for invalid lengths
-        assert!(gen_secret_key(0).is_err(), "Should fail for length 0");
-        assert!(gen_secret_key(7).is_err(), "Should fail for length 7");
-
-        // Verify error message
-        let error = gen_secret_key(5).unwrap_err();
-        assert_eq!(error.to_string(), "secret key length is too short");
-    }
-
     #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
     struct Claims {
         sub: String,
@@ -370,26 +213,4 @@ mod tests {
 
         assert_eq!(decoded.claims, special_claims);
     }
-
-    #[test]
-    fn test_access_key_length_boundaries() {
-        // Test boundary conditions for access key length
-        assert!(gen_access_key(3).is_ok(), "Length 3 should be valid (minimum)");
-        assert!(gen_access_key(1000).is_ok(), "Large length should be valid");
-
-        // Test that minimum length is enforced
-        let min_key = gen_access_key(3).unwrap();
-        assert_eq!(min_key.len(), 3);
-    }
-
-    #[test]
-    fn test_secret_key_length_boundaries() {
-        // Test boundary conditions for secret key length
-        assert!(gen_secret_key(8).is_ok(), "Length 8 should be valid (minimum)");
-        assert!(gen_secret_key(1000).is_ok(), "Large length should be valid");
-
-        // Test that minimum length is enforced
-        let result = gen_secret_key(8);
-        assert!(result.is_ok(), "Minimum valid length should work");
-    }
 }

crates/io-metrics/README.md

@@ -143,6 +143,47 @@ record_backpressure_event("warning", 0.85);
 record_timeout_event("GetObject", Duration::from_secs(30));
 ```
 
+### Internode Transport Metrics
+
+Internode metrics are recorded by `src/internode_metrics.rs`. Aggregate metrics
+remain unlabeled for compatibility with existing dashboards:
+
+| Metric | Meaning |
+| --- | --- |
+| `rustfs_system_network_internode_sent_bytes_total` | Total internode bytes sent by this node. |
+| `rustfs_system_network_internode_recv_bytes_total` | Total internode bytes received by this node. |
+| `rustfs_system_network_internode_requests_outgoing_total` | Total outgoing internode requests. |
+| `rustfs_system_network_internode_requests_incoming_total` | Total incoming internode requests. |
+| `rustfs_system_network_internode_errors_total` | Total internode errors. |
+| `rustfs_system_network_internode_dial_errors_total` | Failed internode connection attempts. |
+| `rustfs_system_network_internode_dial_avg_time_nanos` | Average internode dial duration. |
+
+Operation-level metrics use the same low-cardinality label set:
+
+| Metric | Labels | Meaning |
+| --- | --- | --- |
+| `rustfs_system_network_internode_operation_sent_bytes_total` | `operation`, `backend` | Bytes sent for an internode operation. |
+| `rustfs_system_network_internode_operation_recv_bytes_total` | `operation`, `backend` | Bytes received for an internode operation. |
+| `rustfs_system_network_internode_operation_requests_outgoing_total` | `operation`, `backend` | Outgoing request attempts for an internode operation. |
+| `rustfs_system_network_internode_operation_requests_incoming_total` | `operation`, `backend` | Incoming request attempts for an internode operation. |
+| `rustfs_system_network_internode_operation_errors_total` | `operation`, `backend` | Failed internode operation attempts. |
+
+Current `operation` values are `read_file_stream`, `put_file_stream`,
+`walk_dir`, `grpc_read_all`, and `grpc_write_all`. Current `backend` values are
+`tcp-http` for the `InternodeDataTransport` TCP/HTTP path and `grpc` for the
+remaining gRPC byte paths. The compatibility wrapper uses `unknown` only for
+callers that have not been classified yet.
+
+Success/failure is intentionally not a high-cardinality label today. Failures
+are represented by `rustfs_system_network_internode_operation_errors_total`;
+successful completions are not emitted as a dedicated result-labeled metric.
+Adding completion/result labels is a follow-up once stream completion semantics
+are defined consistently for request setup, body transfer, and shutdown.
+
+`scripts/run_internode_transport_baseline.sh --metrics-url ...` records metric
+deltas with `operation` and `backend` columns, so the TCP baseline can attribute
+bytes and request/error counts to `tcp-http` transport operations.
+
 ### Unified Configuration
 
 Centralized configuration management:
@@ -181,6 +222,7 @@ rustfs-io-metrics/
 │   ├── deadlock_metrics.rs  # Deadlock metrics
 │   ├── lock_metrics.rs      # Lock metrics
 │   ├── timeout_metrics.rs   # Timeout metrics
+│   ├── internode_metrics.rs # Internode transport metrics
 │   ├── bandwidth.rs         # Bandwidth monitoring
 │   ├── global_metrics.rs    # Global metrics
 │   └── performance.rs       # Performance metrics

crates/io-metrics/src/internode_metrics.rs

@@ -24,14 +24,49 @@ pub const INTERNODE_OPERATION_PUT_FILE_STREAM: &str = "put_file_stream";
 pub const INTERNODE_OPERATION_WALK_DIR: &str = "walk_dir";
 pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all";
 pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all";
+pub const INTERNODE_TRANSPORT_BACKEND_TCP_HTTP: &str = "tcp-http";
+pub const INTERNODE_TRANSPORT_BACKEND_GRPC: &str = "grpc";
+pub const INTERNODE_TRANSPORT_BACKEND_UNKNOWN: &str = "unknown";
 
 const OPERATION_LABEL: &str = "operation";
+const BACKEND_LABEL: &str = "backend";
 const INTERNODE_OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
 const INTERNODE_OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
 const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
 const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
 const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct InternodeOperationMetricDescriptor {
+    pub name: &'static str,
+    pub labels: &'static [&'static str],
+}
+
+const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL];
+
+pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_SENT_BYTES_TOTAL,
+        labels: OPERATION_BACKEND_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_RECV_BYTES_TOTAL,
+        labels: OPERATION_BACKEND_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
+        labels: OPERATION_BACKEND_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
+        labels: OPERATION_BACKEND_LABELS,
+    },
+    InternodeOperationMetricDescriptor {
+        name: INTERNODE_OPERATION_ERRORS_TOTAL,
+        labels: OPERATION_BACKEND_LABELS,
+    },
+];
+
 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
 pub struct InternodeMetricsSnapshot {
     pub sent_bytes_total: u64,
@@ -68,13 +103,17 @@ impl InternodeMetrics {
     }
 
     pub fn record_sent_bytes_for_operation(&self, operation: &'static str, bytes: usize) {
+        self.record_sent_bytes_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN, bytes);
+    }
+
+    pub fn record_sent_bytes_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, bytes: usize) {
         self.record_sent_bytes(bytes);
 
         let bytes = bytes as u64;
         if bytes == 0 {
             return;
         }
-        counter!(INTERNODE_OPERATION_SENT_BYTES_TOTAL, OPERATION_LABEL => operation).increment(bytes);
+        counter!(INTERNODE_OPERATION_SENT_BYTES_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(bytes);
     }
 
     pub fn record_recv_bytes(&self, bytes: usize) {
@@ -87,13 +126,17 @@ impl InternodeMetrics {
     }
 
     pub fn record_recv_bytes_for_operation(&self, operation: &'static str, bytes: usize) {
+        self.record_recv_bytes_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN, bytes);
+    }
+
+    pub fn record_recv_bytes_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, bytes: usize) {
         self.record_recv_bytes(bytes);
 
         let bytes = bytes as u64;
         if bytes == 0 {
             return;
         }
-        counter!(INTERNODE_OPERATION_RECV_BYTES_TOTAL, OPERATION_LABEL => operation).increment(bytes);
+        counter!(INTERNODE_OPERATION_RECV_BYTES_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(bytes);
     }
 
     pub fn record_outgoing_request(&self) {
@@ -102,8 +145,13 @@ impl InternodeMetrics {
     }
 
     pub fn record_outgoing_request_for_operation(&self, operation: &'static str) {
+        self.record_outgoing_request_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN);
+    }
+
+    pub fn record_outgoing_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_outgoing_request();
-        counter!(INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL, OPERATION_LABEL => operation).increment(1);
+        counter!(INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
+            .increment(1);
     }
 
     pub fn record_incoming_request(&self) {
@@ -112,8 +160,13 @@ impl InternodeMetrics {
     }
 
     pub fn record_incoming_request_for_operation(&self, operation: &'static str) {
+        self.record_incoming_request_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN);
+    }
+
+    pub fn record_incoming_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_incoming_request();
-        counter!(INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL, OPERATION_LABEL => operation).increment(1);
+        counter!(INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
+            .increment(1);
     }
 
     pub fn record_error(&self) {
@@ -122,8 +175,12 @@ impl InternodeMetrics {
     }
 
     pub fn record_error_for_operation(&self, operation: &'static str) {
+        self.record_error_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_UNKNOWN);
+    }
+
+    pub fn record_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
         self.record_error();
-        counter!(INTERNODE_OPERATION_ERRORS_TOTAL, OPERATION_LABEL => operation).increment(1);
+        counter!(INTERNODE_OPERATION_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
     }
 
     pub fn record_dial_result(&self, duration: Duration, success: bool) {
@@ -216,11 +273,25 @@ mod tests {
     fn operation_metrics_also_update_aggregate_snapshot() {
         let metrics = InternodeMetrics::default();
 
-        metrics.record_sent_bytes_for_operation(INTERNODE_OPERATION_READ_FILE_STREAM, 128);
-        metrics.record_recv_bytes_for_operation(INTERNODE_OPERATION_PUT_FILE_STREAM, 256);
-        metrics.record_outgoing_request_for_operation(INTERNODE_OPERATION_GRPC_WRITE_ALL);
-        metrics.record_incoming_request_for_operation(INTERNODE_OPERATION_GRPC_READ_ALL);
-        metrics.record_error_for_operation(INTERNODE_OPERATION_WALK_DIR);
+        metrics.record_sent_bytes_for_operation_and_backend(
+            INTERNODE_OPERATION_READ_FILE_STREAM,
+            INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+            128,
+        );
+        metrics.record_recv_bytes_for_operation_and_backend(
+            INTERNODE_OPERATION_PUT_FILE_STREAM,
+            INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+            256,
+        );
+        metrics.record_outgoing_request_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_WRITE_ALL,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+        );
+        metrics.record_incoming_request_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_READ_ALL,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+        );
+        metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
 
         let snapshot = metrics.snapshot();
         assert_eq!(snapshot.sent_bytes_total, 128);
@@ -229,4 +300,51 @@ mod tests {
         assert_eq!(snapshot.incoming_requests_total, 1);
         assert_eq!(snapshot.errors_total, 1);
     }
+
+    #[test]
+    fn operation_metric_descriptors_include_backend_and_operation_labels() {
+        assert_eq!(INTERNODE_OPERATION_METRICS.len(), 5);
+        for metric in INTERNODE_OPERATION_METRICS {
+            assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
+        }
+    }
+
+    #[test]
+    fn operation_metric_names_and_low_cardinality_values_are_stable() {
+        assert_eq!(INTERNODE_OPERATION_READ_FILE_STREAM, "read_file_stream");
+        assert_eq!(INTERNODE_OPERATION_PUT_FILE_STREAM, "put_file_stream");
+        assert_eq!(INTERNODE_OPERATION_WALK_DIR, "walk_dir");
+        assert_eq!(INTERNODE_OPERATION_GRPC_READ_ALL, "grpc_read_all");
+        assert_eq!(INTERNODE_OPERATION_GRPC_WRITE_ALL, "grpc_write_all");
+
+        assert_eq!(INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, "tcp-http");
+        assert_eq!(INTERNODE_TRANSPORT_BACKEND_GRPC, "grpc");
+        assert_eq!(INTERNODE_TRANSPORT_BACKEND_UNKNOWN, "unknown");
+
+        assert_eq!(
+            INTERNODE_OPERATION_METRICS,
+            &[
+                InternodeOperationMetricDescriptor {
+                    name: "rustfs_system_network_internode_operation_sent_bytes_total",
+                    labels: &[OPERATION_LABEL, BACKEND_LABEL],
+                },
+                InternodeOperationMetricDescriptor {
+                    name: "rustfs_system_network_internode_operation_recv_bytes_total",
+                    labels: &[OPERATION_LABEL, BACKEND_LABEL],
+                },
+                InternodeOperationMetricDescriptor {
+                    name: "rustfs_system_network_internode_operation_requests_outgoing_total",
+                    labels: &[OPERATION_LABEL, BACKEND_LABEL],
+                },
+                InternodeOperationMetricDescriptor {
+                    name: "rustfs_system_network_internode_operation_requests_incoming_total",
+                    labels: &[OPERATION_LABEL, BACKEND_LABEL],
+                },
+                InternodeOperationMetricDescriptor {
+                    name: "rustfs_system_network_internode_operation_errors_total",
+                    labels: &[OPERATION_LABEL, BACKEND_LABEL],
+                },
+            ]
+        );
+    }
 }

crates/madmin/src/user.rs

@@ -500,6 +500,22 @@ where
     Ok(Option::<Vec<String>>::deserialize(deserializer)?.unwrap_or_default())
 }
 
+fn parse_service_account_expiration(expiration: &str) -> Result<OffsetDateTime, time::Error> {
+    const LEGACY_SERVICE_ACCOUNT_EXPIRATION_FORMAT: &[time::format_description::BorrowedFormatItem<'_>] = time::macros::format_description!(
+        "[year]-[month]-[day] [hour]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]"
+    );
+
+    OffsetDateTime::parse(expiration, &Rfc3339)
+        .or_else(|_| OffsetDateTime::parse(expiration, LEGACY_SERVICE_ACCOUNT_EXPIRATION_FORMAT).map_err(Into::into))
+}
+
+fn serialize_optional_service_account_expiration<S>(expiration: &Option<OffsetDateTime>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    time::serde::rfc3339::option::serialize(expiration, serializer)
+}
+
 fn deserialize_optional_service_account_expiration<'de, D>(deserializer: D) -> Result<Option<OffsetDateTime>, D::Error>
 where
     D: Deserializer<'de>,
@@ -513,7 +529,7 @@ where
         return Ok(None);
     }
 
-    let expiration = OffsetDateTime::parse(expiration, &Rfc3339).map_err(D::Error::custom)?;
+    let expiration = parse_service_account_expiration(expiration).map_err(D::Error::custom)?;
     if expiration.unix_timestamp() == 0 {
         return Ok(None);
     }
@@ -549,6 +565,7 @@ pub struct SRSvcAccCreate {
     #[serde(
         default,
         skip_serializing_if = "Option::is_none",
+        serialize_with = "serialize_optional_service_account_expiration",
         deserialize_with = "deserialize_optional_service_account_expiration"
     )]
     pub expiration: Option<OffsetDateTime>,
@@ -706,6 +723,7 @@ mod tests {
     use super::*;
     use serde_json;
     use time::OffsetDateTime;
+    use time::macros::datetime;
 
     #[test]
     fn test_account_status_default() {
@@ -1295,4 +1313,50 @@ mod tests {
         let svc: SRSvcAccCreate = serde_json::from_str(payload).unwrap();
         assert!(svc.expiration.is_none());
     }
+
+    #[test]
+    fn test_sr_svc_acc_create_serializes_expiration_as_rfc3339() {
+        let svc = SRSvcAccCreate {
+            parent: "useralpha".to_string(),
+            access_key: "svcalpha".to_string(),
+            secret_key: "svcAlphaSecret123".to_string(),
+            groups: Vec::new(),
+            claims: HashMap::new(),
+            session_policy: SRSessionPolicy::default(),
+            status: "on".to_string(),
+            name: "uploaderKey".to_string(),
+            description: "alpha upload key".to_string(),
+            expiration: Some(datetime!(9999-01-01 00:00:00 UTC)),
+            api_version: None,
+        };
+
+        let json = serde_json::to_string(&svc).unwrap();
+        assert!(
+            json.contains(r#""expiration":"9999-01-01T00:00:00Z""#),
+            "service account export expiration should be RFC3339; got: {json}"
+        );
+        assert!(!json.contains("+00:00:00"), "export must not use the legacy offset format: {json}");
+
+        let decoded: SRSvcAccCreate = serde_json::from_str(&json).unwrap();
+        assert_eq!(decoded.expiration, svc.expiration);
+    }
+
+    #[test]
+    fn test_sr_svc_acc_create_deserializes_legacy_exported_expiration() {
+        let payload = r#"{
+            "parent": "useralpha",
+            "accessKey": "svcalpha",
+            "secretKey": "svcAlphaSecret123",
+            "groups": [],
+            "claims": {},
+            "sessionPolicy": null,
+            "status": "on",
+            "name": "uploaderKey",
+            "description": "alpha upload key",
+            "expiration": "9999-01-01 00:00:00.0 +00:00:00"
+        }"#;
+
+        let svc: SRSvcAccCreate = serde_json::from_str(payload).unwrap();
+        assert_eq!(svc.expiration, Some(datetime!(9999-01-01 00:00:00 UTC)));
+    }
 }

crates/protocols/Cargo.toml

@@ -27,7 +27,7 @@ categories = ["network-programming", "filesystem"]
 
 [features]
 default = []
-ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls"]
+ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime"]
 swift = [
     "dep:rustfs-keystone",
     "dep:rustfs-ecstore",
@@ -53,7 +53,7 @@ swift = [
     "dep:base64",
     "dep:async-compression",
 ]
-webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding"]
+webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime"]
 sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"]
 
 [dependencies]
@@ -63,6 +63,7 @@ rustfs-credentials = { workspace = true }
 rustfs-policy = { workspace = true }
 rustfs-utils = { workspace = true }
 rustfs-config = { workspace = true }
+rustfs-tls-runtime = { workspace = true, optional = true }
 
 # Async dependencies
 tokio = { workspace = true, features = ["fs", "io-util", "sync", "time","io-uring"] }
@@ -128,7 +129,6 @@ socket2 = { workspace = true, optional = true }
 [dev-dependencies]
 tempfile = { workspace = true }
 proptest = "1"
-rcgen = { workspace = true }
 tracing-subscriber = { workspace = true }
 
 [package.metadata.docs.rs]

crates/protocols/src/ftps/server.rs

@@ -17,12 +17,14 @@ use super::driver::FtpsDriver;
 use crate::common::client::s3::StorageBackend;
 use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
 use crate::constants::{network::DEFAULT_SOURCE_IP, paths::ROOT_PATH};
-use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop};
 use libunftp::options::FtpsRequired;
+use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
+use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
 use std::fmt::{Debug, Display, Formatter};
 use std::net::IpAddr;
 use std::path::Path;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::broadcast;
 use tokio::sync::watch;
 use tracing::{debug, error, info, warn};
@@ -68,6 +70,14 @@ impl<S> FtpsServer<S>
 where
     S: StorageBackend + Clone + Send + Sync + 'static + Debug,
 {
+    fn tls_reload_options() -> TlsReloadOptions {
+        TlsReloadOptions {
+            enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE),
+            interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)),
+            ..TlsReloadOptions::default()
+        }
+    }
+
     /// Create a new FTPS server
     pub async fn new(config: FtpsConfig, storage: S) -> Result<Self, FtpsInitError> {
         config.validate().await?;
@@ -114,9 +124,14 @@ where
             if let Some(cert_dir) = &self.config.cert_dir {
                 debug!("Enabling FTPS with multi-certificate support from directory: {}", cert_dir);
 
-                let resolver = ReloadableCertResolver::load_from_directory(cert_dir)
+                let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
                     .map_err(|e| FtpsInitError::InvalidConfig(format!("Failed to create certificate resolver: {}", e)))?;
-                let _reload_task = spawn_cert_reload_loop("ftps", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone());
+                let _reload_task = spawn_server_cert_reload_loop(
+                    "ftps",
+                    resolver.clone(),
+                    Self::tls_reload_options(),
+                    reload_shutdown_rx.clone(),
+                );
 
                 // Build ServerConfig with SNI support
                 let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();

crates/protocols/src/lib.rs

@@ -17,9 +17,6 @@
 pub mod common;
 pub mod constants;
 
-#[cfg(any(feature = "ftps", feature = "webdav"))]
-mod tls_hot_reload;
-
 #[cfg(feature = "ftps")]
 pub mod ftps;
 

crates/protocols/src/sftp/errors.rs

@@ -18,6 +18,7 @@
 
 use super::constants::{http_error_codes, s3_error_codes};
 use russh_sftp::protocol::{Status, StatusCode};
+use russh_sftp::server::StatusReply;
 use s3s::{S3Error, S3ErrorCode};
 use std::{any::Any, fmt::Display};
 
@@ -31,6 +32,12 @@ impl From<SftpError> for StatusCode {
     }
 }
 
+impl From<SftpError> for StatusReply {
+    fn from(err: SftpError) -> Self {
+        StatusReply::new(err.0)
+    }
+}
+
 impl SftpError {
     pub(super) fn code(code: StatusCode) -> Self {
         Self(code)

crates/protocols/src/webdav/server.rs

@@ -16,7 +16,6 @@ use super::config::{WebDavConfig, WebDavInitError};
 use super::driver::WebDavDriver;
 use crate::common::client::s3::StorageBackend;
 use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
-use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop};
 use bytes::Bytes;
 use dav_server::DavHandler;
 use dav_server::fakels::FakeLs;
@@ -25,10 +24,13 @@ use hyper::server::conn::http1;
 use hyper::service::service_fn;
 use hyper::{Request, Response, StatusCode};
 use hyper_util::rt::TokioIo;
+use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
+use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
 use rustls::ServerConfig;
 use std::convert::Infallible;
 use std::net::IpAddr;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::net::TcpListener;
 use tokio::sync::{broadcast, watch};
 use tokio_rustls::TlsAcceptor;
@@ -49,6 +51,14 @@ impl<S> WebDavServer<S>
 where
     S: StorageBackend + Clone + Send + Sync + 'static + std::fmt::Debug,
 {
+    fn tls_reload_options() -> TlsReloadOptions {
+        TlsReloadOptions {
+            enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE),
+            interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)),
+            ..TlsReloadOptions::default()
+        }
+    }
+
     /// Create a new WebDAV server
     pub async fn new(config: WebDavConfig, storage: S) -> Result<Self, WebDavInitError> {
         config.validate().await?;
@@ -68,10 +78,14 @@ where
             if let Some(cert_dir) = &self.config.cert_dir {
                 debug!("Enabling WebDAV TLS with certificates from: {}", cert_dir);
 
-                let resolver = ReloadableCertResolver::load_from_directory(cert_dir)
+                let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
                     .map_err(|e| WebDavInitError::Tls(format!("Failed to create certificate resolver: {}", e)))?;
-                let _reload_task =
-                    spawn_cert_reload_loop("webdav", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone());
+                let _reload_task = spawn_server_cert_reload_loop(
+                    "webdav",
+                    resolver.clone(),
+                    Self::tls_reload_options(),
+                    reload_shutdown_rx.clone(),
+                );
 
                 let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
 

crates/protos/Cargo.toml

@@ -36,14 +36,15 @@ path = "src/main.rs"
 rustfs-common.workspace = true
 rustfs-io-metrics.workspace = true
 rustfs-config.workspace = true
+rustfs-tls-runtime.workspace = true
 rustfs-utils.workspace = true
 flatbuffers = { workspace = true }
 prost = { workspace = true }
-tonic = { workspace = true, features = ["transport"] }
+tonic = { workspace = true, features = ["transport", "tls-native-roots", "tls-aws-lc"] }
 tonic-prost = { workspace = true }
 tonic-prost-build = { workspace = true }
+tokio = { workspace = true, features = ["sync"] }
 tracing = { workspace = true }
 
 [lib]
-test = false
 doctest = false

crates/protos/src/lib.rs

@@ -18,12 +18,16 @@
 mod generated;
 
 use proto_gen::node_service::node_service_client::NodeServiceClient;
-use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection};
+use rustfs_common::{GLOBAL_CONN_MAP, evict_connection};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
+use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_consumer_stale_generation};
 use std::{
+    collections::HashMap,
     error::Error,
+    sync::LazyLock,
     time::{Duration, Instant},
 };
+use tokio::sync::Mutex;
 use tonic::{
     Request, Status,
     service::interceptor::InterceptedService,
@@ -46,6 +50,21 @@ pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024;
 /// It is used to identify HTTPS URLs.
 /// Default value: https://
 const RUSTFS_HTTPS_PREFIX: &str = "https://";
+const TLS_GENERATION_CACHE_MAX_SIZE: usize = 512;
+static TLS_GENERATION_CACHE: LazyLock<Mutex<HashMap<String, u64>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+
+fn enforce_tls_generation_cache_bound(generation_cache: &mut HashMap<String, u64>, generation: u64, addr: &str) {
+    if generation_cache.len() < TLS_GENERATION_CACHE_MAX_SIZE || generation_cache.contains_key(addr) {
+        return;
+    }
+
+    generation_cache.retain(|_, g| *g == generation);
+    if generation_cache.len() >= TLS_GENERATION_CACHE_MAX_SIZE
+        && let Some(victim) = generation_cache.keys().next().cloned()
+    {
+        generation_cache.remove(&victim);
+    }
+}
 
 fn internode_connect_timeout() -> Duration {
     Duration::from_secs(rustfs_utils::get_env_u64(
@@ -114,14 +133,19 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
         // Overall timeout for any RPC - fail fast on unresponsive peers
         .timeout(rpc_timeout);
 
-    let root_cert = GLOBAL_ROOT_CERT.read().await;
-    if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
-        if root_cert.is_none() {
-            debug!("No custom root certificate configured; using system roots for TLS: {}", addr);
-            // If no custom root cert is configured, try to use system roots.
-            connector = connector.tls_config(ClientTlsConfig::new())?;
+    let outbound_tls = load_global_outbound_tls_state().await;
+    let generation = outbound_tls.generation.0;
+    let mut stale_generation = false;
+    {
+        let generation_cache = TLS_GENERATION_CACHE.lock().await;
+        if let Some(cached_generation) = generation_cache.get(addr)
+            && *cached_generation != generation
+        {
+            stale_generation = true;
         }
-        if let Some(cert_pem) = root_cert.as_ref() {
+    }
+    if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
+        if let Some(cert_pem) = outbound_tls.root_ca_pem.as_ref() {
             let ca = Certificate::from_pem(cert_pem);
             // Derive the hostname from the HTTPS URL for TLS hostname verification.
             let domain = addr
@@ -134,8 +158,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
                 .unwrap_or("");
             let tls = if !domain.is_empty() {
                 let mut cfg = ClientTlsConfig::new().ca_certificate(ca).domain_name(domain);
-                let mtls_identity = GLOBAL_MTLS_IDENTITY.read().await;
-                if let Some(id) = mtls_identity.as_ref() {
+                if let Some(id) = outbound_tls.mtls_identity.as_ref() {
                     let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone());
                     cfg = cfg.identity(identity);
                 }
@@ -147,9 +170,10 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
             connector = connector.tls_config(tls)?;
             debug!("Configured TLS with custom root certificate for: {}", addr);
         } else {
-            return Err(std::io::Error::other(
-                "HTTPS requested but no trusted roots are configured. Provide tls/ca.crt (or enable system roots via RUSTFS_TRUST_SYSTEM_CA=true)."
-            ).into());
+            // No custom root CA published — fall back to system roots.
+            // This is the expected path when no TLS path is configured.
+            debug!("No custom root certificate configured; using system roots for TLS: {}", addr);
+            connector = connector.tls_config(ClientTlsConfig::new())?;
         }
     }
 
@@ -168,6 +192,14 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
     {
         GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel.clone());
     }
+    {
+        let mut generation_cache = TLS_GENERATION_CACHE.lock().await;
+        enforce_tls_generation_cache_bound(&mut generation_cache, generation, addr);
+        generation_cache.insert(addr.to_string(), generation);
+    }
+    if stale_generation {
+        record_tls_consumer_stale_generation("protos_grpc_channel");
+    }
 
     debug!("Successfully created and cached gRPC channel to: {}", addr);
     Ok(channel)
@@ -178,4 +210,21 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
 pub async fn evict_failed_connection(addr: &str) {
     warn!("Evicting failed gRPC connection: {}", addr);
     evict_connection(addr).await;
+    TLS_GENERATION_CACHE.lock().await.remove(addr);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn enforce_tls_generation_cache_bound_evicts_when_retained_entries_still_full() {
+        let mut cache = HashMap::new();
+        for i in 0..TLS_GENERATION_CACHE_MAX_SIZE {
+            cache.insert(format!("node-{i}"), 42);
+        }
+
+        enforce_tls_generation_cache_bound(&mut cache, 42, "new-node");
+        assert_eq!(cache.len(), TLS_GENERATION_CACHE_MAX_SIZE - 1);
+    }
 }

crates/rio/Cargo.toml

@@ -38,12 +38,14 @@ pin-project-lite.workspace = true
 serde = { workspace = true }
 bytes.workspace = true
 reqwest.workspace = true
+rustls-pki-types.workspace = true
 tokio-util.workspace = true
 faster-hex.workspace = true
 futures.workspace = true
 rustfs-config = { workspace = true, features = ["constants"] }
 rustfs-io-metrics.workspace = true
-rustfs-utils = { workspace = true, features = ["io", "hash", "compress", "tls"] }
+rustfs-tls-runtime.workspace = true
+rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
 serde_json.workspace = true
 md-5 = { workspace = true }
 tracing.workspace = true

crates/rio/src/http_reader.rs

@@ -20,9 +20,14 @@ use pin_project_lite::pin_project;
 use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
 use rustfs_io_metrics::internode_metrics::{
     INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
-    global_internode_metrics,
+    INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
+};
+use rustfs_tls_runtime::{
+    load_cert_bundle_der_bytes, load_global_outbound_tls_generation, load_global_outbound_tls_state,
+    record_tls_consumer_stale_generation,
 };
 use rustfs_utils::get_env_opt_str;
+use rustls_pki_types::pem::PemObject;
 use std::io::IoSlice;
 use std::io::{self, Error};
 use std::net::IpAddr;
@@ -32,76 +37,36 @@ use std::sync::LazyLock;
 use std::task::{Context, Poll};
 use std::time::Duration;
 use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
-use tokio::sync::mpsc;
+use tokio::sync::{Mutex, mpsc};
 use tokio::time::{self, Sleep};
 use tokio_util::io::StreamReader;
 use tokio_util::sync::PollSender;
-use tracing::error;
+use tracing::{error, warn};
 
 const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
 const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
 const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
 
-/// Get the TLS path from the RUSTFS_TLS_PATH environment variable.
-/// If the variable is not set, return None.
-fn tls_path() -> Option<&'static std::path::PathBuf> {
-    static TLS_PATH: LazyLock<Option<std::path::PathBuf>> =
-        LazyLock::new(|| get_env_opt_str("RUSTFS_TLS_PATH").and_then(|s| if s.is_empty() { None } else { Some(s.into()) }));
-    TLS_PATH.as_ref()
-}
-
-/// Load CA root certificates from the RUSTFS_TLS_PATH directory.
-/// The CA certificates should be in PEM format and stored in the file
-/// specified by the RUSTFS_CA_CERT constant.
-/// If the file does not exist or cannot be read, return the builder unchanged.
-fn load_ca_roots_from_tls_path(builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
-    let Some(tp) = tls_path() else {
-        return builder;
-    };
-    let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT);
-    if !ca_path.exists() {
-        return builder;
-    }
-
-    let Ok(certs_der) = rustfs_utils::load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default()) else {
-        return builder;
-    };
-
+fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[Vec<u8>]) -> reqwest::ClientBuilder {
     let mut b = builder;
     for der in certs_der {
-        if let Ok(cert) = Certificate::from_der(&der) {
+        if let Ok(cert) = Certificate::from_der(der) {
             b = b.add_root_certificate(cert);
         }
     }
     b
 }
 
-/// Load optional mTLS identity from the RUSTFS_TLS_PATH directory.
-/// The client certificate and private key should be in PEM format and stored in the files
-/// specified by RUSTFS_CLIENT_CERT_FILENAME and RUSTFS_CLIENT_KEY_FILENAME constants.
-/// If the files do not exist or cannot be read, return None.
-fn load_optional_mtls_identity_from_tls_path() -> Option<Identity> {
-    let tp = tls_path()?;
-    let cert = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME)).ok()?;
-    let key = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME)).ok()?;
-
-    let mut pem = Vec::with_capacity(cert.len() + key.len() + 1);
-    pem.extend_from_slice(&cert);
-    if !pem.ends_with(b"\n") {
-        pem.push(b'\n');
-    }
-    pem.extend_from_slice(&key);
-
-    match Identity::from_pem(&pem) {
-        Ok(id) => Some(id),
-        Err(e) => {
-            error!("Failed to load mTLS identity from PEM: {e}");
-            None
-        }
-    }
+#[derive(Clone)]
+struct CachedClients {
+    generation: u64,
+    client: Client,
+    local_client: Client,
 }
 
-fn build_http_client(disable_proxy: bool) -> Client {
+static CLIENT_CACHE: LazyLock<Mutex<Option<CachedClients>>> = LazyLock::new(|| Mutex::new(None));
+
+async fn build_http_client(disable_proxy: bool, outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState) -> Client {
     let mut builder = Client::builder()
         .connect_timeout(std::time::Duration::from_secs(5))
         .tcp_keepalive(std::time::Duration::from_secs(10))
@@ -113,9 +78,51 @@ fn build_http_client(disable_proxy: bool) -> Client {
         builder = builder.no_proxy();
     }
 
-    builder = load_ca_roots_from_tls_path(builder);
-    if let Some(id) = load_optional_mtls_identity_from_tls_path() {
-        builder = builder.identity(id);
+    if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
+        let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
+        match rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader).collect::<Result<Vec<_>, _>>() {
+            Ok(certs_der) => {
+                let certs_der = certs_der.into_iter().map(|cert| cert.to_vec()).collect::<Vec<_>>();
+                builder = add_root_certificates_from_der(builder, &certs_der);
+            }
+            Err(err) => {
+                warn!("Failed to parse published outbound root CA PEM; falling back to default trust roots: {err}");
+            }
+        }
+    } else if let Some(tp) = get_env_opt_str(rustfs_config::ENV_RUSTFS_TLS_PATH).and_then(|s| {
+        if s.is_empty() {
+            None
+        } else {
+            Some(std::path::PathBuf::from(s))
+        }
+    }) {
+        let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT);
+        if ca_path.exists()
+            && let Some(ca_path_str) = ca_path.to_str()
+        {
+            match load_cert_bundle_der_bytes(ca_path_str) {
+                Ok(certs_der) => {
+                    builder = add_root_certificates_from_der(builder, &certs_der);
+                }
+                Err(err) => {
+                    warn!("Failed to parse fallback root CA bundle '{}': {}", ca_path.display(), err);
+                }
+            }
+        }
+    }
+
+    if let Some(identity) = outbound_tls.mtls_identity.as_ref() {
+        let mut pem = Vec::with_capacity(identity.cert_pem.len() + identity.key_pem.len() + 1);
+        pem.extend_from_slice(&identity.cert_pem);
+        if !pem.ends_with(b"\n") {
+            pem.push(b'\n');
+        }
+        pem.extend_from_slice(&identity.key_pem);
+
+        match Identity::from_pem(&pem) {
+            Ok(id) => builder = builder.identity(id),
+            Err(e) => error!("Failed to load mTLS identity from PEM: {e}"),
+        }
     }
 
     builder.build().expect("Failed to create global HTTP client")
@@ -133,17 +140,53 @@ fn should_bypass_proxy_for_url(url: &str) -> bool {
     host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|addr| addr.is_loopback())
 }
 
-fn get_http_client(url: &str) -> Client {
+async fn get_http_client(url: &str) -> Client {
     // Reuse HTTP connection pools while keeping loopback traffic away from
     // system proxies so local RPC/tests do not leak to proxy listeners.
-    static CLIENT: LazyLock<Client> = LazyLock::new(|| build_http_client(false));
-    static LOCAL_CLIENT: LazyLock<Client> = LazyLock::new(|| build_http_client(true));
+    let disable_proxy = should_bypass_proxy_for_url(url);
 
-    if should_bypass_proxy_for_url(url) {
-        return LOCAL_CLIENT.clone();
+    // Fast path: check generation first (cheap atomic read) to avoid cloning
+    // the full PEM + identity bytes when the TLS state hasn't changed.
+    let generation = load_global_outbound_tls_generation().0;
+
+    let guard = CLIENT_CACHE.lock().await;
+    if let Some(cached) = guard.as_ref() {
+        if cached.generation == generation {
+            return if disable_proxy {
+                cached.local_client.clone()
+            } else {
+                cached.client.clone()
+            };
+        }
+        record_tls_consumer_stale_generation("rio_http_reader");
     }
+    drop(guard);
 
-    CLIENT.clone()
+    // Cache miss or stale generation — load full outbound TLS state.
+    let outbound_tls = load_global_outbound_tls_state().await;
+
+    let client = build_http_client(false, &outbound_tls).await;
+    let local_client = build_http_client(true, &outbound_tls).await;
+    let cached = CachedClients {
+        generation,
+        client,
+        local_client,
+    };
+
+    let return_client = if disable_proxy {
+        cached.local_client.clone()
+    } else {
+        cached.client.clone()
+    };
+
+    let mut guard = CLIENT_CACHE.lock().await;
+    // Guard against races: only overwrite the cache if it is empty or
+    // contains an older generation, so a slower task cannot regress the
+    // TLS state after a faster task already cached a newer generation.
+    if guard.as_ref().is_none_or(|c| c.generation <= generation) {
+        *guard = Some(cached);
+    }
+    return_client
 }
 
 pin_project! {
@@ -197,7 +240,7 @@ impl HttpReader {
     ) -> io::Result<Self> {
         let track_internode_metrics = is_internode_rpc_url(&url);
         let internode_operation = internode_rpc_operation(&url);
-        let client = get_http_client(&url);
+        let client = get_http_client(&url).await;
         let mut request: RequestBuilder = client.request(method.clone(), url.clone()).headers(headers.clone());
         if let Some(body) = body {
             request = request.body(body);
@@ -379,7 +422,7 @@ impl HttpWriter {
             //     "[HttpWriter::spawn] sending HTTP request: url={url_clone}, method={method_clone:?}, headers={headers_clone:?}"
             // );
 
-            let client = get_http_client(&url_clone);
+            let client = get_http_client(&url_clone).await;
             let request = client
                 .request(method_clone, url_clone.clone())
                 .headers(headers_clone.clone())
@@ -459,7 +502,8 @@ fn record_internode_outgoing_request(track: bool, operation: Option<&'static str
     }
 
     match operation {
-        Some(operation) => global_internode_metrics().record_outgoing_request_for_operation(operation),
+        Some(operation) => global_internode_metrics()
+            .record_outgoing_request_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP),
         None => global_internode_metrics().record_outgoing_request(),
     }
 }
@@ -470,7 +514,11 @@ fn record_internode_sent_bytes(track: bool, operation: Option<&'static str>, byt
     }
 
     match operation {
-        Some(operation) => global_internode_metrics().record_sent_bytes_for_operation(operation, bytes),
+        Some(operation) => global_internode_metrics().record_sent_bytes_for_operation_and_backend(
+            operation,
+            INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+            bytes,
+        ),
         None => global_internode_metrics().record_sent_bytes(bytes),
     }
 }
@@ -481,7 +529,11 @@ fn record_internode_recv_bytes(track: bool, operation: Option<&'static str>, byt
     }
 
     match operation {
-        Some(operation) => global_internode_metrics().record_recv_bytes_for_operation(operation, bytes),
+        Some(operation) => global_internode_metrics().record_recv_bytes_for_operation_and_backend(
+            operation,
+            INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+            bytes,
+        ),
         None => global_internode_metrics().record_recv_bytes(bytes),
     }
 }
@@ -492,7 +544,9 @@ fn record_internode_error(track: bool, operation: Option<&'static str>) {
     }
 
     match operation {
-        Some(operation) => global_internode_metrics().record_error_for_operation(operation),
+        Some(operation) => {
+            global_internode_metrics().record_error_for_operation_and_backend(operation, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP)
+        }
         None => global_internode_metrics().record_error(),
     }
 }

crates/s3select-api/src/object_store.rs

@@ -18,30 +18,34 @@ use chrono::Utc;
 use futures::pin_mut;
 use futures::{Stream, StreamExt, future::ready, stream};
 use futures_core::stream::BoxStream;
-use http::HeaderMap;
+use http::{HeaderMap, HeaderValue, header::HeaderName};
 use object_store::{
-    Attributes, CopyOptions, Error as o_Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
-    PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, path::Path,
+    Attributes, CopyOptions, Error as o_Error, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload,
+    ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, path::Path,
 };
 use pin_project_lite::pin_project;
 use rustfs_common::DEFAULT_DELIMITER;
+use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
 use rustfs_ecstore::new_object_layer_fn;
 use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
 use rustfs_ecstore::store::ECStore;
-use rustfs_ecstore::store_api::ObjectIO;
-use rustfs_ecstore::store_api::ObjectOptions;
+use rustfs_ecstore::store_api::{GetObjectReader, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectOptions};
 use s3s::S3Result;
 use s3s::dto::SelectObjectContentInput;
+use s3s::header::{
+    X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
+    X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
+};
 use s3s::s3_error;
+use std::collections::VecDeque;
 use std::ops::Range;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::Poll;
 use std::task::ready;
-use tokio::io::AsyncRead;
 use tokio::io::AsyncReadExt;
+use tokio::io::{AsyncRead, ReadBuf};
 use tokio_util::io::ReaderStream;
-use tracing::info;
 use transform_stream::AsyncTryStream;
 
 /// Maximum allowed object size for JSON DOCUMENT mode.
@@ -58,6 +62,8 @@ use transform_stream::AsyncTryStream;
 /// Default: 128 MiB.  This matches the AWS S3 Select limit for JSON DOCUMENT
 /// inputs.
 pub const MAX_JSON_DOCUMENT_BYTES: u64 = 128 * 1024 * 1024;
+pub const INVALID_SCAN_RANGE_MESSAGE: &str =
+    "The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again.";
 
 #[derive(Debug)]
 pub struct EcObjectStore {
@@ -75,6 +81,16 @@ pub struct EcObjectStore {
 
     store: Arc<ECStore>,
 }
+
+#[derive(Clone, Copy, Debug)]
+struct SelectScanRange {
+    start: u64,
+    end: u64,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct InvalidScanRange;
+
 impl EcObjectStore {
     pub fn new(input: Arc<SelectObjectContentInput>) -> S3Result<Self> {
         let Some(store) = new_object_layer_fn() else {
@@ -123,6 +139,97 @@ impl EcObjectStore {
             store,
         })
     }
+
+    fn object_options(&self, options: &GetOptions) -> ObjectOptions {
+        ObjectOptions {
+            version_id: options.version.clone(),
+            ..Default::default()
+        }
+    }
+
+    fn read_headers(&self) -> HeaderMap {
+        select_read_headers(&self.input)
+    }
+
+    fn scan_range(&self, object_size: u64) -> Result<Option<SelectScanRange>> {
+        let Some(scan_range) = self.input.request.scan_range.as_ref() else {
+            return Ok(None);
+        };
+        scan_range_from_bounds(scan_range.start, scan_range.end, object_size)
+    }
+
+    fn record_delimiter(&self) -> Vec<u8> {
+        self.input
+            .request
+            .input_serialization
+            .csv
+            .as_ref()
+            .and_then(|csv| csv.record_delimiter.as_ref())
+            .map(|delimiter| delimiter.as_bytes().to_vec())
+            .unwrap_or_else(|| b"\n".to_vec())
+    }
+
+    fn csv_has_header(&self) -> bool {
+        self.input
+            .request
+            .input_serialization
+            .csv
+            .as_ref()
+            .and_then(|csv| csv.file_header_info.as_ref())
+            .is_some_and(|info| matches!(info.as_str(), "USE" | "IGNORE"))
+    }
+
+    async fn object_info(&self, opts: &ObjectOptions) -> Result<rustfs_ecstore::store_api::ObjectInfo> {
+        self.store
+            .get_object_info(&self.input.bucket, &self.input.key, opts)
+            .await
+            .map_err(|err| map_storage_error(&self.input.bucket, &self.input.key, err))
+    }
+
+    async fn object_reader(&self, range: Option<HTTPRangeSpec>, opts: &ObjectOptions) -> Result<GetObjectReader> {
+        let h = self.read_headers();
+        self.store
+            .get_object_reader(&self.input.bucket, &self.input.key, range, h, opts)
+            .await
+            .map_err(|err| map_storage_error(&self.input.bucket, &self.input.key, err))
+    }
+
+    async fn read_raw_range_with_opts(&self, range: Range<u64>, opts: &ObjectOptions) -> Result<Bytes> {
+        if range.is_empty() {
+            return Ok(Bytes::new());
+        }
+        let reader = self.object_reader(Some(http_range_spec_from_range(range)), opts).await?;
+        let mut reader = reader.stream;
+        let mut bytes = Vec::new();
+        reader.read_to_end(&mut bytes).await.map_err(|err| o_Error::Generic {
+            store: "EcObjectStore",
+            source: Box::new(err),
+        })?;
+        Ok(Bytes::from(bytes))
+    }
+
+    async fn read_raw_range(&self, range: Range<u64>) -> Result<Bytes> {
+        self.read_raw_range_with_opts(range, &self.object_options(&GetOptions::new()))
+            .await
+    }
+
+    async fn read_header_record(&self, object_size: u64, delimiter: &[u8], opts: &ObjectOptions) -> Result<Bytes> {
+        if object_size == 0 {
+            return Ok(Bytes::new());
+        }
+
+        let mut end = (DEFAULT_READ_BUFFER_SIZE as u64).min(object_size);
+        loop {
+            let bytes = self.read_raw_range_with_opts(0..end, opts).await?;
+            if let Some(pos) = find_delimiter(&bytes, delimiter) {
+                return Ok(bytes.slice(0..pos + delimiter.len()));
+            }
+            if end == object_size {
+                return Ok(bytes);
+            }
+            end = end.saturating_mul(2).min(object_size);
+        }
+    }
 }
 
 impl std::fmt::Display for EcObjectStore {
@@ -141,6 +248,150 @@ fn unsupported_store_error(op: &str) -> o_Error {
     }
 }
 
+fn insert_header(headers: &mut HeaderMap, name: HeaderName, value: Option<&str>) {
+    if let Some(value) = value
+        && let Ok(value) = HeaderValue::from_str(value)
+    {
+        headers.insert(name, value);
+    }
+}
+
+fn select_read_headers(input: &SelectObjectContentInput) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    insert_header(
+        &mut headers,
+        X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
+        input.sse_customer_algorithm.as_deref(),
+    );
+    insert_header(&mut headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, input.sse_customer_key.as_deref());
+    insert_header(
+        &mut headers,
+        X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
+        input.sse_customer_key_md5.as_deref(),
+    );
+    headers
+}
+
+fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
+    match range {
+        GetRange::Bounded(range) => http_range_spec_from_range(range.clone()),
+        GetRange::Offset(start) => HTTPRangeSpec {
+            is_suffix_length: false,
+            start: *start as i64,
+            end: -1,
+        },
+        GetRange::Suffix(length) => HTTPRangeSpec {
+            is_suffix_length: true,
+            start: *length as i64,
+            end: -1,
+        },
+    }
+}
+
+fn http_range_spec_from_range(range: Range<u64>) -> HTTPRangeSpec {
+    HTTPRangeSpec {
+        is_suffix_length: false,
+        start: range.start as i64,
+        end: range.end.saturating_sub(1) as i64,
+    }
+}
+
+fn http_range_spec_from_start(start: u64) -> HTTPRangeSpec {
+    HTTPRangeSpec {
+        is_suffix_length: false,
+        start: start as i64,
+        end: -1,
+    }
+}
+
+fn scan_range_read_start(scan_range: SelectScanRange, delimiter: &[u8]) -> u64 {
+    scan_range.start.saturating_sub(delimiter.len() as u64)
+}
+
+fn find_delimiter(bytes: &[u8], delimiter: &[u8]) -> Option<usize> {
+    if delimiter.is_empty() {
+        return None;
+    }
+    bytes.windows(delimiter.len()).position(|window| window == delimiter)
+}
+
+fn map_storage_error(bucket: &str, object: &str, err: StorageError) -> o_Error {
+    if is_err_bucket_not_found(&err) || is_err_object_not_found(&err) || is_err_version_not_found(&err) {
+        return o_Error::NotFound {
+            path: format!("{bucket}/{object}"),
+            source: err.to_string().into(),
+        };
+    }
+    o_Error::Generic {
+        store: "EcObjectStore",
+        source: Box::new(err),
+    }
+}
+
+fn scan_range_from_bounds(start: Option<i64>, end: Option<i64>, object_size: u64) -> Result<Option<SelectScanRange>> {
+    parse_scan_range_from_bounds(start, end, object_size).map_err(|_| invalid_scan_range_store_error())
+}
+
+pub fn validate_scan_range_bounds(
+    start: Option<i64>,
+    end: Option<i64>,
+    object_size: u64,
+) -> std::result::Result<(), InvalidScanRange> {
+    parse_scan_range_from_bounds(start, end, object_size).map(|_| ())
+}
+
+fn parse_scan_range_from_bounds(
+    start: Option<i64>,
+    end: Option<i64>,
+    object_size: u64,
+) -> std::result::Result<Option<SelectScanRange>, InvalidScanRange> {
+    if start.is_none() && end.is_none() {
+        return Ok(None);
+    }
+    if start.is_some_and(|value| value < 0) || end.is_some_and(|value| value < 0) {
+        return Err(InvalidScanRange);
+    }
+    if let (Some(start), Some(end)) = (start, end)
+        && start > end
+    {
+        return Err(InvalidScanRange);
+    }
+    if let Some(start) = start {
+        let start = start as u64;
+        if object_size == 0 {
+            if start > 0 {
+                return Err(InvalidScanRange);
+            }
+            return Ok(Some(SelectScanRange { start: 0, end: 0 }));
+        }
+        if start >= object_size {
+            return Err(InvalidScanRange);
+        }
+    }
+    if object_size == 0 {
+        return Ok(Some(SelectScanRange { start: 0, end: 0 }));
+    }
+
+    let last_byte = object_size - 1;
+    let (start, end) = match (start, end) {
+        (Some(start), Some(end)) => (start as u64, (end as u64).min(last_byte)),
+        (Some(start), None) => (start as u64, last_byte),
+        (None, Some(suffix_len)) => {
+            let suffix_len = suffix_len as u64;
+            (object_size.saturating_sub(suffix_len), last_byte)
+        }
+        (None, None) => return Ok(None),
+    };
+    Ok(Some(SelectScanRange { start, end }))
+}
+
+fn invalid_scan_range_store_error() -> o_Error {
+    o_Error::Generic {
+        store: "EcObjectStore",
+        source: format!("ScanRange: {INVALID_SCAN_RANGE_MESSAGE}").into(),
+    }
+}
+
 #[async_trait]
 impl ObjectStore for EcObjectStore {
     async fn put_opts(&self, _location: &Path, _payload: PutPayload, _opts: PutOptions) -> Result<PutResult> {
@@ -151,24 +402,50 @@ impl ObjectStore for EcObjectStore {
         Err(unsupported_store_error("put_multipart_opts"))
     }
 
-    async fn get_opts(&self, location: &Path, _options: GetOptions) -> Result<GetResult> {
-        info!("{:?}", location);
-        let opts = ObjectOptions::default();
-        let h = HeaderMap::new();
-        let reader = self
-            .store
-            .get_object_reader(&self.input.bucket, &self.input.key, None, h, &opts)
-            .await
-            .map_err(|_| o_Error::NotFound {
-                path: format!("{}/{}", self.input.bucket, self.input.key),
-                source: "can not get object info".into(),
-            })?;
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
+        let opts = self.object_options(&options);
+        let needs_scan_context = options.range.is_none() && !options.head && self.input.request.scan_range.is_some();
+        let source_size = if needs_scan_context {
+            Some(self.object_info(&opts).await?.size as u64)
+        } else {
+            None
+        };
+        let scan_context = if needs_scan_context {
+            let original_size = source_size.expect("source size is loaded when scan range is present");
+            self.scan_range(original_size)?.map(|scan_range| (original_size, scan_range))
+        } else {
+            None
+        };
 
-        let original_size = reader.object_info.size as u64;
+        let range = options.range.as_ref().map(http_range_spec_from_get_range);
+        let reader = if let Some((original_size, scan_range)) = scan_context.as_ref() {
+            let delimiter = self.record_delimiter();
+            let read_start = scan_range_read_start(*scan_range, &delimiter);
+            let range = (*original_size > 0).then(|| http_range_spec_from_start(read_start));
+            self.object_reader(range, &opts).await?
+        } else {
+            self.object_reader(range, &opts).await?
+        };
+
+        let original_size = source_size.unwrap_or(reader.object_info.size as u64);
         let etag = reader.object_info.etag;
+        let version = reader.object_info.version_id.map(|version| version.to_string());
         let attributes = Attributes::default();
+        let result_range = match options.range.as_ref() {
+            Some(range) => range.as_range(original_size).map_err(|err| o_Error::Generic {
+                store: "EcObjectStore",
+                source: Box::new(err),
+            })?,
+            None => 0..original_size,
+        };
 
-        let (payload, size) = if self.is_json_document {
+        let payload = if options.head {
+            GetResultPayload::Stream(stream::empty().boxed())
+        } else if options.range.is_some() {
+            let size = (result_range.end - result_range.start) as usize;
+            let stream = bytes_stream(ReaderStream::with_capacity(reader.stream, DEFAULT_READ_BUFFER_SIZE), size).boxed();
+            GetResultPayload::Stream(stream)
+        } else if self.is_json_document {
             // JSON DOCUMENT mode: gate on object size before doing any I/O.
             //
             // Small files (<= MAX_JSON_DOCUMENT_BYTES): build a lazy stream
@@ -195,41 +472,68 @@ impl ObjectStore for EcObjectStore {
                 });
             }
             let stream = json_document_ndjson_stream(reader.stream, original_size, self.json_sub_path.clone());
-            (object_store::GetResultPayload::Stream(stream), original_size)
+            GetResultPayload::Stream(stream)
+        } else if let Some((_, scan_range)) = scan_context {
+            let delimiter = self.record_delimiter();
+            let include_header = self.csv_has_header();
+            let read_start = scan_range_read_start(scan_range, &delimiter);
+            let header = if include_header && scan_range.start > 0 {
+                Some(self.read_header_record(original_size, &delimiter, &opts).await?)
+            } else {
+                None
+            };
+            let stream = scan_range_stream(
+                ReaderStream::with_capacity(reader.stream, DEFAULT_READ_BUFFER_SIZE),
+                delimiter,
+                scan_range,
+                include_header && header.is_none(),
+                read_start,
+            )
+            .boxed();
+            let stream = if let Some(header) = header {
+                stream::once(ready(Ok(header))).chain(stream).boxed()
+            } else {
+                stream
+            };
+            GetResultPayload::Stream(convert_field_delimiter_stream(stream, self.need_convert.then(|| self.delimiter.clone())))
         } else if self.need_convert {
             let stream = bytes_stream(
                 ReaderStream::with_capacity(ConvertStream::new(reader.stream, self.delimiter.clone()), DEFAULT_READ_BUFFER_SIZE),
                 original_size as usize,
             )
             .boxed();
-            (object_store::GetResultPayload::Stream(stream), original_size)
+            GetResultPayload::Stream(stream)
         } else {
             let stream = bytes_stream(
                 ReaderStream::with_capacity(reader.stream, DEFAULT_READ_BUFFER_SIZE),
                 original_size as usize,
             )
             .boxed();
-            (object_store::GetResultPayload::Stream(stream), original_size)
+            GetResultPayload::Stream(stream)
         };
 
         let meta = ObjectMeta {
             location: location.clone(),
             last_modified: Utc::now(),
-            size,
+            size: original_size,
             e_tag: etag,
-            version: None,
+            version,
         };
 
         Ok(GetResult {
             payload,
             meta,
-            range: 0..size,
+            range: result_range,
             attributes,
         })
     }
 
-    async fn get_ranges(&self, _location: &Path, _ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
-        Err(unsupported_store_error("get_ranges"))
+    async fn get_ranges(&self, _location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
+        let mut out = Vec::with_capacity(ranges.len());
+        for range in ranges {
+            out.push(self.read_raw_range(range.clone()).await?);
+        }
+        Ok(out)
     }
 
     fn delete_stream(&self, _locations: BoxStream<'static, Result<Path>>) -> BoxStream<'static, Result<Path>> {
@@ -252,7 +556,11 @@ impl ObjectStore for EcObjectStore {
 pin_project! {
     struct ConvertStream<R> {
         inner: R,
-        delimiter: Vec<u8>,
+        converter: DelimiterConverter,
+        read_buf: Vec<u8>,
+        pending: Vec<u8>,
+        pending_pos: usize,
+        eof: bool,
     }
 }
 
@@ -260,29 +568,120 @@ impl<R> ConvertStream<R> {
     fn new(inner: R, delimiter: String) -> Self {
         ConvertStream {
             inner,
-            delimiter: delimiter.as_bytes().to_vec(),
+            converter: DelimiterConverter::new(delimiter.into_bytes()),
+            read_buf: vec![0; DEFAULT_READ_BUFFER_SIZE],
+            pending: Vec::new(),
+            pending_pos: 0,
+            eof: false,
         }
     }
 }
 
 impl<R: AsyncRead + Unpin> AsyncRead for ConvertStream<R> {
     #[tracing::instrument(level = "debug", skip_all)]
-    fn poll_read(
-        self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> Poll<std::io::Result<()>> {
-        let me = self.project();
-        ready!(Pin::new(&mut *me.inner).poll_read(cx, buf))?;
-        let bytes = buf.filled();
-        let replaced = replace_symbol(me.delimiter, bytes);
-        buf.clear();
-        buf.put_slice(&replaced);
-        Poll::Ready(Ok(()))
+    fn poll_read(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        if buf.remaining() == 0 {
+            return Poll::Ready(Ok(()));
+        }
+        let this = self.project();
+        loop {
+            if drain_pending(this.pending, this.pending_pos, buf) || *this.eof {
+                return Poll::Ready(Ok(()));
+            }
+
+            let read_len = DEFAULT_READ_BUFFER_SIZE.min(buf.remaining().max(1));
+            let bytes_read = {
+                let mut read_buf = ReadBuf::new(&mut this.read_buf[..read_len]);
+                ready!(Pin::new(&mut *this.inner).poll_read(cx, &mut read_buf))?;
+                read_buf.filled().len()
+            };
+            if bytes_read == 0 {
+                *this.eof = true;
+                *this.pending = this.converter.finish();
+                *this.pending_pos = 0;
+                continue;
+            }
+
+            *this.pending = this.converter.convert_chunk(&this.read_buf[..bytes_read]);
+            *this.pending_pos = 0;
+        }
     }
 }
 
+struct DelimiterConverter {
+    delimiter: Vec<u8>,
+    carry: Vec<u8>,
+}
+
+impl DelimiterConverter {
+    fn new(delimiter: Vec<u8>) -> Self {
+        Self {
+            delimiter,
+            carry: Vec::new(),
+        }
+    }
+
+    fn convert_chunk(&mut self, chunk: &[u8]) -> Vec<u8> {
+        if self.delimiter.is_empty() {
+            return chunk.to_vec();
+        }
+
+        let mut combined = Vec::with_capacity(self.carry.len() + chunk.len());
+        combined.extend_from_slice(&self.carry);
+        combined.extend_from_slice(chunk);
+
+        let safe_end = combined.len().saturating_sub(self.delimiter.len().saturating_sub(1));
+        let mut converted = Vec::with_capacity(combined.len());
+        let mut pos = 0;
+        while pos < safe_end {
+            if combined[pos..].starts_with(&self.delimiter) {
+                converted.push(DEFAULT_DELIMITER);
+                pos += self.delimiter.len();
+            } else {
+                converted.push(combined[pos]);
+                pos += 1;
+            }
+        }
+        self.carry.clear();
+        self.carry.extend_from_slice(&combined[pos..]);
+        converted
+    }
+
+    fn finish(&mut self) -> Vec<u8> {
+        if self.delimiter.is_empty() {
+            return std::mem::take(&mut self.carry);
+        }
+        let converted = replace_symbol(&self.delimiter, &self.carry);
+        self.carry.clear();
+        converted
+    }
+}
+
+fn drain_pending(pending: &mut Vec<u8>, pending_pos: &mut usize, buf: &mut ReadBuf<'_>) -> bool {
+    if *pending_pos >= pending.len() {
+        pending.clear();
+        *pending_pos = 0;
+        return false;
+    }
+    if buf.remaining() == 0 {
+        return false;
+    }
+
+    let len = buf.remaining().min(pending.len() - *pending_pos);
+    buf.put_slice(&pending[*pending_pos..*pending_pos + len]);
+    *pending_pos += len;
+    if *pending_pos >= pending.len() {
+        pending.clear();
+        *pending_pos = 0;
+    }
+    true
+}
+
 fn replace_symbol(delimiter: &[u8], slice: &[u8]) -> Vec<u8> {
+    if delimiter.is_empty() {
+        return slice.to_vec();
+    }
+
     let mut result = Vec::with_capacity(slice.len());
     let mut i = 0;
     while i < slice.len() {
@@ -297,6 +696,148 @@ fn replace_symbol(delimiter: &[u8], slice: &[u8]) -> Vec<u8> {
     result
 }
 
+fn convert_field_delimiter_stream<S>(stream: S, delimiter: Option<String>) -> BoxStream<'static, Result<Bytes>>
+where
+    S: Stream<Item = Result<Bytes>> + Send + 'static,
+{
+    let Some(delimiter) = delimiter else {
+        return stream.boxed();
+    };
+    AsyncTryStream::<Bytes, o_Error, _>::new(|mut y| async move {
+        let mut converter = DelimiterConverter::new(delimiter.into_bytes());
+        pin_mut!(stream);
+        while let Some(result) = stream.next().await {
+            let bytes = result?;
+            let converted = converter.convert_chunk(&bytes);
+            if !converted.is_empty() {
+                y.yield_ok(Bytes::from(converted)).await;
+            }
+        }
+        let converted = converter.finish();
+        if !converted.is_empty() {
+            y.yield_ok(Bytes::from(converted)).await;
+        }
+        Ok(())
+    })
+    .boxed()
+}
+
+struct ScanRangeState<S> {
+    stream: S,
+    delimiter: Vec<u8>,
+    range: SelectScanRange,
+    include_header: bool,
+    offset: u64,
+    record_start: u64,
+    record: Vec<u8>,
+    pending: VecDeque<Bytes>,
+    done: bool,
+}
+
+fn scan_range_stream<S>(
+    stream: S,
+    delimiter: Vec<u8>,
+    range: SelectScanRange,
+    include_header: bool,
+    base_offset: u64,
+) -> BoxStream<'static, Result<Bytes>>
+where
+    S: Stream<Item = std::io::Result<Bytes>> + Send + Unpin + 'static,
+{
+    let state = ScanRangeState {
+        stream,
+        delimiter,
+        range,
+        include_header,
+        offset: base_offset,
+        record_start: base_offset,
+        record: Vec::new(),
+        pending: VecDeque::new(),
+        done: false,
+    };
+
+    stream::unfold(state, |mut state| async move {
+        loop {
+            if let Some(bytes) = state.pending.pop_front() {
+                return Some((Ok(bytes), state));
+            }
+            if state.done {
+                return None;
+            }
+            match state.stream.next().await {
+                Some(Ok(bytes)) => state.push_chunk(&bytes),
+                Some(Err(err)) => {
+                    state.done = true;
+                    return Some((
+                        Err(o_Error::Generic {
+                            store: "EcObjectStore",
+                            source: Box::new(err),
+                        }),
+                        state,
+                    ));
+                }
+                None => {
+                    state.finish_pending_record();
+                    state.done = true;
+                }
+            }
+        }
+    })
+    .boxed()
+}
+
+impl<S> ScanRangeState<S> {
+    fn push_chunk(&mut self, bytes: &[u8]) {
+        if bytes.is_empty() || self.done {
+            return;
+        }
+        if self.record.is_empty() {
+            self.record_start = self.offset;
+        }
+        let search_start = self.record.len().saturating_sub(self.delimiter.len().saturating_sub(1));
+        self.record.extend_from_slice(bytes);
+        self.offset = self.offset.saturating_add(bytes.len() as u64);
+
+        let mut search_start = search_start;
+        while let Some(pos) = find_delimiter(&self.record[search_start..], &self.delimiter) {
+            let record_end = search_start + pos + self.delimiter.len();
+            self.finish_record(record_end);
+            if self.done {
+                break;
+            }
+            search_start = 0;
+        }
+    }
+
+    fn finish_record(&mut self, record_end: usize) {
+        let record = self.record.drain(..record_end).collect::<Vec<_>>();
+        let record_start = self.record_start;
+        self.record_start = self.record_start.saturating_add(record_end as u64);
+        self.push_record(record, record_start);
+    }
+
+    fn finish_pending_record(&mut self) {
+        if self.record.is_empty() {
+            return;
+        }
+        let record = std::mem::take(&mut self.record);
+        let record_start = self.record_start;
+        self.push_record(record, record_start);
+    }
+
+    fn push_record(&mut self, record: Vec<u8>, record_start: u64) {
+        let include_header = self.include_header && record_start == 0;
+        let include_record = record_start >= self.range.start && record_start <= self.range.end;
+        if include_header || include_record {
+            self.pending.push_back(Bytes::from(record));
+        } else {
+            if record_start > self.range.end {
+                self.done = true;
+            }
+        }
+    }
+}
+
 /// Extract the JSON sub-path from a SQL expression's FROM clause.
 ///
 /// Given `SELECT e.name FROM s3object.employees e WHERE …` this returns
@@ -508,19 +1049,226 @@ where
 
 #[cfg(test)]
 mod test {
-    use super::{extract_json_sub_path_from_expression, flatten_json_document_to_ndjson, replace_symbol};
+    use super::{
+        ConvertStream, SelectScanRange, convert_field_delimiter_stream, extract_json_sub_path_from_expression, find_delimiter,
+        flatten_json_document_to_ndjson, http_range_spec_from_get_range, replace_symbol, scan_range_from_bounds,
+        scan_range_read_start, scan_range_stream, select_read_headers,
+    };
+    use bytes::Bytes;
+    use futures::{StreamExt, stream};
+    use object_store::GetRange;
+    use s3s::dto::{
+        CSVInput, CSVOutput, ExpressionType, InputSerialization, OutputSerialization, SelectObjectContentInput,
+        SelectObjectContentRequest,
+    };
+    use s3s::header::{
+        X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
+        X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
+    };
+    use tokio::io::AsyncReadExt;
+    use tokio_util::io::StreamReader;
 
     #[test]
     fn test_replace() {
-        let ss = String::from("dandan&&is&&best");
-        let slice = ss.as_bytes();
-        let delimiter = b"&&";
-        println!("len: {}", "╦".len());
-        let result = replace_symbol(delimiter, slice);
-        match String::from_utf8(result) {
-            Ok(s) => println!("slice: {s}"),
-            Err(e) => eprintln!("Error converting to string: {e}"),
+        let result = replace_symbol(b"&&", b"dandan&&is&&best");
+        assert_eq!(result, b"dandan,is,best");
+    }
+
+    #[tokio::test]
+    async fn test_convert_stream_replaces_delimiter_across_chunks() {
+        let chunks = stream::iter(vec![
+            Ok::<_, std::io::Error>(Bytes::from_static(b"a&")),
+            Ok::<_, std::io::Error>(Bytes::from_static(b"&b&&c")),
+        ]);
+        let reader = StreamReader::new(chunks);
+        let mut reader = ConvertStream::new(reader, "&&".to_string());
+        let mut output = Vec::new();
+        reader.read_to_end(&mut output).await.unwrap();
+        assert_eq!(output, b"a,b,c");
+    }
+
+    #[tokio::test]
+    async fn test_convert_stream_replaces_delimiter_at_stream_end() {
+        let chunks = stream::iter(vec![
+            Ok::<_, std::io::Error>(Bytes::from_static(b"a&")),
+            Ok::<_, std::io::Error>(Bytes::from_static(b"&")),
+        ]);
+        let reader = StreamReader::new(chunks);
+        let mut reader = ConvertStream::new(reader, "&&".to_string());
+        let mut output = Vec::new();
+        reader.read_to_end(&mut output).await.unwrap();
+        assert_eq!(output, b"a,");
+    }
+
+    #[tokio::test]
+    async fn test_scan_range_stream_keeps_header_and_selected_record() {
+        let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"h1,h2\n1,a\n2,b\n3,c\n"))]);
+        let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange { start: 10, end: 11 }, true, 0);
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
         }
+        assert_eq!(output, b"h1,h2\n2,b\n");
+    }
+
+    #[tokio::test]
+    async fn test_scan_range_stream_skips_record_when_start_is_in_middle() {
+        let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"1,a\n2,b\n3,c\n"))]);
+        let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange { start: 2, end: 7 }, false, 0);
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
+        }
+        assert_eq!(output, b"2,b\n");
+    }
+
+    #[tokio::test]
+    async fn test_scan_range_stream_keeps_record_when_end_is_in_middle() {
+        let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"1,a\n2,b\n3,c\n"))]);
+        let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange { start: 0, end: 5 }, false, 0);
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
+        }
+        assert_eq!(output, b"1,a\n2,b\n");
+    }
+
+    #[tokio::test]
+    async fn test_scan_range_stream_uses_base_offset_for_range_reader() {
+        let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"\n2,b\n3,c\n"))]);
+        let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange { start: 4, end: 7 }, false, 3);
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
+        }
+        assert_eq!(output, b"2,b\n");
+    }
+
+    #[tokio::test]
+    async fn test_scan_range_stream_handles_delimiter_split_across_chunks() {
+        let chunks = stream::iter(vec![
+            Ok::<_, std::io::Error>(Bytes::from_static(b"h1,h2\r")),
+            Ok::<_, std::io::Error>(Bytes::from_static(b"\n1,a\r\n2,b\r")),
+            Ok::<_, std::io::Error>(Bytes::from_static(b"\n3,c\r\n")),
+        ]);
+        let mut stream = scan_range_stream(chunks, b"\r\n".to_vec(), SelectScanRange { start: 12, end: 14 }, true, 0);
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
+        }
+        assert_eq!(output, b"h1,h2\r\n2,b\r\n");
+    }
+
+    #[test]
+    fn test_scan_range_read_start_keeps_full_delimiter_boundary() {
+        let range = SelectScanRange { start: 10, end: 20 };
+        assert_eq!(scan_range_read_start(range, b"\n"), 9);
+        assert_eq!(scan_range_read_start(range, b"\r\n"), 8);
+        assert_eq!(scan_range_read_start(range, b"abcdef"), 4);
+    }
+
+    #[test]
+    fn test_find_delimiter_handles_multi_byte_delimiter() {
+        assert_eq!(find_delimiter(b"one\r\ntwo", b"\r\n"), Some(3));
+        assert_eq!(find_delimiter(b"one\ntwo", b"\r\n"), None);
+    }
+
+    #[test]
+    fn test_scan_range_end_only_uses_aws_suffix_semantics() {
+        let range = scan_range_from_bounds(None, Some(35), 100).unwrap().unwrap();
+        assert_eq!(range.start, 65);
+        assert_eq!(range.end, 99);
+    }
+
+    #[test]
+    fn test_scan_range_start_after_object_is_rejected_before_reader() {
+        let err = scan_range_from_bounds(Some(100), None, 100).unwrap_err();
+        assert!(err.to_string().contains("ScanRange"));
+    }
+
+    #[test]
+    fn test_scan_range_start_after_end_is_rejected() {
+        let err = scan_range_from_bounds(Some(20), Some(10), 100).unwrap_err();
+        assert!(err.to_string().contains("ScanRange"));
+    }
+
+    #[test]
+    fn test_get_range_conversion_for_parquet_bounded_ranges() {
+        let range = http_range_spec_from_get_range(&GetRange::Bounded(10..20));
+        assert!(!range.is_suffix_length);
+        assert_eq!(range.start, 10);
+        assert_eq!(range.end, 19);
+    }
+
+    #[test]
+    fn test_select_read_headers_preserves_ssec_context() {
+        let input = SelectObjectContentInput {
+            bucket: "bucket".to_string(),
+            expected_bucket_owner: None,
+            key: "object.csv".to_string(),
+            sse_customer_algorithm: Some("AES256".to_string()),
+            sse_customer_key: Some("customer-key".to_string()),
+            sse_customer_key_md5: Some("customer-key-md5".to_string()),
+            request: SelectObjectContentRequest {
+                expression: "SELECT * FROM s3object".to_string(),
+                expression_type: ExpressionType::from_static(ExpressionType::SQL),
+                input_serialization: InputSerialization {
+                    csv: Some(CSVInput::default()),
+                    ..Default::default()
+                },
+                output_serialization: OutputSerialization {
+                    csv: Some(CSVOutput::default()),
+                    ..Default::default()
+                },
+                request_progress: None,
+                scan_range: None,
+            },
+        };
+
+        let headers = select_read_headers(&input);
+        assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM).unwrap(), "AES256");
+        assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY).unwrap(), "customer-key");
+        assert_eq!(headers.get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5).unwrap(), "customer-key-md5");
+    }
+
+    #[tokio::test]
+    async fn test_scan_range_output_can_convert_field_delimiter() {
+        let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"a&&1\nb&&2\n"))]);
+        let stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange { start: 0, end: 10 }, false, 0);
+        let mut stream = convert_field_delimiter_stream(stream, Some("&&".to_string()));
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
+        }
+        assert_eq!(output, b"a,1\nb,2\n");
+    }
+
+    #[tokio::test]
+    async fn test_field_delimiter_stream_converts_delimiter_split_across_chunks() {
+        let chunks = stream::iter(vec![
+            Ok::<_, object_store::Error>(Bytes::from_static(b"a&")),
+            Ok::<_, object_store::Error>(Bytes::from_static(b"&1\nb&&2\n")),
+        ]);
+        let mut stream = convert_field_delimiter_stream(chunks, Some("&&".to_string()));
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
+        }
+        assert_eq!(output, b"a,1\nb,2\n");
+    }
+
+    #[tokio::test]
+    async fn test_field_delimiter_stream_converts_delimiter_at_stream_end() {
+        let chunks = stream::iter(vec![
+            Ok::<_, object_store::Error>(Bytes::from_static(b"a&")),
+            Ok::<_, object_store::Error>(Bytes::from_static(b"&")),
+        ]);
+        let mut stream = convert_field_delimiter_stream(chunks, Some("&&".to_string()));
+        let mut output = Vec::new();
+        while let Some(bytes) = stream.next().await {
+            output.extend_from_slice(&bytes.unwrap());
+        }
+        assert_eq!(output, b"a,");
     }
 
     /// A JSON array is split into one NDJSON line per element.

crates/s3select-api/src/query/execution.rs

@@ -117,6 +117,15 @@ impl Output {
         }
     }
 
+    pub fn into_record_batch_stream(self) -> QueryResult<SendableRecordBatchStream> {
+        match self {
+            Self::StreamData(stream) => Ok(stream),
+            Self::Nil(_) => Err(QueryError::NotImplemented {
+                err: "empty select output stream".to_string(),
+            }),
+        }
+    }
+
     pub async fn num_rows(self) -> usize {
         match self.chunk_result().await {
             Ok(rb) => rb.iter().map(|e| e.num_rows()).sum(),

crates/s3select-api/src/query/session.rs

@@ -15,7 +15,13 @@
 use crate::query::Context;
 use crate::{QueryError, QueryResult, object_store::EcObjectStore};
 use datafusion::{
+    arrow::{
+        array::{Int32Array, StringArray},
+        datatypes::{DataType, Field, Schema},
+        record_batch::RecordBatch,
+    },
     execution::{SessionStateBuilder, context::SessionState, runtime_env::RuntimeEnvBuilder},
+    parquet::arrow::ArrowWriter,
     prelude::SessionContext,
 };
 use object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path};
@@ -66,7 +72,9 @@ impl SessionCtxFactory {
             let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
 
             // Choose test data format based on what the request serialization specifies.
-            let data_bytes: &[u8] = if context.input.request.input_serialization.json.is_some() {
+            let data_bytes: Vec<u8> = if context.input.request.input_serialization.parquet.is_some() {
+                test_parquet_bytes()?
+            } else if context.input.request.input_serialization.json.is_some() {
                 // NDJSON: one JSON object per line — usable for both LINES and DOCUMENT
                 // requests (DOCUMENT inputs are converted to NDJSON by EcObjectStore, but
                 // in test mode we bypass EcObjectStore, so we put NDJSON here directly).
@@ -80,6 +88,7 @@ impl SessionCtxFactory {
                    {\"id\":8,\"name\":\"Henry\",\"age\":32,\"department\":\"IT\",\"salary\":6200}\n\
                    {\"id\":9,\"name\":\"Ivy\",\"age\":24,\"department\":\"Marketing\",\"salary\":4800}\n\
                    {\"id\":10,\"name\":\"Jack\",\"age\":38,\"department\":\"Finance\",\"salary\":7500}\n"
+                    .to_vec()
             } else {
                 b"id,name,age,department,salary
                 1,Alice,25,HR,5000
@@ -92,6 +101,7 @@ impl SessionCtxFactory {
                 8,Henry,32,IT,6200
                 9,Ivy,24,Marketing,4800
                 10,Jack,38,Finance,7500"
+                    .to_vec()
             };
 
             let path = Path::from(context.input.key.clone());
@@ -112,3 +122,35 @@ impl SessionCtxFactory {
         Ok(df_session_ctx)
     }
 }
+
+fn test_parquet_bytes() -> QueryResult<Vec<u8>> {
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new("name", DataType::Utf8, false),
+        Field::new("age", DataType::Int32, false),
+        Field::new("department", DataType::Utf8, false),
+        Field::new("salary", DataType::Int32, false),
+    ]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])),
+            Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie", "Diana", "Eve"])),
+            Arc::new(Int32Array::from(vec![25, 30, 35, 22, 28])),
+            Arc::new(StringArray::from(vec!["HR", "IT", "Finance", "Marketing", "IT"])),
+            Arc::new(Int32Array::from(vec![5000, 6000, 7000, 4500, 5500])),
+        ],
+    )
+    .map_err(|e| QueryError::StoreError { e: e.to_string() })?;
+
+    let mut bytes = Vec::new();
+    {
+        let mut writer =
+            ArrowWriter::try_new(&mut bytes, schema, None).map_err(|e| QueryError::StoreError { e: e.to_string() })?;
+        writer
+            .write(&batch)
+            .map_err(|e| QueryError::StoreError { e: e.to_string() })?;
+        writer.close().map_err(|e| QueryError::StoreError { e: e.to_string() })?;
+    }
+    Ok(bytes)
+}

crates/s3select-query/src/dispatcher/manager.rs

@@ -31,6 +31,7 @@ use datafusion::{
     },
     error::Result as DFResult,
     execution::{RecordBatchStream, SendableRecordBatchStream},
+    sql::sqlparser::parser::ParserError,
 };
 use futures::{Stream, StreamExt};
 use rustfs_s3select_api::{
@@ -106,7 +107,11 @@ impl QueryDispatcher for SimpleQueryDispatcher {
 
         let stmt = match statements.front() {
             Some(stmt) => stmt.clone(),
-            None => return Ok(None),
+            None => {
+                return Err(QueryError::Parser {
+                    source: ParserError::ParserError("empty SQL expression".to_string()),
+                });
+            }
         };
 
         let logical_plan = self

crates/s3select-query/src/test/error_handling_test.rs

@@ -178,15 +178,10 @@ mod error_handling_tests {
             let query = Query::new(Context { input: Arc::new(input) }, sql.to_string());
 
             let result = db.execute(&query).await;
-            // Empty queries might be handled differently by the parser
-            match result {
-                Ok(_) => {
-                    // Some parsers might accept empty queries
-                }
-                Err(_) => {
-                    // Expected to fail for empty SQL
-                }
-            }
+            assert!(
+                matches!(result, Err(QueryError::Parser { .. })),
+                "Expected parser error for empty SQL: {sql:?}"
+            );
         }
     }
 

crates/s3select-query/src/test/integration_test.rs

@@ -21,7 +21,7 @@ mod integration_tests {
     };
     use s3s::dto::{
         CSVInput, CSVOutput, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput, JSONType,
-        OutputSerialization, SelectObjectContentInput, SelectObjectContentRequest,
+        OutputSerialization, ParquetInput, SelectObjectContentInput, SelectObjectContentRequest,
     };
     use std::sync::Arc;
 
@@ -83,6 +83,31 @@ mod integration_tests {
         }
     }
 
+    fn create_test_parquet_input(sql: &str) -> SelectObjectContentInput {
+        SelectObjectContentInput {
+            bucket: "test-bucket".to_string(),
+            expected_bucket_owner: None,
+            key: "test.parquet".to_string(),
+            sse_customer_algorithm: None,
+            sse_customer_key: None,
+            sse_customer_key_md5: None,
+            request: SelectObjectContentRequest {
+                expression: sql.to_string(),
+                expression_type: ExpressionType::from_static("SQL"),
+                input_serialization: InputSerialization {
+                    parquet: Some(ParquetInput {}),
+                    ..Default::default()
+                },
+                output_serialization: OutputSerialization {
+                    json: Some(JSONOutput::default()),
+                    ..Default::default()
+                },
+                request_progress: None,
+                scan_range: None,
+            },
+        }
+    }
+
     #[tokio::test]
     async fn test_database_creation() {
         let input = create_test_input("SELECT * FROM S3Object");
@@ -290,6 +315,21 @@ mod integration_tests {
         assert!(output.is_ok());
     }
 
+    #[tokio::test]
+    async fn test_simple_select_query_parquet() {
+        let sql = "SELECT name, age FROM S3Object WHERE age > 25";
+        let input = create_test_parquet_input(sql);
+        let db = get_global_db(input.clone(), true).await.unwrap();
+        let query = Query::new(Context { input: Arc::new(input) }, sql.to_string());
+
+        let result = db.execute(&query).await;
+        assert!(result.is_ok());
+
+        let output = result.unwrap().result().chunk_result().await.unwrap();
+        let total_rows: usize = output.iter().map(|batch| batch.num_rows()).sum();
+        assert_eq!(total_rows, 3);
+    }
+
     #[tokio::test]
     async fn test_select_with_where_clause_json() {
         let sql = "SELECT name, age FROM S3Object WHERE age > 30";

crates/scanner/src/scanner.rs

@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH};
-use crate::scanner_folder::data_usage_update_dir_cycles;
+use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
 use crate::scanner_io::ScannerIO;
 use crate::sleeper::SCANNER_SLEEPER;
 use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError};
@@ -23,7 +23,10 @@ use chrono::{DateTime, Utc};
 use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics};
 use rustfs_config::ScannerSpeed;
-use rustfs_config::{DEFAULT_SCANNER_SPEED, ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
+use rustfs_config::{
+    DEFAULT_SCANNER_BITROT_CYCLE_SECS, DEFAULT_SCANNER_SPEED, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CYCLE,
+    ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS,
+};
 use rustfs_ecstore::StorageAPI as _;
 use rustfs_ecstore::config::com::{read_config, save_config};
 use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
@@ -80,7 +83,7 @@ fn initial_scanner_delay() -> Duration {
 fn initial_scanner_delay_for(start_delay_secs: Option<u64>) -> Duration {
     start_delay_secs
         .map(|secs| randomized_cycle_delay_for(Duration::from_secs(secs)))
-        .unwrap_or_else(|| Duration::from_secs(rand::random::<u64>() % 5))
+        .unwrap_or_else(randomized_cycle_delay)
 }
 
 pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
@@ -111,9 +114,60 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
     });
 }
 
-fn get_cycle_scan_mode(_current_cycle: u64, _bitrot_start_cycle: u64, _bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode {
-    // TODO: from config
-    HealScanMode::Normal
+fn bitrot_scan_cycle() -> Option<Duration> {
+    let Ok(value) = std::env::var(ENV_SCANNER_BITROT_CYCLE_SECS) else {
+        return Some(Duration::from_secs(DEFAULT_SCANNER_BITROT_CYCLE_SECS));
+    };
+
+    match value.trim().to_ascii_lowercase().as_str() {
+        "0" | "true" | "on" | "yes" => Some(Duration::ZERO),
+        "false" | "off" | "no" | "disabled" => None,
+        value => value.parse::<u64>().ok().map(Duration::from_secs).or_else(|| {
+            warn!(
+                env = ENV_SCANNER_BITROT_CYCLE_SECS,
+                value,
+                default_secs = DEFAULT_SCANNER_BITROT_CYCLE_SECS,
+                "Invalid scanner bitrot cycle, using default"
+            );
+            Some(Duration::from_secs(DEFAULT_SCANNER_BITROT_CYCLE_SECS))
+        }),
+    }
+}
+
+fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode {
+    let Some(bitrot_cycle) = bitrot_scan_cycle() else {
+        return HealScanMode::Normal;
+    };
+
+    if bitrot_cycle.is_zero() {
+        return HealScanMode::Deep;
+    }
+
+    if current_cycle.saturating_sub(bitrot_start_cycle) < heal_object_select_prob() as u64 {
+        return HealScanMode::Deep;
+    }
+
+    let Some(bitrot_start_time) = bitrot_start_time else {
+        return HealScanMode::Deep;
+    };
+
+    let elapsed = Utc::now()
+        .signed_duration_since(bitrot_start_time)
+        .to_std()
+        .unwrap_or(Duration::ZERO);
+    if elapsed >= bitrot_cycle {
+        HealScanMode::Deep
+    } else {
+        HealScanMode::Normal
+    }
+}
+
+fn retain_recent_cycle_completions(cycle_completed: &mut Vec<DateTime<Utc>>) {
+    let keep = data_usage_update_dir_cycles() as usize;
+    if cycle_completed.len() > keep {
+        let drop_count = cycle_completed.len() - keep;
+        cycle_completed.drain(..drop_count);
+    }
 }
 
 /// Background healing information
@@ -238,9 +292,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
 
     info!(duration = ?now.elapsed(), cycles_total=cycle_info.cycle_completed.len(), "Success run data scanner cycle");
 
-    if cycle_info.cycle_completed.len() >= data_usage_update_dir_cycles() as usize {
-        cycle_info.cycle_completed = cycle_info.cycle_completed.split_off(data_usage_update_dir_cycles() as usize);
-    }
+    retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
 
     global_metrics().set_cycle(Some(cycle_info.clone())).await;
 
@@ -351,6 +403,8 @@ pub async fn store_data_usage_in_backend(
         // Save main configuration
         if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
             error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
+        } else {
+            rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
         }
 
         attempts += 1;
@@ -382,6 +436,16 @@ mod tests {
         assert!(delay <= Duration::from_secs(132));
     }
 
+    #[test]
+    #[serial]
+    fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() {
+        with_var(ENV_SCANNER_CYCLE, Some("120"), || {
+            let delay = initial_scanner_delay_for(None);
+            assert!(delay >= Duration::from_secs(108));
+            assert!(delay <= Duration::from_secs(132));
+        });
+    }
+
     #[test]
     #[serial]
     fn test_cycle_interval_prefers_explicit_cycle_override() {
@@ -426,4 +490,46 @@ mod tests {
         assert!(delay >= Duration::from_secs(1), "expected delay >= 1s");
         assert!(delay < Duration::from_secs(2), "expected delay < 2s");
     }
+
+    #[test]
+    #[serial]
+    fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
+        with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
+            let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()));
+            assert_eq!(mode, HealScanMode::Deep);
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
+        with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
+            let recent = Utc::now() - chrono::Duration::minutes(30);
+            let old = Utc::now() - chrono::Duration::hours(2);
+
+            assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent)), HealScanMode::Normal);
+            assert_eq!(get_cycle_scan_mode(2048, 0, Some(old)), HealScanMode::Deep);
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
+        with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
+            assert_eq!(get_cycle_scan_mode(1, 0, None), HealScanMode::Normal);
+        });
+    }
+
+    #[test]
+    fn test_retain_recent_cycle_completions_keeps_last_entries() {
+        let base = Utc::now();
+        let keep = data_usage_update_dir_cycles() as usize;
+        let mut completed: Vec<_> = (0..keep + 2).map(|i| base + chrono::Duration::seconds(i as i64)).collect();
+
+        retain_recent_cycle_completions(&mut completed);
+
+        assert_eq!(completed.len(), keep);
+        assert_eq!(completed.first().copied(), Some(base + chrono::Duration::seconds(2)));
+        assert_eq!(completed.last().copied(), Some(base + chrono::Duration::seconds((keep + 1) as i64)));
+    }
 }

crates/scanner/src/scanner_folder.rs

@@ -69,9 +69,13 @@ const ENV_FAILED_OBJECTS_MAX: &str = "RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX";
 const DEFAULT_FAILED_OBJECT_TTL_SECS: u32 = 86_400;
 const DEFAULT_FAILED_OBJECTS_MAX: u32 = 10_000;
 const METRIC_SCANNER_INLINE_HEAL_TOTAL: &str = "rustfs_scanner_inline_heal_total";
+const METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL: &str = "rustfs_scanner_excess_object_versions_total";
+const METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL: &str = "rustfs_scanner_excess_object_version_size_total";
+const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders_total";
 
 static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
 static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new();
+static SCANNER_ALERT_METRICS_ONCE: Once = Once::new();
 
 pub fn data_usage_update_dir_cycles() -> u32 {
     rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
@@ -102,6 +106,50 @@ fn ensure_scanner_inline_heal_metric_registered() {
     });
 }
 
+fn ensure_scanner_alert_metrics_registered() {
+    SCANNER_ALERT_METRICS_ONCE.call_once(|| {
+        describe_counter!(
+            METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL,
+            "Total scanner alerts for objects with too many retained versions."
+        );
+        describe_counter!(
+            METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL,
+            "Total scanner alerts for objects whose retained versions exceed the cumulative size threshold."
+        );
+        describe_counter!(
+            METRIC_SCANNER_EXCESS_FOLDERS_TOTAL,
+            "Total scanner alerts for folders with too many direct subfolders."
+        );
+    });
+}
+
+fn scanner_excess_versions_threshold() -> u64 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS,
+        rustfs_config::DEFAULT_SCANNER_ALERT_EXCESS_VERSIONS,
+    )
+}
+
+fn scanner_excess_version_size_threshold() -> u64 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE,
+        rustfs_config::DEFAULT_SCANNER_ALERT_EXCESS_VERSION_SIZE,
+    )
+}
+
+fn scanner_excess_folders_threshold() -> u64 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS,
+        rustfs_config::DEFAULT_SCANNER_ALERT_EXCESS_FOLDERS,
+    )
+}
+
+fn should_alert_excessive_versions(remaining_versions: usize, cumulative_size: i64) -> (bool, bool) {
+    let too_many_versions = remaining_versions as u64 >= scanner_excess_versions_threshold();
+    let too_large_versions = cumulative_size > 0 && cumulative_size as u64 >= scanner_excess_version_size_threshold();
+    (too_many_versions, too_large_versions)
+}
+
 fn warn_inline_heal_compat_requested() {
     if !scanner_inline_heal_enabled() {
         return;
@@ -503,7 +551,7 @@ impl ScannerItem {
         };
 
         let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
-        if oi.delete_marker || oi.version_purge_status.is_empty() {
+        if !Self::should_account_replication_stats(oi) {
             return;
         }
 
@@ -545,6 +593,10 @@ impl ScannerItem {
         }
     }
 
+    fn should_account_replication_stats(oi: &ObjectInfo) -> bool {
+        !oi.delete_marker && oi.version_purge_status.is_empty()
+    }
+
     async fn enqueue_heal(&mut self, oi: &ObjectInfo) {
         let done_heal = Metrics::time(Metric::HealAbandonedObject);
         debug!(
@@ -588,8 +640,38 @@ impl ScannerItem {
         done_heal();
     }
 
-    fn alert_excessive_versions(&self, _object_infos_length: usize, _cumulative_size: i64) {
-        // TODO: Implement alerting for excessive versions
+    fn alert_excessive_versions(&self, remaining_versions: usize, cumulative_size: i64) {
+        ensure_scanner_alert_metrics_registered();
+        let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size);
+        if too_many_versions {
+            counter!(
+                METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL,
+                "bucket" => self.bucket.clone()
+            )
+            .increment(1);
+            warn!(
+                bucket = %self.bucket,
+                object = %self.object_path(),
+                versions = remaining_versions,
+                threshold = scanner_excess_versions_threshold(),
+                "scanner detected object with excessive retained versions"
+            );
+        }
+        if too_large_versions {
+            counter!(
+                METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL,
+                "bucket" => self.bucket.clone()
+            )
+            .increment(1);
+            warn!(
+                bucket = %self.bucket,
+                object = %self.object_path(),
+                versions = remaining_versions,
+                cumulative_size,
+                threshold = scanner_excess_version_size_threshold(),
+                "scanner detected object with excessive retained version size"
+            );
+        }
     }
 }
 
@@ -694,6 +776,27 @@ impl FolderScanner {
         }
     }
 
+    fn alert_excessive_folders(&self, folder: &str, total_folders: usize) {
+        let threshold = scanner_excess_folders_threshold();
+        if total_folders as u64 <= threshold {
+            return;
+        }
+
+        ensure_scanner_alert_metrics_registered();
+        counter!(
+            METRIC_SCANNER_EXCESS_FOLDERS_TOTAL,
+            "root" => self.root.clone()
+        )
+        .increment(1);
+        warn!(
+            root = %self.root,
+            folder,
+            folders = total_folders,
+            threshold,
+            "scanner detected folder with excessive direct subfolders"
+        );
+    }
+
     pub async fn should_heal(&self) -> bool {
         if self.skip_heal.load(std::sync::atomic::Ordering::Relaxed) {
             return false;
@@ -1011,7 +1114,8 @@ impl FolderScanner {
                 && existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
                 || existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
 
-            // TODO: Check for excess folders and send events
+            let total_folders = existing_folders.len() + new_folders.len();
+            self.alert_excessive_folders(&folder.name, total_folders);
 
             if !into.compacted && should_compact {
                 into.compacted = true;
@@ -1557,10 +1661,12 @@ mod tests {
 
     use super::*;
     use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
+    use rustfs_filemeta::VersionPurgeStatusType;
     use serial_test::serial;
     #[cfg(unix)]
     use std::os::unix::fs::{PermissionsExt, symlink};
     use std::sync::atomic::AtomicBool;
+    use temp_env::with_var;
     use uuid::Uuid;
 
     async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
@@ -1663,6 +1769,45 @@ mod tests {
         assert!(!scanner.should_skip_failed("path2"));
     }
 
+    #[test]
+    fn test_should_account_replication_stats_only_for_live_object_versions() {
+        let live = ObjectInfo::default();
+        assert!(ScannerItem::should_account_replication_stats(&live));
+
+        let delete_marker = ObjectInfo {
+            delete_marker: true,
+            ..Default::default()
+        };
+        assert!(!ScannerItem::should_account_replication_stats(&delete_marker));
+
+        let purge_version = ObjectInfo {
+            version_purge_status: VersionPurgeStatusType::Pending,
+            ..Default::default()
+        };
+        assert!(!ScannerItem::should_account_replication_stats(&purge_version));
+    }
+
+    #[test]
+    #[serial]
+    fn test_excessive_version_alert_thresholds_use_env() {
+        with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS, Some("3"), || {
+            with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, Some("100"), || {
+                assert_eq!(should_alert_excessive_versions(2, 99), (false, false));
+                assert_eq!(should_alert_excessive_versions(3, 99), (true, false));
+                assert_eq!(should_alert_excessive_versions(2, 100), (false, true));
+                assert_eq!(should_alert_excessive_versions(3, 100), (true, true));
+            });
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_excessive_folders_threshold_uses_env() {
+        with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, Some("3"), || {
+            assert_eq!(scanner_excess_folders_threshold(), 3);
+        });
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_record_failed_prunes_to_max_entries() {

crates/targets/Cargo.toml

@@ -14,20 +14,23 @@ documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
 [dependencies]
 rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] }
 rustfs-ecstore = { workspace = true }
-rustfs-utils = { workspace = true, features = ["notify", "tls"] }
+rustfs-tls-runtime = { workspace = true }
 rustfs-s3-types = { workspace = true }
 async-trait = { workspace = true }
 async-nats = { workspace = true }
 deadpool-postgres = { workspace = true }
+hyper = { workspace = true }
 hyper-rustls = { workspace = true }
 lapin = { workspace = true }
+libc = { workspace = true }
 pulsar = { workspace = true }
+regex = { workspace = true }
 reqwest = { workspace = true }
 rumqttc = { workspace = true }
 redis = { workspace = true }
 rustls = { workspace = true }
 rustls-native-certs = { workspace = true }
-rustls-pki-types = { workspace = true }
+s3s = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 snap = { workspace = true }
@@ -45,6 +48,8 @@ mysql_async = { workspace = true }
 chrono = { workspace = true }
 parking_lot = { workspace = true }
 hashbrown = { workspace = true }
+arc-swap = { workspace = true }
+metrics = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true }

crates/targets/src/check.rs

@@ -64,8 +64,8 @@ pub async fn check_mqtt_broker_available_with_tls(
     use crate::target::mqtt::build_mqtt_options;
     use rumqttc::{AsyncClient, QoS};
 
-    let url = rustfs_utils::parse_url(broker_url)
-        .map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
+    let url =
+        crate::parse_url(broker_url).map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
     let url = url.url();
 
     // build_mqtt_options returns TargetError directly; Configuration variants propagate as-is.

crates/targets/src/lib.rs

@@ -20,6 +20,7 @@ pub mod control_plane;
 pub mod domain;
 pub mod error;
 pub mod manifest;
+mod net;
 pub mod plugin;
 pub mod runtime;
 pub mod store;
@@ -48,6 +49,7 @@ pub use manifest::{
     TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginPackaging,
     TargetPluginRuntimeTransport, builtin_target_marketplace_manifest, installable_target_marketplace_manifest,
 };
+pub use net::*;
 pub use plugin::{
     BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetPluginRegistry,
     TargetRequestValidator, boxed_target,

crates/targets/src/net.rs

@@ -1,18 +1,21 @@
-//  Copyright 2024 RustFS Team
+// Copyright 2024 RustFS Team
 //
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-//      http://www.apache.org/licenses/LICENSE-2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
+use hashbrown::HashMap;
+use hyper::HeaderMap;
 use regex::Regex;
+use s3s::{S3Request, S3Response};
 use serde::{Deserialize, Serialize};
 use std::net::IpAddr;
 use std::path::Path;
@@ -20,7 +23,6 @@ use std::sync::LazyLock;
 use thiserror::Error;
 use url::Url;
 
-// Lazy static for the host label regex.
 static HOST_LABEL_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$").unwrap());
 
 /// NetError represents errors that can occur in network operations.
@@ -41,21 +43,17 @@ pub enum NetError {
 }
 
 /// Host represents a network host with IP/name and port.
-/// Similar to Go's net.Host structure.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Host {
     pub name: String,
-    pub port: Option<u16>, // Using Option<u16> to represent if port is set, similar to IsPortSet.
+    pub port: Option<u16>,
 }
 
-// Implementation of Host methods.
 impl Host {
-    // is_empty returns true if the host name is empty.
     pub fn is_empty(&self) -> bool {
         self.name.is_empty()
     }
 
-    // equal checks if two hosts are equal by comparing their string representations.
     pub fn equal(&self, other: &Host) -> bool {
         self.to_string() == other.to_string()
     }
@@ -70,13 +68,122 @@ impl std::fmt::Display for Host {
     }
 }
 
-// parse_host parses a string into a Host, with validation similar to Go's ParseHost.
+/// Extract request parameters from S3Request, mainly header information.
+pub fn extract_req_params<T>(req: &S3Request<T>) -> HashMap<String, String> {
+    extract_params_header(&req.headers)
+}
+
+/// Extract request parameters from hyper::HeaderMap, mainly header information.
+/// This function is useful when you have a raw HTTP request and need to extract parameters.
+#[deprecated(since = "0.1.0", note = "Use extract_params_header instead")]
+pub fn extract_req_params_header(head: &HeaderMap) -> HashMap<String, String> {
+    extract_params_header(head)
+}
+
+/// Extract parameters from hyper::HeaderMap, mainly header information.
+pub fn extract_params_header(head: &HeaderMap) -> HashMap<String, String> {
+    let mut params = HashMap::new();
+    for (key, value) in head.iter() {
+        if let Ok(val_str) = value.to_str() {
+            params.insert(key.as_str().to_string(), val_str.to_string());
+        }
+    }
+    params
+}
+
+/// Extract response elements from S3Response, mainly header information.
+pub fn extract_resp_elements<T>(resp: &S3Response<T>) -> HashMap<String, String> {
+    extract_params_header(&resp.headers)
+}
+
+/// Get host from header information.
+pub fn get_request_host(headers: &HeaderMap) -> String {
+    headers
+        .get("host")
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or_default()
+        .to_string()
+}
+
+/// Get Port from header information.
+/// Priority:
+/// 1. x-forwarded-port
+/// 2. host header (parse port)
+/// 3. x-forwarded-proto inferred default (http=80, https=443) when host has no explicit port
+/// 4. port header
+pub fn get_request_port(headers: &HeaderMap) -> u16 {
+    if let Some(port) = headers
+        .get("x-forwarded-port")
+        .and_then(|v| v.to_str().ok())
+        .and_then(|s| s.parse::<u16>().ok())
+    {
+        return port;
+    }
+
+    if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) {
+        if let Some(idx) = host.rfind(':') {
+            let valid_colon = match host.rfind(']') {
+                Some(close_bracket_idx) => idx > close_bracket_idx,
+                None => true,
+            };
+
+            if valid_colon
+                && let Ok(port) = host[idx + 1..].parse::<u16>()
+                && port > 0
+            {
+                return port;
+            }
+        }
+
+        if let Some(proto) = headers.get("x-forwarded-proto").and_then(|v| v.to_str().ok()) {
+            match proto {
+                "http" => return 80,
+                "https" => return 443,
+                _ => {}
+            }
+        }
+    }
+
+    headers
+        .get("port")
+        .and_then(|v| v.to_str().ok())
+        .and_then(|s| s.parse::<u16>().ok())
+        .unwrap_or(0)
+}
+
+/// Get content-length from header information.
+pub fn get_request_content_length(headers: &HeaderMap) -> u64 {
+    headers
+        .get("content-length")
+        .and_then(|v| v.to_str().ok())
+        .and_then(|s| s.parse::<u64>().ok())
+        .unwrap_or(0)
+}
+
+/// Get referer from header information.
+pub fn get_request_referer(headers: &HeaderMap) -> String {
+    headers
+        .get("referer")
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or_default()
+        .to_string()
+}
+
+/// Get user-agent from header information.
+pub fn get_request_user_agent(headers: &HeaderMap) -> String {
+    headers
+        .get("user-agent")
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or_default()
+        .to_string()
+}
+
+/// parse_host parses a string into a Host, with validation similar to Go's ParseHost.
 pub fn parse_host(s: &str) -> Result<Host, NetError> {
     if s.is_empty() {
         return Err(NetError::InvalidArgument);
     }
 
-    // is_valid_host validates the host string, checking for IP or hostname validity.
     let is_valid_host = |host: &str| -> bool {
         if host.is_empty() {
             return true;
@@ -122,9 +229,6 @@ pub fn parse_host(s: &str) -> Result<Host, NetError> {
             return Err(NetError::MissingBracket);
         }
 
-        // A host with multiple colons is an IPv6 literal, optionally with a
-        // zone identifier. Unbracketed IPv6 with port is ambiguous, so callers
-        // must use the standard bracketed form when they need a port.
         let (host_str, port_str) = if s.matches(':').count() > 1 {
             (s, "")
         } else {
@@ -139,7 +243,6 @@ pub fn parse_host(s: &str) -> Result<Host, NetError> {
         (trim_ipv6(host_str)?, port)
     };
 
-    // Handle IPv6 zone identifier.
     let trimmed_host = host.split('%').next().unwrap_or(&host);
 
     if !is_valid_host(trimmed_host) {
@@ -149,7 +252,6 @@ pub fn parse_host(s: &str) -> Result<Host, NetError> {
     Ok(Host { name: host, port })
 }
 
-// trim_ipv6 removes square brackets from IPv6 addresses, similar to Go's trimIPv6.
 fn trim_ipv6(host: &str) -> Result<String, NetError> {
     if host.ends_with(']') {
         if !host.starts_with('[') {
@@ -162,37 +264,18 @@ fn trim_ipv6(host: &str) -> Result<String, NetError> {
 }
 
 /// URL is a wrapper around url::Url for custom handling.
-/// Provides methods similar to Go's URL struct.
 #[derive(Debug, Clone)]
 pub struct ParsedURL(pub Url);
 
 impl ParsedURL {
-    /// is_empty returns true if the URL is empty or "about:blank".
-    ///
-    /// # Arguments
-    /// * `&self` - Reference to the ParsedURL instance.
-    ///
-    /// # Returns
-    /// * `bool` - True if the URL is empty or "about:blank", false otherwise.
-    ///
     pub fn is_empty(&self) -> bool {
         self.0.as_str() == "" || (self.0.scheme() == "about" && self.0.path() == "blank")
     }
 
-    /// hostname returns the hostname of the URL.
-    ///
-    /// # Returns
-    /// * `String` - The hostname of the URL, or an empty string if not set.
-    ///
     pub fn hostname(&self) -> String {
         self.0.host_str().unwrap_or("").to_string()
     }
 
-    /// port returns the port of the URL as a string, defaulting to "80" for http and "443" for https if not set.
-    ///
-    /// # Returns
-    /// * `String` - The port of the URL as a string.
-    ///
     pub fn port(&self) -> String {
         match self.0.port() {
             Some(p) => p.to_string(),
@@ -204,20 +287,10 @@ impl ParsedURL {
         }
     }
 
-    /// scheme returns the scheme of the URL.
-    ///
-    /// # Returns
-    /// * `&str` - The scheme of the URL.
-    ///
     pub fn scheme(&self) -> &str {
         self.0.scheme()
     }
 
-    /// url returns a reference to the underlying Url.
-    ///
-    /// # Returns
-    /// * `&Url` - Reference to the underlying Url.
-    ///
     pub fn url(&self) -> &Url {
         &self.0
     }
@@ -235,7 +308,6 @@ impl std::fmt::Display for ParsedURL {
         }
         let mut s = url.to_string();
 
-        // If the URL ends with a slash and the path is just "/", remove the trailing slash.
         if s.ends_with('/') && url.path() == "/" {
             s.pop();
         }
@@ -268,17 +340,6 @@ impl<'de> serde::Deserialize<'de> for ParsedURL {
 }
 
 /// parse_url parses a string into a ParsedURL, with host validation and path cleaning.
-///
-/// # Arguments
-/// * `s` - The URL string to parse.
-///
-/// # Returns
-/// * `Ok(ParsedURL)` - If parsing is successful.
-/// * `Err(NetError)` - If parsing fails or host is invalid.
-///
-/// # Errors
-/// Returns NetError if parsing fails or host is invalid.
-///
 pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
     if let Some(scheme_end) = s.find("://")
         && s[scheme_end + 3..].starts_with('/')
@@ -303,13 +364,11 @@ pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
 
         if !port_str.is_empty() {
             let host_port = format!("{}:{}", uu.host_str().unwrap(), port_str);
-            parse_host(&host_port)?; // Validate host.
+            parse_host(&host_port)?;
         }
     }
 
-    // Clean path: Use Url's path_segments to normalize.
     if !uu.path().is_empty() {
-        // Url automatically cleans paths, but we ensure trailing slash if original had it.
         let mut cleaned_path = String::new();
         for comp in Path::new(uu.path()).components() {
             use std::path::Component;
@@ -337,15 +396,6 @@ pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
 }
 
 #[allow(dead_code)]
-/// parse_http_url parses a string into a ParsedURL, ensuring the scheme is http or https.
-///
-/// # Arguments
-/// * `s` - The URL string to parse.
-///
-/// # Returns
-/// * `Ok(ParsedURL)` - If parsing is successful and scheme is http/https.
-/// * `Err(NetError)` - If parsing fails or scheme is not http/https.
-///
 pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
     let u = parse_url(s)?;
     match u.0.scheme() {
@@ -355,20 +405,10 @@ pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
 }
 
 #[allow(dead_code)]
-/// is_network_or_host_down checks if an error indicates network or host down, considering timeouts.
-///
-/// # Arguments
-/// * `err` - The std::io::Error to check.
-/// * `expect_timeouts` - Whether timeouts are expected.
-///
-/// # Returns
-/// * `bool` - True if the error indicates network or host down, false otherwise.
-///
 pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool {
     if err.kind() == std::io::ErrorKind::TimedOut {
         return !expect_timeouts;
     }
-    // Simplified checks based on Go logic; adapt for Rust as needed
     let err_str = err.to_string().to_lowercase();
     err_str.contains("connection reset by peer")
         || err_str.contains("connection timed out")
@@ -377,27 +417,11 @@ pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> b
 }
 
 #[allow(dead_code)]
-/// is_conn_reset_err checks if an error indicates a connection reset by peer.
-///
-/// # Arguments
-/// * `err` - The std::io::Error to check.
-///
-/// # Returns
-/// * `bool` - True if the error indicates connection reset, false otherwise.
-///
 pub fn is_conn_reset_err(err: &std::io::Error) -> bool {
     err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET))
 }
 
 #[allow(dead_code)]
-/// is_conn_refused_err checks if an error indicates a connection refused.
-///
-/// # Arguments
-/// * `err` - The std::io::Error to check.
-///
-/// # Returns
-/// * `bool` - True if the error indicates connection refused, false otherwise.
-///
 pub fn is_conn_refused_err(err: &std::io::Error) -> bool {
     err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED))
 }
@@ -405,6 +429,51 @@ pub fn is_conn_refused_err(err: &std::io::Error) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use hyper::header::HeaderValue;
+
+    #[test]
+    fn test_get_request_port() {
+        let mut headers = HeaderMap::new();
+
+        assert_eq!(get_request_port(&headers), 0);
+
+        headers.insert("port", HeaderValue::from_static("8080"));
+        assert_eq!(get_request_port(&headers), 8080);
+
+        headers.remove("port");
+        headers.insert("host", HeaderValue::from_static("example.com:9000"));
+        assert_eq!(get_request_port(&headers), 9000);
+
+        headers.insert("host", HeaderValue::from_static("example.com"));
+        assert_eq!(get_request_port(&headers), 0);
+
+        headers.insert("host", HeaderValue::from_static("[::1]:9001"));
+        assert_eq!(get_request_port(&headers), 9001);
+
+        headers.insert("host", HeaderValue::from_static("[::1]"));
+        assert_eq!(get_request_port(&headers), 0);
+
+        headers.insert("x-forwarded-port", HeaderValue::from_static("7000"));
+        assert_eq!(get_request_port(&headers), 7000);
+
+        headers.remove("x-forwarded-port");
+        headers.insert("host", HeaderValue::from_static("example.com"));
+        headers.insert("x-forwarded-proto", HeaderValue::from_static("http"));
+        assert_eq!(get_request_port(&headers), 80);
+
+        headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
+        assert_eq!(get_request_port(&headers), 443);
+
+        headers.insert("x-forwarded-proto", HeaderValue::from_static("ftp"));
+        assert_eq!(get_request_port(&headers), 0);
+
+        headers.insert("host", HeaderValue::from_static("example.com:0"));
+        headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
+        assert_eq!(get_request_port(&headers), 443);
+
+        headers.remove("x-forwarded-proto");
+        assert_eq!(get_request_port(&headers), 0);
+    }
 
     #[test]
     fn parse_host_with_empty_string_returns_error() {
@@ -457,6 +526,15 @@ mod tests {
         assert_eq!(host.port, None);
     }
 
+    #[test]
+    fn parse_host_with_bracketed_ipv6_zone_and_port() {
+        let result = parse_host("[fe80::1%eth0]:9000");
+        assert!(result.is_ok());
+        let host = result.unwrap();
+        assert_eq!(host.name, "fe80::1%eth0");
+        assert_eq!(host.port, Some(9000));
+    }
+
     #[test]
     fn parse_host_with_bracketed_ipv6_without_port() {
         let result = parse_host("[::1]");
@@ -523,32 +601,6 @@ mod tests {
         assert_eq!(host.to_string(), "example.com");
     }
 
-    #[test]
-    fn host_equal_when_same() {
-        let host1 = Host {
-            name: "example.com".to_string(),
-            port: Some(80),
-        };
-        let host2 = Host {
-            name: "example.com".to_string(),
-            port: Some(80),
-        };
-        assert!(host1.equal(&host2));
-    }
-
-    #[test]
-    fn host_not_equal_when_different() {
-        let host1 = Host {
-            name: "example.com".to_string(),
-            port: Some(80),
-        };
-        let host2 = Host {
-            name: "example.com".to_string(),
-            port: Some(443),
-        };
-        assert!(!host1.equal(&host2));
-    }
-
     #[test]
     fn parse_url_with_valid_http_url() {
         let result = parse_url("http://example.com/path");
@@ -556,100 +608,35 @@ mod tests {
         let parsed = result.unwrap();
         assert_eq!(parsed.hostname(), "example.com");
         assert_eq!(parsed.port(), "80");
+        assert_eq!(parsed.scheme(), "http");
+        assert_eq!(parsed.to_string(), "http://example.com/path");
     }
 
     #[test]
-    fn parse_url_with_valid_https_url() {
+    fn parse_url_with_explicit_default_https_port() {
         let result = parse_url("https://example.com:443/path");
         assert!(result.is_ok());
         let parsed = result.unwrap();
-        assert_eq!(parsed.hostname(), "example.com");
-        assert_eq!(parsed.port(), "443");
+        assert_eq!(parsed.to_string(), "https://example.com/path");
     }
 
     #[test]
-    fn parse_url_with_scheme_but_empty_host() {
+    fn parse_url_with_empty_host_returns_error() {
         let result = parse_url("http:///path");
         assert!(matches!(result, Err(NetError::SchemeWithEmptyHost)));
     }
 
     #[test]
-    fn parse_url_with_invalid_host() {
+    fn parse_url_with_invalid_host_returns_error() {
         let result = parse_url("http://invalid..host/path");
         assert!(matches!(result, Err(NetError::InvalidHost)));
     }
 
     #[test]
-    fn parse_url_with_path_cleaning() {
+    fn parse_url_normalizes_path() {
         let result = parse_url("http://example.com//path/../path/");
         assert!(result.is_ok());
         let parsed = result.unwrap();
-        assert_eq!(parsed.0.path(), "/path/");
-    }
-
-    #[test]
-    fn parse_http_url_with_http_scheme() {
-        let result = parse_http_url("http://example.com");
-        assert!(result.is_ok());
-    }
-
-    #[test]
-    fn parse_http_url_with_https_scheme() {
-        let result = parse_http_url("https://example.com");
-        assert!(result.is_ok());
-    }
-
-    #[test]
-    fn parse_http_url_with_invalid_scheme() {
-        let result = parse_http_url("ftp://example.com");
-        assert!(matches!(result, Err(NetError::UnexpectedScheme(_))));
-    }
-
-    #[test]
-    fn parsed_url_is_empty_when_url_is_empty() {
-        let url = ParsedURL(Url::parse("about:blank").unwrap());
-        assert!(url.is_empty());
-    }
-
-    #[test]
-    fn parsed_url_hostname() {
-        let url = ParsedURL(Url::parse("http://example.com:8080").unwrap());
-        assert_eq!(url.hostname(), "example.com");
-    }
-
-    #[test]
-    fn parsed_url_port() {
-        let url = ParsedURL(Url::parse("http://example.com:8080").unwrap());
-        assert_eq!(url.port(), "8080");
-    }
-
-    #[test]
-    fn parsed_url_to_string_removes_default_ports() {
-        let url = ParsedURL(Url::parse("http://example.com:80").unwrap());
-        assert_eq!(url.to_string(), "http://example.com");
-    }
-
-    #[test]
-    fn is_network_or_host_down_with_timeout() {
-        let err = std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout");
-        assert!(is_network_or_host_down(&err, false));
-    }
-
-    #[test]
-    fn is_network_or_host_down_with_expected_timeout() {
-        let err = std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout");
-        assert!(!is_network_or_host_down(&err, true));
-    }
-
-    #[test]
-    fn is_conn_reset_err_with_reset_message() {
-        let err = std::io::Error::other("connection reset by peer");
-        assert!(is_conn_reset_err(&err));
-    }
-
-    #[test]
-    fn is_conn_refused_err_with_refused_message() {
-        let err = std::io::Error::other("connection refused");
-        assert!(is_conn_refused_err(&err));
+        assert_eq!(parsed.to_string(), "http://example.com/path/");
     }
 }

crates/targets/src/runtime/mod.rs

@@ -15,6 +15,7 @@
 pub mod adapter;
 pub mod sidecar;
 pub mod sidecar_protocol;
+pub mod tls;
 
 use crate::Target;
 use crate::arn::TargetID;

crates/targets/src/runtime/tls/adapter.rs

@@ -0,0 +1,217 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! `TlsReloadAdapter<M>` — the single entry-point that connects a target to
+//! the TLS reload coordinator.  Each target holds an `Option<TlsReloadAdapter<M>>`
+//! and calls `current_material()` on the hot path.  When `None`, the target
+//! falls back to its legacy inline fingerprint logic.
+
+use super::config::TlsReloadOptions;
+use super::coordinator::TargetTlsReloadCoordinator;
+use super::state::{TargetTlsRuntimeState, TargetTlsStatusSnapshot};
+use super::r#trait::ReloadableTargetTls;
+use std::sync::Arc;
+use tracing::warn;
+
+/// Bridges a `ReloadableTargetTls` implementor and the reload coordinator.
+///
+/// Created via [`TlsReloadAdapter::try_register`].  Holds the coordinator-
+/// managed runtime state and exposes a zero-cost `current_material()` accessor
+/// for the send hot-path.
+pub struct TlsReloadAdapter<M> {
+    runtime_state: Arc<TargetTlsRuntimeState<M>>,
+    options: TlsReloadOptions,
+}
+
+impl<M> std::fmt::Debug for TlsReloadAdapter<M> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TlsReloadAdapter")
+            .field("target_label", &self.runtime_state.inputs.target_label)
+            .finish_non_exhaustive()
+    }
+}
+
+impl<M> Clone for TlsReloadAdapter<M> {
+    fn clone(&self) -> Self {
+        Self {
+            runtime_state: Arc::clone(&self.runtime_state),
+            options: self.options.clone(),
+        }
+    }
+}
+
+impl<M: Send + Sync + 'static> TlsReloadAdapter<M> {
+    /// Registers `target` with the coordinator and returns an adapter.
+    ///
+    /// On success the coordinator has:
+    /// - built initial TLS material
+    /// - spawned a background poll loop
+    ///
+    /// On failure returns `None` (the caller should keep its inline fallback
+    /// path intact — the target continues to work, just without coordinator
+    /// support).
+    pub async fn try_register<T: ReloadableTargetTls<Material = M>>(
+        target: Arc<T>,
+        options: TlsReloadOptions,
+        coordinator: &TargetTlsReloadCoordinator,
+    ) -> Option<Self> {
+        let label = target.tls_input_set().target_label.clone();
+        match coordinator.register(target, options.clone()).await {
+            Ok(runtime_state) => {
+                tracing::info!(target = %label, "TLS reload adapter registered");
+                Some(Self { runtime_state, options })
+            }
+            Err(err) => {
+                warn!(target = %label, error = %err, "TLS reload adapter registration failed; target will use inline fallback");
+                None
+            }
+        }
+    }
+
+    /// Hot-path accessor: returns the current TLS material managed by the
+    /// coordinator.  The returned `Arc<M>` is cheap to clone.
+    #[inline]
+    pub fn current_material(&self) -> Arc<M> {
+        Arc::clone(&self.runtime_state.current.load().material)
+    }
+
+    /// Returns the active generation counter.
+    #[inline]
+    pub fn generation(&self) -> u64 {
+        self.runtime_state.current.load().generation.0
+    }
+
+    /// Returns a read-only status snapshot for admin/observability.
+    pub fn status_snapshot(&self) -> TargetTlsStatusSnapshot {
+        TargetTlsReloadCoordinator::build_status_snapshot(&self.runtime_state, &self.options)
+    }
+
+    /// Returns the underlying runtime state (for `close()` cleanup etc.).
+    pub fn runtime_state(&self) -> &Arc<TargetTlsRuntimeState<M>> {
+        &self.runtime_state
+    }
+
+    /// Unregisters from the coordinator (stops the poll loop).
+    pub async fn unregister(&self, coordinator: &TargetTlsReloadCoordinator) {
+        let label = &self.runtime_state.inputs.target_label;
+        if let Err(err) = coordinator.unregister(label).await {
+            warn!(target = %label, error = %err, "Failed to unregister TLS reload adapter");
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::error::TargetError;
+    use async_trait::async_trait;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+    struct FakeTarget {
+        label: String,
+        build_calls: AtomicUsize,
+        should_fail: AtomicBool,
+    }
+
+    impl FakeTarget {
+        fn new(label: &str) -> Self {
+            Self {
+                label: label.to_string(),
+                build_calls: AtomicUsize::new(0),
+                should_fail: AtomicBool::new(false),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl ReloadableTargetTls for FakeTarget {
+        type Material = String;
+
+        fn tls_input_set(&self) -> super::super::state::TargetTlsInputSet {
+            super::super::state::TargetTlsInputSet {
+                ca_path: String::new(),
+                client_cert_path: String::new(),
+                client_key_path: String::new(),
+                target_label: self.label.clone(),
+            }
+        }
+
+        async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+            self.build_calls.fetch_add(1, Ordering::SeqCst);
+            if self.should_fail.load(Ordering::SeqCst) {
+                return Err(TargetError::Configuration("fail".to_string()));
+            }
+            Ok("material".to_string())
+        }
+
+        async fn apply_tls_material(
+            &self,
+            _generation: super::super::fingerprint::TargetTlsGeneration,
+            _material: Arc<Self::Material>,
+            _mode: super::super::config::ReloadApplyMode,
+        ) -> Result<(), TargetError> {
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn try_register_returns_adapter_on_success() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(FakeTarget::new("test:fake"));
+        let options = TlsReloadOptions::default();
+
+        let adapter = TlsReloadAdapter::try_register(target.clone(), options, &coordinator).await;
+        assert!(adapter.is_some());
+        assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
+
+        let a = adapter.unwrap();
+        assert_eq!(*a.current_material(), "material");
+    }
+
+    #[tokio::test]
+    async fn try_register_returns_none_on_failure() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(FakeTarget::new("test:fail"));
+        target.should_fail.store(true, Ordering::SeqCst);
+        let options = TlsReloadOptions::default();
+
+        let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await;
+        assert!(adapter.is_none());
+    }
+
+    #[tokio::test]
+    async fn adapter_is_clone_and_shares_state() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(FakeTarget::new("test:clone"));
+        let options = TlsReloadOptions::default();
+
+        let a = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
+        let b = a.clone();
+
+        assert_eq!(*a.current_material(), *b.current_material());
+        assert_eq!(a.generation(), b.generation());
+    }
+
+    #[tokio::test]
+    async fn status_snapshot_contains_label() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(FakeTarget::new("test:snap"));
+        let options = TlsReloadOptions::default();
+
+        let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
+        let snap = adapter.status_snapshot();
+        assert_eq!(snap.target_label, "test:snap");
+        assert!(snap.reload_enabled);
+    }
+}

crates/targets/src/runtime/tls/config.rs

@@ -0,0 +1,20 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Target-level TLS reload configuration.
+//!
+//! Re-exports the shared types from `rustfs_tls_runtime::config` so that
+//! targets and their callers use a single source of truth.
+
+pub use rustfs_tls_runtime::config::{ReloadApplyHint as ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};

crates/targets/src/runtime/tls/coordinator.rs

@@ -0,0 +1,656 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Target TLS reload coordinator. Manages per-target background poll loops
+//! that periodically check TLS material fingerprints and drive safe reload.
+
+use super::config::{ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};
+#[cfg(test)]
+use super::fingerprint::TargetTlsFingerprint;
+use super::fingerprint::{TargetTlsGeneration, build_target_tls_fingerprint};
+use super::metrics::{record_target_tls_publication_fail, record_target_tls_reload_result, record_target_tls_reload_skipped};
+#[cfg(test)]
+use super::state::TargetTlsInputSet;
+use super::state::{TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
+use super::r#trait::ReloadableTargetTls;
+use super::validate::validate_tls_material;
+use crate::error::TargetError;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::sync::RwLock;
+use tokio::task::JoinHandle;
+use tracing::{debug, info, warn};
+
+struct TargetReloadEntry {
+    #[expect(dead_code)]
+    target_label: String,
+    cancel_tx: tokio::sync::mpsc::Sender<()>,
+    poll_handle: JoinHandle<()>,
+}
+
+/// The top-level coordinator that manages TLS reload for all registered targets.
+///
+/// Typically one instance per process, held alongside `TargetRuntimeManager`.
+/// Each registered target gets its own background poll loop that periodically
+/// checks TLS fingerprints and drives the build/apply cycle.
+pub struct TargetTlsReloadCoordinator {
+    entries: RwLock<HashMap<String, TargetReloadEntry>>,
+}
+
+impl Default for TargetTlsReloadCoordinator {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl TargetTlsReloadCoordinator {
+    pub fn new() -> Self {
+        Self {
+            entries: RwLock::new(HashMap::new()),
+        }
+    }
+
+    /// Register a target for coordinated TLS reload. Spawns a background poll loop.
+    ///
+    /// Returns the initial runtime state that the target should hold for
+    /// accessing the current TLS material via `ArcSwap`.
+    pub async fn register<T: ReloadableTargetTls>(
+        &self,
+        target: Arc<T>,
+        options: TlsReloadOptions,
+    ) -> Result<Arc<TargetTlsRuntimeState<T::Material>>, TargetError> {
+        if !options.enabled {
+            return Err(TargetError::Configuration("TLS reload is disabled".to_string()));
+        }
+
+        let inputs = target.tls_input_set();
+        let target_label = inputs.target_label.clone();
+
+        // Build initial material
+        let initial_material = Arc::new(target.build_tls_material().await?);
+        let initial_fingerprint =
+            build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
+
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(1),
+            fingerprint: initial_fingerprint,
+            material: initial_material,
+            loaded_at_unix_ms: unix_time_ms(),
+        });
+
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state.clone(), inputs));
+
+        if options.detect_mode == ReloadDetectMode::Poll || options.detect_mode == ReloadDetectMode::Hybrid {
+            let (cancel_tx, cancel_rx) = tokio::sync::mpsc::channel(1);
+            let poll_handle = tokio::spawn(spawn_target_poll_loop(target, Arc::clone(&runtime_state), options, cancel_rx));
+
+            let mut entries = self.entries.write().await;
+            entries.insert(
+                target_label.clone(),
+                TargetReloadEntry {
+                    target_label: target_label.clone(),
+                    cancel_tx,
+                    poll_handle,
+                },
+            );
+
+            info!(target = %target_label, "Registered target for TLS reload coordinator");
+        }
+
+        Ok(runtime_state)
+    }
+
+    /// Unregister a target and stop its poll loop.
+    pub async fn unregister(&self, target_label: &str) -> Result<(), TargetError> {
+        let mut entries = self.entries.write().await;
+        if let Some(entry) = entries.remove(target_label) {
+            let _ = entry.cancel_tx.send(()).await;
+            entry.poll_handle.abort();
+            info!(target = %target_label, "Unregistered target from TLS reload coordinator");
+        }
+        Ok(())
+    }
+
+    /// Force an immediate reload check for a specific target.
+    /// Used by admin endpoints and test harnesses.
+    pub async fn force_reload<T: ReloadableTargetTls>(
+        &self,
+        target: &T,
+        runtime_state: &TargetTlsRuntimeState<T::Material>,
+        options: &TlsReloadOptions,
+    ) -> Result<TargetTlsGeneration, TargetError> {
+        reload_target_once(target, runtime_state, options).await
+    }
+
+    /// Stop all poll loops.
+    pub async fn shutdown(&self) {
+        let mut entries = self.entries.write().await;
+        for (label, entry) in entries.drain() {
+            let _ = entry.cancel_tx.send(()).await;
+            entry.poll_handle.abort();
+            debug!(target = %label, "Stopped TLS reload poll loop");
+        }
+    }
+
+    /// Collect status snapshots from all registered targets.
+    /// The caller must provide the runtime states separately since the
+    /// coordinator does not hold type-erased references to them.
+    pub fn build_status_snapshot<M>(
+        runtime_state: &TargetTlsRuntimeState<M>,
+        options: &TlsReloadOptions,
+    ) -> TargetTlsStatusSnapshot {
+        let current = runtime_state.current.load();
+        let last_attempt = runtime_state.last_attempt_unix_ms();
+        let last_success = runtime_state.last_success_unix_ms();
+        let last_error = runtime_state.last_error.read().clone();
+
+        TargetTlsStatusSnapshot {
+            target_label: runtime_state.inputs.target_label.clone(),
+            generation: current.generation.0,
+            reload_enabled: options.enabled,
+            detect_mode: match options.detect_mode {
+                ReloadDetectMode::Poll => "poll",
+                ReloadDetectMode::Watch => "watch",
+                ReloadDetectMode::Hybrid => "hybrid",
+            },
+            apply_mode: match options.apply_hint {
+                ReloadApplyMode::Lazy => "lazy",
+                ReloadApplyMode::SoftReconnect => "soft_reconnect",
+            },
+            last_attempt_time: if last_attempt > 0 { Some(last_attempt) } else { None },
+            last_success_time: if last_success > 0 { Some(last_success) } else { None },
+            last_error,
+            ca_path: runtime_state.inputs.ca_path.clone(),
+            client_cert_path: runtime_state.inputs.client_cert_path.clone(),
+            client_key_path: runtime_state.inputs.client_key_path.clone(),
+        }
+    }
+}
+
+/// Background poll loop for a single target.
+async fn spawn_target_poll_loop<T: ReloadableTargetTls>(
+    target: Arc<T>,
+    runtime_state: Arc<TargetTlsRuntimeState<T::Material>>,
+    options: TlsReloadOptions,
+    mut cancel_rx: tokio::sync::mpsc::Receiver<()>,
+) {
+    let mut interval = tokio::time::interval(options.interval);
+    interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+    interval.tick().await; // skip the immediate first tick
+
+    let label = &runtime_state.inputs.target_label;
+    let debounce = options.debounce;
+    debug!(target = %label, interval_secs = options.interval.as_secs(), "TLS reload poll loop started");
+
+    loop {
+        tokio::select! {
+            biased;
+            _ = cancel_rx.recv() => {
+                info!(target = %label, "TLS reload poll loop stopped");
+                return;
+            }
+            _ = interval.tick() => {
+                // Enforce minimum stable age: if the last attempt was too recent
+                // (e.g. a rapid succession of ticks), wait one debounce period
+                // before reading files again to avoid picking up half-written certs.
+                let last_attempt = runtime_state.last_attempt_unix_ms();
+                if last_attempt > 0 {
+                    let elapsed_since_last = unix_time_ms().saturating_sub(last_attempt);
+                    if elapsed_since_last < debounce.as_millis() as u64 {
+                        continue;
+                    }
+                }
+
+                if let Err(err) = reload_target_once(target.as_ref(), runtime_state.as_ref(), &options).await {
+                    warn!(target = %label, error = %err, "TLS reload poll check failed (will retry)");
+                }
+            }
+        }
+    }
+}
+
+/// Single reload cycle: read → compare → validate → build → apply → publish.
+///
+/// Returns the new generation on success, or an error on failure.
+/// On failure the current generation and material are untouched.
+async fn reload_target_once<T: ReloadableTargetTls>(
+    target: &T,
+    runtime_state: &TargetTlsRuntimeState<T::Material>,
+    options: &TlsReloadOptions,
+) -> Result<TargetTlsGeneration, TargetError> {
+    let now = unix_time_ms();
+    runtime_state.mark_attempt(now);
+    let started_at = std::time::Instant::now();
+    let label = &runtime_state.inputs.target_label;
+
+    // 1. Read TLS files and compute fingerprint
+    let inputs = &runtime_state.inputs;
+    let next_fingerprint =
+        build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
+
+    // 2. Compare with current — skip if unchanged
+    let current = runtime_state.current.load();
+    if current.fingerprint == next_fingerprint {
+        record_target_tls_reload_skipped(label, "unchanged");
+        return Ok(current.generation);
+    }
+
+    // 3. Validate TLS files (cert/key pairing, CA parseable)
+    if let Err(err) = validate_tls_material(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path) {
+        *runtime_state.last_error.write() = Some(err.to_string());
+        record_target_tls_publication_fail(label);
+        return Err(err);
+    }
+
+    // Also call target-specific validation
+    if let Err(err) = target.validate_tls_files().await {
+        *runtime_state.last_error.write() = Some(err.to_string());
+        record_target_tls_publication_fail(label);
+        return Err(err);
+    }
+
+    // 4. Build new material (does not touch current state yet)
+    let new_material = match target.build_tls_material().await {
+        Ok(m) => Arc::new(m),
+        Err(err) => {
+            *runtime_state.last_error.write() = Some(err.to_string());
+            record_target_tls_publication_fail(label);
+            return Err(err);
+        }
+    };
+
+    // 5. Bump generation and apply
+    let new_generation = runtime_state.bump_generation();
+    if let Err(err) = target
+        .apply_tls_material(new_generation, Arc::clone(&new_material), options.apply_hint)
+        .await
+    {
+        *runtime_state.last_error.write() = Some(err.to_string());
+        record_target_tls_publication_fail(label);
+        return Err(err);
+    }
+
+    // 6. Publish new state
+    let published = Arc::new(TargetTlsPublishedState {
+        generation: new_generation,
+        fingerprint: next_fingerprint,
+        material: new_material,
+        loaded_at_unix_ms: now,
+    });
+    runtime_state.current.store(published.clone());
+    runtime_state.last_good.store(published);
+    runtime_state.mark_success(now);
+    *runtime_state.last_error.write() = None;
+
+    record_target_tls_reload_result(label, "ok", started_at.elapsed().as_secs_f64(), new_generation.0);
+
+    debug!(target = %label, generation = new_generation.0, "TLS reload successful");
+    Ok(new_generation)
+}
+
+fn unix_time_ms() -> u64 {
+    SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+    struct MockTarget {
+        inputs: TargetTlsInputSet,
+        build_calls: AtomicUsize,
+        apply_calls: AtomicUsize,
+        validate_calls: AtomicUsize,
+        should_fail_build: AtomicBool,
+        should_fail_apply: AtomicBool,
+        should_fail_validate: AtomicBool,
+    }
+
+    impl MockTarget {
+        fn new(label: &str) -> Self {
+            Self {
+                inputs: TargetTlsInputSet {
+                    ca_path: String::new(),
+                    client_cert_path: String::new(),
+                    client_key_path: String::new(),
+                    target_label: label.to_string(),
+                },
+                build_calls: AtomicUsize::new(0),
+                apply_calls: AtomicUsize::new(0),
+                validate_calls: AtomicUsize::new(0),
+                should_fail_build: AtomicBool::new(false),
+                should_fail_apply: AtomicBool::new(false),
+                should_fail_validate: AtomicBool::new(false),
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl ReloadableTargetTls for MockTarget {
+        type Material = String;
+
+        fn tls_input_set(&self) -> TargetTlsInputSet {
+            self.inputs.clone()
+        }
+
+        async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+            self.build_calls.fetch_add(1, Ordering::SeqCst);
+            if self.should_fail_build.load(Ordering::SeqCst) {
+                return Err(TargetError::Configuration("build failed".to_string()));
+            }
+            Ok("mock-material".to_string())
+        }
+
+        async fn apply_tls_material(
+            &self,
+            _generation: TargetTlsGeneration,
+            _material: Arc<Self::Material>,
+            _mode: ReloadApplyMode,
+        ) -> Result<(), TargetError> {
+            self.apply_calls.fetch_add(1, Ordering::SeqCst);
+            if self.should_fail_apply.load(Ordering::SeqCst) {
+                return Err(TargetError::Configuration("apply failed".to_string()));
+            }
+            Ok(())
+        }
+
+        async fn validate_tls_files(&self) -> Result<(), TargetError> {
+            self.validate_calls.fetch_add(1, Ordering::SeqCst);
+            if self.should_fail_validate.load(Ordering::SeqCst) {
+                return Err(TargetError::Configuration("validate failed".to_string()));
+            }
+            Ok(())
+        }
+    }
+
+    fn default_options() -> TlsReloadOptions {
+        TlsReloadOptions {
+            enabled: true,
+            detect_mode: ReloadDetectMode::Poll,
+            interval: std::time::Duration::from_secs(1),
+            debounce: std::time::Duration::from_secs(1),
+            min_stable_age: std::time::Duration::from_millis(100),
+            apply_hint: ReloadApplyMode::Lazy,
+        }
+    }
+
+    #[tokio::test]
+    async fn register_builds_initial_material() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(MockTarget::new("test:webhook"));
+
+        let options = TlsReloadOptions {
+            detect_mode: ReloadDetectMode::Watch, // no poll loop for this test
+            ..default_options()
+        };
+        let state = coordinator.register(target.clone(), options).await.unwrap();
+
+        assert_eq!(state.current.load().generation, TargetTlsGeneration(1));
+        assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn register_disabled_returns_error() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(MockTarget::new("test:webhook"));
+
+        let options = TlsReloadOptions {
+            enabled: false,
+            ..default_options()
+        };
+        let result = coordinator.register(target, options).await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn shutdown_stops_all_loops() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(MockTarget::new("test:webhook"));
+
+        let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
+        assert_eq!(coordinator.entries.read().await.len(), 1);
+
+        coordinator.shutdown().await;
+        assert!(coordinator.entries.read().await.is_empty());
+    }
+
+    #[tokio::test]
+    async fn force_reload_calls_build_and_apply() {
+        let target = MockTarget::new("test:webhook");
+        let initial_material = Arc::new("initial".to_string());
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(1),
+            fingerprint: TargetTlsFingerprint::default(),
+            material: initial_material,
+            loaded_at_unix_ms: 0,
+        });
+        let inputs = target.tls_input_set();
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, inputs));
+        let options = default_options();
+
+        // Force reload should succeed since MockTarget uses empty paths
+        // and the fingerprint won't change from default
+        let result = reload_target_once(&target, &runtime_state, &options).await.unwrap();
+        // Since fingerprint is unchanged (empty paths), generation stays at 1
+        assert_eq!(result, TargetTlsGeneration(1));
+        // Build should NOT be called because fingerprint unchanged
+        assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn build_failure_preserves_old_generation() {
+        let target = MockTarget::new("test:webhook");
+        target.should_fail_build.store(true, Ordering::SeqCst);
+
+        // Use a non-default fingerprint so the reload will detect a change
+        // (empty paths → default fingerprint ≠ initial fingerprint)
+        let initial_material = Arc::new("initial".to_string());
+        let initial_fingerprint = TargetTlsFingerprint {
+            ca_sha256: Some([1; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(1),
+            fingerprint: initial_fingerprint,
+            material: initial_material,
+            loaded_at_unix_ms: 0,
+        });
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
+        let options = default_options();
+
+        // Empty paths produce default fingerprint which differs from initial →
+        // validate passes (empty paths), then build is called and fails.
+        let result = reload_target_once(&target, &runtime_state, &options).await;
+        assert!(result.is_err());
+        // Generation should remain at 1
+        assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
+        assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn status_snapshot_reflects_state() {
+        let target = MockTarget::new("test:webhook");
+        let initial_material = Arc::new("initial".to_string());
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(1),
+            fingerprint: TargetTlsFingerprint::default(),
+            material: initial_material,
+            loaded_at_unix_ms: 0,
+        });
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
+        let options = default_options();
+
+        let snapshot = TargetTlsReloadCoordinator::build_status_snapshot(runtime_state.as_ref(), &options);
+        assert_eq!(snapshot.target_label, "test:webhook");
+        assert_eq!(snapshot.generation, 1);
+        assert!(snapshot.reload_enabled);
+        assert_eq!(snapshot.detect_mode, "poll");
+        assert_eq!(snapshot.apply_mode, "lazy");
+        assert!(snapshot.last_attempt_time.is_none());
+        assert!(snapshot.last_error.is_none());
+    }
+
+    #[tokio::test]
+    async fn apply_failure_preserves_old_generation() {
+        let target = MockTarget::new("test:webhook");
+        target.should_fail_apply.store(true, Ordering::SeqCst);
+
+        // Use a non-default fingerprint so reload detects a change
+        let initial_material = Arc::new("initial".to_string());
+        let initial_fingerprint = TargetTlsFingerprint {
+            ca_sha256: Some([42; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(3),
+            fingerprint: initial_fingerprint,
+            material: initial_material,
+            loaded_at_unix_ms: 0,
+        });
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
+        let options = default_options();
+
+        let result = reload_target_once(&target, &runtime_state, &options).await;
+        assert!(result.is_err());
+        // Generation should remain at 3
+        assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(3));
+        assert!(runtime_state.last_error.read().is_some());
+    }
+
+    #[tokio::test]
+    async fn validate_failure_prevents_build() {
+        let target = MockTarget::new("test:kafka");
+        target.should_fail_validate.store(true, Ordering::SeqCst);
+
+        // Non-default fingerprint to trigger reload
+        let initial_material = Arc::new("initial".to_string());
+        let initial_fingerprint = TargetTlsFingerprint {
+            ca_sha256: Some([99; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(1),
+            fingerprint: initial_fingerprint,
+            material: initial_material,
+            loaded_at_unix_ms: 0,
+        });
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
+        let options = default_options();
+
+        let result = reload_target_once(&target, &runtime_state, &options).await;
+        assert!(result.is_err());
+        // Build should NOT have been called
+        assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
+        // Validate was called
+        assert!(target.validate_calls.load(Ordering::SeqCst) > 0);
+        assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
+    }
+
+    #[tokio::test]
+    async fn error_is_cleared_on_successful_reload() {
+        let target = MockTarget::new("test:nats");
+
+        let initial_material = Arc::new("initial".to_string());
+        let initial_fingerprint = TargetTlsFingerprint {
+            ca_sha256: Some([1; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(1),
+            fingerprint: initial_fingerprint,
+            material: initial_material,
+            loaded_at_unix_ms: 0,
+        });
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
+        let options = default_options();
+
+        // First: fail the reload
+        target.should_fail_build.store(true, Ordering::SeqCst);
+        let _ = reload_target_once(&target, &runtime_state, &options).await;
+        assert!(runtime_state.last_error.read().is_some());
+
+        // Now succeed (fingerprint still different from default)
+        target.should_fail_build.store(false, Ordering::SeqCst);
+        let result = reload_target_once(&target, &runtime_state, &options).await;
+        assert!(result.is_ok());
+        assert!(runtime_state.last_error.read().is_none());
+        assert!(runtime_state.last_success_unix_ms() > 0);
+    }
+
+    #[tokio::test]
+    async fn last_good_is_never_overwritten_by_failed_reload() {
+        let target = MockTarget::new("test:amqp");
+
+        let initial_material = Arc::new("good".to_string());
+        let initial_fingerprint = TargetTlsFingerprint {
+            ca_sha256: Some([5; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let initial_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(2),
+            fingerprint: initial_fingerprint.clone(),
+            material: initial_material.clone(),
+            loaded_at_unix_ms: 100,
+        });
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
+
+        // Verify last_good matches initial
+        let good = runtime_state.last_good.load();
+        assert_eq!(good.generation, TargetTlsGeneration(2));
+
+        // Fail a reload
+        target.should_fail_build.store(true, Ordering::SeqCst);
+        let _ = reload_target_once(&target, &runtime_state, &default_options()).await;
+
+        // last_good should still be the initial state
+        let good_after = runtime_state.last_good.load();
+        assert_eq!(good_after.generation, TargetTlsGeneration(2));
+        assert_eq!(good_after.fingerprint, initial_fingerprint);
+    }
+
+    #[tokio::test]
+    async fn unregister_stops_target_poll_loop() {
+        let coordinator = TargetTlsReloadCoordinator::new();
+        let target = Arc::new(MockTarget::new("test:pulsar"));
+
+        let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
+        assert_eq!(coordinator.entries.read().await.len(), 1);
+
+        coordinator.unregister("test:pulsar").await.unwrap();
+        assert!(coordinator.entries.read().await.is_empty());
+    }
+
+    #[tokio::test]
+    async fn bump_generation_saturates_at_max() {
+        let target = MockTarget::new("test:saturation");
+        let initial_material = Arc::new("initial".to_string());
+        let max_gen_state = Arc::new(TargetTlsPublishedState {
+            generation: TargetTlsGeneration(u64::MAX),
+            fingerprint: TargetTlsFingerprint::default(),
+            material: initial_material,
+            loaded_at_unix_ms: 0,
+        });
+        let runtime_state = Arc::new(TargetTlsRuntimeState::new(max_gen_state, target.tls_input_set()));
+
+        let bumped = runtime_state.bump_generation();
+        assert_eq!(bumped, TargetTlsGeneration(u64::MAX)); // saturating add
+    }
+}

crates/targets/src/runtime/tls/fingerprint.rs

@@ -0,0 +1,160 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! TLS fingerprint types for per-target certificate hot-reload detection.
+
+use crate::error::TargetError;
+
+/// SHA256 digest per TLS file component used to detect certificate changes.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct TargetTlsFingerprint {
+    pub ca_sha256: Option<[u8; 32]>,
+    pub client_cert_sha256: Option<[u8; 32]>,
+    pub client_key_sha256: Option<[u8; 32]>,
+}
+
+/// Monotonically increasing generation counter bumped on each successful reload.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct TargetTlsGeneration(pub u64);
+
+/// Combined TLS state held per-target for tracking reload progress.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct TargetTlsState {
+    pub generation: TargetTlsGeneration,
+    pub fingerprint: Option<TargetTlsFingerprint>,
+}
+
+impl TargetTlsState {
+    /// Compares `next_fingerprint` with the current one. If different, bumps
+    /// generation and stores the new fingerprint. Returns `true` when changed.
+    pub fn refresh(&mut self, next_fingerprint: TargetTlsFingerprint) -> bool {
+        if self.fingerprint.as_ref() == Some(&next_fingerprint) {
+            return false;
+        }
+
+        self.generation = TargetTlsGeneration(self.generation.0.saturating_add(1));
+        self.fingerprint = Some(next_fingerprint);
+        true
+    }
+
+    /// Checks whether `candidate` differs from the stored fingerprint without
+    /// mutating state. Use this to gate a rebuild, then call `refresh` only
+    /// after the rebuild succeeds.
+    pub fn needs_update(&self, candidate: &TargetTlsFingerprint) -> bool {
+        self.fingerprint.as_ref() != Some(candidate)
+    }
+
+    /// Resets state to default (generation 0, no fingerprint).
+    pub fn reset(&mut self) {
+        *self = Self::default();
+    }
+}
+
+/// Reads the three TLS material files from disk and returns a fingerprint
+/// computed from their SHA256 digests. Empty paths produce `None` digests.
+pub async fn build_target_tls_fingerprint(
+    ca_path: &str,
+    client_cert_path: &str,
+    client_key_path: &str,
+) -> Result<TargetTlsFingerprint, TargetError> {
+    async fn load_optional_digest(path: &str) -> Result<Option<[u8; 32]>, TargetError> {
+        if path.is_empty() {
+            return Ok(None);
+        }
+
+        let bytes = tokio::fs::read(path)
+            .await
+            .map_err(|e| TargetError::Configuration(format!("Failed to read TLS material '{path}': {e}")))?;
+        let digest = rustfs_tls_runtime::TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None).server_sha256;
+        Ok(digest)
+    }
+
+    Ok(TargetTlsFingerprint {
+        ca_sha256: load_optional_digest(ca_path).await?,
+        client_cert_sha256: load_optional_digest(client_cert_path).await?,
+        client_key_sha256: load_optional_digest(client_key_path).await?,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState};
+
+    #[test]
+    fn refresh_increments_generation_only_when_fingerprint_changes() {
+        let mut state = TargetTlsState::default();
+        let first = TargetTlsFingerprint {
+            ca_sha256: Some([1; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let second = TargetTlsFingerprint {
+            ca_sha256: Some([2; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+
+        assert!(state.refresh(first.clone()));
+        assert_eq!(state.generation, TargetTlsGeneration(1));
+        assert!(!state.refresh(first));
+        assert_eq!(state.generation, TargetTlsGeneration(1));
+        assert!(state.refresh(second));
+        assert_eq!(state.generation, TargetTlsGeneration(2));
+    }
+
+    #[test]
+    fn reset_clears_generation_and_fingerprint() {
+        let mut state = TargetTlsState {
+            generation: TargetTlsGeneration(5),
+            fingerprint: Some(TargetTlsFingerprint {
+                ca_sha256: Some([9; 32]),
+                client_cert_sha256: None,
+                client_key_sha256: None,
+            }),
+        };
+
+        state.reset();
+        assert_eq!(state, TargetTlsState::default());
+    }
+
+    #[test]
+    fn fingerprint_eq_when_all_fields_match() {
+        let a = TargetTlsFingerprint {
+            ca_sha256: Some([42; 32]),
+            client_cert_sha256: Some([1; 32]),
+            client_key_sha256: None,
+        };
+        let b = TargetTlsFingerprint {
+            ca_sha256: Some([42; 32]),
+            client_cert_sha256: Some([1; 32]),
+            client_key_sha256: None,
+        };
+        assert_eq!(a, b);
+    }
+
+    #[test]
+    fn fingerprint_ne_when_ca_differs() {
+        let a = TargetTlsFingerprint {
+            ca_sha256: Some([1; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let b = TargetTlsFingerprint {
+            ca_sha256: Some([2; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        assert_ne!(a, b);
+    }
+}

crates/targets/src/runtime/tls/metrics.rs

@@ -0,0 +1,63 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Target-level TLS reload metrics.
+
+use ::metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
+
+const TARGET_TLS_RELOAD_TOTAL: &str = "rustfs_target_tls_reload_total";
+const TARGET_TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_target_tls_reload_skipped_total";
+const TARGET_TLS_GENERATION: &str = "rustfs_target_tls_generation";
+const TARGET_TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_target_tls_reload_duration_seconds";
+const TARGET_TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_target_tls_publication_fail_total";
+const TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL: &str = "rustfs_target_tls_active_generation_mismatch_total";
+
+/// Describes all target TLS metrics. Call once during initialization.
+pub fn init_target_tls_metrics() {
+    describe_counter!(TARGET_TLS_RELOAD_TOTAL, "Total number of TLS reload attempts per target");
+    describe_counter!(
+        TARGET_TLS_RELOAD_SKIPPED_TOTAL,
+        "Number of TLS reloads skipped per target (unchanged, etc.)"
+    );
+    describe_gauge!(TARGET_TLS_GENERATION, "Current TLS generation per target");
+    describe_histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "Duration of TLS reload attempts per target");
+    describe_counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "Number of TLS reload publication failures per target");
+    describe_counter!(
+        TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL,
+        "Number of times a target's active connection used a stale TLS generation"
+    );
+}
+
+/// Records a TLS reload result (success or failure).
+pub fn record_target_tls_reload_result(target: &str, result: &str, duration_secs: f64, generation: u64) {
+    counter!(TARGET_TLS_RELOAD_TOTAL, "target_id" => target.to_string(), "result" => result.to_string()).increment(1);
+    histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "target_id" => target.to_string(), "result" => result.to_string())
+        .record(duration_secs);
+    gauge!(TARGET_TLS_GENERATION, "target_id" => target.to_string()).set(generation as f64);
+}
+
+/// Records a skipped reload (typically because fingerprint was unchanged).
+pub fn record_target_tls_reload_skipped(target: &str, reason: &str) {
+    counter!(TARGET_TLS_RELOAD_SKIPPED_TOTAL, "target_id" => target.to_string(), "reason" => reason.to_string()).increment(1);
+}
+
+/// Records a TLS reload publication failure.
+pub fn record_target_tls_publication_fail(target: &str) {
+    counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "target_id" => target.to_string()).increment(1);
+}
+
+/// Records that a target used a stale TLS generation (active ≠ latest published).
+pub fn record_target_tls_stale_generation(target: &str) {
+    counter!(TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL, "target_id" => target.to_string()).increment(1);
+}

crates/targets/src/runtime/tls/mod.rs

@@ -0,0 +1,41 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Unified TLS hot-reload infrastructure for notification targets.
+//!
+//! This module provides:
+//! - Fingerprint-based change detection (`fingerprint`)
+//! - Per-target reload configuration (`config`)
+//! - Runtime state tracking with atomic timestamps (`state`)
+//! - The `ReloadableTargetTls` trait protocol (`trait`)
+//! - TLS material validation helpers (`validate`)
+//! - The reload coordinator with background poll loops (`coordinator`)
+//! - Target-level reload metrics (`metrics`)
+
+pub mod adapter;
+pub mod config;
+pub mod coordinator;
+pub mod fingerprint;
+pub mod metrics;
+pub mod state;
+pub mod r#trait;
+pub mod validate;
+
+pub use adapter::TlsReloadAdapter;
+pub use coordinator::TargetTlsReloadCoordinator;
+pub use fingerprint::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState, build_target_tls_fingerprint};
+pub use metrics::init_target_tls_metrics;
+pub use state::{TargetTlsInputSet, TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
+pub use r#trait::ReloadableTargetTls;
+pub use validate::validate_tls_material;

crates/targets/src/runtime/tls/state.rs

@@ -0,0 +1,127 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-target TLS reload runtime state with atomic timestamps and error tracking.
+
+use super::fingerprint::{TargetTlsFingerprint, TargetTlsGeneration};
+use ::arc_swap::ArcSwap;
+use serde::Serialize;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Describes which TLS files a target reads.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TargetTlsInputSet {
+    pub ca_path: String,
+    pub client_cert_path: String,
+    pub client_key_path: String,
+    /// Human-readable label for logging and metrics (e.g. "webhook:primary").
+    pub target_label: String,
+}
+
+impl TargetTlsInputSet {
+    /// Returns `true` when no TLS paths are configured (no CA, cert, or key).
+    pub fn is_empty(&self) -> bool {
+        self.ca_path.is_empty() && self.client_cert_path.is_empty() && self.client_key_path.is_empty()
+    }
+}
+
+/// Immutable snapshot of a successfully published TLS material generation.
+pub struct TargetTlsPublishedState<M> {
+    pub generation: TargetTlsGeneration,
+    pub fingerprint: TargetTlsFingerprint,
+    pub material: Arc<M>,
+    pub loaded_at_unix_ms: u64,
+}
+
+/// Per-target TLS reload runtime state. Owns the current published material
+/// and tracks timestamps and the last error for observability.
+pub struct TargetTlsRuntimeState<M> {
+    /// The currently active TLS material generation.
+    pub current: ArcSwap<TargetTlsPublishedState<M>>,
+    /// The last known-good generation (never overwritten by a failed reload).
+    pub last_good: ArcSwap<TargetTlsPublishedState<M>>,
+    /// Unix-millis timestamp of the last reload *attempt* (success or failure).
+    pub last_attempt_unix_ms: AtomicU64,
+    /// Unix-millis timestamp of the last *successful* reload.
+    pub last_success_unix_ms: AtomicU64,
+    /// Last reload error message, if any.
+    pub last_error: parking_lot::RwLock<Option<String>>,
+    /// The TLS file paths this state watches.
+    pub inputs: TargetTlsInputSet,
+}
+
+impl<M> TargetTlsRuntimeState<M> {
+    /// Creates a new runtime state with the given initial published state.
+    pub fn new(initial: Arc<TargetTlsPublishedState<M>>, inputs: TargetTlsInputSet) -> Self {
+        Self {
+            current: ArcSwap::from(initial.clone()),
+            last_good: ArcSwap::from(initial),
+            last_attempt_unix_ms: AtomicU64::new(0),
+            last_success_unix_ms: AtomicU64::new(0),
+            last_error: parking_lot::RwLock::new(None),
+            inputs,
+        }
+    }
+
+    /// Returns the generation of the currently active material.
+    pub fn current_generation(&self) -> TargetTlsGeneration {
+        self.current.load().generation
+    }
+
+    /// Atomically bumps and returns the next generation.
+    pub fn bump_generation(&self) -> TargetTlsGeneration {
+        // Load the current generation from the arc-swap, compute next,
+        // and return it. The caller is responsible for publishing the new state.
+        let current = self.current.load();
+        TargetTlsGeneration(current.generation.0.saturating_add(1))
+    }
+
+    /// Records the timestamp of a reload attempt.
+    pub fn mark_attempt(&self, unix_ms: u64) {
+        self.last_attempt_unix_ms.store(unix_ms, Ordering::Release);
+    }
+
+    /// Records the timestamp of a successful reload.
+    pub fn mark_success(&self, unix_ms: u64) {
+        self.last_success_unix_ms.store(unix_ms, Ordering::Release);
+    }
+
+    /// Returns the last attempt timestamp.
+    pub fn last_attempt_unix_ms(&self) -> u64 {
+        self.last_attempt_unix_ms.load(Ordering::Acquire)
+    }
+
+    /// Returns the last success timestamp.
+    pub fn last_success_unix_ms(&self) -> u64 {
+        self.last_success_unix_ms.load(Ordering::Acquire)
+    }
+}
+
+/// Read-only status snapshot for admin/debug visibility.
+#[derive(Debug, Clone, Serialize)]
+pub struct TargetTlsStatusSnapshot {
+    pub target_label: String,
+    pub generation: u64,
+    pub reload_enabled: bool,
+    pub detect_mode: &'static str,
+    pub apply_mode: &'static str,
+    pub last_attempt_time: Option<u64>,
+    pub last_success_time: Option<u64>,
+    pub last_error: Option<String>,
+    /// TLS file paths this target watches (for admin diagnostics).
+    pub ca_path: String,
+    pub client_cert_path: String,
+    pub client_key_path: String,
+}

crates/targets/src/runtime/tls/trait.rs

@@ -0,0 +1,68 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The `ReloadableTargetTls` trait — the public protocol each TLS-capable
+//! target implements to participate in coordinated hot-reload.
+
+use crate::error::TargetError;
+use async_trait::async_trait;
+use std::sync::Arc;
+
+use super::config::ReloadApplyMode;
+use super::fingerprint::TargetTlsGeneration;
+use super::state::TargetTlsInputSet;
+
+/// Protocol that each TLS-capable target implements so the reload coordinator
+/// can drive certificate hot-reload without knowing the target's internals.
+///
+/// The target is responsible for:
+/// - Declaring which TLS files it reads (`tls_input_set`)
+/// - Building a new client/pool/connector from current files (`build_tls_material`)
+/// - Atomically swapping the active connection state (`apply_tls_material`)
+///
+/// The coordinator is responsible for:
+/// - Deciding *when* to check
+/// - Detecting *whether* material changed
+/// - Ensuring *safety* (validate, build-then-apply, fallback on failure)
+#[async_trait]
+pub trait ReloadableTargetTls: Send + Sync + 'static {
+    /// The rebuilt connection/client/pool object this target uses.
+    type Material: Send + Sync + 'static;
+
+    /// Returns the TLS file paths this target reads.
+    fn tls_input_set(&self) -> TargetTlsInputSet;
+
+    /// Build a fresh TLS material object from current files on disk.
+    ///
+    /// Called by the coordinator on the reload path only — never on the send hot path.
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError>;
+
+    /// Atomically apply new TLS material, replacing the current active connection state.
+    ///
+    /// On success, the target's internal state must point to the new material.
+    /// On failure, the target must keep its current state unchanged.
+    async fn apply_tls_material(
+        &self,
+        generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        mode: ReloadApplyMode,
+    ) -> Result<(), TargetError>;
+
+    /// Optional pre-check: validate that TLS files on disk are self-consistent
+    /// (cert/key pair parseable, CA loadable) before attempting `build_tls_material`.
+    /// Default implementation returns `Ok(())`.
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        Ok(())
+    }
+}

crates/targets/src/runtime/tls/validate.rs

@@ -0,0 +1,58 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! TLS material validation helpers used by the reload coordinator before
+//! attempting to build new client/pool objects.
+
+use crate::error::TargetError;
+use rustfs_tls_runtime::{load_certs, load_private_key};
+
+/// Validates that a client certificate and private key file can be loaded
+/// and paired together. Returns `Ok(())` if both files parse successfully,
+/// or `Ok(())` if both paths are empty (no mTLS configured).
+pub fn validate_cert_key_pairing(cert_path: &str, key_path: &str) -> Result<(), TargetError> {
+    if cert_path.is_empty() && key_path.is_empty() {
+        return Ok(());
+    }
+
+    if cert_path.is_empty() || key_path.is_empty() {
+        return Err(TargetError::Configuration(
+            "Client certificate and key must both be specified or both be empty".to_string(),
+        ));
+    }
+
+    load_certs(cert_path).map_err(|e| TargetError::Configuration(format!("Invalid client certificate '{cert_path}': {e}")))?;
+
+    load_private_key(key_path).map_err(|e| TargetError::Configuration(format!("Invalid client key '{key_path}': {e}")))?;
+
+    Ok(())
+}
+
+/// Validates that a CA certificate file can be loaded. Returns `Ok(())`
+/// if the path is empty (no custom CA) or if the file parses successfully.
+pub fn validate_ca_file(ca_path: &str) -> Result<(), TargetError> {
+    if ca_path.is_empty() {
+        return Ok(());
+    }
+
+    load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Invalid CA certificate '{ca_path}': {e}")))?;
+
+    Ok(())
+}
+
+/// Validates all three TLS material files in one call.
+pub fn validate_tls_material(ca_path: &str, cert_path: &str, key_path: &str) -> Result<(), TargetError> {
+    validate_ca_file(ca_path)?;
+    validate_cert_key_pairing(cert_path, key_path)
+}

crates/targets/src/target/amqp.rs

@@ -22,11 +22,15 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
+        validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload_with_records, is_connectivity_error, open_target_queue_store,
-        persist_queued_payload_to_store,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
+        open_target_queue_store, persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
@@ -37,6 +41,7 @@ use lapin::{
 };
 use parking_lot::Mutex;
 use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY};
+use rustfs_tls_runtime::load_cert_bundle_der_bytes;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::fmt;
@@ -196,16 +201,24 @@ async fn build_tls_config(args: &AMQPArgs) -> Result<OwnedTLSConfig, TargetError
     let cert_chain = if args.tls_ca.is_empty() {
         None
     } else {
-        Some(
-            tokio::fs::read_to_string(&args.tls_ca)
-                .await
-                .map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?,
-        )
+        let certs_der = load_cert_bundle_der_bytes(&args.tls_ca)
+            .map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CA}: {e}")))?;
+        if certs_der.is_empty() {
+            return Err(TargetError::Configuration(format!(
+                "{AMQP_TLS_CA} did not contain any parsable certificates"
+            )));
+        }
+        let pem = tokio::fs::read_to_string(&args.tls_ca)
+            .await
+            .map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?;
+        Some(pem)
     };
 
     let identity = if args.tls_client_cert.is_empty() {
         None
     } else {
+        let _ = load_cert_bundle_der_bytes(&args.tls_client_cert)
+            .map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CLIENT_CERT}: {e}")))?;
         let pem = tokio::fs::read(&args.tls_client_cert)
             .await
             .map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_CERT}: {e}")))?;
@@ -290,6 +303,9 @@ where
     id: TargetID,
     args: AMQPArgs,
     connection: Arc<Mutex<Option<Arc<AMQPConnection>>>>,
+    tls_state: Arc<Mutex<TargetTlsState>>,
+    /// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
+    tls_adapter: Option<TlsReloadAdapter<AMQPConnection>>,
     connect_lock: Arc<AsyncMutex<()>>,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     delivery_counters: Arc<TargetDeliveryCounters>,
@@ -305,6 +321,8 @@ where
             id: self.id.clone(),
             args: self.args.clone(),
             connection: Arc::clone(&self.connection),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
             connect_lock: Arc::clone(&self.connect_lock),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             delivery_counters: Arc::clone(&self.delivery_counters),
@@ -329,6 +347,8 @@ where
             id: target_id,
             args,
             connection: Arc::new(Mutex::new(None)),
+            tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
+            tls_adapter: None,
             connect_lock: Arc::new(AsyncMutex::new(())),
             store: queue_store,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -341,6 +361,27 @@ where
     }
 
     async fn get_or_connect(&self) -> Result<Arc<AMQPConnection>, TargetError> {
+        // When a TLS reload adapter is attached, it drives connection rebuilds
+        // in the background. The inline per-send fingerprint check is skipped.
+        if let Some(adapter) = &self.tls_adapter {
+            let material = adapter.current_material();
+            if material.connection.status().connected() && material.channel.status().connected() {
+                return Ok(material);
+            }
+            self.clear_connection_handle();
+        } else {
+            let next_fingerprint =
+                build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
+            let tls_changed = {
+                let tls_state_guard = self.tls_state.lock();
+                tls_state_guard.needs_update(&next_fingerprint)
+            };
+            if tls_changed {
+                self.clear_connection_handle();
+                self.tls_state.lock().refresh(next_fingerprint);
+            }
+        }
+
         if let Some(connection) = self.connection.lock().clone()
             && connection.connection.status().connected()
             && connection.channel.status().connected()
@@ -362,10 +403,19 @@ where
         Ok(connection)
     }
 
-    fn clear_connection(&self) {
+    fn clear_connection_handle(&self) {
         *self.connection.lock() = None;
     }
 
+    fn clear_connection_cache(&self) {
+        self.clear_connection_handle();
+        self.tls_state.lock().reset();
+    }
+
+    fn clear_connection(&self) {
+        self.clear_connection_cache();
+    }
+
     async fn send_body(&self, body: &[u8]) -> Result<(), TargetError> {
         let connection = self.get_or_connect().await?;
         let publish = connection
@@ -407,6 +457,46 @@ where
     }
 }
 
+/// Coordinated TLS hot-reload implementation for AMQP targets.
+///
+/// The coordinator calls these methods on a background poll loop to detect
+/// TLS file changes and rebuild the connection without restarting.
+#[async_trait]
+impl<E> ReloadableTargetTls for AMQPTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = AMQPConnection;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls_ca.clone(),
+            client_cert_path: self.args.tls_client_cert.clone(),
+            client_key_path: self.args.tls_client_key.clone(),
+            target_label: format!("amqp:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        connect_amqp(&self.args).await
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        let mut guard = self.connection.lock();
+        *guard = Some(material);
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
+    }
+}
+
 #[async_trait]
 impl<E> Target<E> for AMQPTarget<E>
 where
@@ -458,6 +548,12 @@ where
                 .await
                 .map_err(|e| map_lapin_error(e, "Failed to close AMQP connection"))?;
         }
+        self.tls_state.lock().reset();
+        // If a TLS reload adapter is attached, reset its error tracking
+        // so that a future re-init does not inherit stale failure state.
+        if let Some(adapter) = &self.tls_adapter {
+            *adapter.runtime_state().last_error.write() = None;
+        }
         info!(target_id = %self.id, "AMQP target closed");
         Ok(())
     }

crates/targets/src/target/kafka.rs

@@ -16,16 +16,21 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
+        validate::validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, open_target_queue_store,
-        persist_queued_payload_to_store,
+        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
+        open_target_queue_store, persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
 use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
 use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
+use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::{marker::PhantomData, sync::Arc, time::Duration};
@@ -104,6 +109,11 @@ where
     args: KafkaArgs,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    tls_state: Arc<Mutex<TargetTlsState>>,
+    /// Adapter that bridges this target to the TLS reload coordinator.
+    /// When `Some`, the target uses coordinator-managed material; when `None`,
+    /// it falls back to inline fingerprint-based change detection.
+    tls_adapter: Option<TlsReloadAdapter<Arc<AsyncProducer>>>,
     delivery_counters: Arc<TargetDeliveryCounters>,
     _phantom: PhantomData<E>,
 }
@@ -144,6 +154,8 @@ where
             args,
             store: queue_store,
             producer: Arc::new(Mutex::new(None)),
+            tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
+            tls_adapter: None,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
             _phantom: PhantomData,
         })
@@ -164,9 +176,27 @@ where
         if self.args.tls_enable {
             let mut security = SecurityConfig::new();
             if !self.args.tls_ca.is_empty() {
+                let certs = load_cert_bundle_der_bytes(&self.args.tls_ca)
+                    .map_err(|e| Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_ca"))?;
+                if certs.is_empty() {
+                    return Err(TargetError::Configuration(
+                        "Kafka tls_ca did not contain any parsable certificates".to_string(),
+                    ));
+                }
                 security = security.with_ca_cert(self.args.tls_ca.clone());
             }
             if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
+                let certs = load_cert_bundle_der_bytes(&self.args.tls_client_cert).map_err(|e| {
+                    Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_cert")
+                })?;
+                if certs.is_empty() {
+                    return Err(TargetError::Configuration(
+                        "Kafka tls_client_cert did not contain any parsable certificates".to_string(),
+                    ));
+                }
+                let _ = load_private_key(&self.args.tls_client_key).map_err(|e| {
+                    Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_key")
+                })?;
                 security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone());
             }
             config = config.with_security(security);
@@ -178,6 +208,31 @@ where
     }
 
     async fn get_or_build_producer(&self) -> Result<Arc<AsyncProducer>, TargetError> {
+        // Adapter-managed path: use the material directly from the TLS reload adapter.
+        if let Some(adapter) = &self.tls_adapter {
+            let producer: Arc<AsyncProducer> = (*adapter.current_material()).clone();
+
+            // Ensure the producer is also stored locally so that close() can drain it.
+            {
+                let mut guard = self.producer.lock().await;
+                *guard = Some(Arc::clone(&producer));
+            }
+            return Ok(producer);
+        }
+
+        // Inline fingerprint fallback path (no coordinator).
+        let next_fingerprint =
+            build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
+        let tls_changed = {
+            let tls_state_guard = self.tls_state.lock().await;
+            tls_state_guard.needs_update(&next_fingerprint)
+        };
+        if tls_changed {
+            let mut cached = self.producer.lock().await;
+            *cached = None;
+            self.tls_state.lock().await.refresh(next_fingerprint);
+        }
+
         let mut cached = self.producer.lock().await;
         if let Some(producer) = cached.as_ref() {
             return Ok(Arc::clone(producer));
@@ -191,6 +246,7 @@ where
     async fn invalidate_cached_producer(&self) {
         let mut cached = self.producer.lock().await;
         *cached = None;
+        self.tls_state.lock().await.reset();
     }
 
     /// Serializes the event and builds a QueuedPayload
@@ -230,6 +286,8 @@ where
             args: self.args.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             producer: Arc::clone(&self.producer),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
             delivery_counters: Arc::clone(&self.delivery_counters),
             _phantom: PhantomData,
         })
@@ -292,6 +350,13 @@ where
     }
 
     async fn close(&self) -> Result<(), TargetError> {
+        {
+            let mut guard = self.producer.lock().await;
+            *guard = None;
+        }
+
+        self.tls_state.lock().await.reset();
+
         info!("Kafka target closed: {}", self.id);
         Ok(())
     }
@@ -318,6 +383,47 @@ where
     }
 }
 
+/// Coordinated TLS hot-reload implementation for Kafka targets.
+///
+/// The coordinator calls these methods on a background poll loop to detect
+/// TLS file changes and rebuild the producer without restarting.
+#[async_trait]
+impl<E> ReloadableTargetTls for KafkaTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = Arc<AsyncProducer>;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls_ca.clone(),
+            client_cert_path: self.args.tls_client_cert.clone(),
+            client_key_path: self.args.tls_client_key.clone(),
+            target_label: format!("kafka:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        let producer = self.build_producer().await?;
+        Ok(Arc::new(producer))
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        let mut guard = self.producer.lock().await;
+        *guard = Some((*material).clone());
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/targets/src/target/mod.rs

@@ -37,6 +37,13 @@ pub mod pulsar;
 pub mod redis;
 pub mod webhook;
 
+#[cfg(test)]
+pub(crate) use crate::runtime::tls::fingerprint::TargetTlsFingerprint as TargetTlsFingerprintState;
+#[cfg(test)]
+pub(crate) use crate::runtime::tls::fingerprint::TargetTlsGeneration;
+pub(crate) use crate::runtime::tls::fingerprint::TargetTlsState;
+pub(crate) use crate::runtime::tls::fingerprint::build_target_tls_fingerprint;
+
 /// A read-only snapshot of delivery counters for a target.
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct TargetDeliverySnapshot {
@@ -531,6 +538,48 @@ pub(crate) fn ensure_rustls_provider_installed() {
     }
 }
 
+#[cfg(test)]
+mod tls_state_tests {
+    use super::{TargetTlsFingerprintState, TargetTlsGeneration, TargetTlsState};
+
+    #[test]
+    fn refresh_increments_generation_only_when_fingerprint_changes() {
+        let mut state = TargetTlsState::default();
+        let first = TargetTlsFingerprintState {
+            ca_sha256: Some([1; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+        let second = TargetTlsFingerprintState {
+            ca_sha256: Some([2; 32]),
+            client_cert_sha256: None,
+            client_key_sha256: None,
+        };
+
+        assert!(state.refresh(first.clone()));
+        assert_eq!(state.generation, TargetTlsGeneration(1));
+        assert!(!state.refresh(first));
+        assert_eq!(state.generation, TargetTlsGeneration(1));
+        assert!(state.refresh(second));
+        assert_eq!(state.generation, TargetTlsGeneration(2));
+    }
+
+    #[test]
+    fn reset_clears_generation_and_fingerprint() {
+        let mut state = TargetTlsState {
+            generation: TargetTlsGeneration(5),
+            fingerprint: Some(TargetTlsFingerprintState {
+                ca_sha256: Some([9; 32]),
+                client_cert_sha256: None,
+                client_key_sha256: None,
+            }),
+        };
+
+        state.reset();
+        assert_eq!(state, TargetTlsState::default());
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/targets/src/target/mqtt.rs

@@ -16,6 +16,10 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TargetTlsState, TlsReloadAdapter, config::ReloadApplyMode,
+        validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -23,6 +27,7 @@ use crate::{
         persist_queued_payload_to_store,
     },
 };
+use arc_swap::ArcSwap;
 use async_trait::async_trait;
 use hyper_rustls::ConfigBuilderExt;
 use rumqttc::{
@@ -32,6 +37,7 @@ use rumqttc::{
 use rustfs_config::{
     EnableState, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_WS_PATH_ALLOWLIST,
 };
+use rustfs_tls_runtime::{load_certs, load_private_key};
 use rustls::ClientConfig;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
@@ -185,8 +191,7 @@ fn validate_path_is_absolute(path: &str, field: &str) -> Result<(), TargetError>
 }
 
 fn build_root_store(ca_path: &str, trust_leaf_as_ca: bool) -> Result<rustls::RootCertStore, TargetError> {
-    let certs =
-        rustfs_utils::load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
+    let certs = load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
     let mut store = rustls::RootCertStore::empty();
 
     if trust_leaf_as_ca {
@@ -222,9 +227,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
             if tls.client_cert_path.is_empty() {
                 builder.with_no_client_auth()
             } else {
-                let certs = rustfs_utils::load_certs(&tls.client_cert_path)
+                let certs = load_certs(&tls.client_cert_path)
                     .map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
-                let key = rustfs_utils::load_private_key(&tls.client_key_path)
+                let key = load_private_key(&tls.client_key_path)
                     .map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
                 builder
                     .with_client_auth_cert(certs, key)
@@ -237,9 +242,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
             if tls.client_cert_path.is_empty() {
                 builder.with_no_client_auth()
             } else {
-                let certs = rustfs_utils::load_certs(&tls.client_cert_path)
+                let certs = load_certs(&tls.client_cert_path)
                     .map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
-                let key = rustfs_utils::load_private_key(&tls.client_key_path)
+                let key = load_private_key(&tls.client_key_path)
                     .map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
                 builder
                     .with_client_auth_cert(certs, key)
@@ -490,6 +495,12 @@ where
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     connected: Arc<AtomicBool>,
     bg_task_manager: Arc<BgTaskManager>,
+    /// TLS fingerprint tracking for inline fallback path.
+    tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
+    /// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
+    tls_adapter: Option<TlsReloadAdapter<MqttOptions>>,
+    /// Updated MqttOptions from coordinator for use on next reconnection.
+    pending_mqtt_options: Arc<ArcSwap<MqttOptions>>,
     delivery_counters: Arc<TargetDeliveryCounters>,
     _phantom: PhantomData<E>,
 }
@@ -519,6 +530,17 @@ where
             initial_cancel_rx: Mutex::new(Some(cancel_rx)),
         });
 
+        // Build the initial MqttOptions for TLS reload support.
+        let initial_mqtt_options = build_mqtt_options(
+            format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
+            &args.broker,
+            Some(args.username.as_str()),
+            Some(args.password.as_str()),
+            &args.tls,
+            args.keep_alive,
+            Some(MAX_MQTT_PACKET_SIZE_BYTES),
+        )?;
+
         info!(target_id = %target_id, "MQTT target created");
         Ok(MQTTTarget::<E> {
             id: target_id,
@@ -527,6 +549,9 @@ where
             store: queue_store,
             connected: Arc::new(AtomicBool::new(false)),
             bg_task_manager,
+            tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
+            tls_adapter: None,
+            pending_mqtt_options: Arc::new(ArcSwap::from(Arc::new(initial_mqtt_options))),
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
             _phantom: PhantomData,
         })
@@ -544,20 +569,15 @@ where
         let connected_arc = Arc::clone(&self.connected);
         let target_id_clone = self.id.clone();
         let args_clone = self.args.clone();
+        let pending_mqtt_options = Arc::clone(&self.pending_mqtt_options);
 
         let _ = bg_task_manager
             .init_cell
             .get_or_try_init(|| async {
                 debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
-                let mqtt_options = build_mqtt_options(
-                    format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
-                    &args_clone.broker,
-                    Some(args_clone.username.as_str()),
-                    Some(args_clone.password.as_str()),
-                    &args_clone.tls,
-                    args_clone.keep_alive,
-                    Some(MAX_MQTT_PACKET_SIZE_BYTES),
-                )?;
+
+                // Use the latest MqttOptions (may have been updated by TLS reload coordinator).
+                let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
 
                 let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
 
@@ -662,12 +682,66 @@ where
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             connected: self.connected.clone(),
             bg_task_manager: self.bg_task_manager.clone(),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
+            pending_mqtt_options: Arc::clone(&self.pending_mqtt_options),
             delivery_counters: self.delivery_counters.clone(),
             _phantom: PhantomData,
         })
     }
 }
 
+/// Coordinated TLS hot-reload implementation for MQTT targets.
+///
+/// MQTT uses `MqttOptions` as the material type. The coordinator rebuilds
+/// `MqttOptions` on TLS file changes, and `apply_tls_material` stores it in
+/// an `ArcSwap` for use on the next reconnection. The running event loop is
+/// not interrupted; rumqttc handles reconnection internally.
+#[async_trait]
+impl<E> ReloadableTargetTls for MQTTTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = MqttOptions;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls.ca_path.clone(),
+            client_cert_path: self.args.tls.client_cert_path.clone(),
+            client_key_path: self.args.tls.client_key_path.clone(),
+            target_label: format!("mqtt:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        build_mqtt_options(
+            format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
+            &self.args.broker,
+            Some(self.args.username.as_str()),
+            Some(self.args.password.as_str()),
+            &self.args.tls,
+            self.args.keep_alive,
+            Some(MAX_MQTT_PACKET_SIZE_BYTES),
+        )
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        // Store the new MqttOptions for use on next reconnection.
+        // The running event loop is not interrupted; rumqttc handles reconnection.
+        self.pending_mqtt_options.store(material);
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
+    }
+}
+
 async fn run_mqtt_event_loop(
     mut eventloop: EventLoop,
     connected_status: Arc<AtomicBool>,
@@ -968,6 +1042,13 @@ where
             }
         }
 
+        self.tls_state.lock().reset();
+        // If a TLS reload adapter is attached, reset its error tracking
+        // so that a future re-init does not inherit stale failure state.
+        if let Some(adapter) = &self.tls_adapter {
+            *adapter.runtime_state().last_error.write() = None;
+        }
+
         self.connected.store(false, Ordering::SeqCst);
         info!(target_id = %self.id, "MQTT target close method finished.");
         Ok(())

crates/targets/src/target/mysql.rs

@@ -16,6 +16,10 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
+        validate::validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -26,6 +30,7 @@ use crate::{
 use async_trait::async_trait;
 use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable};
 use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY};
+use rustfs_tls_runtime::{load_certs, load_private_key};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::marker::PhantomData;
@@ -478,6 +483,11 @@ where
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     /// Lazily-initialized MySQL connection pool
     pool: Arc<Mutex<Option<Pool>>>,
+    /// TLS fingerprint tracking for hot reload (inline fallback path)
+    tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
+    /// When present, the adapter provides coordinator-managed TLS material;
+    /// otherwise the inline fingerprint path is used as a fallback.
+    tls_adapter: Option<TlsReloadAdapter<Pool>>,
     /// Success/failure counters exposed via `delivery_snapshot`
     delivery_counters: Arc<TargetDeliveryCounters>,
     /// Zero-sized marker for the event type `E`
@@ -489,6 +499,9 @@ where
     E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
 {
     /// Creates a new MySqlTarget.
+    ///
+    /// The target starts without a TLS reload coordinator. Use
+    /// `TlsReloadAdapter::try_register` to opt into coordinated TLS hot-reload.
     pub fn new(id: String, args: MySqlArgs) -> Result<Self, TargetError> {
         args.validate()?;
 
@@ -511,6 +524,8 @@ where
             store: queue_store,
             // Pool is lazily initialized on first use to avoid unnecessary connections at startup and allow for better error handling
             pool: Arc::new(Mutex::new(None)),
+            tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
+            tls_adapter: None,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
             _phantom: PhantomData,
         })
@@ -518,6 +533,10 @@ where
 
     /// Returns or lazily initializes the MySQL connection pool.
     ///
+    /// When `tls_adapter` is present (coordinator-managed), the pool
+    /// is sourced from the coordinator's published material.
+    /// Otherwise, the inline fingerprint-based path is used as a fallback.
+    ///
     /// # Errors
     ///
     /// | Scenario | Error variant |
@@ -528,6 +547,31 @@ where
     /// | Existing table has incompatible schema | `Initialization` |
     /// | DSN parse failure / invalid config | `Configuration` |
     async fn get_or_init_pool(&self) -> Result<Pool, TargetError> {
+        // Adapter-managed path: use the material directly from the coordinator.
+        if let Some(adapter) = &self.tls_adapter {
+            let pool: Pool = (*adapter.current_material()).clone();
+
+            // Ensure the pool is also stored locally so that close() can drain it.
+            {
+                let mut guard = self.pool.lock().await;
+                *guard = Some(pool.clone());
+            }
+            return Ok(pool);
+        }
+
+        // Inline fingerprint fallback path (no coordinator).
+        let next_fingerprint =
+            super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
+        let tls_changed = {
+            let tls_state_guard = self.tls_state.lock();
+            tls_state_guard.needs_update(&next_fingerprint)
+        };
+        if tls_changed {
+            let mut guard = self.pool.lock().await;
+            *guard = None;
+            self.tls_state.lock().refresh(next_fingerprint);
+        }
+
         {
             let guard = self.pool.lock().await;
             if let Some(pool) = guard.as_ref() {
@@ -535,68 +579,7 @@ where
             }
         }
 
-        let dsn = MySqlDsn::parse(&self.args.dsn_string)?;
-
-        let mut builder = OptsBuilder::default()
-            .user(Some(dsn.user.clone()))
-            .pass(Some(dsn.password.clone()))
-            .ip_or_hostname(dsn.host.clone())
-            .tcp_port(dsn.port)
-            .db_name(Some(dsn.database.clone()));
-
-        if dsn.tls {
-            super::ensure_rustls_provider_installed();
-            let mut ssl_opts = SslOpts::default();
-            if !self.args.tls_ca.is_empty() {
-                ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(self.args.tls_ca.clone()).into()]);
-            }
-            if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
-                let identity = mysql_async::ClientIdentity::new(
-                    PathBuf::from(self.args.tls_client_cert.clone()).into(),
-                    PathBuf::from(self.args.tls_client_key.clone()).into(),
-                );
-                ssl_opts = ssl_opts.with_client_identity(Some(identity));
-            }
-            builder = builder.ssl_opts(Some(ssl_opts));
-        } else {
-            warn!(
-                "MySQL target '{}' is configured without TLS. This is insecure and should not be used in production.",
-                self.id
-            );
-        }
-
-        // When max_open_connections is 0, no explicit upper bound is set —
-        // mysql_async uses its default pool constraints (10–100).
-        if self.args.max_open_connections > 0 {
-            let constraints = PoolConstraints::new(1, self.args.max_open_connections).ok_or_else(|| {
-                TargetError::Configuration(format!(
-                    "MySQL max_open_connections must be >= 1, got {}",
-                    self.args.max_open_connections
-                ))
-            })?;
-            builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
-        }
-
-        let opts = Opts::from(builder);
-        let pool = Pool::new(opts);
-
-        // Uses a double-check pattern: the mutex guard is only held for
-        // short reads/writes to the pool cache. All I/O (connecting,
-        // DDL, schema validation) happens outside the lock so that
-        // concurrent callers are not blocked by a slow MySQL server.
-        let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
-
-        conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
-
-        let ddl = format!(
-            "CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
-            quote_table_name(&self.args.table)?
-        );
-        conn.query_drop(ddl)
-            .await
-            .map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
-
-        validate_existing_schema(&mut conn, &self.args.table).await?;
+        let pool = build_mysql_pool_from_args(&self.args).await?;
 
         // Double-check: another caller may have initialized the pool
         // while we were doing I/O.
@@ -653,12 +636,87 @@ where
             args: self.args.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             pool: Arc::clone(&self.pool),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
             delivery_counters: Arc::clone(&self.delivery_counters),
             _phantom: PhantomData,
         })
     }
 }
 
+/// Builds a MySQL connection pool from the given args, including TLS setup,
+/// DDL table creation, and schema validation.
+///
+/// This is a standalone function so it can be called both from
+/// `get_or_init_pool` (inline fallback) and from `build_tls_material`
+/// (coordinator path).
+async fn build_mysql_pool_from_args(args: &MySqlArgs) -> Result<Pool, TargetError> {
+    let dsn = MySqlDsn::parse(&args.dsn_string)?;
+
+    let mut builder = OptsBuilder::default()
+        .user(Some(dsn.user.clone()))
+        .pass(Some(dsn.password.clone()))
+        .ip_or_hostname(dsn.host.clone())
+        .tcp_port(dsn.port)
+        .db_name(Some(dsn.database.clone()));
+
+    if dsn.tls {
+        super::ensure_rustls_provider_installed();
+        let mut ssl_opts = SslOpts::default();
+        if !args.tls_ca.is_empty() {
+            let _ =
+                load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_ca: {e}")))?;
+            ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(args.tls_ca.clone()).into()]);
+        }
+        if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
+            let _ = load_certs(&args.tls_client_cert)
+                .map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_cert: {e}")))?;
+            let _ = load_private_key(&args.tls_client_key)
+                .map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_key: {e}")))?;
+            let identity = mysql_async::ClientIdentity::new(
+                PathBuf::from(args.tls_client_cert.clone()).into(),
+                PathBuf::from(args.tls_client_key.clone()).into(),
+            );
+            ssl_opts = ssl_opts.with_client_identity(Some(identity));
+        }
+        builder = builder.ssl_opts(Some(ssl_opts));
+    } else {
+        warn!("MySQL target is configured without TLS. This is insecure and should not be used in production.");
+    }
+
+    // When max_open_connections is 0, no explicit upper bound is set —
+    // mysql_async uses its default pool constraints (10–100).
+    if args.max_open_connections > 0 {
+        let constraints = PoolConstraints::new(1, args.max_open_connections).ok_or_else(|| {
+            TargetError::Configuration(format!("MySQL max_open_connections must be >= 1, got {}", args.max_open_connections))
+        })?;
+        builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
+    }
+
+    let opts = Opts::from(builder);
+    let pool = Pool::new(opts);
+
+    // Uses a double-check pattern: the mutex guard is only held for
+    // short reads/writes to the pool cache. All I/O (connecting,
+    // DDL, schema validation) happens outside the lock so that
+    // concurrent callers are not blocked by a slow MySQL server.
+    let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
+
+    conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
+
+    let ddl = format!(
+        "CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
+        quote_table_name(&args.table)?
+    );
+    conn.query_drop(ddl)
+        .await
+        .map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
+
+    validate_existing_schema(&mut conn, &args.table).await?;
+
+    Ok(pool)
+}
+
 /// Maps a mysql_async error to `TargetError`:
 /// - `Io`/`Driver` → `NotConnected` (connection lost, fixed-delay retry)
 /// - `Server(1213|1205|1040)` → `Timeout` (deadlock/lock timeout/too
@@ -793,6 +851,8 @@ where
                 .map_err(|err| TargetError::Network(format!("Failed to disconnect MySQL pool: {err}")))?;
         }
 
+        // Adapter cleanup is done by the coordinator; no local state to reset.
+
         info!("MySQL target closed: {}", self.id);
         Ok(())
     }
@@ -828,6 +888,46 @@ where
     }
 }
 
+/// Coordinated TLS hot-reload implementation for MySQL targets.
+///
+/// The coordinator calls these methods on a background poll loop to detect
+/// TLS file changes and rebuild the connection pool without restarting.
+#[async_trait]
+impl<E> ReloadableTargetTls for MySqlTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = Pool;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls_ca.clone(),
+            client_cert_path: self.args.tls_client_cert.clone(),
+            client_key_path: self.args.tls_client_key.clone(),
+            target_label: format!("mysql:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        build_mysql_pool_from_args(&self.args).await
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        let mut guard = self.pool.lock().await;
+        *guard = Some((*material).clone());
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/targets/src/target/nats.rs

@@ -16,10 +16,15 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
+        validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
+        persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
@@ -28,9 +33,10 @@ use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
+use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex};
-use tracing::{info, instrument};
+use tokio::sync::Mutex;
+use tracing::{info, instrument, warn};
 
 #[derive(Debug, Clone)]
 pub struct NATSArgs {
@@ -168,7 +174,12 @@ where
 {
     id: TargetID,
     args: NATSArgs,
-    client: Mutex<Option<async_nats::Client>>,
+    client: Arc<Mutex<Option<async_nats::Client>>>,
+    tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
+    /// Adapter that bridges this target to the TLS reload coordinator.
+    /// When `Some`, the target uses coordinator-managed material; when `None`,
+    /// it falls back to inline fingerprint-based change detection.
+    tls_adapter: Option<TlsReloadAdapter<async_nats::Client>>,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     connected: AtomicBool,
     delivery_counters: Arc<TargetDeliveryCounters>,
@@ -183,7 +194,9 @@ where
         Box::new(NATSTarget::<E> {
             id: self.id.clone(),
             args: self.args.clone(),
-            client: Mutex::new(self.client.lock().unwrap().clone()),
+            client: Arc::clone(&self.client),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
             delivery_counters: Arc::clone(&self.delivery_counters),
@@ -207,7 +220,9 @@ where
         Ok(Self {
             id: target_id,
             args,
-            client: Mutex::new(None),
+            client: Arc::new(Mutex::new(None)),
+            tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
+            tls_adapter: None,
             store: queue_store,
             connected: AtomicBool::new(false),
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -215,11 +230,42 @@ where
         })
     }
 
+    async fn invalidate_cached_client_connection(&self) {
+        *self.client.lock().await = None;
+    }
+
     async fn get_or_connect(&self) -> Result<async_nats::Client, TargetError> {
-        if let Some(client) = self.client.lock().unwrap().clone() {
+        // Adapter-managed path: use the material directly from the TLS reload adapter.
+        if let Some(adapter) = &self.tls_adapter {
+            let client: async_nats::Client = (*adapter.current_material()).clone();
+
+            // Ensure the client is also stored locally so that close() can drain it.
+            {
+                let mut guard = self.client.lock().await;
+                *guard = Some(client.clone());
+            }
             return Ok(client);
         }
 
+        // Inline fingerprint fallback path (no coordinator).
+        let next_fingerprint =
+            build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
+        let tls_changed = {
+            let tls_state_guard = self.tls_state.lock();
+            tls_state_guard.needs_update(&next_fingerprint)
+        };
+        if tls_changed {
+            self.invalidate_cached_client_connection().await;
+            self.tls_state.lock().refresh(next_fingerprint);
+        }
+
+        {
+            let guard = self.client.lock().await;
+            if let Some(client) = guard.as_ref() {
+                return Ok(client.clone());
+            }
+        }
+
         let client = connect_nats(&self.args).await?;
         client
             .flush()
@@ -227,7 +273,7 @@ where
             .map_err(|e| TargetError::Network(format!("Failed to flush NATS connection: {e}")))?;
         self.connected.store(true, Ordering::SeqCst);
 
-        let mut guard = self.client.lock().unwrap();
+        let mut guard = self.client.lock().await;
         let shared = guard.get_or_insert_with(|| client.clone()).clone();
         Ok(shared)
     }
@@ -294,7 +340,11 @@ where
     }
 
     async fn close(&self) -> Result<(), TargetError> {
-        let client = self.client.lock().unwrap().take();
+        let client = {
+            let mut guard = self.client.lock().await;
+            guard.take()
+        };
+        self.tls_state.lock().reset();
         self.connected.store(false, Ordering::SeqCst);
         if let Some(client) = client {
             client
@@ -335,3 +385,87 @@ where
         self.delivery_counters.record_final_failure();
     }
 }
+
+/// Coordinated TLS hot-reload implementation for NATS targets.
+///
+/// The coordinator calls these methods on a background poll loop to detect
+/// TLS file changes and rebuild the NATS client without restarting.
+#[async_trait]
+impl<E> ReloadableTargetTls for NATSTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = async_nats::Client;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls_ca.clone(),
+            client_cert_path: self.args.tls_client_cert.clone(),
+            client_key_path: self.args.tls_client_key.clone(),
+            target_label: format!("nats:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        connect_nats(&self.args).await
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        let mut guard = self.client.lock().await;
+        *guard = Some((*material).clone());
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn base_args() -> NATSArgs {
+        NATSArgs {
+            enable: true,
+            address: "nats://127.0.0.1:4222".to_string(),
+            subject: "rustfs.events".to_string(),
+            username: String::new(),
+            password: String::new(),
+            token: String::new(),
+            credentials_file: String::new(),
+            tls_ca: String::new(),
+            tls_client_cert: String::new(),
+            tls_client_key: String::new(),
+            tls_required: false,
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
+    #[test]
+    fn validate_nats_rejects_multiple_auth_methods() {
+        let args = NATSArgs {
+            token: "abc".to_string(),
+            username: "user".to_string(),
+            password: "pass".to_string(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn validate_nats_rejects_relative_queue_dir() {
+        let args = NATSArgs {
+            queue_dir: "relative/path".to_string(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+}

crates/targets/src/target/postgres.rs

@@ -29,6 +29,10 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
+        validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -38,12 +42,10 @@ use crate::{
 use async_trait::async_trait;
 use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod};
 use rustfs_config::{POSTGRES_DSN_STRING, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY};
-use rustls_pki_types::pem::PemObject;
-use rustls_pki_types::{CertificateDer, PrivateKeyDer};
+use rustfs_tls_runtime::{load_certs, load_private_key};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::fmt;
-use std::io::BufReader;
 use std::path::Path;
 use std::sync::Arc;
 use tokio_postgres::Config;
@@ -425,11 +427,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
             let _ = root_store.add(cert);
         }
     } else {
-        let pem = std::fs::read(&args.tls_ca)
-            .map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CA}: {e}")))?;
-        let mut reader = BufReader::new(pem.as_slice());
-        for cert in CertificateDer::pem_reader_iter(&mut reader) {
-            let cert = cert.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
+        let certs =
+            load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
+        for cert in certs {
             root_store
                 .add(cert)
                 .map_err(|e| TargetError::Configuration(format!("failed to add CA cert: {e}")))?;
@@ -439,16 +439,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
     let builder = rustls::ClientConfig::builder().with_root_certificates(root_store);
 
     let client_config = if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
-        let cert_pem = std::fs::read(&args.tls_client_cert)
-            .map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
-        let key_pem = std::fs::read(&args.tls_client_key)
-            .map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
-
-        let certs: Vec<_> = CertificateDer::pem_reader_iter(&mut BufReader::new(cert_pem.as_slice()))
-            .collect::<Result<_, _>>()
+        let certs = load_certs(&args.tls_client_cert)
             .map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
-
-        let key = PrivateKeyDer::from_pem_reader(&mut BufReader::new(key_pem.as_slice()))
+        let key = load_private_key(&args.tls_client_key)
             .map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
 
         builder
@@ -548,13 +541,22 @@ fn resolve_payload_key(payload: &serde_json::Value, meta: &QueuedPayloadMeta) ->
 /// so that `clone_box` does not duplicate connection state. The optional
 /// `QueueStore` provides at-least-once delivery semantics consistent with the
 /// other built-in targets.
+///
+/// When `tls_adapter` is `Some`, the target participates in the
+/// coordinated TLS hot-reload system driven by `TlsReloadAdapter`,
+/// and the inline fingerprint check in `send_body` is skipped. When `None`,
+/// the legacy inline fingerprint check is used as a fallback.
 pub struct PostgresTarget<E>
 where
     E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
 {
     id: TargetID,
     args: PostgresArgs,
-    pool: Pool,
+    pool: Arc<parking_lot::Mutex<Pool>>,
+    tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
+    /// When present, the adapter provides coordinator-managed TLS material;
+    /// otherwise the inline fingerprint path is used as a fallback.
+    tls_adapter: Option<TlsReloadAdapter<Pool>>,
     namespace_sql: String,
     access_sql: String,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
@@ -570,7 +572,9 @@ where
         Box::new(PostgresTarget::<E> {
             id: self.id.clone(),
             args: self.args.clone(),
-            pool: self.pool.clone(),
+            pool: Arc::clone(&self.pool),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
             namespace_sql: self.namespace_sql.clone(),
             access_sql: self.access_sql.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
@@ -599,7 +603,9 @@ where
             namespace_sql: namespace_upsert_sql(&args.schema, &args.table),
             access_sql: access_insert_sql(&args.schema, &args.table),
             args,
-            pool,
+            pool: Arc::new(parking_lot::Mutex::new(pool)),
+            tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
+            tls_adapter: None,
             store: queue_store,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
             _phantom: std::marker::PhantomData,
@@ -611,8 +617,25 @@ where
     /// Identifier validation has already happened in `PostgresArgs::validate()`,
     /// so `qualified_table` cannot produce a malformed SQL string here.
     async fn send_body(&self, body: &[u8], event_id: &str, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
-        let client = self
-            .pool
+        // When a TLS reload adapter is attached, it drives pool rebuilds in
+        // the background. The inline per-send fingerprint check is skipped.
+        if self.tls_adapter.is_none() {
+            let next_fingerprint =
+                super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
+                    .await?;
+            let tls_changed = {
+                let tls_state_guard = self.tls_state.lock();
+                tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
+            };
+            if tls_changed {
+                let new_pool = build_pool(&self.args)?;
+                *self.pool.lock() = new_pool;
+                self.tls_state.lock().refresh(next_fingerprint);
+            }
+        }
+
+        let pool = self.pool.lock().clone();
+        let client = pool
             .get()
             .await
             .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
@@ -645,8 +668,8 @@ where
     /// Probes the table from `init()`. Failure is non-fatal when a queue is
     /// configured: events buffer in the store until the schema is fixed.
     async fn probe_table(&self) -> Result<(), TargetError> {
-        let client = self
-            .pool
+        let pool = self.pool.lock().clone();
+        let client = pool
             .get()
             .await
             .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed during init probe"))?;
@@ -659,6 +682,41 @@ where
     }
 }
 
+#[async_trait]
+impl<E> ReloadableTargetTls for PostgresTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = Pool;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls_ca.clone(),
+            client_cert_path: self.args.tls_client_cert.clone(),
+            client_key_path: self.args.tls_client_key.clone(),
+            target_label: format!("postgres:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        build_pool(&self.args)
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        *self.pool.lock() = (*material).clone();
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
+    }
+}
+
 #[async_trait]
 impl<E> Target<E> for PostgresTarget<E>
 where
@@ -674,8 +732,8 @@ where
         }
 
         match tokio::time::timeout(std::time::Duration::from_secs(10), async {
-            let client = self
-                .pool
+            let pool = self.pool.lock().clone();
+            let client = pool
                 .get()
                 .await
                 .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
@@ -728,7 +786,8 @@ where
     }
 
     async fn close(&self) -> Result<(), TargetError> {
-        self.pool.close();
+        self.pool.lock().close();
+        // Adapter cleanup is done by the coordinator; no local state to reset.
         info!(target_id = %self.id, "PostgreSQL target closed");
         Ok(())
     }

crates/targets/src/target/pulsar.rs

@@ -16,14 +16,20 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
+        validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
+        persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
 use pulsar::{Authentication, Producer, Pulsar, TokioExecutor};
+use rustfs_tls_runtime::load_cert_bundle_der_bytes;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::path::Path;
@@ -136,6 +142,13 @@ pub async fn connect_pulsar(args: &PulsarArgs) -> Result<Pulsar<TokioExecutor>,
     }
 
     if !args.tls_ca.is_empty() {
+        let certs = load_cert_bundle_der_bytes(&args.tls_ca)
+            .map_err(|e| TargetError::Configuration(format!("Failed to parse Pulsar tls_ca: {e}")))?;
+        if certs.is_empty() {
+            return Err(TargetError::Configuration(
+                "Pulsar tls_ca did not contain any parsable certificates".to_string(),
+            ));
+        }
         builder = builder
             .with_certificate_chain_file(&args.tls_ca)
             .map_err(|e| TargetError::Configuration(format!("Failed to load Pulsar tls_ca: {e}")))?;
@@ -158,6 +171,9 @@ where
     id: TargetID,
     args: PulsarArgs,
     client: Mutex<Option<Pulsar<TokioExecutor>>>,
+    tls_state: Mutex<TargetTlsState>,
+    /// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
+    tls_adapter: Option<TlsReloadAdapter<Pulsar<TokioExecutor>>>,
     producer: AsyncMutex<Option<Producer<TokioExecutor>>>,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     connected: AtomicBool,
@@ -174,6 +190,8 @@ where
             id: self.id.clone(),
             args: self.args.clone(),
             client: Mutex::new(self.client.lock().unwrap().clone()),
+            tls_state: Mutex::new(self.tls_state.lock().unwrap().clone()),
+            tls_adapter: self.tls_adapter.clone(),
             producer: AsyncMutex::new(None),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
@@ -199,6 +217,8 @@ where
             id: target_id,
             args,
             client: Mutex::new(None),
+            tls_state: Mutex::new(TargetTlsState::default()),
+            tls_adapter: None,
             producer: AsyncMutex::new(None),
             store: queue_store,
             connected: AtomicBool::new(false),
@@ -207,7 +227,36 @@ where
         })
     }
 
+    fn clear_cached_client_connection(&self) {
+        self.client.lock().unwrap().take();
+    }
+
+    fn clear_cached_client(&self) {
+        self.clear_cached_client_connection();
+        self.tls_state.lock().unwrap().reset();
+    }
+
     async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
+        // When a TLS reload adapter is attached, it drives client rebuilds
+        // in the background. The inline per-send fingerprint check is skipped.
+        if let Some(adapter) = &self.tls_adapter {
+            let material = adapter.current_material();
+            {
+                let mut guard = self.client.lock().unwrap();
+                *guard = Some((*material).clone());
+            }
+        } else {
+            let next_fingerprint = build_target_tls_fingerprint(&self.args.tls_ca, "", "").await?;
+            let tls_changed = {
+                let tls_state_guard = self.tls_state.lock().unwrap();
+                tls_state_guard.needs_update(&next_fingerprint)
+            };
+            if tls_changed {
+                self.clear_cached_client_connection();
+                self.tls_state.lock().unwrap().refresh(next_fingerprint);
+            }
+        }
+
         if let Some(client) = self.client.lock().unwrap().clone() {
             return Ok(client);
         }
@@ -262,6 +311,56 @@ where
     }
 }
 
+/// Coordinated TLS hot-reload implementation for Pulsar targets.
+///
+/// Pulsar only uses a CA certificate (no client cert/key).
+/// The coordinator calls these methods on a background poll loop to detect
+/// TLS file changes and rebuild the client without restarting.
+#[async_trait]
+impl<E> ReloadableTargetTls for PulsarTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = Pulsar<TokioExecutor>;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls_ca.clone(),
+            client_cert_path: String::new(),
+            client_key_path: String::new(),
+            target_label: format!("pulsar:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        connect_pulsar(&self.args).await
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        // Pulsar client is Clone, so we clone from the Arc and store it.
+        {
+            let mut guard = self.client.lock().unwrap();
+            *guard = Some((*material).clone());
+        }
+        // Producer is bound to the old client; clear it so next send rebuilds.
+        {
+            let mut producer = self.producer.lock().await;
+            *producer = None;
+        }
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        // Pulsar only uses CA, no client cert/key.
+        validate_tls_material(&self.args.tls_ca, "", "")
+    }
+}
+
 #[async_trait]
 impl<E> Target<E> for PulsarTarget<E>
 where
@@ -321,8 +420,13 @@ where
                 .map_err(|e| TargetError::Network(format!("Failed to close Pulsar producer: {e}")))?;
         }
         *producer = None;
-        self.client.lock().unwrap().take();
+        self.clear_cached_client();
         self.connected.store(false, Ordering::SeqCst);
+        // If a TLS reload adapter is attached, reset its error tracking
+        // so that a future re-init does not inherit stale failure state.
+        if let Some(adapter) = &self.tls_adapter {
+            *adapter.runtime_state().last_error.write() = None;
+        }
         info!(target_id = %self.id, "Pulsar target closed");
         Ok(())
     }
@@ -355,3 +459,45 @@ where
         self.delivery_counters.record_final_failure();
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn base_args() -> PulsarArgs {
+        PulsarArgs {
+            enable: true,
+            broker: "pulsar://127.0.0.1:6650".to_string(),
+            topic: "persistent://public/default/rustfs-events".to_string(),
+            auth_token: String::new(),
+            username: String::new(),
+            password: String::new(),
+            tls_ca: String::new(),
+            tls_allow_insecure: false,
+            tls_hostname_verification: true,
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
+    #[test]
+    fn validate_pulsar_rejects_mixed_auth_methods() {
+        let args = PulsarArgs {
+            auth_token: "token".to_string(),
+            username: "user".to_string(),
+            password: "pass".to_string(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn validate_pulsar_rejects_relative_queue_dir() {
+        let args = PulsarArgs {
+            queue_dir: "relative/path".to_string(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+}

crates/targets/src/target/redis.rs

@@ -16,6 +16,10 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
+        validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -31,9 +35,12 @@ use redis::{
     io::tcp::{TcpSettings, socket2},
 };
 use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY};
+use rustls::pki_types::CertificateDer;
+use rustls::pki_types::pem::PemObject;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::fmt;
+use std::io::BufReader;
 use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -297,7 +304,8 @@ where
 {
     id: TargetID,
     args: RedisArgs,
-    publisher_client: Client,
+    /// Redis client, wrapped in a lock so TLS hot-reload can atomically replace it.
+    publisher_client: Arc<parking_lot::Mutex<Client>>,
     publisher: Arc<Mutex<Option<ConnectionManager>>>,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     /// Business-level liveness flag.
@@ -306,6 +314,12 @@ where
     /// publish exhausted retries, or the target was explicitly closed). Temporary reconnectable
     /// errors only invalidate the cached publisher so that a later request can lazily rebuild it.
     connected: Arc<AtomicBool>,
+    /// TLS fingerprint tracking for hot reload (inline fallback path).
+    tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
+    /// Adapter that bridges this target to the TLS reload coordinator.
+    /// When `Some`, the target uses coordinator-managed material; when `None`,
+    /// it falls back to inline fingerprint-based change detection.
+    tls_adapter: Option<TlsReloadAdapter<Client>>,
     delivery_counters: Arc<TargetDeliveryCounters>,
     _phantom: std::marker::PhantomData<E>,
 }
@@ -334,10 +348,12 @@ where
         Ok(Self {
             id: target_id,
             args,
-            publisher_client,
+            publisher_client: Arc::new(parking_lot::Mutex::new(publisher_client)),
             publisher: Arc::new(Mutex::new(None)),
             store: queue_store,
             connected: Arc::new(AtomicBool::new(false)),
+            tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
+            tls_adapter: None,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
             _phantom: std::marker::PhantomData,
         })
@@ -347,23 +363,61 @@ where
         Box::new(Self {
             id: self.id.clone(),
             args: self.args.clone(),
-            publisher_client: self.publisher_client.clone(),
+            publisher_client: Arc::clone(&self.publisher_client),
             publisher: Arc::clone(&self.publisher),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             connected: Arc::clone(&self.connected),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
             delivery_counters: Arc::clone(&self.delivery_counters),
             _phantom: std::marker::PhantomData,
         })
     }
 
     async fn get_or_create_publisher(&self) -> Result<ConnectionManager, TargetError> {
+        // Adapter-managed path: use the material directly from the TLS reload adapter.
+        if let Some(adapter) = &self.tls_adapter {
+            let client: Client = (*adapter.current_material()).clone();
+
+            // Ensure the client is also stored locally so close() can drain it.
+            *self.publisher_client.lock() = client.clone();
+
+            let manager = client
+                .get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
+                .map_err(map_redis_error)?;
+
+            *self.publisher.lock().await = Some(manager.clone());
+            return Ok(manager);
+        }
+
+        // Inline fingerprint fallback path (no coordinator).
+        let secure_scheme = matches!(self.args.url.scheme(), "rediss" | "valkeys");
+        if secure_scheme {
+            let next_fingerprint = super::build_target_tls_fingerprint(
+                &self.args.tls.ca_path,
+                &self.args.tls.client_cert_path,
+                &self.args.tls.client_key_path,
+            )
+            .await?;
+            let tls_changed = {
+                let tls_state_guard = self.tls_state.lock();
+                tls_state_guard.needs_update(&next_fingerprint)
+            };
+            if tls_changed {
+                let new_client = build_redis_client(&self.args)?;
+                *self.publisher_client.lock() = new_client;
+                self.invalidate_cached_publisher().await;
+                self.tls_state.lock().refresh(next_fingerprint);
+            }
+        }
+
         let mut guard = self.publisher.lock().await;
         if let Some(manager) = guard.clone() {
             return Ok(manager);
         }
 
-        let manager = self
-            .publisher_client
+        let client = self.publisher_client.lock().clone();
+        let manager = client
             .get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
             .map_err(map_redis_error)?;
 
@@ -475,7 +529,8 @@ where
             return Ok(false);
         }
 
-        match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&self.publisher_client, &self.args)).await {
+        let client = self.publisher_client.lock().clone();
+        match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&client, &self.args)).await {
             Ok(Ok(())) => {
                 self.connected.store(true, Ordering::SeqCst);
                 Ok(true)
@@ -557,6 +612,7 @@ where
 
     async fn close(&self) -> Result<(), TargetError> {
         self.invalidate_cached_publisher().await;
+        self.tls_state.lock().reset();
         self.connected.store(false, Ordering::SeqCst);
         info!(target_id = %self.id, "Redis target closed");
         Ok(())
@@ -696,9 +752,20 @@ fn read_root_cert(tls: &RedisTlsConfig) -> Result<Option<Vec<u8>>, TargetError>
         return Ok(None);
     }
 
-    std::fs::read(&tls.ca_path)
-        .map(Some)
-        .map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))
+    let pem =
+        std::fs::read(&tls.ca_path).map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))?;
+    let mut reader = BufReader::new(pem.as_slice());
+    let certs_der = CertificateDer::pem_reader_iter(&mut reader)
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| TargetError::Configuration(format!("Failed to parse Redis root CA cert: {e}")))?;
+
+    if certs_der.is_empty() {
+        return Err(TargetError::Configuration(
+            "Redis root CA cert did not contain any parsable certificates".to_string(),
+        ));
+    }
+
+    Ok(Some(pem))
 }
 
 fn map_redis_error(err: RedisError) -> TargetError {
@@ -722,6 +789,46 @@ fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration)
     min_delay.saturating_mul(factor).min(max_delay)
 }
 
+/// Coordinated TLS hot-reload implementation for Redis targets.
+///
+/// The coordinator calls these methods on a background poll loop to detect
+/// TLS file changes and rebuild the Redis client without restarting.
+#[async_trait]
+impl<E> ReloadableTargetTls for RedisTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = Client;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.tls.ca_path.clone(),
+            client_cert_path: self.args.tls.client_cert_path.clone(),
+            client_key_path: self.args.tls.client_key_path.clone(),
+            target_label: format!("redis:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        build_redis_client(&self.args)
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        *self.publisher_client.lock() = (*material).clone();
+        self.invalidate_cached_publisher().await;
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/targets/src/target/webhook.rs

@@ -16,14 +16,21 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
+    runtime::tls::{
+        ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
+        validate::validate_tls_material,
+    },
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store,
+        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, open_target_queue_store,
+        persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
+use parking_lot::Mutex;
 use reqwest::{Client, StatusCode, Url};
+use rustfs_tls_runtime::load_cert_bundle_der_bytes;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::{
@@ -104,7 +111,11 @@ where
     id: TargetID,
     args: WebhookArgs,
     health_check_url: Option<Url>,
-    http_client: Arc<Client>,
+    http_client: Arc<Mutex<Client>>,
+    tls_state: Arc<Mutex<TargetTlsState>>,
+    /// When present, the adapter provides coordinator-managed TLS material;
+    /// otherwise the inline fingerprint path is used as a fallback.
+    tls_adapter: Option<TlsReloadAdapter<Client>>,
     // Add Send + Sync constraints to ensure thread safety
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     initialized: AtomicBool,
@@ -124,6 +135,8 @@ where
             args: self.args.clone(),
             health_check_url: self.health_check_url.clone(),
             http_client: Arc::clone(&self.http_client),
+            tls_state: Arc::clone(&self.tls_state),
+            tls_adapter: self.tls_adapter.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
             cancel_sender: self.cancel_sender.clone(),
@@ -146,7 +159,7 @@ where
         };
 
         // Build HTTP client using the helper function
-        let http_client = Arc::new(Self::build_http_client(&args)?);
+        let http_client = Arc::new(Mutex::new(Self::build_http_client(&args)?));
 
         let queue_store = open_target_queue_store(
             &args.queue_dir,
@@ -165,6 +178,8 @@ where
             args,
             health_check_url,
             http_client,
+            tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
+            tls_adapter: None,
             store: queue_store,
             initialized: AtomicBool::new(false),
             cancel_sender,
@@ -188,11 +203,18 @@ where
             );
         } else if !args.client_ca.is_empty() {
             // Use user-provided custom CA certificate
-            let ca_cert_pem = std::fs::read(&args.client_ca)
-                .map_err(|e| TargetError::Configuration(format!("Failed to read root CA cert: {e}")))?;
-            let ca_cert = reqwest::Certificate::from_pem(&ca_cert_pem)
+            let certs_der = load_cert_bundle_der_bytes(&args.client_ca)
                 .map_err(|e| TargetError::Configuration(format!("Failed to parse root CA cert: {e}")))?;
-            client_builder = client_builder.add_root_certificate(ca_cert);
+            if certs_der.is_empty() {
+                return Err(TargetError::Configuration(
+                    "Webhook client_ca did not contain any parsable certificates".to_string(),
+                ));
+            }
+            for cert_der in certs_der {
+                let ca_cert = reqwest::Certificate::from_der(&cert_der)
+                    .map_err(|e| TargetError::Configuration(format!("Failed to load root CA cert: {e}")))?;
+                client_builder = client_builder.add_root_certificate(ca_cert);
+            }
         }
         // If neither is set, use the system's default trust store
 
@@ -213,6 +235,29 @@ where
             .map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}")))
     }
 
+    async fn refresh_tls(&self) -> Result<(), TargetError> {
+        let next_fingerprint =
+            build_target_tls_fingerprint(&self.args.client_ca, &self.args.client_cert, &self.args.client_key).await?;
+        let tls_changed = {
+            let tls_state_guard = self.tls_state.lock();
+            tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
+        };
+        if !tls_changed {
+            return Ok(());
+        }
+
+        let new_client = Self::build_http_client(&self.args)?;
+        {
+            let mut tls_state_guard = self.tls_state.lock();
+            if tls_state_guard.fingerprint.as_ref() == Some(&next_fingerprint) {
+                return Ok(());
+            }
+            *self.http_client.lock() = new_client;
+            tls_state_guard.refresh(next_fingerprint);
+        }
+        Ok(())
+    }
+
     fn health_check_url(endpoint: &Url) -> Result<Url, TargetError> {
         endpoint
             .host()
@@ -230,7 +275,8 @@ where
             return Ok(false);
         };
 
-        match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(health_check_url.as_str()).send()).await {
+        let client = self.http_client.lock().clone();
+        match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
             Ok(Ok(resp)) => {
                 debug!(
                     target = %self.id,
@@ -299,8 +345,14 @@ where
             "Sending webhook payload"
         );
 
-        let mut req_builder = self
-            .http_client
+        // When a TLS reload adapter is attached, it drives client rebuilds in
+        // the background. The inline per-send fingerprint check is skipped.
+        if self.tls_adapter.is_none() {
+            self.refresh_tls().await?;
+        }
+
+        let client = self.http_client.lock().clone();
+        let mut req_builder = client
             .post(self.args.endpoint.as_str())
             .header("Content-Type", meta.content_type.as_str());
 
@@ -425,6 +477,7 @@ where
     async fn close(&self) -> Result<(), TargetError> {
         // Send cancel signal to background tasks
         let _ = self.cancel_sender.try_send(());
+        // Adapter cleanup is done by the coordinator; no local state to reset.
         info!("Webhook target closed: {}", self.id);
         Ok(())
     }
@@ -460,6 +513,48 @@ where
     }
 }
 
+/// Coordinated TLS hot-reload implementation for Webhook targets.
+///
+/// The coordinator calls these methods on a background poll loop to detect
+/// TLS file changes and rebuild the HTTP client without restarting.
+#[async_trait]
+impl<E> ReloadableTargetTls for WebhookTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    type Material = Client;
+
+    fn tls_input_set(&self) -> TargetTlsInputSet {
+        TargetTlsInputSet {
+            ca_path: self.args.client_ca.clone(),
+            client_cert_path: self.args.client_cert.clone(),
+            client_key_path: self.args.client_key.clone(),
+            target_label: format!("webhook:{}", self.id.id),
+        }
+    }
+
+    async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
+        // build_http_client is synchronous (reads files + configures reqwest).
+        // The coordinator already runs this in a background task, so the
+        // synchronous file I/O does not block the send path.
+        Self::build_http_client(&self.args)
+    }
+
+    async fn apply_tls_material(
+        &self,
+        _generation: TargetTlsGeneration,
+        material: Arc<Self::Material>,
+        _mode: ReloadApplyMode,
+    ) -> Result<(), TargetError> {
+        *self.http_client.lock() = (*material).clone();
+        Ok(())
+    }
+
+    async fn validate_tls_files(&self) -> Result<(), TargetError> {
+        validate_tls_material(&self.args.client_ca, &self.args.client_cert, &self.args.client_key)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::{WebhookArgs, WebhookTarget};

crates/tls-runtime/Cargo.toml

@@ -0,0 +1,50 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-tls-runtime"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+description = "Project-wide TLS runtime foundation for RustFS."
+keywords = ["tls", "runtime", "rustls", "hot-reload", "rustfs"]
+categories = ["network-programming", "web-programming", "development-tools"]
+
+[lints]
+workspace = true
+
+[dependencies]
+rustfs-common.workspace = true
+rustfs-config.workspace = true
+arc-swap.workspace = true
+metrics.workspace = true
+rustls.workspace = true
+rustls-pki-types.workspace = true
+serde = { workspace = true, features = ["derive"] }
+sha2.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
+tracing.workspace = true
+
+[dev-dependencies]
+rcgen.workspace = true
+serde_json.workspace = true
+tempfile.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+
+[lib]
+doctest = false

crates/tls-runtime/src/certs.rs

@@ -25,7 +25,6 @@ use std::sync::Arc;
 use std::{fs, io};
 use tracing::{debug, warn};
 
-/// Options for loading certificate/key pairs from a directory tree.
 #[derive(Debug, Clone)]
 pub struct CertDirectoryLoadOptions {
     dir_path: PathBuf,
@@ -34,7 +33,6 @@ pub struct CertDirectoryLoadOptions {
 }
 
 impl CertDirectoryLoadOptions {
-    /// Create a builder with explicit certificate and private key filenames.
     pub fn builder(
         dir_path: impl Into<PathBuf>,
         cert_filename: impl Into<String>,
@@ -58,7 +56,6 @@ impl CertDirectoryLoadOptions {
     }
 }
 
-/// Builder for [`CertDirectoryLoadOptions`].
 #[derive(Debug, Clone)]
 pub struct CertDirectoryLoadOptionsBuilder {
     dir_path: PathBuf,
@@ -67,19 +64,16 @@ pub struct CertDirectoryLoadOptionsBuilder {
 }
 
 impl CertDirectoryLoadOptionsBuilder {
-    /// Override the certificate filename searched in the directory.
     pub fn cert_filename(mut self, cert_filename: impl Into<String>) -> Self {
         self.cert_filename = cert_filename.into();
         self
     }
 
-    /// Override the private key filename searched in the directory.
     pub fn key_filename(mut self, key_filename: impl Into<String>) -> Self {
         self.key_filename = key_filename.into();
         self
     }
 
-    /// Build the load options value.
     pub fn build(self) -> CertDirectoryLoadOptions {
         CertDirectoryLoadOptions {
             dir_path: self.dir_path,
@@ -89,7 +83,6 @@ impl CertDirectoryLoadOptionsBuilder {
     }
 }
 
-/// Options for building an mTLS WebPki client verifier.
 #[derive(Debug, Clone)]
 pub struct WebPkiClientVerifierOptions {
     tls_path: PathBuf,
@@ -99,7 +92,6 @@ pub struct WebPkiClientVerifierOptions {
 }
 
 impl WebPkiClientVerifierOptions {
-    /// Create a builder with explicit CA bundle filenames.
     pub fn builder(
         tls_path: impl Into<PathBuf>,
         client_ca_cert_filename: impl Into<String>,
@@ -114,7 +106,6 @@ impl WebPkiClientVerifierOptions {
     }
 }
 
-/// Builder for [`WebPkiClientVerifierOptions`].
 #[derive(Debug, Clone)]
 pub struct WebPkiClientVerifierOptionsBuilder {
     tls_path: PathBuf,
@@ -124,25 +115,21 @@ pub struct WebPkiClientVerifierOptionsBuilder {
 }
 
 impl WebPkiClientVerifierOptionsBuilder {
-    /// Set whether mTLS verification should be enabled.
     pub fn enabled(mut self, enabled: bool) -> Self {
         self.enabled = enabled;
         self
     }
 
-    /// Override the preferred client CA bundle filename.
     pub fn client_ca_cert_filename(mut self, client_ca_cert_filename: impl Into<String>) -> Self {
         self.client_ca_cert_filename = client_ca_cert_filename.into();
         self
     }
 
-    /// Override the fallback CA bundle filename.
     pub fn fallback_ca_cert_filename(mut self, fallback_ca_cert_filename: impl Into<String>) -> Self {
         self.fallback_ca_cert_filename = fallback_ca_cert_filename.into();
         self
     }
 
-    /// Build the verifier options value.
     pub fn build(self) -> WebPkiClientVerifierOptions {
         WebPkiClientVerifierOptions {
             tls_path: self.tls_path,
@@ -153,21 +140,10 @@ impl WebPkiClientVerifierOptionsBuilder {
     }
 }
 
-/// Load public certificate from file.
-/// This function loads a public certificate from the specified file.
-///
-/// # Arguments
-///  * `filename` - A string slice that holds the name of the file containing the public certificate.
-///
-/// # Returns
-/// * An io::Result containing a vector of CertificateDer if successful, or an io::Error if an error occurs during loading.
-///
 pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
-    // Open certificate file.
     let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
     let mut reader = io::BufReader::new(cert_file);
 
-    // Load and return certificate.
     let certs = CertificateDer::pem_reader_iter(&mut reader)
         .collect::<Result<Vec<_>, _>>()
         .map_err(|e| certs_error(format!("certificate file {filename} format error:{e:?}")))?;
@@ -177,16 +153,6 @@ pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
     Ok(certs)
 }
 
-/// Load a PEM certificate bundle and return each certificate as DER bytes.
-///
-/// This is a low-level helper intended for TLS clients (reqwest/hyper-rustls) that
-/// need to add root certificates one-by-one.
-///
-/// - Input: a PEM file that may contain multiple cert blocks.
-/// - Output: Vec of DER-encoded cert bytes, one per cert.
-///
-/// NOTE: This intentionally returns raw bytes to avoid forcing downstream crates
-/// to depend on rustls types.
 pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
     let pem = fs::read(path)?;
     let mut reader = io::BufReader::new(&pem[..]);
@@ -198,15 +164,6 @@ pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
     Ok(certs.into_iter().map(|c| c.to_vec()).collect())
 }
 
-/// Builds a WebPkiClientVerifier for mTLS when enabled by the caller.
-///
-/// # Arguments
-/// * `options` - mTLS verifier options, including the TLS directory and CA bundle filenames
-///
-/// # Returns
-/// * `Ok(Some(verifier))` if mTLS is enabled and CA certs are found
-/// * `Ok(None)` if mTLS is disabled
-/// * `Err` if mTLS is enabled but configuration is invalid
 pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io::Result<Option<Arc<dyn ClientCertVerifier>>> {
     if !options.enabled {
         return Ok(None);
@@ -243,7 +200,6 @@ pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io:
     Ok(Some(verifier))
 }
 
-/// Locate the mTLS client CA bundle in the specified TLS path
 fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option<PathBuf> {
     let p1 = options.tls_path.join(&options.client_ca_cert_filename);
     if p1.exists() {
@@ -256,34 +212,14 @@ fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option<PathBuf>
     None
 }
 
-/// Load private key from file.
-/// This function loads a private key from the specified file.
-///
-/// # Arguments
-///  * `filename` - A string slice that holds the name of the file containing the private key.
-///
-/// # Returns
-/// * An io::Result containing the PrivateKeyDer if successful, or an io::Error if an error occurs during loading.
-///
 pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
-    // Open keyfile.
     let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
     let mut reader = io::BufReader::new(keyfile);
 
-    // Load and return a single private key.
     PrivateKeyDer::from_pem_reader(&mut reader)
         .map_err(|e| certs_error(format!("failed to parse private key in {filename}: {e}")))
 }
 
-/// error function
-/// This function creates a new io::Error with the provided error message.
-///
-/// # Arguments
-///  * `err` - A string containing the error message.
-///
-/// # Returns
-///  * An io::Error instance with the specified error message.
-///
 pub fn certs_error(err: String) -> Error {
     Error::other(err)
 }
@@ -292,17 +228,6 @@ fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool {
     !domain_name.starts_with('.')
 }
 
-/// Load all certificates and private keys in the directory
-/// This function loads all certificate and private key pairs from the specified directory.
-/// It looks for files named `options.cert_filename` and `options.key_filename` in each subdirectory.
-/// The root directory can also contain a default certificate/private key pair.
-///
-/// # Arguments
-/// * `options` - Directory and filename options for discovering certificates and private keys.
-///
-/// # Returns
-/// * An io::Result containing a HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec<CertificateDer>, PrivateKeyDer). If no valid certificate/private key pairs are found, an io::Error is returned.
-///
 pub fn load_all_certs_from_directory(
     options: CertDirectoryLoadOptions,
 ) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
@@ -318,7 +243,6 @@ pub fn load_all_certs_from_directory(
         )));
     }
 
-    // 1. First check whether there is a certificate/private key pair in the root directory
     let root_cert_path = dir.join(&options.cert_filename);
     let root_key_path = dir.join(&options.key_filename);
 
@@ -332,7 +256,6 @@ pub fn load_all_certs_from_directory(
             .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {root_key_path:?}")))?;
         match load_cert_key_pair(root_cert_str, root_key_str) {
             Ok((certs, key)) => {
-                // The root directory certificate is used as the default certificate and is stored using special keys.
                 cert_key_pairs.insert("default".to_string(), (certs, key));
             }
             Err(e) => {
@@ -341,7 +264,6 @@ pub fn load_all_certs_from_directory(
         }
     }
 
-    // 2.iterate through all folders in the directory
     for entry in fs::read_dir(dir)? {
         let entry = entry?;
         let path = entry.path();
@@ -356,9 +278,8 @@ pub fn load_all_certs_from_directory(
                 continue;
             }
 
-            // find certificate and private key files
-            let cert_path = path.join(&options.cert_filename); // e.g., rustfs_cert.pem
-            let key_path = path.join(&options.key_filename); // e.g., rustfs_key.pem
+            let cert_path = path.join(&options.cert_filename);
+            let key_path = path.join(&options.key_filename);
 
             if cert_path.exists() && key_path.exists() {
                 debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path);
@@ -391,43 +312,21 @@ pub fn load_all_certs_from_directory(
     }
 
     if cert_key_pairs.is_empty() {
-        return Err(certs_error(format!(
-            "No valid certificate/private key pair found in directory {}",
-            dir.display()
-        )));
+        return Err(io::Error::new(
+            io::ErrorKind::NotFound,
+            format!("No valid certificate/private key pair found in directory {}", dir.display()),
+        ));
     }
 
     Ok(cert_key_pairs)
 }
 
-/// loading a single certificate private key pair
-/// This function loads a certificate and private key from the specified paths.
-/// It returns a tuple containing the certificate and private key.
-///
-/// # Arguments
-/// * `cert_path` - A string slice that holds the path to the certificate file.
-/// * `key_path` - A string slice that holds the path to the private key file
-///
-/// # Returns
-/// * An io::Result containing a tuple of (Vec<CertificateDer>, PrivateKeyDer) if successful, or an io::Error if an error occurs during loading.
-///
 fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)> {
     let certs = load_certs(cert_path)?;
     let key = load_private_key(key_path)?;
     Ok((certs, key))
 }
 
-/// Create a multi-cert resolver
-/// This function loads all certificates and private keys from the specified directory.
-/// It uses the first certificate/private key pair found in the root directory as the default certificate.
-/// The rest of the certificates/private keys are used for SNI resolution.
-///
-/// # Arguments
-/// * `cert_key_pairs` - A HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec<CertificateDer>, PrivateKeyDer).
-///
-/// # Returns
-/// * An io::Result containing an implementation of ResolvesServerCert if successful, or an io::Error if an error occurs during loading.
-///
 pub fn create_multi_cert_resolver(
     cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
 ) -> io::Result<impl ResolvesServerCert> {
@@ -436,14 +335,13 @@ pub fn create_multi_cert_resolver(
         cert_resolver: ResolvesServerCertUsingSni,
         default_cert: Option<Arc<CertifiedKey>>,
     }
+
     impl ResolvesServerCert for MultiCertResolver {
-        fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
-            // try matching certificates with sni
+        fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
             if let Some(cert) = self.cert_resolver.resolve(client_hello) {
                 return Some(cert);
             }
 
-            // If there is no matching SNI certificate, use the default certificate
             self.default_cert.clone()
         }
     }
@@ -452,16 +350,13 @@ pub fn create_multi_cert_resolver(
     let mut default_cert = None;
 
     for (domain, (certs, key)) in cert_key_pairs {
-        // create a signature
         let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
             .map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?;
 
-        // create a CertifiedKey
         let certified_key = CertifiedKey::new(certs, signing_key);
         if domain == "default" {
             default_cert = Some(Arc::new(certified_key.clone()));
         } else {
-            // add certificate to resolver
             resolver
                 .add(&domain, certified_key)
                 .map_err(|e| certs_error(format!("failed to add a domain name certificate:{domain},err: {e:?}")))?;
@@ -479,6 +374,7 @@ mod tests {
     use super::*;
     use std::fs;
     use std::io::ErrorKind;
+    use std::path::PathBuf;
     use tempfile::TempDir;
 
     fn default_load_options(path: impl Into<PathBuf>) -> CertDirectoryLoadOptions {
@@ -487,9 +383,9 @@ mod tests {
 
     fn write_test_cert_pair(dir: &std::path::Path) {
         let rcgen::CertifiedKey { cert, signing_key } =
-            rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).unwrap();
-        fs::write(dir.join("rustfs_cert.pem"), cert.pem()).unwrap();
-        fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).unwrap();
+            rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).expect("cert should generate");
+        fs::write(dir.join("rustfs_cert.pem"), cert.pem()).expect("cert should write");
+        fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).expect("key should write");
     }
 
     #[test]
@@ -506,7 +402,7 @@ mod tests {
         let result = load_certs("non_existent_file.pem");
         assert!(result.is_err());
 
-        let error = result.unwrap_err();
+        let error = result.expect_err("missing cert should error");
         assert_eq!(error.kind(), ErrorKind::Other);
         assert!(error.to_string().contains("failed to open"));
     }
@@ -516,252 +412,40 @@ mod tests {
         let result = load_private_key("non_existent_key.pem");
         assert!(result.is_err());
 
-        let error = result.unwrap_err();
+        let error = result.expect_err("missing key should error");
         assert_eq!(error.kind(), ErrorKind::Other);
         assert!(error.to_string().contains("failed to open"));
     }
 
-    #[test]
-    fn test_load_certs_empty_file() {
-        let temp_dir = TempDir::new().unwrap();
-        let cert_path = temp_dir.path().join("empty.pem");
-        fs::write(&cert_path, "").unwrap();
-
-        let result = load_certs(cert_path.to_str().unwrap());
-        assert!(result.is_err());
-
-        let error = result.unwrap_err();
-        assert!(error.to_string().contains("No valid certificate was found"));
-    }
-
-    #[test]
-    fn test_load_certs_invalid_format() {
-        let temp_dir = TempDir::new().unwrap();
-        let cert_path = temp_dir.path().join("invalid.pem");
-        fs::write(&cert_path, "invalid certificate content").unwrap();
-
-        let result = load_certs(cert_path.to_str().unwrap());
-        assert!(result.is_err());
-
-        let error = result.unwrap_err();
-        assert!(error.to_string().contains("No valid certificate was found"));
-    }
-
-    #[test]
-    fn test_load_private_key_empty_file() {
-        let temp_dir = TempDir::new().unwrap();
-        let key_path = temp_dir.path().join("empty_key.pem");
-        fs::write(&key_path, "").unwrap();
-
-        let result = load_private_key(key_path.to_str().unwrap());
-        assert!(result.is_err());
-
-        let error = result.unwrap_err();
-        assert!(error.to_string().contains("failed to parse private key in"));
-    }
-
-    #[test]
-    fn test_load_private_key_invalid_format() {
-        let temp_dir = TempDir::new().unwrap();
-        let key_path = temp_dir.path().join("invalid_key.pem");
-        fs::write(&key_path, "invalid private key content").unwrap();
-
-        let result = load_private_key(key_path.to_str().unwrap());
-        assert!(result.is_err());
-
-        let error = result.unwrap_err();
-        assert!(error.to_string().contains("failed to parse private key in"));
-    }
-
-    #[test]
-    fn test_load_all_certs_from_directory_not_exists() {
-        let result = load_all_certs_from_directory(default_load_options("/non/existent/directory"));
-        assert!(result.is_err());
-
-        let error = result.unwrap_err();
-        assert!(error.to_string().contains("does not exist or is not a directory"));
-    }
-
     #[test]
     fn test_load_all_certs_from_directory_empty() {
-        let temp_dir = TempDir::new().unwrap();
-
+        let temp_dir = TempDir::new().expect("tempdir should create");
         let result = load_all_certs_from_directory(default_load_options(temp_dir.path()));
         assert!(result.is_err());
-
-        let error = result.unwrap_err();
+        let error = result.expect_err("empty directory should error");
+        assert_eq!(error.kind(), ErrorKind::NotFound);
         assert!(error.to_string().contains("No valid certificate/private key pair found"));
     }
 
-    #[test]
-    fn test_load_all_certs_from_directory_file_instead_of_dir() {
-        let temp_dir = TempDir::new().unwrap();
-        let file_path = temp_dir.path().join("not_a_directory.txt");
-        fs::write(&file_path, "content").unwrap();
-
-        let result = load_all_certs_from_directory(default_load_options(&file_path));
-        assert!(result.is_err());
-
-        let error = result.unwrap_err();
-        assert!(error.to_string().contains("does not exist or is not a directory"));
-    }
-
-    #[test]
-    fn test_load_cert_key_pair_missing_cert() {
-        let temp_dir = TempDir::new().unwrap();
-        let key_path = temp_dir.path().join("test_key.pem");
-        fs::write(&key_path, "dummy key content").unwrap();
-
-        let result = load_cert_key_pair("non_existent_cert.pem", key_path.to_str().unwrap());
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn test_load_cert_key_pair_missing_key() {
-        let temp_dir = TempDir::new().unwrap();
-        let cert_path = temp_dir.path().join("test_cert.pem");
-        fs::write(&cert_path, "dummy cert content").unwrap();
-
-        let result = load_cert_key_pair(cert_path.to_str().unwrap(), "non_existent_key.pem");
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn test_create_multi_cert_resolver_empty_map() {
-        let empty_map = HashMap::new();
-        let result = create_multi_cert_resolver(empty_map);
-
-        // Should succeed even with empty map
-        assert!(result.is_ok());
-    }
-
-    #[test]
-    fn test_error_message_formatting() {
-        let test_cases = vec![
-            ("file not found", "failed to open test.pem: file not found"),
-            ("permission denied", "failed to open key.pem: permission denied"),
-            ("invalid format", "certificate file cert.pem format error:invalid format"),
-        ];
-
-        for (input, _expected_pattern) in test_cases {
-            let error1 = certs_error(format!("failed to open test.pem: {input}"));
-            assert!(error1.to_string().contains(input));
-
-            let error2 = certs_error(format!("failed to open key.pem: {input}"));
-            assert!(error2.to_string().contains(input));
-        }
-    }
-
-    #[test]
-    fn test_path_handling_edge_cases() {
-        // Test with various path formats
-        let path_cases = vec![
-            "",               // Empty path
-            ".",              // Current directory
-            "..",             // Parent directory
-            "/",              // Root directory (Unix)
-            "relative/path",  // Relative path
-            "/absolute/path", // Absolute path
-        ];
-
-        for path in path_cases {
-            let result = load_all_certs_from_directory(default_load_options(path));
-            // All should fail since these are not valid cert directories
-            assert!(result.is_err());
-        }
-    }
-
-    #[test]
-    fn test_directory_structure_validation() {
-        let temp_dir = TempDir::new().unwrap();
-
-        // Create a subdirectory without certificates
-        let sub_dir = temp_dir.path().join("example.com");
-        fs::create_dir(&sub_dir).unwrap();
-
-        // Should fail because no certificates found
-        let result = load_all_certs_from_directory(default_load_options(temp_dir.path()));
-        assert!(result.is_err());
-        assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("No valid certificate/private key pair found")
-        );
-    }
-
     #[test]
     fn test_load_all_certs_skips_kubernetes_secret_projection_dirs() {
-        let temp_dir = TempDir::new().unwrap();
+        let temp_dir = TempDir::new().expect("tempdir should create");
         write_test_cert_pair(temp_dir.path());
 
         let domain_dir = temp_dir.path().join("example.com");
-        fs::create_dir(&domain_dir).unwrap();
+        fs::create_dir(&domain_dir).expect("domain dir should create");
         write_test_cert_pair(&domain_dir);
 
         for internal_dir_name in ["..data", "..2026_04_28_18_33_53.4209048473"] {
             let internal_dir = temp_dir.path().join(internal_dir_name);
-            fs::create_dir(&internal_dir).unwrap();
+            fs::create_dir(&internal_dir).expect("internal dir should create");
             write_test_cert_pair(&internal_dir);
         }
 
-        let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).unwrap();
-
+        let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).expect("certs should load");
         assert!(certs.contains_key("default"));
         assert!(certs.contains_key("example.com"));
         assert!(!certs.contains_key("..data"));
-        assert!(!certs.contains_key("..2026_04_28_18_33_53.4209048473"));
         assert_eq!(certs.len(), 2);
     }
-
-    #[test]
-    fn test_unicode_path_handling() {
-        let temp_dir = TempDir::new().unwrap();
-
-        // Create directory with Unicode characters
-        let unicode_dir = temp_dir.path().join("test_directory");
-        fs::create_dir(&unicode_dir).unwrap();
-
-        let result = load_all_certs_from_directory(default_load_options(&unicode_dir));
-        assert!(result.is_err());
-        assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("No valid certificate/private key pair found")
-        );
-    }
-
-    #[test]
-    fn test_concurrent_access_safety() {
-        use std::sync::Arc;
-        use std::thread;
-
-        let temp_dir = TempDir::new().unwrap();
-        let dir_path = Arc::new(temp_dir.path().to_string_lossy().to_string());
-
-        let handles: Vec<_> = (0..5)
-            .map(|_| {
-                let path = Arc::clone(&dir_path);
-                thread::spawn(move || {
-                    let result = load_all_certs_from_directory(default_load_options(path.as_str()));
-                    // All should fail since directory is empty
-                    assert!(result.is_err());
-                })
-            })
-            .collect();
-
-        for handle in handles {
-            handle.join().expect("Thread should complete successfully");
-        }
-    }
-
-    #[test]
-    fn test_memory_efficiency() {
-        let error = certs_error("test".to_string());
-        let error_size = std::mem::size_of_val(&error);
-
-        // Error should not be excessively large
-        assert!(error_size < 1024, "Error size should be reasonable, got {error_size} bytes");
-    }
 }

crates/tls-runtime/src/config.rs

@@ -0,0 +1,51 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReloadDetectMode {
+    Poll,
+    Watch,  // TODO: implement fs::watch-based reload
+    Hybrid, // TODO: implement poll + fs::watch hybrid
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReloadApplyHint {
+    Lazy,
+    SoftReconnect,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TlsReloadOptions {
+    pub enabled: bool,
+    pub detect_mode: ReloadDetectMode,
+    pub interval: Duration,
+    pub debounce: Duration,
+    pub min_stable_age: Duration,
+    pub apply_hint: ReloadApplyHint,
+}
+
+impl Default for TlsReloadOptions {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            detect_mode: ReloadDetectMode::Poll,
+            interval: Duration::from_secs(15),
+            debounce: Duration::from_secs(2),
+            min_stable_age: Duration::from_secs(1),
+            apply_hint: ReloadApplyHint::Lazy,
+        }
+    }
+}

crates/tls-runtime/src/coordinator.rs

@@ -0,0 +1,226 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::config::{ReloadDetectMode, TlsReloadOptions};
+use crate::error::TlsRuntimeError;
+use crate::material::TlsMaterialSnapshot;
+use crate::metrics::{
+    TLS_RUNTIME_FOUNDATION_CONSUMER, record_tls_generation, record_tls_publication_fail, record_tls_reload_result,
+    record_tls_reload_skipped,
+};
+use crate::source::TlsSource;
+use crate::state::{
+    TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeConsumerSection, TlsRuntimeOutboundSection,
+    TlsRuntimeRuntimeSection, TlsRuntimeServerSection, TlsRuntimeStatusSnapshot, detect_mode_label,
+};
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::task::JoinHandle;
+use tracing::{debug, info, warn};
+
+pub trait TlsConsumer<M>: Send + Sync + 'static {
+    fn on_publish(&self, generation: TlsGeneration, state: Arc<TlsPublishedState<M>>) -> Result<(), TlsRuntimeError>;
+}
+
+#[derive(Debug)]
+pub struct TlsReloadCoordinator {
+    source: TlsSource,
+    options: TlsReloadOptions,
+}
+
+impl TlsReloadCoordinator {
+    pub fn new(source: TlsSource, options: TlsReloadOptions) -> Self {
+        Self { source, options }
+    }
+
+    pub fn source(&self) -> &TlsSource {
+        &self.source
+    }
+
+    pub fn options(&self) -> &TlsReloadOptions {
+        &self.options
+    }
+
+    pub async fn status_snapshot(&self, runtime_state: &TlsReloadRuntimeState<TlsMaterialSnapshot>) -> TlsRuntimeStatusSnapshot {
+        let current = runtime_state.current.load();
+        let last_attempt = runtime_state.last_attempt_unix_ms();
+        let last_success = runtime_state.last_success_unix_ms();
+
+        TlsRuntimeStatusSnapshot {
+            runtime: TlsRuntimeRuntimeSection {
+                generation: current.generation.0,
+                reload_enabled: self.options.enabled,
+                detect_mode: detect_mode_label(self.options.detect_mode),
+                last_attempt_time: (last_attempt != 0).then_some(last_attempt),
+                last_success_time: (last_success != 0).then_some(last_success),
+                last_error: runtime_state.last_error.read().await.clone(),
+                source_path: self.source.base_dir.display().to_string(),
+            },
+            outbound: TlsRuntimeOutboundSection {
+                has_roots: !current.material.outbound.root_ca_pem.is_empty(),
+                has_mtls_identity: current.material.outbound.mtls_identity.is_some(),
+            },
+            server: TlsRuntimeServerSection {
+                has_material: current.material.server.is_some(),
+            },
+            consumer: TlsRuntimeConsumerSection { stale_generation: false },
+        }
+    }
+
+    pub async fn load_initial_snapshot(&self) -> Result<TlsMaterialSnapshot, TlsRuntimeError> {
+        TlsMaterialSnapshot::load(&self.source).await
+    }
+
+    pub async fn publish_initial_state(&self, snapshot: TlsMaterialSnapshot) -> Arc<TlsPublishedState<TlsMaterialSnapshot>> {
+        let published = Arc::new(TlsPublishedState {
+            generation: TlsGeneration(1),
+            fingerprint: snapshot.fingerprint.clone(),
+            material: Arc::new(snapshot),
+            loaded_at_unix_ms: unix_time_ms(),
+        });
+        record_tls_generation(TLS_RUNTIME_FOUNDATION_CONSUMER, published.generation.0);
+        published
+    }
+
+    pub async fn reload_once<C>(
+        &self,
+        runtime_state: &TlsReloadRuntimeState<TlsMaterialSnapshot>,
+        consumer: &C,
+    ) -> Result<Option<Arc<TlsPublishedState<TlsMaterialSnapshot>>>, TlsRuntimeError>
+    where
+        C: TlsConsumer<TlsMaterialSnapshot>,
+    {
+        runtime_state.mark_attempt(unix_time_ms());
+        let started_at = std::time::Instant::now();
+
+        let snapshot = self.load_initial_snapshot().await?;
+        let current = runtime_state.current.load();
+        if current.fingerprint == snapshot.fingerprint {
+            debug!(source = %self.source.base_dir.display(), "TLS material unchanged; skipping publication");
+            record_tls_reload_skipped(TLS_RUNTIME_FOUNDATION_CONSUMER, "unchanged");
+            return Ok(None);
+        }
+
+        let published = Arc::new(TlsPublishedState {
+            generation: runtime_state.bump_generation(),
+            fingerprint: snapshot.fingerprint.clone(),
+            material: Arc::new(snapshot),
+            loaded_at_unix_ms: unix_time_ms(),
+        });
+
+        if let Err(err) = consumer.on_publish(published.generation, published.clone()) {
+            record_tls_publication_fail(TLS_RUNTIME_FOUNDATION_CONSUMER);
+            return Err(err);
+        }
+        runtime_state.current.store(published.clone());
+        runtime_state.last_good.store(published.clone());
+        runtime_state.mark_success(unix_time_ms());
+        *runtime_state.last_error.write().await = None;
+        record_tls_reload_result(
+            TLS_RUNTIME_FOUNDATION_CONSUMER,
+            "ok",
+            Some(started_at.elapsed().as_secs_f64()),
+            Some(published.generation.0),
+        );
+
+        Ok(Some(published))
+    }
+
+    pub fn spawn_poll_loop<C>(
+        self: Arc<Self>,
+        runtime_state: Arc<TlsReloadRuntimeState<TlsMaterialSnapshot>>,
+        consumer: Arc<C>,
+    ) -> Option<JoinHandle<()>>
+    where
+        C: TlsConsumer<TlsMaterialSnapshot>,
+    {
+        if !self.options.enabled {
+            debug!(source = %self.source.base_dir.display(), "TLS reload disabled; poll loop not started");
+            return None;
+        }
+
+        if !matches!(self.options.detect_mode, ReloadDetectMode::Poll | ReloadDetectMode::Hybrid) {
+            debug!(source = %self.source.base_dir.display(), "TLS poll loop skipped for non-poll detect mode");
+            return None;
+        }
+
+        let interval_duration = self.options.interval;
+        info!(
+            source = %self.source.base_dir.display(),
+            interval_secs = interval_duration.as_secs(),
+            "TLS poll reload loop enabled"
+        );
+
+        Some(tokio::spawn(async move {
+            let mut interval = tokio::time::interval(interval_duration);
+            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            interval.tick().await;
+
+            loop {
+                interval.tick().await;
+                if let Err(err) = self.reload_once(runtime_state.as_ref(), consumer.as_ref()).await {
+                    warn!(
+                        source = %self.source.base_dir.display(),
+                        error = %err,
+                        "TLS reload failed (will retry)"
+                    );
+                    *runtime_state.last_error.write().await = Some(err.to_string());
+                }
+            }
+        }))
+    }
+}
+
+fn unix_time_ms() -> u64 {
+    SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::source::TlsSource;
+    use std::sync::Arc;
+
+    struct NopConsumer;
+
+    impl TlsConsumer<crate::material::TlsMaterialSnapshot> for NopConsumer {
+        fn on_publish(
+            &self,
+            _generation: TlsGeneration,
+            _state: Arc<TlsPublishedState<crate::material::TlsMaterialSnapshot>>,
+        ) -> Result<(), TlsRuntimeError> {
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn reload_once_skips_when_fingerprint_unchanged() {
+        let temp = tempfile::tempdir().expect("tempdir");
+        let source = TlsSource::from_directory(temp.path().to_path_buf());
+        let options = TlsReloadOptions::default();
+        let coordinator = TlsReloadCoordinator::new(source.clone(), options);
+
+        // Load the actual snapshot from the (empty) temp dir so its fingerprint
+        // matches what reload_once will observe on the next load.
+        let initial_snapshot = coordinator.load_initial_snapshot().await.expect("initial load");
+        let initial = coordinator.publish_initial_state(initial_snapshot).await;
+        let runtime_state = TlsReloadRuntimeState::new(initial);
+
+        let consumer = NopConsumer;
+        let result = coordinator.reload_once(&runtime_state, &consumer).await;
+        // Fingerprint has not changed → should skip and return Ok(None).
+        assert!(result.is_ok(), "reload_once should succeed: {:?}", result.err());
+        assert!(result.unwrap().is_none(), "should skip when fingerprint unchanged");
+    }
+}

crates/tls-runtime/src/debug.rs

@@ -0,0 +1,106 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::state::TlsRuntimeStatusSnapshot;
+use serde::Serialize;
+
+#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
+pub struct TlsConsumerStatusItem {
+    pub consumer: &'static str,
+    pub generation: u64,
+    pub has_root_ca: bool,
+    pub has_mtls_identity: bool,
+}
+
+#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
+pub struct TlsDebugStatusResponse {
+    pub foundation: TlsRuntimeStatusSnapshot,
+    pub consumers: Vec<TlsConsumerStatusItem>,
+}
+
+#[derive(Debug, Clone)]
+pub struct TlsDebugStatusResponseBuilder {
+    foundation: TlsRuntimeStatusSnapshot,
+    consumers: Vec<TlsConsumerStatusItem>,
+}
+
+impl TlsDebugStatusResponse {
+    pub fn builder(foundation: TlsRuntimeStatusSnapshot) -> TlsDebugStatusResponseBuilder {
+        TlsDebugStatusResponseBuilder {
+            foundation,
+            consumers: Vec::new(),
+        }
+    }
+}
+
+impl TlsDebugStatusResponseBuilder {
+    pub fn push_consumers<I>(mut self, sources: I) -> Self
+    where
+        I: IntoIterator<Item = TlsConsumerStatusItem>,
+    {
+        self.consumers.extend(sources);
+        self
+    }
+
+    pub fn build(self) -> TlsDebugStatusResponse {
+        TlsDebugStatusResponse {
+            foundation: self.foundation,
+            consumers: self.consumers,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{TlsConsumerStatusItem, TlsDebugStatusResponse};
+    use crate::state::{
+        TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection,
+        TlsRuntimeStatusSnapshot,
+    };
+
+    #[test]
+    fn builder_produces_structured_response() {
+        let foundation = TlsRuntimeStatusSnapshot {
+            runtime: TlsRuntimeRuntimeSection {
+                generation: 5,
+                reload_enabled: true,
+                detect_mode: "poll",
+                last_attempt_time: Some(1),
+                last_success_time: Some(2),
+                last_error: None,
+                source_path: "/tmp/tls".to_string(),
+            },
+            outbound: TlsRuntimeOutboundSection {
+                has_roots: true,
+                has_mtls_identity: false,
+            },
+            server: TlsRuntimeServerSection { has_material: true },
+            consumer: TlsRuntimeConsumerSection { stale_generation: false },
+        };
+
+        let response = TlsDebugStatusResponse::builder(foundation)
+            .push_consumers([TlsConsumerStatusItem {
+                consumer: "test_consumer",
+                generation: 7,
+                has_root_ca: true,
+                has_mtls_identity: false,
+            }])
+            .build();
+
+        let json = serde_json::to_value(response).expect("response should serialize");
+        assert!(json.get("foundation").is_some());
+        assert!(json.get("consumers").is_some());
+        assert!(json["consumers"].is_array());
+    }
+}

crates/tls-runtime/src/error.rs

@@ -0,0 +1,32 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::PathBuf;
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum TlsRuntimeError {
+    #[error("TLS source path is empty")]
+    EmptySourcePath,
+    #[error("TLS directory does not exist: {path}")]
+    DirectoryNotFound { path: PathBuf },
+    #[error("TLS path is not a directory: {path}")]
+    NotADirectory { path: PathBuf },
+    #[error("TLS material error: {0}")]
+    Material(String),
+    #[error("TLS publication error: {0}")]
+    Publication(String),
+    #[error("I/O error: {0}")]
+    Io(#[from] std::io::Error),
+}