refactor: update binary field types and conversions in RPC and protofiles (#2619)

Signed-off-by: 唐小鸭 <tangtang1251@qq.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <hello@rustfs.com>

.docker/observability/grafana/dashboards/rustfs.json

@@ -2641,7 +2641,7 @@
       },
       "gridPos": {
         "h": 7,
-        "w": 24,
+        "w": 12,
         "x": 0,
         "y": 73
       },
@@ -2670,7 +2670,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_network_host_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
           "legendFormat": "RX - {{job}}",
           "range": true,
           "refId": "C"
@@ -2681,7 +2681,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_network_host_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
           "legendFormat": "TX - {{job}}",
           "range": true,
           "refId": "D"
@@ -2690,6 +2690,115 @@
       "title": "Network",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 73
+      },
+      "id": 500,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job, interface) (rate(rustfs_system_network_host_network_io_per_interface{job=~\"$job\", direction=\"received\"}[5m]))",
+          "legendFormat": "RX {{interface}} - {{job}}",
+          "range": true,
+          "refId": "E"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job, interface) (rate(rustfs_system_network_host_network_io_per_interface{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "legendFormat": "TX {{interface}} - {{job}}",
+          "range": true,
+          "refId": "F"
+        }
+      ],
+      "title": "Network by Interface",
+      "type": "timeseries"
+    },
     {
       "datasource": {
         "type": "prometheus",

.github/AGENTS.md

@@ -25,6 +25,12 @@ Current `test-and-lint` gate includes:
 
 - `cargo nextest run --all --exclude e2e_test`
 - `cargo test --all --doc`
+- `cargo test -p rustfs get_object_chunk_fast_path`
+- `cargo test -p rustfs materialize_chunk_stream_before_commit`
+- `touch rustfs/build.rs`
+- `cargo build -p rustfs --bins --jobs 2`
+- `cargo test -p e2e_test archive_multipart_roundtrip_preserves_bytes`
+- `cargo test -p e2e_test presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path`
 - `cargo fmt --all --check`
 - `cargo clippy --all-targets --all-features -- -D warnings`
 - `./scripts/check_layer_dependencies.sh`

.github/workflows/ci.yml

@@ -53,6 +53,8 @@ on:
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
       - ".github/workflows/performance.yml"
+  merge_group:
+    types: [checks_requested]
   schedule:
     - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
   workflow_dispatch:

.github/workflows/cla.yml

@@ -17,6 +17,8 @@ name: CLA Check
 on:
   pull_request_target:
     types: [opened, synchronize, reopened]
+  merge_group:
+    types: [checks_requested]
   issue_comment:
     types: [created, edited]
 
@@ -31,7 +33,26 @@ jobs:
     if: ${{ github.event_name != 'issue_comment' || github.event.issue.pull_request }}
     runs-on: ubuntu-latest
     steps:
+      - name: Report CLA result for merge queue
+        if: github.event_name == 'merge_group'
+        uses: actions/github-script@v8
+        with:
+          script: |
+            await github.rest.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'CLA Check',
+              head_sha: context.sha,
+              status: 'completed',
+              conclusion: 'success',
+              output: {
+                title: 'CLA requirements satisfied for merge queue',
+                summary: 'Queued pull requests must satisfy the required CLA check before they enter the merge queue. This reports the existing CLA result on the merge-group SHA.'
+              }
+            });
+
       - name: Create token for rustfs/cla
+        if: github.event_name != 'merge_group'
         id: registry-token
         uses: actions/create-github-app-token@v3
         with:
@@ -42,6 +63,7 @@ jobs:
           permission-contents: write
 
       - name: Run CLA Bot
+        if: github.event_name != 'merge_group'
         uses: overtrue/cla-bot@v0.0.9
         with:
           github-token: ${{ github.token }}

AGENTS.md

@@ -6,24 +6,39 @@ Use the nearest subdirectory `AGENTS.md` for path-specific guidance.
 ## Rule Precedence
 
 1. System/developer instructions.
-2. This file (global defaults).
-3. The nearest `AGENTS.md` in the current path (more specific scope wins).
+2. Current user/task instructions.
+3. The nearest `AGENTS.md` in the current path.
+4. This file (global defaults).
 
 If repo-level instructions conflict, follow the nearest file and keep behavior aligned with CI.
 
+## Execution Discipline
+
+- Read the relevant existing code, tests, and local guidance before changing behavior.
+- State assumptions when they affect the implementation or verification path.
+- If a task has multiple plausible interpretations, list the options briefly and choose the narrowest reasonable path; ask when the ambiguity would make the change risky.
+- For multi-step work, keep the plan minimal and tied to verifiable outcomes.
+- Avoid redundant file reads, repeated commands, and unnecessary exploratory work once enough context is available.
+- A good result is a minimal diff with clear assumptions, no over-engineering, and independent verification.
+
 ## Communication and Language
 
 - Respond in the same language used by the requester.
 - Keep source code, comments, commit messages, and PR title/body in English.
+- Be concise. Avoid sycophantic openers, closing fluff, and verbose status reporting.
 
 ## Change Style for Existing Logic
 
 - Prefer direct, local code over extracting one-off helpers.
 - Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow.
+- Solve only the requested problem; do not add speculative features, configurability, or adjacent improvements.
+- Prefer editing existing code over rewriting files or reshaping unrelated logic.
+- Modify only what is required and remove only artifacts introduced by your own changes.
 - Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths.
 - Do not refactor existing code only to make it easier to unit test.
 - Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths.
 - Keep code elegant, concise, and direct. Prefer minimal, readable implementations over over-engineering and excessive abstraction. Use comments to clarify non-obvious intent and invariants, not to compensate for unclear code.
+- Mention unrelated issues when useful, but do not fix them as part of a narrow task.
 
 ## Constant and String Usage
 
@@ -44,6 +59,8 @@ Reference the source files above instead.
 
 ## Verification Before PR
 
+Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion.
+
 For code changes, run and pass the following before opening a PR:
 
 ```bash
@@ -69,6 +86,7 @@ Do not open a PR with code changes when the required checks fail.
 - Use `N/A` for non-applicable template sections.
 - Include verification commands in the PR description.
 - When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
+- Do not include the literal sequence `\n` in any GitHub issue, pull request, or discussion comment.
 - After fixing code review comments or CI findings, always mark corresponding review
   comments/threads as resolved before returning to the user.
 - In handling review comments, confirm the underlying issue before changing code.

ARCHITECTURE.md

@@ -0,0 +1,376 @@
+# ARCHITECTURE.md
+
+> Last updated: 2026-04-13 · Revision: 1 (draft)
+>
+> This document describes the high-level architecture of RustFS.
+> If you want to familiarize yourself with the code base, you are in the right place!
+>
+> See also [CONTRIBUTING.md](CONTRIBUTING.md) for development workflow.
+
+## Bird's Eye View
+
+RustFS is a high-performance, S3-compatible distributed object storage system written
+in Rust. It uses erasure coding for data durability, supports multi-tenancy through
+IAM/STS, and provides a web-based admin console.
+
+A running RustFS node exposes:
+
+- **S3 API** (port 9000) — the primary data path for object CRUD
+- **Admin API** (port 9000, `/minio/` prefix) — cluster management, IAM, metrics
+- **Console** (port 9001) — web UI backed by the Admin API
+- **Inter-node RPC** (gRPC/tonic) — cluster communication for distributed mode
+
+The core data flow for a PUT request looks like:
+
+```
+HTTP request
+  → server (TLS, auth, routing, compression)
+    → app/object_usecase (validation, policy, lifecycle)
+      → storage/ecfs (erasure coding, encryption, checksums)
+        → ecstore (disk pool selection, data distribution)
+          → rio (reader pipeline: encrypt → compress → hash → write)
+            → io-core (zero-copy I/O, buffer pool, direct I/O)
+              → local disk / remote disk via RPC
+```
+
+## Code Map
+
+The repository is a Cargo workspace with a flat `crates/` layout:
+
+```
+rustfs/                      # Workspace root (virtual manifest)
+├── rustfs/                  # Main binary + library crate (75K lines)
+│   └── src/
+│       ├── main.rs          # Entry point, startup sequence
+│       ├── lib.rs           # Module tree root
+│       ├── server/          # HTTP server, TLS, routing, middleware
+│       ├── admin/           # Admin API handlers and console
+│       ├── app/             # Use-case layer (object, bucket, multipart)
+│       ├── storage/         # Storage engine interface and implementation
+│       ├── auth.rs          # S3 request authentication
+│       ├── config/          # CLI args, config parsing, workload profiles
+│       └── ...
+├── crates/                  # 39 library crates
+│   ├── ecstore/             # Erasure-coded storage engine (⚠️ 87K lines)
+│   ├── rio/                 # Reader I/O pipeline (encrypt, compress, hash)
+│   ├── io-core/             # Zero-copy I/O, scheduling, buffer pool
+│   ├── io-metrics/          # I/O metrics collection
+│   ├── common/              # Shared runtime state, globals, data usage types
+│   ├── config/              # Configuration types and parsing
+│   ├── utils/               # Pure utility functions
+│   ├── ...                  # (see "Crate Reference" below)
+│   └── e2e_test/            # End-to-end integration tests
+└── docs/                    # Design documents and analysis
+```
+
+### Main Crate Layers (`rustfs/src/`)
+
+The main crate is organized in layers, top to bottom:
+
+| Layer | Directory | Responsibility |
+|-------|-----------|----------------|
+| **Server** | `server/` | HTTP listener, TLS, CORS, compression, middleware, graceful shutdown |
+| **Admin** | `admin/` | Admin API routing, 30+ handler modules, web console |
+| **App** | `app/` | Use-case orchestration: object_usecase, bucket_usecase, multipart_usecase |
+| **Storage** | `storage/` | S3 API translation, erasure-coded FS, SSE encryption, RPC, concurrency |
+| **Auth** | `auth.rs` | S3 signature verification, credential validation |
+| **Config** | `config/` | CLI parsing, config struct, workload profiles |
+
+A request flows **downward** through the layers. No layer should reach upward
+(e.g., storage must not import from admin).
+
+### Crate Reference
+
+Crates are organized in a dependency DAG with 9 depth levels (0 = leaf, 8 = top):
+
+```
+Depth 0 — LEAF (no internal deps):
+  appauth, checksums, config, credentials, crypto, io-metrics,
+  madmin, s3-common, workers, zip
+
+Depth 1:
+  io-core (→ io-metrics)
+  policy (→ config, credentials, crypto)
+  utils (→ config)                        ⚠️ inverted: utils should be leaf
+
+Depth 2:
+  concurrency, filemeta, keystone, kms, lock, obs,
+  signer, targets, trusted-proxies
+
+Depth 3:
+  common (→ filemeta, madmin)             ⚠️ inverted: common should be leaf
+
+Depth 4:
+  object-capacity, protos, rio
+
+Depth 5 — CORE:
+  ecstore (16 internal deps, 11 dependents — the architectural heart)
+
+Depth 6:
+  audit, heal, iam, metrics, notify, s3select-api, scanner
+
+Depth 7:
+  object-io, protocols, s3select-query
+
+Depth 8 — TOP:
+  rustfs (35 internal deps — the binary, depends on almost everything)
+```
+
+#### By Domain
+
+**Core Infrastructure:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `config` | 3.3K | Configuration types and environment parsing |
+| `utils` | 8.7K | Pure utilities (paths, compression, network, retry) |
+| `common` | 4.4K | Shared runtime state, globals, data usage types, metrics |
+| `madmin` | 5.5K | Admin API request/response types |
+
+**I/O Pipeline:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `io-core` | 6.5K | Zero-copy I/O, buffer pool, direct I/O, scheduling, backpressure |
+| `io-metrics` | 4.5K | I/O operation metrics and counters |
+| `rio` | 6.9K | Composable reader chain (encrypt → compress → hash → limit) |
+| `object-io` | 2.4K | High-level object read/write using rio + ecstore |
+| `concurrency` | 1.8K | Concurrency control wrappers over io-core |
+
+**Storage Engine:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `ecstore` | 87K | ⚠️ Erasure-coded storage: disks, pools, buckets, replication, lifecycle |
+| `filemeta` | 10K | File/object metadata types and versioning |
+| `checksums` | 732 | Checksum computation |
+| `lock` | 7.1K | Distributed lock manager |
+| `heal` | 5.9K | Data healing / bitrot repair |
+| `scanner` | 5.4K | Background data usage scanner |
+| `object-capacity` | 2.5K | Capacity tracking and management |
+
+**Security & Auth:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `crypto` | 1.6K | Encryption primitives |
+| `credentials` | 713 | Credential types (access key / secret key) |
+| `signer` | 1.4K | S3 v4 request signing |
+| `iam` | 9.0K | Identity and access management |
+| `policy` | 8.8K | Policy engine (S3 bucket/IAM policies) |
+| `kms` | 8.1K | Key management service integration |
+| `keystone` | 1.9K | OpenStack Keystone auth |
+| `appauth` | 143 | Application-level auth tokens |
+
+**Protocol & API:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `protos` | 5.7K | Protobuf/gRPC definitions for inter-node RPC |
+| `protocols` | 18K | FTP/FTPS, WebDAV, Swift API support |
+| `s3-common` | 738 | Shared S3 types |
+| `s3select-api` | 1.9K | S3 Select interface |
+| `s3select-query` | 3.6K | S3 Select query engine |
+
+**Observability:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `metrics` | 8.4K | Prometheus metric collectors |
+| `io-metrics` | 4.5K | I/O-specific metrics |
+| `obs` | 5.6K | OpenTelemetry tracing and telemetry |
+| `audit` | 2.4K | Audit logging |
+
+**Events:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `notify` | 5.5K | Event notification system |
+| `targets` | 3.2K | Notification targets (Kafka, AMQP, webhook, etc.) |
+
+**Other:**
+
+| Crate | Lines | Purpose |
+|-------|-------|---------|
+| `trusted-proxies` | 4.0K | Trusted proxy / IP forwarding |
+| `zip` | 986 | ZIP archive support for bulk downloads |
+| `workers` | 136 | Simple worker abstraction |
+
+## Architecture Invariants
+
+> These are rules that the codebase should follow. Some are currently violated
+> (marked with ⚠️). Documenting them here makes the violations explicit and
+> trackable.
+
+1. **Layers flow downward.** Server → Admin/App → Storage → ecstore → rio/io-core.
+   No upward imports.
+
+2. **Leaf crates have zero internal dependencies.** `config`, `credentials`, `crypto`,
+   `io-metrics`, `madmin`, `s3-common` should depend only on external crates.
+   - ⚠️ VIOLATED: `utils` depends on `config`, `common` depends on `filemeta` and `madmin`.
+
+3. **Each type has exactly one definition.** Types shared across crates must be defined
+   in one crate and re-exported or imported by others.
+   - ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
+     `BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
+
+4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
+   storage-level abstractions (objects, buckets, disks, pools).
+
+5. **The `rustfs` binary crate is the only place that wires everything together.**
+   Individual crates should be testable in isolation.
+
+6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
+   not bare `Error`).
+   - ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
+      `heal` use `anyhow` in library code.
+
+## Known Structural Issues
+
+> This section documents known problems in the current architecture.
+> It exists so the team can track and address them deliberately.
+
+### Critical
+
+- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
+  but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
+  types instead of importing them.
+
+- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
+  bucket management, erasure coding, replication, lifecycle, RPC, and configuration
+  — all in one crate. It should be decomposed along its existing subdirectories.
+
+### High
+
+- **Dependency inversions.** `utils → config` and `common → filemeta/madmin` break
+  the layering model. These need to be untangled.
+
+- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
+  concurrency, and rustfs/storage. Should be defined once with builder/composition.
+
+### Medium
+
+- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
+  mixed naming (bare `Error` vs descriptive names).
+
+- **Ambiguous common vs utils boundary.** Both described as "utilities and data
+  structures." Need clear ownership rules.
+
+## Cross-Cutting Concerns
+
+### Error Handling
+
+The project convention is `thiserror` for typed errors with descriptive names.
+See `AGENTS.md`: "Prefer thiserror for library-facing error types."
+
+```rust
+// GOOD
+#[derive(Debug, thiserror::Error)]
+pub enum StorageError {
+    #[error("disk not found: {0}")]
+    DiskNotFound(String),
+}
+
+// AVOID
+pub enum Error { ... }        // too generic
+anyhow::Result<T>             // in library code (OK in tests/CLI)
+```
+
+### Logging & Tracing
+
+- Use `tracing` crate (`info!`, `warn!`, `error!`, `debug!`, `trace!`)
+- Structured fields: `tracing::info!(bucket = %name, "created bucket")`
+- Spans for request-scoped context
+
+### Metrics
+
+- Prometheus-style metrics via `rustfs-obs` runtime and schema
+- I/O-specific counters via `rustfs-io-metrics`
+- Registration happens at crate level, collection/reporting in `rustfs-obs`
+
+### Testing
+
+- Unit tests: `#[cfg(test)] mod tests` in the same file
+- Integration tests: inside respective crates (not top-level `tests/`)
+- E2E tests: `crates/e2e_test/` — tests against a running server
+- Run all: `make test` or `cargo nextest run`
+
+## Startup Sequence
+
+The binary (`main.rs`) boots in this order:
+
+1. Environment variable compatibility (`MINIO_*` → `RUSTFS_*`)
+2. Tokio runtime construction
+3. CLI argument parsing
+4. License, observability, TLS, trusted proxies initialization
+5. Config parsing, server address resolution
+6. Credentials, endpoints, local disks, lock client initialization
+7. Capacity management initialization
+8. HTTP server start (S3 API + optional console)
+9. ECStore initialization (erasure coding storage engine)
+10. Global config, background replication, KMS
+11. Optional: FTP/FTPS/WebDAV servers
+12. Event notifier, audit system, deadlock detector
+13. Bucket metadata, IAM, Keystone, OIDC
+14. Scanner and heal manager
+15. Metrics system, mark `FullReady`
+16. Wait for shutdown signal → graceful shutdown
+
+## Dependency Diagram (Simplified)
+
+```
+                            ┌─────────┐
+                            │  rustfs │  (binary + lib, 75K lines)
+                            │  main   │
+                            └────┬────┘
+                                 │
+                 ┌───────────────┼───────────────┐
+                 │               │               │
+            ┌────▼────┐    ┌────▼────┐    ┌─────▼─────┐
+            │ server  │    │  admin  │    │    app     │
+            │ (HTTP)  │    │(console)│    │(use-cases) │
+            └────┬────┘    └────┬────┘    └─────┬─────┘
+                 │               │               │
+                 └───────────────┼───────────────┘
+                                 │
+                          ┌──────▼──────┐
+                          │   storage   │
+                          │ (ecfs, SSE, │
+                          │  RPC, ACL)  │
+                          └──────┬──────┘
+                                 │
+              ┌──────────────────┼──────────────────┐
+              │                  │                  │
+        ┌─────▼─────┐    ┌──────▼──────┐    ┌──────▼──────┐
+        │  ecstore   │    │     rio     │    │   io-core   │
+        │ (87K,core) │    │  (readers)  │    │ (zero-copy) │
+        └─────┬──────┘    └─────────────┘    └─────────────┘
+              │
+    ┌─────┬──┼──┬─────┬──────┐
+    │     │  │  │     │      │
+ common utils config policy filemeta ...
+```
+
+## How to Navigate
+
+- **"Where does S3 PutObject go?"**
+  `server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
+  `ecstore` distributes → `rio` encrypts/compresses → `io-core` writes
+
+- **"Where are bucket policies enforced?"**
+  `app/bucket_usecase` calls into `crates/policy/`
+
+- **"Where is replication configured?"**
+  `admin/handlers/replication.rs` and `admin/handlers/site_replication.rs` for API,
+  `ecstore/src/bucket/replication/` for engine
+
+- **"Where do I add a new admin endpoint?"**
+  Add handler in `admin/handlers/`, register in `admin/router.rs`
+
+- **"Where do I add a new metric?"**
+  Define descriptor/collector in `crates/obs/src/metrics/`, expose via `/minio/v2/metrics`
+
+---
+
+*Inspired by [matklad's ARCHITECTURE.md](https://matklad.github.io/2021/02/06/ARCHITECTURE.md.html)
+and [rust-analyzer's architecture.md](https://github.com/rust-analyzer/rust-analyzer/blob/master/docs/book/src/contributing/architecture.md).*

Cargo.toml

@@ -31,8 +31,6 @@ members = [
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
     "crates/madmin", # Management dashboard and admin API interface
-    "crates/mcp", # MCP server for S3 operations
-    "crates/metrics", # Metrics collection and reporting
     "crates/notify", # Notification system for events
     "crates/obs", # Observability utilities
     "crates/object-capacity", # Capacity scan and refresh core
@@ -40,7 +38,7 @@ members = [
     "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
     "crates/protos", # Protocol buffer definitions
     "crates/rio", # Rust I/O utilities and abstractions
-    "crates/concurrency", # Rust I/O utilities and abstractions
+    "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
     "crates/s3-common", # Common utilities and data structures for S3 compatibility
     "crates/s3select-api", # S3 Select API interface
     "crates/s3select-query", # S3 Select query engine
@@ -52,7 +50,6 @@ members = [
     "crates/workers", # Worker thread pools and task scheduling
     "crates/io-metrics", # Zero-copy metrics collection for performance analysis
     "crates/io-core", # Zero-copy core reader and writer implementations
-    "crates/object-io", # Object I/O policy and zero-copy helper primitives
     "crates/zip", # ZIP file handling and compression
 ]
 resolver = "3"
@@ -74,6 +71,8 @@ unsafe_code = "deny"
 
 [workspace.lints.clippy]
 all = "warn"
+needless_collect = "warn"
+redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
@@ -94,16 +93,14 @@ rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
 rustfs-kms = { path = "crates/kms", version = "0.0.5" }
 rustfs-lock = { path = "crates/lock", version = "0.0.5" }
 rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
-rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
-rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
 rustfs-notify = { path = "crates/notify", version = "0.0.5" }
 rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
 rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
-rustfs-object-io = { path = "crates/object-io", version = "0.0.5" }
 rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
 rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
 rustfs-protos = { path = "crates/protos", version = "0.0.5" }
+rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 rustfs-rio = { path = "crates/rio", version = "0.0.5" }
 rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
 rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
@@ -115,27 +112,28 @@ rustfs-targets = { path = "crates/targets", version = "0.0.5" }
 rustfs-utils = { path = "crates/utils", version = "0.0.5" }
 rustfs-workers = { path = "crates/workers", version = "0.0.5" }
 rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
-rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async-compression = { version = "0.4.41" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
-axum = "0.8.8"
+async-nats = "0.47.0"
+axum = "0.8.9"
 futures = "0.3.32"
 futures-core = "0.3.32"
 futures-util = "0.3.32"
 pollster = "0.4.0"
+pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
 hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
-hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
+hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
 hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 socket2 = { version = "0.6.3", features = ["all"] }
-tokio = { version = "1.51.1", features = ["fs", "rt-multi-thread"] }
+tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
@@ -154,18 +152,16 @@ flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
 quick-xml = "0.39.2"
-rmcp = { version = "1.3.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
-schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
 argon2 = { version = "0.6.0-rc.8" }
-blake2 = "0.11.0-rc.5"
+blake2 = "0.11.0-rc.6"
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0" }
@@ -173,7 +169,7 @@ jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 openidconnect = { version = "4.0", default-features = false }
 pbkdf2 = "0.13.0-rc.10"
 rsa = { version = "0.10.0-rc.17" }
-rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
+rustls = { version = "0.23.38", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types = "1.14.0"
 sha1 = "0.11.0"
 sha2 = "0.11.0"
@@ -202,7 +198,7 @@ base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.6.0", features = ["derive", "env"] }
+clap = { version = "4.6.1", features = ["derive", "env"] }
 const-str = { version = "1.1.0", features = ["std", "proc"] }
 convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
@@ -210,21 +206,21 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "53.0.0"
+datafusion = "53.1.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
 flate2 = "1.1.9"
 glob = "0.3.3"
-google-cloud-storage = "1.10.0"
-google-cloud-auth = "1.8.0"
-hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
+google-cloud-storage = "1.11.0"
+google-cloud-auth = "1.9.0"
+hashbrown = { version = "0.17.0", features = ["serde", "rayon"] }
 hex = "0.4.3"
 hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.184"
+libc = "0.2.185"
 libsystemd = "0.7.2"
 local-ip-address = "0.6.11"
 memmap2 = "0.9.10"
@@ -244,17 +240,17 @@ path-clean = "1.0.1"
 percent-encoding = "2.3.2"
 pin-project-lite = "0.2.17"
 pretty_assertions = "1.4.1"
-rand = { version = "0.10.0", features = ["serde"] }
+rand = { version = "0.10.1", features = ["serde"] }
 ratelimit = "0.10.1"
-rayon = "1.11.0"
+rayon = "1.12.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
-rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] }
+rumqttc = { package = "rumqttc-next", version = "0.30.0", features = ["websocket"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
-s3s = { git = "https://github.com/rustfs/s3s", rev = "010ae03bdd8c93a9c046108190987bb0e6cb0692", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
 serial_test = "3.4.0"
 shadow-rs = { version = "1.7.1", default-features = false }
 siphasher = "1.0.2"
@@ -270,14 +266,14 @@ tempfile = "3.27.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
-tracing-appender = "0.2.4"
+tracing-appender = "0.2.5"
 tracing-error = "0.2.1"
 tracing-opentelemetry = "0.32.1"
 tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.23.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
@@ -288,7 +284,7 @@ zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
-dial9-tokio-telemetry = { version = "0.2", git = "https://github.com/dial9-rs/dial9-tokio-telemetry.git", rev = "60502082601b647c4a51962595721f631b7bbce1" }
+dial9-tokio-telemetry = "0.3"
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
 opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
@@ -320,7 +316,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
 pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] }
 
 [workspace.metadata.cargo-shear]
-ignored = ["rustfs", "rustfs-mcp"]
+ignored = ["rustfs"]
 
 [profile.release]
 opt-level = 3

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM alpine:3.23 AS build
+FROM alpine:3.23.4 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
@@ -54,7 +54,7 @@ RUN set -eux; \
     rm -rf rustfs.zip /build/.tmp || true
 
 
-FROM alpine:3.23
+FROM alpine:3.23.4
 
 ARG RELEASE=latest
 ARG BUILD_DATE
@@ -73,7 +73,7 @@ LABEL name="RustFS" \
       license="Apache-2.0"
 
 RUN apk update && \
-    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
+    apk add --no-cache ca-certificates coreutils curl
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

README.md

@@ -100,28 +100,28 @@ To get started with RustFS, follow these steps:
 curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
 ```
 
-### 2\. Docker Quick Start (Option 2)
+### 2. Docker Quick Start (Option 2)
 
 The RustFS container runs as a non-root user `rustfs` (UID `10001`). If you run Docker with `-v` to mount a host directory, please ensure the host directory owner is set to `10001`, otherwise you will encounter permission denied errors.
 
 ```bash
- # Create data and logs directories
- mkdir -p data logs
+# Create data and logs directories
+mkdir -p data logs
 
- # Change the owner of these directories
- chown -R 10001:10001 data logs
+# Change the owner of these directories
+chown -R 10001:10001 data logs
 
- # Using latest version
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
+# Using latest version
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
- # Using specific version
- docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
+# Using specific version
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-alpha.76
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
 
 ```bash
- podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
+podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 ```
 
 You can also use Docker Compose. Using the `docker-compose.yml` file in the root directory:
@@ -219,6 +219,48 @@ rustfs --help
 
 **NOTE**: To access the RustFS instance via `https`, please refer to the [TLS Configuration Docs](https://docs.rustfs.com/integration/tls-configured.html).
 
+### OIDC Roles Claim (Microsoft Entra ID)
+
+RustFS supports mapping an OIDC claim containing role values into the existing
+authorization pipeline. The `roles_claim` setting is **optional**: when unset or
+empty, only the `groups` claim contributes to authorization (same as older
+RustFS releases). For Microsoft Entra ID app roles, set `roles_claim=roles` so
+both console admin checks and bucket IAM policies can evaluate those roles.
+
+Example environment configuration (opt-in roles claim):
+
+```bash
+RUSTFS_IDENTITY_OPENID_ENABLE=on
+RUSTFS_IDENTITY_OPENID_CONFIG_URL="https://login.microsoftonline.com/<tenant-id>/v2.0/.well-known/openid-configuration"
+RUSTFS_IDENTITY_OPENID_CLIENT_ID="<client-id>"
+RUSTFS_IDENTITY_OPENID_CLIENT_SECRET="<client-secret>"
+RUSTFS_IDENTITY_OPENID_SCOPES="openid,profile,email"
+RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM="groups"
+RUSTFS_IDENTITY_OPENID_ROLES_CLAIM="roles"
+```
+
+Policy condition example (evaluate app roles directly with `jwt:roles`; when
+`roles_claim` is configured, RustFS also merges those values into `jwt:groups`
+for backward compatibility with older policies):
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["admin:*"],
+      "Resource": ["arn:aws:s3:::*"],
+      "Condition": {
+        "ForAnyValue:StringEquals": {
+          "jwt:roles": ["RustFS.ConsoleAdmin"]
+        }
+      }
+    }
+  ]
+}
+```
+
 ## Documentation
 
 For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).

_typos.toml

@@ -39,7 +39,7 @@ abd = "abd"
 mak = "mak"
 gae = "gae"
 GAE = "GAE"
-writeable = "writeable"
+thr = "thr"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
 consts = "consts"

crates/audit/src/factory.rs

@@ -14,28 +14,19 @@
 
 use crate::AuditEntry;
 use async_trait::async_trait;
-use hashbrown::HashSet;
-use rumqttc::QoS;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_WEBHOOK_KEYS, ENV_AUDIT_MQTT_KEYS, ENV_AUDIT_WEBHOOK_KEYS};
-use rustfs_config::{
-    AUDIT_DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
-    MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
-    MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT,
-    WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR,
-    WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
-};
+use rustfs_config::AUDIT_DEFAULT_DIR;
+use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
 use rustfs_ecstore::config::KVS;
 use rustfs_targets::{
     Target,
-    error::TargetError,
-    target::{
-        mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
-        webhook::WebhookArgs,
+    config::{
+        build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
+        validate_pulsar_config, validate_webhook_config,
     },
+    error::TargetError,
+    target::TargetType,
 };
-use std::time::Duration;
-use tracing::{debug, warn};
-use url::Url;
+use std::collections::HashSet;
 
 /// Trait for creating targets from configuration
 #[async_trait]
@@ -49,10 +40,6 @@ pub trait TargetFactory: Send + Sync {
     /// Returns a set of valid configuration field names for this target type.
     /// This is used to filter environment variables.
     fn get_valid_fields(&self) -> HashSet<String>;
-
-    /// Returns a set of valid configuration env field names for this target type.
-    /// This is used to filter environment variables.
-    fn get_valid_env_fields(&self) -> HashSet<String>;
 }
 
 /// Factory for creating Webhook targets
@@ -61,71 +48,18 @@ pub struct WebhookTargetFactory;
 #[async_trait]
 impl TargetFactory for WebhookTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        // All config values are now read directly from the merged `config` KVS.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        let parsed_endpoint = endpoint.trim();
-        let endpoint_url = Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let args = WebhookArgs {
-            enable: true, // If we are here, it's already enabled.
-            endpoint: endpoint_url,
-            auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
-            queue_dir: config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(WEBHOOK_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            client_cert: config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default(),
-            client_key: config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default(),
-            client_ca: config.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
-            skip_tls_verify: config
-                .lookup(WEBHOOK_SKIP_TLS_VERIFY)
-                .and_then(|v| v.parse::<bool>().ok())
-                .unwrap_or(RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        // Validation also uses the merged `config` KVS directly.
-        let endpoint = config
-            .lookup(WEBHOOK_ENDPOINT)
-            .ok_or_else(|| TargetError::Configuration("Missing webhook endpoint".to_string()))?;
-        debug!("endpoint: {}", endpoint);
-        let parsed_endpoint = endpoint.trim();
-        Url::parse(parsed_endpoint)
-            .map_err(|e| TargetError::Configuration(format!("Invalid endpoint URL: {e} (value: '{parsed_endpoint}')")))?;
-
-        let client_cert = config.lookup(WEBHOOK_CLIENT_CERT).unwrap_or_default();
-        let client_key = config.lookup(WEBHOOK_CLIENT_KEY).unwrap_or_default();
-
-        if client_cert.is_empty() != client_key.is_empty() {
-            return Err(TargetError::Configuration(
-                "Both client_cert and client_key must be specified together".to_string(),
-            ));
-        }
-
-        let queue_dir = config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string());
-        if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
-            return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
-        }
-
-        Ok(())
+        validate_webhook_config(config, AUDIT_DEFAULT_DIR)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
     }
-
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
-    }
 }
 
 /// Factory for creating MQTT targets
@@ -134,112 +68,54 @@ pub struct MQTTTargetFactory;
 #[async_trait]
 impl TargetFactory for MQTTTargetFactory {
     async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let broker_url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let topic = config
-            .lookup(MQTT_TOPIC)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT topic".to_string()))?;
-
-        let args = MQTTArgs {
-            enable: true, // Assumed enabled.
-            broker: broker_url,
-            topic,
-            qos: config
-                .lookup(MQTT_QOS)
-                .and_then(|v| v.parse::<u8>().ok())
-                .map(|q| match q {
-                    0 => QoS::AtMostOnce,
-                    1 => QoS::AtLeastOnce,
-                    2 => QoS::ExactlyOnce,
-                    _ => QoS::AtLeastOnce,
-                })
-                .unwrap_or(QoS::AtLeastOnce),
-            username: config.lookup(MQTT_USERNAME).unwrap_or_default(),
-            password: config.lookup(MQTT_PASSWORD).unwrap_or_default(),
-            max_reconnect_interval: config
-                .lookup(MQTT_RECONNECT_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(5)),
-            keep_alive: config
-                .lookup(MQTT_KEEP_ALIVE_INTERVAL)
-                .and_then(|v| v.parse::<u64>().ok())
-                .map(Duration::from_secs)
-                .unwrap_or_else(|| Duration::from_secs(30)),
-            tls: MQTTTlsConfig::from_values(
-                config.lookup(MQTT_TLS_POLICY).as_deref(),
-                config.lookup(MQTT_TLS_CA).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-                config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-                config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-                config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-            )?,
-            queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
-            queue_limit: config
-                .lookup(MQTT_QUEUE_LIMIT)
-                .and_then(|v| v.parse::<u64>().ok())
-                .unwrap_or(DEFAULT_LIMIT),
-            target_type: rustfs_targets::target::TargetType::AuditLog,
-        };
-
+        let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
         let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
         Ok(Box::new(target))
     }
 
     fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
-        let broker = config
-            .lookup(MQTT_BROKER)
-            .ok_or_else(|| TargetError::Configuration("Missing MQTT broker".to_string()))?;
-        let url = Url::parse(&broker)
-            .map_err(|e| TargetError::Configuration(format!("Invalid broker URL: {e} (value: '{broker}')")))?;
-
-        let tls = MQTTTlsConfig::from_values(
-            config.lookup(MQTT_TLS_POLICY).as_deref(),
-            config.lookup(MQTT_TLS_CA).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_CERT).as_deref(),
-            config.lookup(MQTT_TLS_CLIENT_KEY).as_deref(),
-            config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
-            config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
-        )?;
-        validate_mqtt_broker_url(&url, &tls)?;
-
-        if config.lookup(MQTT_TOPIC).is_none() {
-            return Err(TargetError::Configuration("Missing MQTT topic".to_string()));
-        }
-
-        if let Some(qos_str) = config.lookup(MQTT_QOS) {
-            let qos = qos_str
-                .parse::<u8>()
-                .map_err(|_| TargetError::Configuration("Invalid QoS value".to_string()))?;
-            if qos > 2 {
-                return Err(TargetError::Configuration("QoS must be 0, 1, or 2".to_string()));
-            }
-        }
-
-        let queue_dir = config.lookup(MQTT_QUEUE_DIR).unwrap_or_default();
-        if !queue_dir.is_empty() {
-            if !std::path::Path::new(&queue_dir).is_absolute() {
-                return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
-            }
-            if let Some(qos_str) = config.lookup(MQTT_QOS)
-                && qos_str == "0"
-            {
-                warn!("Using queue_dir with QoS 0 may result in event loss");
-            }
-        }
-
-        Ok(())
+        validate_mqtt_config(config)
     }
 
     fn get_valid_fields(&self) -> HashSet<String> {
         AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
     }
+}
 
-    fn get_valid_env_fields(&self) -> HashSet<String> {
-        ENV_AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
+pub struct NATSTargetFactory;
+
+#[async_trait]
+impl TargetFactory for NATSTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_nats_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_NATS_KEYS.iter().map(|s| s.to_string()).collect()
+    }
+}
+
+pub struct PulsarTargetFactory;
+
+#[async_trait]
+impl TargetFactory for PulsarTargetFactory {
+    async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
+        let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+        let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
+        Ok(Box::new(target))
+    }
+
+    fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
+        validate_pulsar_config(config, AUDIT_DEFAULT_DIR)
+    }
+
+    fn get_valid_fields(&self) -> HashSet<String> {
+        AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
     }
 }

crates/audit/src/global.rs

@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{AuditEntry, AuditResult, AuditSystem};
+use crate::{AuditEntry, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
 use rustfs_ecstore::config::Config;
 use std::sync::{Arc, OnceLock};
 use tracing::{debug, error, trace, warn};
@@ -89,6 +89,15 @@ pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
     with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
 }
 
+/// Returns per-target audit delivery metrics for Prometheus collection.
+pub async fn audit_target_metrics() -> Vec<AuditTargetMetricSnapshot> {
+    if let Some(system) = audit_system() {
+        system.snapshot_target_metrics().await
+    } else {
+        Vec::new()
+    }
+}
+
 /// Check if the global audit system is running
 pub async fn is_audit_system_running() -> bool {
     if let Some(system) = audit_system() {

crates/audit/src/lib.rs

@@ -31,4 +31,4 @@ pub use error::{AuditError, AuditResult};
 pub use global::*;
 pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation};
 pub use registry::AuditRegistry;
-pub use system::AuditSystem;
+pub use system::{AuditSystem, AuditTargetMetricSnapshot};

crates/audit/src/registry.rs

@@ -14,18 +14,17 @@
 
 use crate::{
     AuditEntry, AuditError, AuditResult,
-    factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory},
+    factory::{MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory},
 };
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
-use hashbrown::{HashMap, HashSet};
-use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, audit::AUDIT_ROUTE_PREFIX};
+use hashbrown::HashMap;
+use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_targets::arn::TargetID;
-use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
-use std::str::FromStr;
+use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
 use std::sync::Arc;
-use tracing::{debug, error, info, warn};
+use tracing::{error, info};
 
 /// Registry for managing audit targets
 pub struct AuditRegistry {
@@ -52,6 +51,8 @@ impl AuditRegistry {
         // Register built-in factories
         registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
         registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
+        registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
+        registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
 
         registry
     }
@@ -105,143 +106,22 @@ impl AuditRegistry {
         &self,
         config: &Config,
     ) -> AuditResult<Vec<Box<dyn Target<AuditEntry> + Send + Sync>>> {
-        // Collect only environment variables with the relevant prefix to reduce memory usage
-        let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
-        // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
             info!("Start working on target types...");
-
-            // 2. Prepare the configuration source
-            // 2.1. Get the configuration segment in the file, e.g. 'audit_webhook'
-            let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
-            let file_configs = config.0.get(&section_name).cloned().unwrap_or_default();
-            // 2.2. Get the default configuration for that type
-            let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
-            debug!(?default_cfg, "Get the default configuration");
-
-            // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
-            debug!(?valid_fields, "Get the legitimate configuration fields");
-
-            // 3. Resolve instance IDs and configuration overrides from environment variables
-            let mut instance_ids_from_env = HashSet::new();
-            // 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
-            let enable_prefix =
-                format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
-                    .to_uppercase();
-            for (key, value) in &all_env {
-                if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
-                    && let Some(id) = key.strip_prefix(&enable_prefix)
-                    && !id.is_empty()
-                {
-                    instance_ids_from_env.insert(id.to_lowercase());
-                }
-            }
-
-            // 3.2. Parse all relevant environment variable configurations
-            // 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
-            let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
-            // 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
-            let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
-            for (key, value) in &all_env {
-                if let Some(rest) = key.strip_prefix(&env_prefix) {
-                    // Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
-                    // Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
-                    let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
-
-                    // The first part from the right is INSTANCE_ID
-                    let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
-                    // The remaining part is FIELD_NAME
-                    let field_name_part = parts.next();
-
-                    let (field_name, instance_id) = match field_name_part {
-                        // Case 1: The format is <FIELD_NAME>_<INSTANCE_ID>
-                        // e.g., rest = "ENDPOINT_PRIMARY" -> field_name="ENDPOINT", instance_id="PRIMARY"
-                        Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
-                        // Case 2: The format is <FIELD_NAME> (without INSTANCE_ID)
-                        // e.g., rest = "ENABLE" -> field_name="ENABLE", instance_id="" (Universal configuration `_ DEFAULT_DELIMITER`)
-                        None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
-                    };
-
-                    // *** Optimization point 2: Verify whether the parsed field_name is legal ***
-                    if !field_name.is_empty() && valid_fields.contains(&field_name) {
-                        debug!(
-                            instance_id = %if instance_id.is_empty() { DEFAULT_DELIMITER } else { &instance_id },
-                            %field_name,
-                            %value,
-                            "Parsing to environment variables"
-                        );
-                        env_overrides
-                            .entry(instance_id)
-                            .or_default()
-                            .insert(field_name, value.clone());
-                    } else {
-                        // Ignore illegal field names
-                        warn!(
-                            field_name = %field_name,
-                            "Ignore environment variable fields, not found in the list of valid fields for target type {}",
-                            target_type
-                        );
-                    }
-                }
-            }
-            debug!(?env_overrides, "Complete the environment variable analysis");
-
-            // 4. Determine all instance IDs that need to be processed
-            let mut all_instance_ids: HashSet<String> =
-                file_configs.keys().filter(|k| *k != DEFAULT_DELIMITER).cloned().collect();
-            all_instance_ids.extend(instance_ids_from_env);
-            debug!(?all_instance_ids, "Determine all instance IDs");
-
-            // 5. Merge configurations and create tasks for each instance
-            for id in all_instance_ids {
-                // 5.1. Merge configuration, priority: Environment variables > File instance configuration > File default configuration
-                let mut merged_config = default_cfg.clone();
-                // Instance-specific configuration in application files
-                if let Some(file_instance_cfg) = file_configs.get(&id) {
-                    merged_config.extend(file_instance_cfg.clone());
-                }
-                // Application instance-specific environment variable configuration
-                if let Some(env_instance_cfg) = env_overrides.get(&id) {
-                    // Convert HashMap<String, String> to KVS
-                    let mut kvs_from_env = KVS::new();
-                    for (k, v) in env_instance_cfg {
-                        kvs_from_env.insert(k.clone(), v.clone());
-                    }
-                    merged_config.extend(kvs_from_env);
-                }
-                debug!(instance_id = %id, ?merged_config, "Complete configuration merge");
-
-                // 5.2. Check if the instance is enabled
-                let enabled = merged_config
-                    .lookup(ENABLE_KEY)
-                    .map(|v| {
-                        EnableState::from_str(v.as_str())
-                            .ok()
-                            .map(|s| s.is_enabled())
-                            .unwrap_or(false)
-                    })
-                    .unwrap_or(false);
-
-                if enabled {
-                    info!(instance_id = %id, "Target is enabled, ready to create a task");
-                    // 5.3. Create asynchronous tasks for enabled instances
-                    let tid = id.clone();
-                    let merged_config_arc = Arc::new(merged_config);
-                    tasks.push(async move {
-                        let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (tid, result)
-                    });
-                } else {
-                    info!(instance_id = %id, "Skip disabled target");
-                }
+            for (id, merged_config) in collect_target_configs(config, AUDIT_ROUTE_PREFIX, target_type, &valid_fields) {
+                info!(instance_id = %id, "Target is enabled, ready to create a task");
+                let tid = id.clone();
+                let merged_config_arc = Arc::new(merged_config);
+                tasks.push(async move {
+                    let result = factory.create_target(tid.clone(), &merged_config_arc).await;
+                    (tid, result)
+                });
             }
         }
 
-        // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
         while let Some((id, result)) = tasks.next().await {
             match result {

crates/audit/src/system.rs

@@ -24,6 +24,14 @@ use std::sync::Arc;
 use tokio::sync::{Mutex, RwLock, mpsc};
 use tracing::{error, info, warn};
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct AuditTargetMetricSnapshot {
+    pub failed_messages: u64,
+    pub queue_length: u64,
+    pub target_id: String,
+    pub total_messages: u64,
+}
+
 /// State of the audit system
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum AuditSystemState {
@@ -537,8 +545,14 @@ impl AuditSystem {
                                     TargetError::Timeout(_) => {
                                         warn!("Timeout sending to target {}, retrying...", target.id());
                                     }
+                                    TargetError::Dropped(reason) => {
+                                        warn!("Dropped queued payload for target {}: {}", target.id(), reason);
+                                        observability::record_target_failure();
+                                        break;
+                                    }
                                     _ => {
                                         error!("Permanent error for target {}: {}", target.id(), e);
+                                        target.record_final_failure();
                                         observability::record_target_failure();
                                         break;
                                     }
@@ -552,6 +566,7 @@ impl AuditSystem {
 
                     if retries >= MAX_RETRIES && !success {
                         warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id());
+                        target.record_final_failure();
                         observability::record_target_failure();
                     }
                 }
@@ -664,6 +679,25 @@ impl AuditSystem {
         registry.list_target_values()
     }
 
+    /// Returns per-target delivery metrics for Prometheus collection.
+    pub async fn snapshot_target_metrics(&self) -> Vec<AuditTargetMetricSnapshot> {
+        let targets = self.get_target_values().await;
+        let mut snapshots = Vec::with_capacity(targets.len());
+
+        for target in targets {
+            let delivery = target.delivery_snapshot();
+            snapshots.push(AuditTargetMetricSnapshot {
+                failed_messages: delivery.failed_messages,
+                queue_length: delivery.queue_length,
+                target_id: target.id().to_string(),
+                total_messages: delivery.total_messages,
+            });
+        }
+
+        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
+        snapshots
+    }
+
     /// Gets information about a specific target
     ///
     /// # Arguments

crates/common/src/data_usage.rs

@@ -51,14 +51,14 @@ impl AllTierStats {
     pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
         for (tier, st) in tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
     pub fn merge(&mut self, other: AllTierStats) {
         for (tier, st) in other.tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
@@ -673,7 +673,7 @@ impl DataUsageCache {
         let mut leaves = Vec::new();
         let mut remove = total - limit;
         add(self, path, &mut leaves);
-        leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
+        leaves.sort_by_key(|a| a.objects);
 
         while remove > 0 && !leaves.is_empty() {
             let Some(e) = leaves.first() else {
@@ -1097,7 +1097,7 @@ impl DataUsageInfo {
 
     /// Add bucket usage info
     pub fn add_bucket_usage(&mut self, bucket: String, usage: BucketUsageInfo) {
-        self.buckets_usage.insert(bucket.clone(), usage);
+        self.buckets_usage.insert(bucket, usage);
         self.buckets_count = self.buckets_usage.len() as u64;
         self.last_update = Some(SystemTime::now());
     }

crates/common/src/heal_channel.rs

@@ -18,7 +18,7 @@ use std::{
     fmt::{self, Display},
     sync::OnceLock,
 };
-use tokio::sync::{broadcast, mpsc};
+use tokio::sync::{broadcast, mpsc, oneshot};
 use uuid::Uuid;
 
 pub const HEAL_DELETE_DANGLING: bool = true;
@@ -206,11 +206,59 @@ pub struct HealOpts {
     pub set: Option<usize>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionDropReason {
+    QueueFull,
+    PolicyDropped,
+}
+
+impl HealAdmissionDropReason {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::QueueFull => "queue_full",
+            Self::PolicyDropped => "policy_dropped",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionResult {
+    Accepted,
+    Merged,
+    Full,
+    Dropped(HealAdmissionDropReason),
+}
+
+impl HealAdmissionResult {
+    pub fn result_label(self) -> &'static str {
+        match self {
+            Self::Accepted => "accepted",
+            Self::Merged => "merged",
+            Self::Full => "full",
+            Self::Dropped(_) => "dropped",
+        }
+    }
+
+    pub fn reason_label(self) -> &'static str {
+        match self {
+            Self::Dropped(reason) => reason.as_str(),
+            _ => "none",
+        }
+    }
+
+    pub fn is_admitted(self) -> bool {
+        matches!(self, Self::Accepted | Self::Merged)
+    }
+}
+
 /// Heal channel command type
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub enum HealChannelCommand {
     /// Start a new heal task
-    Start(HealChannelRequest),
+    Start {
+        request: HealChannelRequest,
+        response_tx: oneshot::Sender<Result<HealAdmissionResult, String>>,
+    },
     /// Query heal task status
     Query { heal_path: String, client_token: String },
     /// Cancel heal task
@@ -339,9 +387,22 @@ pub fn subscribe_heal_responses() -> broadcast::Receiver<HealChannelResponse> {
     heal_response_sender().subscribe()
 }
 
+/// Send heal start request and wait for structured admission feedback.
+pub async fn send_heal_request_with_admission(request: HealChannelRequest) -> Result<HealAdmissionResult, String> {
+    let (response_tx, response_rx) = oneshot::channel();
+    send_heal_command(HealChannelCommand::Start { request, response_tx }).await?;
+    response_rx
+        .await
+        .map_err(|e| format!("Failed to receive heal admission response: {e}"))?
+}
+
 /// Send heal start request
 pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> {
-    send_heal_command(HealChannelCommand::Start(request)).await
+    match send_heal_request_with_admission(request).await? {
+        HealAdmissionResult::Accepted | HealAdmissionResult::Merged => Ok(()),
+        HealAdmissionResult::Full => Err("Heal request queue is full".to_string()),
+        HealAdmissionResult::Dropped(reason) => Err(format!("Heal request dropped: {}", reason.as_str())),
+    }
 }
 
 /// Send heal query request
@@ -450,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str)
         }
 
         if let Some(e) = &rule.noncurrent_version_expiration {
-            if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
+            if e.noncurrent_days.is_some() {
                 return true;
             }
             if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
@@ -542,6 +603,19 @@ pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPri
 mod tests {
     use super::*;
 
+    #[test]
+    fn heal_admission_result_labels_are_stable() {
+        assert_eq!(HealAdmissionResult::Accepted.result_label(), "accepted");
+        assert_eq!(HealAdmissionResult::Merged.result_label(), "merged");
+        assert_eq!(HealAdmissionResult::Full.result_label(), "full");
+        assert_eq!(
+            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull).reason_label(),
+            "queue_full"
+        );
+        assert!(HealAdmissionResult::Merged.is_admitted());
+        assert!(!HealAdmissionResult::Full.is_admitted());
+    }
+
     #[tokio::test]
     async fn heal_response_broadcast_reaches_subscriber() {
         let mut receiver = subscribe_heal_responses();

crates/common/src/internode_metrics.rs

@@ -101,11 +101,7 @@ impl InternodeMetrics {
     pub fn snapshot(&self) -> InternodeMetricsSnapshot {
         let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
         let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
-        let dial_avg_time_nanos = if dial_samples_total == 0 {
-            0
-        } else {
-            dial_total_time_nanos / dial_samples_total
-        };
+        let dial_avg_time_nanos = dial_total_time_nanos.checked_div(dial_samples_total).unwrap_or(0);
 
         InternodeMetricsSnapshot {
             sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),

crates/common/src/metrics.rs

@@ -685,7 +685,7 @@ pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn,
     };
 
     let done_fn: CloseDiskFn = {
-        let disk = disk_name.clone();
+        let disk = disk_name;
         Arc::new(move || {
             let disk = disk.clone();
             Box::pin(async move {

crates/concurrency/Cargo.toml

@@ -26,7 +26,7 @@ thiserror = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]
-tokio = { workspace = true, features = ["full"] }
+tokio = { workspace = true, features = ["test-util","macros","rt-multi-thread"] }
 
 [features]
 default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"]

crates/config/README.md

@@ -53,8 +53,24 @@ Current guidance:
 
 ## Scanner environment aliases
 
+- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
+- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
 - `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
 - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
+- `RUSTFS_SCANNER_IDLE_MODE` (canonical)
+- `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
+
+## Drive timeout environment variables
+
+- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS`
+
+Legacy compatibility fallback:
+- `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
+  This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
 
 ## 📄 License
 

crates/config/src/audit/mod.rs

@@ -17,9 +17,13 @@
 //! webhook and MQTT audit-related settings.
 
 mod mqtt;
+mod nats;
+mod pulsar;
 mod webhook;
 
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use webhook::*;
 
 use crate::DEFAULT_DELIMITER;
@@ -30,7 +34,14 @@ pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DE
 
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
 pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
+pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
+pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
 
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
 #[allow(dead_code)]
-pub const AUDIT_SUB_SYSTEMS: &[&str] = &[AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS];
+pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
+    AUDIT_MQTT_SUB_SYS,
+    AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_SUB_SYS,
+    AUDIT_WEBHOOK_SUB_SYS,
+];

crates/config/src/audit/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_NATS_ENABLE: &str = "RUSTFS_AUDIT_NATS_ENABLE";
+pub const ENV_AUDIT_NATS_ADDRESS: &str = "RUSTFS_AUDIT_NATS_ADDRESS";
+pub const ENV_AUDIT_NATS_SUBJECT: &str = "RUSTFS_AUDIT_NATS_SUBJECT";
+pub const ENV_AUDIT_NATS_USERNAME: &str = "RUSTFS_AUDIT_NATS_USERNAME";
+pub const ENV_AUDIT_NATS_PASSWORD: &str = "RUSTFS_AUDIT_NATS_PASSWORD";
+pub const ENV_AUDIT_NATS_TOKEN: &str = "RUSTFS_AUDIT_NATS_TOKEN";
+pub const ENV_AUDIT_NATS_CREDENTIALS_FILE: &str = "RUSTFS_AUDIT_NATS_CREDENTIALS_FILE";
+pub const ENV_AUDIT_NATS_TLS_CA: &str = "RUSTFS_AUDIT_NATS_TLS_CA";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
+pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
+pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
+    ENV_AUDIT_NATS_ENABLE,
+    ENV_AUDIT_NATS_ADDRESS,
+    ENV_AUDIT_NATS_SUBJECT,
+    ENV_AUDIT_NATS_USERNAME,
+    ENV_AUDIT_NATS_PASSWORD,
+    ENV_AUDIT_NATS_TOKEN,
+    ENV_AUDIT_NATS_CREDENTIALS_FILE,
+    ENV_AUDIT_NATS_TLS_CA,
+    ENV_AUDIT_NATS_TLS_CLIENT_CERT,
+    ENV_AUDIT_NATS_TLS_CLIENT_KEY,
+    ENV_AUDIT_NATS_TLS_REQUIRED,
+    ENV_AUDIT_NATS_QUEUE_DIR,
+    ENV_AUDIT_NATS_QUEUE_LIMIT,
+];
+
+pub const AUDIT_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_PULSAR_ENABLE: &str = "RUSTFS_AUDIT_PULSAR_ENABLE";
+pub const ENV_AUDIT_PULSAR_BROKER: &str = "RUSTFS_AUDIT_PULSAR_BROKER";
+pub const ENV_AUDIT_PULSAR_TOPIC: &str = "RUSTFS_AUDIT_PULSAR_TOPIC";
+pub const ENV_AUDIT_PULSAR_AUTH_TOKEN: &str = "RUSTFS_AUDIT_PULSAR_AUTH_TOKEN";
+pub const ENV_AUDIT_PULSAR_USERNAME: &str = "RUSTFS_AUDIT_PULSAR_USERNAME";
+pub const ENV_AUDIT_PULSAR_PASSWORD: &str = "RUSTFS_AUDIT_PULSAR_PASSWORD";
+pub const ENV_AUDIT_PULSAR_TLS_CA: &str = "RUSTFS_AUDIT_PULSAR_TLS_CA";
+pub const ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_AUDIT_PULSAR_QUEUE_DIR: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_DIR";
+pub const ENV_AUDIT_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_AUDIT_PULSAR_ENABLE,
+    ENV_AUDIT_PULSAR_BROKER,
+    ENV_AUDIT_PULSAR_TOPIC,
+    ENV_AUDIT_PULSAR_AUTH_TOKEN,
+    ENV_AUDIT_PULSAR_USERNAME,
+    ENV_AUDIT_PULSAR_PASSWORD,
+    ENV_AUDIT_PULSAR_TLS_CA,
+    ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_AUDIT_PULSAR_QUEUE_DIR,
+    ENV_AUDIT_PULSAR_QUEUE_LIMIT,
+];
+
+pub const AUDIT_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/constants/capacity.rs

@@ -39,6 +39,9 @@ pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT";
 /// Environment variable for sample rate
 pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE";
 
+/// Environment variable for metrics logging interval
+pub const ENV_CAPACITY_METRICS_INTERVAL: &str = "RUSTFS_CAPACITY_METRICS_INTERVAL";
+
 /// Environment variable for following symbolic links during capacity calculation
 pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS";
 
@@ -89,6 +92,10 @@ pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3;
 /// Default: 200
 pub const DEFAULT_SAMPLE_RATE: usize = 200;
 
+/// Capacity metrics logging interval in seconds
+/// Default: 600 seconds (10 minutes)
+pub const DEFAULT_CAPACITY_METRICS_INTERVAL_SECS: u64 = 600;
+
 /// Follow symbolic links during capacity calculation
 /// Default: false (disabled for safety)
 pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false;
@@ -130,6 +137,7 @@ mod tests {
         assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD");
         assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT");
         assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE");
+        assert_eq!(ENV_CAPACITY_METRICS_INTERVAL, "RUSTFS_CAPACITY_METRICS_INTERVAL");
         assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS");
         assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH");
         assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT");
@@ -147,6 +155,7 @@ mod tests {
         assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
         assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
         assert_eq!(DEFAULT_SAMPLE_RATE, 200);
+        assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
         assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3);
         assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2);
         assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15);

crates/config/src/constants/drive.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Legacy global drive timeout fallback.
+/// Deprecated in favor of per-operation drive timeout knobs.
+pub const ENV_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
+
+/// Default timeout in seconds for the legacy global drive timeout fallback.
+pub const DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS: u64 = 30;
+
+/// Timeout for metadata-oriented drive operations such as `read_metadata`.
+pub const ENV_DRIVE_METADATA_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_METADATA_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_METADATA_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `disk_info()` calls on local and remote drives.
+pub const ENV_DRIVE_DISK_INFO_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `list_dir()` style metadata listing operations.
+pub const ENV_DRIVE_LIST_DIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS: u64 = 5;
+
+/// Total timeout for `walk_dir()` operations.
+pub const ENV_DRIVE_WALKDIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
+
+/// Maximum time without forward progress while consuming a `walk_dir()` stream.
+pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
+
+/// Number of consecutive failures before a suspect drive is classified as offline.
+pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
+pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
+
+/// Number of consecutive successful recovery probes before a returning drive is considered online again.
+pub const ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD: &str = "RUSTFS_DRIVE_RETURNING_SUCCESS_THRESHOLD";
+pub const DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD: u64 = 3;
+
+/// Probe interval in seconds while a drive is in the recovery path.
+pub const ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS: &str = "RUSTFS_DRIVE_RETURNING_PROBE_INTERVAL_SECS";
+pub const DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS: u64 = 2;
+
+/// Duration in seconds for classifying a recovered drive as a short offline event.
+pub const ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS: &str = "RUSTFS_DRIVE_OFFLINE_GRACE_PERIOD_SECS";
+pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
+
+/// Duration in seconds after which a recovered drive is classified as long offline.
+pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
+pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;

crates/config/src/constants/heal.rs

@@ -56,6 +56,15 @@ pub const ENV_HEAL_TASK_TIMEOUT_SECS: &str = "RUSTFS_HEAL_TASK_TIMEOUT_SECS";
 /// - Note: A higher concurrency limit can speed up healing but may lead to resource contention.
 pub const ENV_HEAL_MAX_CONCURRENT_HEALS: &str = "RUSTFS_HEAL_MAX_CONCURRENT_HEALS";
 
+/// Environment variable name that specifies the maximum number of concurrent heal operations
+/// allowed for a single erasure set.
+///
+/// - Purpose: Prevent one degraded set from consuming all global heal slots.
+/// - Unit: number of operations (usize).
+/// - Valid values: any positive integer.
+/// - Example: `export RUSTFS_HEAL_MAX_CONCURRENT_PER_SET=1`
+pub const ENV_HEAL_MAX_CONCURRENT_PER_SET: &str = "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET";
+
 /// Default value for enabling authentication for heal operations if not specified in the environment variable.
 /// - Value: true (authentication enabled).
 /// - Rationale: Enabling authentication by default enhances security for heal operations.
@@ -86,3 +95,46 @@ pub const DEFAULT_HEAL_TASK_TIMEOUT_SECS: u64 = 300; // 5 minutes
 /// - Rationale: This default concurrency limit helps balance healing speed with resource usage, preventing system overload.
 /// - Adjustments: Users may modify this value via the `RUSTFS_HEAL_MAX_CONCURRENT_HEALS` environment variable based on their system capacity and expected heal workload.
 pub const DEFAULT_HEAL_MAX_CONCURRENT_HEALS: usize = 4;
+
+/// Default maximum number of concurrent heal operations per erasure set.
+///
+/// - Value: 1 concurrent heal operation per set.
+/// - Rationale: Keeps a degraded set from monopolizing the global heal scheduler.
+pub const DEFAULT_HEAL_MAX_CONCURRENT_PER_SET: usize = 1;
+
+/// Environment variable that controls whether low-priority heal requests should merge into
+/// an existing queued request with the same deduplication key.
+pub const ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE: &str = "RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE";
+
+/// Environment variable that allows low-priority heal requests to be dropped when the queue is full.
+pub const ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: &str = "RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL";
+
+/// Environment variable that controls concurrent object heals within a single erasure-set page.
+pub const ENV_HEAL_PAGE_OBJECT_CONCURRENCY: &str = "RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY";
+
+/// Environment variable that toggles notify-driven scheduler wakeups.
+pub const ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: &str = "RUSTFS_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE";
+
+/// Environment variable that toggles per-set bulkhead scheduling.
+pub const ENV_HEAL_SET_BULKHEAD_ENABLE: &str = "RUSTFS_HEAL_SET_BULKHEAD_ENABLE";
+
+/// Environment variable that toggles page-level parallel object healing for erasure-set repair.
+pub const ENV_HEAL_PAGE_PARALLEL_ENABLE: &str = "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE";
+
+/// Default behavior is to merge duplicate low-priority requests.
+pub const DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE: bool = true;
+
+/// Default behavior is to drop low-priority requests instead of blocking when the queue is full.
+pub const DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: bool = true;
+
+/// Default per-page object heal concurrency for erasure-set healing.
+pub const DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY: usize = 8;
+
+/// Default behavior is to keep notify-driven scheduler wakeups enabled.
+pub const DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: bool = true;
+
+/// Default behavior is to keep per-set bulkhead scheduling enabled.
+pub const DEFAULT_HEAL_SET_BULKHEAD_ENABLE: bool = true;
+
+/// Default behavior is to keep erasure-set page parallelism enabled.
+pub const DEFAULT_HEAL_PAGE_PARALLEL_ENABLE: bool = true;

crates/config/src/constants/mod.rs

@@ -17,6 +17,7 @@ pub(crate) mod body_limits;
 pub(crate) mod capacity;
 pub(crate) mod compress;
 pub(crate) mod console;
+pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
 pub(crate) mod object;

crates/config/src/constants/oidc.rs

@@ -17,6 +17,7 @@ pub const OIDC_CONFIG_URL: &str = "config_url";
 pub const OIDC_CLIENT_ID: &str = "client_id";
 pub const OIDC_CLIENT_SECRET: &str = "client_secret";
 pub const OIDC_SCOPES: &str = "scopes";
+pub const OIDC_OTHER_AUDIENCES: &str = "other_audiences";
 pub const OIDC_REDIRECT_URI: &str = "redirect_uri";
 pub const OIDC_REDIRECT_URI_DYNAMIC: &str = "redirect_uri_dynamic";
 pub const OIDC_CLAIM_NAME: &str = "claim_name";
@@ -24,6 +25,7 @@ pub const OIDC_CLAIM_PREFIX: &str = "claim_prefix";
 pub const OIDC_ROLE_POLICY: &str = "role_policy";
 pub const OIDC_DISPLAY_NAME: &str = "display_name";
 pub const OIDC_GROUPS_CLAIM: &str = "groups_claim";
+pub const OIDC_ROLES_CLAIM: &str = "roles_claim";
 pub const OIDC_EMAIL_CLAIM: &str = "email_claim";
 pub const OIDC_USERNAME_CLAIM: &str = "username_claim";
 
@@ -33,6 +35,7 @@ pub const ENV_IDENTITY_OPENID_CONFIG_URL: &str = "RUSTFS_IDENTITY_OPENID_CONFIG_
 pub const ENV_IDENTITY_OPENID_CLIENT_ID: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_ID";
 pub const ENV_IDENTITY_OPENID_CLIENT_SECRET: &str = "RUSTFS_IDENTITY_OPENID_CLIENT_SECRET";
 pub const ENV_IDENTITY_OPENID_SCOPES: &str = "RUSTFS_IDENTITY_OPENID_SCOPES";
+pub const ENV_IDENTITY_OPENID_OTHER_AUDIENCES: &str = "RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI";
 pub const ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: &str = "RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC";
 pub const ENV_IDENTITY_OPENID_CLAIM_NAME: &str = "RUSTFS_IDENTITY_OPENID_CLAIM_NAME";
@@ -40,16 +43,18 @@ pub const ENV_IDENTITY_OPENID_CLAIM_PREFIX: &str = "RUSTFS_IDENTITY_OPENID_CLAIM
 pub const ENV_IDENTITY_OPENID_ROLE_POLICY: &str = "RUSTFS_IDENTITY_OPENID_ROLE_POLICY";
 pub const ENV_IDENTITY_OPENID_DISPLAY_NAME: &str = "RUSTFS_IDENTITY_OPENID_DISPLAY_NAME";
 pub const ENV_IDENTITY_OPENID_GROUPS_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM";
+pub const ENV_IDENTITY_OPENID_ROLES_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_ROLES_CLAIM";
 pub const ENV_IDENTITY_OPENID_EMAIL_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM";
 pub const ENV_IDENTITY_OPENID_USERNAME_CLAIM: &str = "RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM";
 
 /// List of all environment variable keys for an OIDC provider.
-pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
+pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 16] = &[
     ENV_IDENTITY_OPENID_ENABLE,
     ENV_IDENTITY_OPENID_CONFIG_URL,
     ENV_IDENTITY_OPENID_CLIENT_ID,
     ENV_IDENTITY_OPENID_CLIENT_SECRET,
     ENV_IDENTITY_OPENID_SCOPES,
+    ENV_IDENTITY_OPENID_OTHER_AUDIENCES,
     ENV_IDENTITY_OPENID_REDIRECT_URI,
     ENV_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC,
     ENV_IDENTITY_OPENID_CLAIM_NAME,
@@ -57,6 +62,7 @@ pub const ENV_IDENTITY_OPENID_KEYS: &[&str; 14] = &[
     ENV_IDENTITY_OPENID_ROLE_POLICY,
     ENV_IDENTITY_OPENID_DISPLAY_NAME,
     ENV_IDENTITY_OPENID_GROUPS_CLAIM,
+    ENV_IDENTITY_OPENID_ROLES_CLAIM,
     ENV_IDENTITY_OPENID_EMAIL_CLAIM,
     ENV_IDENTITY_OPENID_USERNAME_CLAIM,
 ];
@@ -68,6 +74,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
     OIDC_CLIENT_ID,
     OIDC_CLIENT_SECRET,
     OIDC_SCOPES,
+    OIDC_OTHER_AUDIENCES,
     OIDC_REDIRECT_URI,
     OIDC_REDIRECT_URI_DYNAMIC,
     OIDC_CLAIM_NAME,
@@ -75,6 +82,7 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
     OIDC_ROLE_POLICY,
     OIDC_DISPLAY_NAME,
     OIDC_GROUPS_CLAIM,
+    OIDC_ROLES_CLAIM,
     OIDC_EMAIL_CLAIM,
     OIDC_USERNAME_CLAIM,
     crate::COMMENT_KEY,
@@ -84,6 +92,8 @@ pub const IDENTITY_OPENID_KEYS: &[&str] = &[
 pub const OIDC_DEFAULT_SCOPES: &str = "openid,profile,email";
 pub const OIDC_DEFAULT_CLAIM_NAME: &str = "groups";
 pub const OIDC_DEFAULT_GROUPS_CLAIM: &str = "groups";
+/// Empty means do not merge a secondary claim into groups (legacy behavior). Set to e.g. `roles` to opt in.
+pub const OIDC_DEFAULT_ROLES_CLAIM: &str = "";
 pub const OIDC_DEFAULT_EMAIL_CLAIM: &str = "email";
 pub const OIDC_DEFAULT_USERNAME_CLAIM: &str = "preferred_username";
 

crates/config/src/constants/scanner.rs

@@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS"
 #[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
 pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
 
+/// Environment variable that specifies the scanner cycle interval in seconds.
+/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour)
+pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE";
+
 /// Environment variable that selects the scanner speed preset.
 /// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
 /// Controls the sleep factor, maximum sleep duration, and cycle interval.
@@ -39,9 +45,24 @@ pub const DEFAULT_SCANNER_SPEED: &str = "default";
 /// - Example: `export RUSTFS_SCANNER_IDLE_MODE=false`
 pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
 
+/// Environment variable that controls scanner cache save timeout in seconds.
+/// The scanner enforces a minimum value of `1`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
+pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
+
 /// Default scanner idle mode.
 pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 
+/// Compatibility flag kept for Patch 3 rollback windows.
+///
+/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
+/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
+pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
+
+/// Default inline scanner heal compatibility mode.
+pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
+
 /// Scanner speed preset controlling throttling behavior.
 ///
 /// Each preset defines three parameters:
@@ -56,6 +77,8 @@ pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 /// | `default` | 2x     | 1 second  | 1 minute       |
 /// | `slow`    | 10x    | 15 seconds| 1 minute       |
 /// | `slowest` | 100x   | 15 seconds| 30 minutes     |
+///
+/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum ScannerSpeed {
     Fastest,

crates/config/src/constants/targets.rs

@@ -41,6 +41,30 @@ pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
 pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
 
+pub const NATS_ADDRESS: &str = "address";
+pub const NATS_SUBJECT: &str = "subject";
+pub const NATS_USERNAME: &str = "username";
+pub const NATS_PASSWORD: &str = "password";
+pub const NATS_TOKEN: &str = "token";
+pub const NATS_CREDENTIALS_FILE: &str = "credentials_file";
+pub const NATS_TLS_CA: &str = "tls_ca";
+pub const NATS_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
+pub const NATS_TLS_REQUIRED: &str = "tls_required";
+pub const NATS_QUEUE_DIR: &str = "queue_dir";
+pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
+
+pub const PULSAR_BROKER: &str = "broker";
+pub const PULSAR_TOPIC: &str = "topic";
+pub const PULSAR_AUTH_TOKEN: &str = "auth_token";
+pub const PULSAR_USERNAME: &str = "username";
+pub const PULSAR_PASSWORD: &str = "password";
+pub const PULSAR_TLS_CA: &str = "tls_ca";
+pub const PULSAR_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure";
+pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification";
+pub const PULSAR_QUEUE_DIR: &str = "queue_dir";
+pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit";
+
 /// Environment variable controlling whether target queue files are Snappy-compressed.
 /// Applies to both notify and audit target queue stores.
 pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";

crates/config/src/constants/zero_copy.rs

@@ -17,24 +17,6 @@
 //! This module defines environment variables and default values for zero-copy
 //! read operations, which use memory mapping (mmap) to avoid data copying.
 
-// =============================================================================
-// GET Fast Path Configuration
-// =============================================================================
-
-/// Environment variable for the GetObject chunk fast path master switch.
-///
-/// When disabled, `GetObject` bypasses the chunk-streaming fast path entirely and
-/// always uses the legacy reader path. This provides an operational stopgap for
-/// regressions in the streaming data plane while keeping zero-copy internals
-/// configurable independently for future opt-in validation.
-pub const ENV_OBJECT_GET_CHUNK_FAST_PATH_ENABLE: &str = "RUSTFS_OBJECT_GET_CHUNK_FAST_PATH_ENABLE";
-
-/// Default: GetObject chunk fast path is disabled.
-///
-/// The legacy reader path remains the safe default until the chunk-streaming
-/// path has sufficient regression coverage for full-body delivery semantics.
-pub const DEFAULT_OBJECT_GET_CHUNK_FAST_PATH_ENABLE: bool = false;
-
 // =============================================================================
 // Zero-Copy Configuration
 // =============================================================================
@@ -67,34 +49,6 @@ pub const ENV_OBJECT_ZERO_COPY_ENABLE: &str = "RUSTFS_OBJECT_ZERO_COPY_ENABLE";
 /// to regular I/O without errors.
 pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = true;
 
-/// Environment variable for zero-copy read operating mode.
-///
-/// Supported values:
-/// - `off`: disable mmap-backed chunk fast path and always use the compatibility path
-/// - `conservative`: allow a single mmap window per request
-/// - `balanced`: allow multiple mmap windows with the default size guardrails
-/// - `aggressive`: allow multi-window mmap and relax the small-object cutoff
-pub const ENV_OBJECT_ZERO_COPY_MODE: &str = "RUSTFS_OBJECT_ZERO_COPY_MODE";
-
-/// Default zero-copy read mode.
-pub const DEFAULT_OBJECT_ZERO_COPY_MODE: &str = "balanced";
-
-/// Environment variable for the maximum mmap window size used by the chunk fast path.
-///
-/// This controls the visible bytes per mapped chunk before the implementation emits a new window.
-pub const ENV_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES";
-
-/// Default mmap window size for chunk fast path reads: 8 MiB.
-pub const DEFAULT_OBJECT_ZERO_COPY_MMAP_WINDOW_BYTES: usize = 8 * 1024 * 1024;
-
-/// Environment variable for the maximum total active mmap bytes.
-///
-/// Requests that would exceed this active window budget fall back to the compatibility path.
-pub const ENV_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: &str = "RUSTFS_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES";
-
-/// Default maximum active mmap bytes across concurrent local chunk fast-path reads: 256 MiB.
-pub const DEFAULT_OBJECT_ZERO_COPY_MAX_ACTIVE_MMAP_BYTES: usize = 256 * 1024 * 1024;
-
 // =============================================================================
 // Direct I/O Configuration
 // =============================================================================

crates/config/src/lib.rs

@@ -25,6 +25,8 @@ pub use constants::compress::*;
 #[cfg(feature = "constants")]
 pub use constants::console::*;
 #[cfg(feature = "constants")]
+pub use constants::drive::*;
+#[cfg(feature = "constants")]
 pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::heal::*;

crates/config/src/notify/mod.rs

@@ -14,11 +14,15 @@
 
 mod arn;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod store;
 mod webhook;
 
 pub use arn::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use store::*;
 pub use webhook::*;
 
@@ -64,7 +68,12 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
 pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
 
 #[allow(dead_code)]
-pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
+pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
+    NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_SUB_SYS,
+    NOTIFY_WEBHOOK_SUB_SYS,
+];
 
 #[allow(dead_code)]
 pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
@@ -83,4 +92,5 @@ pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
 pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
 #[allow(dead_code)]
 pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
+pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
 pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";

crates/config/src/notify/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_NATS_ENABLE: &str = "RUSTFS_NOTIFY_NATS_ENABLE";
+pub const ENV_NOTIFY_NATS_ADDRESS: &str = "RUSTFS_NOTIFY_NATS_ADDRESS";
+pub const ENV_NOTIFY_NATS_SUBJECT: &str = "RUSTFS_NOTIFY_NATS_SUBJECT";
+pub const ENV_NOTIFY_NATS_USERNAME: &str = "RUSTFS_NOTIFY_NATS_USERNAME";
+pub const ENV_NOTIFY_NATS_PASSWORD: &str = "RUSTFS_NOTIFY_NATS_PASSWORD";
+pub const ENV_NOTIFY_NATS_TOKEN: &str = "RUSTFS_NOTIFY_NATS_TOKEN";
+pub const ENV_NOTIFY_NATS_CREDENTIALS_FILE: &str = "RUSTFS_NOTIFY_NATS_CREDENTIALS_FILE";
+pub const ENV_NOTIFY_NATS_TLS_CA: &str = "RUSTFS_NOTIFY_NATS_TLS_CA";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
+pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
+pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
+    ENV_NOTIFY_NATS_ENABLE,
+    ENV_NOTIFY_NATS_ADDRESS,
+    ENV_NOTIFY_NATS_SUBJECT,
+    ENV_NOTIFY_NATS_USERNAME,
+    ENV_NOTIFY_NATS_PASSWORD,
+    ENV_NOTIFY_NATS_TOKEN,
+    ENV_NOTIFY_NATS_CREDENTIALS_FILE,
+    ENV_NOTIFY_NATS_TLS_CA,
+    ENV_NOTIFY_NATS_TLS_CLIENT_CERT,
+    ENV_NOTIFY_NATS_TLS_CLIENT_KEY,
+    ENV_NOTIFY_NATS_TLS_REQUIRED,
+    ENV_NOTIFY_NATS_QUEUE_DIR,
+    ENV_NOTIFY_NATS_QUEUE_LIMIT,
+];

crates/config/src/notify/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_PULSAR_ENABLE: &str = "RUSTFS_NOTIFY_PULSAR_ENABLE";
+pub const ENV_NOTIFY_PULSAR_BROKER: &str = "RUSTFS_NOTIFY_PULSAR_BROKER";
+pub const ENV_NOTIFY_PULSAR_TOPIC: &str = "RUSTFS_NOTIFY_PULSAR_TOPIC";
+pub const ENV_NOTIFY_PULSAR_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_PULSAR_AUTH_TOKEN";
+pub const ENV_NOTIFY_PULSAR_USERNAME: &str = "RUSTFS_NOTIFY_PULSAR_USERNAME";
+pub const ENV_NOTIFY_PULSAR_PASSWORD: &str = "RUSTFS_NOTIFY_PULSAR_PASSWORD";
+pub const ENV_NOTIFY_PULSAR_TLS_CA: &str = "RUSTFS_NOTIFY_PULSAR_TLS_CA";
+pub const ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_NOTIFY_PULSAR_QUEUE_DIR: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_DIR";
+pub const ENV_NOTIFY_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_NOTIFY_PULSAR_ENABLE,
+    ENV_NOTIFY_PULSAR_BROKER,
+    ENV_NOTIFY_PULSAR_TOPIC,
+    ENV_NOTIFY_PULSAR_AUTH_TOKEN,
+    ENV_NOTIFY_PULSAR_USERNAME,
+    ENV_NOTIFY_PULSAR_PASSWORD,
+    ENV_NOTIFY_PULSAR_TLS_CA,
+    ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_NOTIFY_PULSAR_QUEUE_DIR,
+    ENV_NOTIFY_PULSAR_QUEUE_LIMIT,
+];

crates/credentials/src/credentials.rs

@@ -20,7 +20,7 @@ use std::collections::HashMap;
 use std::env;
 use std::fmt;
 use std::io::Error;
-use std::sync::OnceLock;
+use std::sync::{LazyLock, OnceLock};
 use time::OffsetDateTime;
 
 /// Global active credentials
@@ -204,10 +204,11 @@ pub fn gen_secret_key(length: usize) -> std::io::Result<String> {
     let mut key = vec![0u8; URL_SAFE_NO_PAD.estimated_decoded_length(length)];
     rng.fill_bytes(&mut key);
 
+    // URL_SAFE_NO_PAD uses "-" and "_" instead of "+" and "/", so "/" never
+    // appears in the output. The .replace("/", "+") was a dead no-op.
     let encoded = URL_SAFE_NO_PAD.encode_to_string(&key);
-    let key_str = encoded.replace("/", "+");
 
-    Ok(key_str)
+    Ok(encoded)
 }
 
 /// Get the RPC authentication token from environment variable
@@ -313,6 +314,17 @@ impl fmt::Debug for Credentials {
 }
 
 impl Credentials {
+    /// Returns a reference to this credential's claims, or a shared empty map
+    /// when the credential has no claims attached. Avoids per-call allocation
+    /// at call sites that need an `&HashMap<String, Value>`.
+    pub fn claims_or_empty(&self) -> &HashMap<String, Value> {
+        static EMPTY: LazyLock<HashMap<String, Value>> = LazyLock::new(HashMap::new);
+        match &self.claims {
+            Some(c) => c,
+            None => &EMPTY,
+        }
+    }
+
     pub fn is_expired(&self) -> bool {
         if self.expiration.is_none() {
             return false;
@@ -447,7 +459,7 @@ mod tests {
             // Initialize
             let test_ak = "test_access_key".to_string();
             let test_sk = "test_secret_key_123456".to_string();
-            init_global_action_credentials(Some(test_ak.clone()), Some(test_sk.clone())).ok();
+            init_global_action_credentials(Some(test_ak), Some(test_sk)).ok();
         }
 
         // Verify the state after initialization
@@ -473,6 +485,16 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_gen_secret_key_uses_url_safe_base64_without_padding() {
+        let key = gen_secret_key(32).expect("secret key should generate");
+
+        assert_eq!(key.len(), 32);
+        assert!(!key.contains('/'));
+        assert!(!key.contains('+'));
+        assert!(!key.contains('='));
+    }
+
     #[test]
     fn test_masked_debug() {
         // Test None

crates/e2e_test/Cargo.toml

@@ -20,11 +20,6 @@ license.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 
-[[bin]]
-name = "small_put_bench"
-path = "src/bin/small_put_bench.rs"
-test = false
-
 [lints]
 workspace = true
 
@@ -46,7 +41,6 @@ serde_json.workspace = true
 tonic = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
-clap = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
 bytes.workspace = true
@@ -58,8 +52,6 @@ async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
 flate2.workspace = true
 http.workspace = true
-http-body.workspace = true
-http-body-util.workspace = true
 reqwest = { workspace = true }
 rustfs-signer.workspace = true
 tracing = { workspace = true }

crates/e2e_test/src/archive_download_integrity_test.rs

@@ -15,6 +15,8 @@
 #[cfg(test)]
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
+    use aws_sdk_s3::Client as S3Client;
+    use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
     use http::header::{CONTENT_TYPE, HOST};
@@ -216,12 +218,157 @@ mod tests {
         Ok(builder.send().await?)
     }
 
+    async fn signed_get_request_with_headers(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        extra_headers: &[(&str, &str)],
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let mut request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+        for (name, value) in extra_headers {
+            request = request.header(*name, *value);
+        }
+
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut builder = client.get(url);
+        for (name, value) in signed.headers() {
+            builder = builder.header(name, value);
+        }
+
+        Ok(builder.send().await?)
+    }
+
+    async fn assert_archive_object_content_encoding(
+        client: &S3Client,
+        bucket: &str,
+        key: &str,
+        expected_content_encoding: Option<&str>,
+        expected_body: &[u8],
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let head_resp = client.head_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(head_resp.content_encoding(), expected_content_encoding);
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(get_resp.content_encoding(), expected_content_encoding);
+        let body = get_resp.body.collect().await?.into_bytes();
+        assert_eq!(body.as_ref(), expected_body);
+
+        Ok(())
+    }
+
+    async fn complete_archive_multipart_upload_with_content_encoding(
+        client: &S3Client,
+        bucket: &str,
+        key: &str,
+        content_encoding: Option<&str>,
+    ) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
+        let payload = random_bytes(MULTIPART_PART_SIZE + 256 * 1024);
+        let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
+        assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
+
+        let mut create_builder = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .content_type("application/zip");
+        if let Some(content_encoding) = content_encoding {
+            create_builder = create_builder.content_encoding(content_encoding);
+        }
+        let create_output = create_builder.send().await?;
+        let upload_id = create_output.upload_id().expect("multipart upload id");
+
+        let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
+        let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
+
+        let upload_part_1 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(first_part))
+            .send()
+            .await?;
+
+        let upload_part_2 = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(2)
+            .body(ByteStream::from(second_part))
+            .send()
+            .await?;
+
+        let completed_upload = CompletedMultipartUpload::builder()
+            .parts(
+                CompletedPart::builder()
+                    .part_number(1)
+                    .e_tag(upload_part_1.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .parts(
+                CompletedPart::builder()
+                    .part_number(2)
+                    .e_tag(upload_part_2.e_tag().unwrap_or_default())
+                    .build(),
+            )
+            .build();
+
+        client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await?;
+
+        Ok(zip_bytes)
+    }
+
     #[tokio::test]
     #[serial]
-    async fn test_archive_put_rejects_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
+    async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
         let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server_without_cleanup(vec![]).await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
+        let response =
+            signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip")
+                .await?;
+
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        let head_resp = client
+            .head_object()
+            .bucket(ARCHIVE_TEST_BUCKET)
+            .key("bundle.zip")
+            .send()
+            .await?;
+        assert_eq!(head_resp.content_encoding(), Some("gzip"));
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
         env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
         let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
         let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
@@ -232,7 +379,145 @@ mod tests {
         assert_eq!(response.status(), StatusCode::BAD_REQUEST);
         let body = response.text().await?;
         assert!(
-            body.contains("InvalidArgument") || body.contains("Content-Encoding"),
+            body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
+            "unexpected error body: {body}"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked-gzip.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked,gzip",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-aws-chunked-gzip.zip",
+            Some("gzip"),
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes.clone(),
+            "application/zip",
+            "aws-chunked",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let client = env.create_s3_client();
+        assert_archive_object_content_encoding(
+            &client,
+            ARCHIVE_TEST_BUCKET,
+            "bundle-strict-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
+
+        let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
+        let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked-gzip.zip");
+        let response = signed_put_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            zip_bytes,
+            "application/zip",
+            "aws-chunked,gzip",
+        )
+        .await?;
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+        let body = response.text().await?;
+        assert!(
+            body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"),
             "unexpected error body: {body}"
         );
 
@@ -400,11 +685,139 @@ mod tests {
 
     #[tokio::test]
     #[serial]
-    async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path()
+    async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let key = "multipart-empty-conditional-headers.zip";
+        let zip_bytes =
+            complete_archive_multipart_upload_with_content_encoding(&client, MULTIPART_ARCHIVE_TEST_BUCKET, key, None).await?;
+        let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, key);
+
+        let response = signed_get_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            &[("if-match", ""), ("if-none-match", "")],
+        )
+        .await?;
+        let status = response.status();
+        let body = response.bytes().await?;
+
+        assert_eq!(
+            status,
+            StatusCode::OK,
+            "unexpected multipart GET status {status}, body: {}",
+            String::from_utf8_lossy(body.as_ref())
+        );
+        assert_eq!(body.as_ref(), zip_bytes.as_slice());
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
     -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
         let mut env = RustFSTestEnvironment::new().await?;
-        start_rustfs_server_with_env(&mut env, &[("RUSTFS_OBJECT_GET_CHUNK_FAST_PATH_ENABLE", "true")]).await?;
+        start_rustfs_server_with_env(&mut env, &[]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-aws-chunked-gzip.zip",
+            Some("aws-chunked,gzip"),
+        )
+        .await?;
+        assert_archive_object_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-aws-chunked-gzip.zip",
+            Some("gzip"),
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let zip_bytes = complete_archive_multipart_upload_with_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-strict-aws-chunked.zip",
+            Some("aws-chunked"),
+        )
+        .await?;
+        assert_archive_object_content_encoding(
+            &client,
+            MULTIPART_ARCHIVE_TEST_BUCKET,
+            "multipart-strict-aws-chunked.zip",
+            None,
+            zip_bytes.as_slice(),
+        )
+        .await?;
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let create_result = client
+            .create_multipart_upload()
+            .bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
+            .key("multipart-strict-aws-chunked-gzip.zip")
+            .content_type("application/zip")
+            .content_encoding("aws-chunked,gzip")
+            .send()
+            .await;
+        let err = create_result.expect_err("strict mode should reject effective archive content encoding");
+        assert_eq!(err.code(), Some("InvalidArgument"));
+        assert!(
+            err.message().is_some_and(|message| {
+                message.contains("Content-Encoding") && message.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING")
+            }),
+            "unexpected error metadata: code={:?}, message={:?}",
+            err.code(),
+            err.message()
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
         env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
 
         let client = env.create_s3_client();

crates/e2e_test/src/bin/small_put_bench.rs

@@ -1,441 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use anyhow::{Context, Result, anyhow, bail};
-use aws_sdk_s3::config::{Credentials, Region};
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{Delete, ObjectIdentifier};
-use aws_sdk_s3::{Client, Config};
-use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
-use bytes::Bytes;
-use clap::Parser;
-use serde::Serialize;
-use std::path::PathBuf;
-use std::time::{Duration, Instant};
-
-#[derive(Parser, Debug)]
-#[command(name = "small_put_bench")]
-#[command(about = "Rust-native small PUT benchmark for RustFS-compatible S3 endpoints")]
-struct Args {
-    #[arg(long, env = "RUSTFS_BENCH_ENDPOINT")]
-    endpoint: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_ACCESS_KEY", default_value = "rustfsadmin")]
-    access_key: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_SECRET_KEY", default_value = "rustfsadmin")]
-    secret_key: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_REGION", default_value = "us-east-1")]
-    region: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_BUCKET", default_value = "small-put-benchmark")]
-    bucket: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_SIZES", default_value = "4KiB,16KiB,64KiB,256KiB,1MiB")]
-    sizes: String,
-
-    #[arg(long, env = "RUSTFS_BENCH_CONCURRENCY", default_value_t = 8)]
-    concurrency: usize,
-
-    #[arg(long, env = "RUSTFS_BENCH_DURATION_SECS", default_value_t = 10)]
-    duration_secs: u64,
-
-    #[arg(long, env = "RUSTFS_BENCH_TIMEOUT_SECS", default_value_t = 15)]
-    timeout_secs: u64,
-
-    #[arg(long, env = "RUSTFS_BENCH_PREFIX")]
-    prefix: Option<String>,
-
-    #[arg(long)]
-    output_json: Option<PathBuf>,
-
-    #[arg(long, default_value_t = false)]
-    cleanup: bool,
-}
-
-#[derive(Clone, Debug)]
-struct SizeSpec {
-    label: String,
-    slug: String,
-    bytes: usize,
-}
-
-#[derive(Debug)]
-struct Sample {
-    ok: bool,
-    duration_ms: f64,
-}
-
-#[derive(Debug, Serialize)]
-struct SizeSummary {
-    label: String,
-    bytes: usize,
-    total: usize,
-    succeeded: usize,
-    failed: usize,
-    wall_secs: f64,
-    object_rate: f64,
-    throughput_mib_per_sec: f64,
-    avg_ms: Option<f64>,
-    p50_ms: Option<f64>,
-    p90_ms: Option<f64>,
-    p99_ms: Option<f64>,
-}
-
-#[derive(Debug, Serialize)]
-struct RunSummary {
-    run_id: String,
-    endpoint: String,
-    bucket: String,
-    concurrency: usize,
-    duration_secs: u64,
-    timeout_secs: u64,
-    sizes: Vec<SizeSummary>,
-}
-
-fn main() -> Result<()> {
-    let runtime = tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()
-        .context("failed to build tokio runtime")?;
-    runtime.block_on(async_main())
-}
-
-async fn async_main() -> Result<()> {
-    let args = Args::parse();
-    validate_args(&args)?;
-
-    let sizes = parse_size_list(&args.sizes)?;
-    let run_id = args.prefix.clone().unwrap_or_else(default_run_id);
-    let client = build_s3_client(&args.endpoint, &args.access_key, &args.secret_key, &args.region);
-
-    ensure_bucket(&client, &args.bucket).await?;
-
-    let mut size_summaries = Vec::with_capacity(sizes.len());
-    for size in &sizes {
-        let summary = run_size_benchmark(
-            client.clone(),
-            args.bucket.clone(),
-            run_id.clone(),
-            size.clone(),
-            args.concurrency,
-            Duration::from_secs(args.duration_secs),
-            Duration::from_secs(args.timeout_secs),
-        )
-        .await?;
-        print_size_summary(&summary);
-        size_summaries.push(summary);
-    }
-
-    if args.cleanup {
-        cleanup_prefix(&client, &args.bucket, &run_id).await?;
-    }
-
-    let summary = RunSummary {
-        run_id,
-        endpoint: args.endpoint,
-        bucket: args.bucket,
-        concurrency: args.concurrency,
-        duration_secs: args.duration_secs,
-        timeout_secs: args.timeout_secs,
-        sizes: size_summaries,
-    };
-
-    if let Some(path) = args.output_json {
-        let json = serde_json::to_vec_pretty(&summary).context("failed to serialize benchmark summary")?;
-        std::fs::write(&path, json).with_context(|| format!("failed to write benchmark summary to {}", path.display()))?;
-        println!("Wrote summary to {}", path.display());
-    }
-
-    Ok(())
-}
-
-fn validate_args(args: &Args) -> Result<()> {
-    if args.concurrency == 0 {
-        bail!("--concurrency must be greater than zero");
-    }
-    if args.duration_secs == 0 {
-        bail!("--duration-secs must be greater than zero");
-    }
-    if args.timeout_secs == 0 {
-        bail!("--timeout-secs must be greater than zero");
-    }
-    Ok(())
-}
-
-fn build_s3_client(endpoint: &str, access_key: &str, secret_key: &str, region: &str) -> Client {
-    let credentials = Credentials::new(access_key, secret_key, None, None, "small-put-bench");
-    let mut config = Config::builder()
-        .credentials_provider(credentials)
-        .region(Region::new(region.to_string()))
-        .endpoint_url(endpoint)
-        .force_path_style(true)
-        .behavior_version_latest();
-
-    if endpoint.starts_with("http://") {
-        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
-    }
-
-    Client::from_conf(config.build())
-}
-
-async fn ensure_bucket(client: &Client, bucket: &str) -> Result<()> {
-    if client.head_bucket().bucket(bucket).send().await.is_ok() {
-        return Ok(());
-    }
-
-    match client.create_bucket().bucket(bucket).send().await {
-        Ok(_) => Ok(()),
-        Err(err) => {
-            let rendered = err.to_string();
-            if rendered.contains("BucketAlreadyOwnedByYou") || rendered.contains("BucketAlreadyExists") {
-                Ok(())
-            } else {
-                Err(err).with_context(|| format!("failed to create benchmark bucket {bucket}"))
-            }
-        }
-    }
-}
-
-async fn run_size_benchmark(
-    client: Client,
-    bucket: String,
-    run_id: String,
-    size: SizeSpec,
-    concurrency: usize,
-    duration: Duration,
-    timeout: Duration,
-) -> Result<SizeSummary> {
-    let payload = Bytes::from(vec![0_u8; size.bytes]);
-    let deadline = Instant::now() + duration;
-    let wall_start = Instant::now();
-
-    let mut handles = Vec::with_capacity(concurrency);
-    for worker in 0..concurrency {
-        let client = client.clone();
-        let bucket = bucket.clone();
-        let payload = payload.clone();
-        let prefix = format!("{run_id}/{}/worker-{worker}", size.slug);
-        handles.push(tokio::spawn(async move {
-            let mut samples = Vec::new();
-            let mut idx = 0usize;
-
-            while Instant::now() < deadline {
-                let key = format!("{prefix}/obj-{idx}.bin");
-                let started_at = Instant::now();
-                let request = client
-                    .put_object()
-                    .bucket(&bucket)
-                    .key(key)
-                    .body(ByteStream::from(payload.clone()))
-                    .content_type("application/octet-stream");
-
-                let ok = matches!(tokio::time::timeout(timeout, request.send()).await, Ok(Ok(_)));
-                samples.push(Sample {
-                    ok,
-                    duration_ms: started_at.elapsed().as_secs_f64() * 1000.0,
-                });
-                idx += 1;
-            }
-
-            samples
-        }));
-    }
-
-    let mut samples = Vec::new();
-    for handle in handles {
-        samples.extend(handle.await.map_err(|err| anyhow!("benchmark worker join error: {err}"))?);
-    }
-
-    Ok(build_size_summary(&size, samples, wall_start.elapsed()))
-}
-
-fn build_size_summary(size: &SizeSpec, mut samples: Vec<Sample>, wall_elapsed: Duration) -> SizeSummary {
-    let total = samples.len();
-    let succeeded = samples.iter().filter(|sample| sample.ok).count();
-    let failed = total.saturating_sub(succeeded);
-    let wall_secs = wall_elapsed.as_secs_f64();
-    let object_rate = if wall_secs > 0.0 { succeeded as f64 / wall_secs } else { 0.0 };
-    let throughput_mib_per_sec = if wall_secs > 0.0 {
-        ((size.bytes * succeeded) as f64 / (1024.0 * 1024.0)) / wall_secs
-    } else {
-        0.0
-    };
-
-    let avg_ms = if total > 0 {
-        Some(samples.iter().map(|sample| sample.duration_ms).sum::<f64>() / total as f64)
-    } else {
-        None
-    };
-
-    samples.sort_by(|lhs, rhs| lhs.duration_ms.total_cmp(&rhs.duration_ms));
-    let durations: Vec<f64> = samples.into_iter().map(|sample| sample.duration_ms).collect();
-
-    SizeSummary {
-        label: size.label.clone(),
-        bytes: size.bytes,
-        total,
-        succeeded,
-        failed,
-        wall_secs,
-        object_rate,
-        throughput_mib_per_sec,
-        avg_ms,
-        p50_ms: percentile(&durations, 0.50),
-        p90_ms: percentile(&durations, 0.90),
-        p99_ms: percentile(&durations, 0.99),
-    }
-}
-
-async fn cleanup_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
-    let mut continuation_token = None;
-    loop {
-        let response = client
-            .list_objects_v2()
-            .bucket(bucket)
-            .prefix(prefix)
-            .set_continuation_token(continuation_token.clone())
-            .send()
-            .await
-            .with_context(|| format!("failed to list objects for cleanup under {bucket}/{prefix}"))?;
-
-        let objects: Vec<ObjectIdentifier> = response
-            .contents
-            .unwrap_or_default()
-            .into_iter()
-            .filter_map(|object| object.key.map(|key| ObjectIdentifier::builder().key(key).build().ok()))
-            .flatten()
-            .collect();
-
-        for chunk in objects.chunks(1_000) {
-            if chunk.is_empty() {
-                continue;
-            }
-
-            client
-                .delete_objects()
-                .bucket(bucket)
-                .delete(
-                    Delete::builder()
-                        .set_objects(Some(chunk.to_vec()))
-                        .quiet(true)
-                        .build()
-                        .context("failed to build delete request")?,
-                )
-                .send()
-                .await
-                .with_context(|| format!("failed to delete cleanup batch under {bucket}/{prefix}"))?;
-        }
-
-        if response.is_truncated.unwrap_or(false) {
-            continuation_token = response.next_continuation_token;
-        } else {
-            break;
-        }
-    }
-
-    Ok(())
-}
-
-fn parse_size_list(input: &str) -> Result<Vec<SizeSpec>> {
-    input
-        .split(',')
-        .map(str::trim)
-        .filter(|item| !item.is_empty())
-        .map(parse_size_spec)
-        .collect()
-}
-
-fn parse_size_spec(input: &str) -> Result<SizeSpec> {
-    let normalized = input.trim();
-    let lower = normalized.to_ascii_lowercase();
-
-    let (number_part, multiplier) = if let Some(value) = lower.strip_suffix("kib") {
-        (value, 1024usize)
-    } else if let Some(value) = lower.strip_suffix("mib") {
-        (value, 1024usize * 1024usize)
-    } else if let Some(value) = lower.strip_suffix('b') {
-        (value, 1usize)
-    } else {
-        (lower.as_str(), 1usize)
-    };
-
-    let value = number_part
-        .trim()
-        .parse::<usize>()
-        .with_context(|| format!("invalid size component: {input}"))?;
-    let bytes = value
-        .checked_mul(multiplier)
-        .ok_or_else(|| anyhow!("size overflow for {input}"))?;
-
-    Ok(SizeSpec {
-        label: normalized.to_string(),
-        slug: normalized
-            .chars()
-            .filter(|ch| ch.is_ascii_alphanumeric())
-            .collect::<String>()
-            .to_ascii_lowercase(),
-        bytes,
-    })
-}
-
-fn percentile(values: &[f64], percentile: f64) -> Option<f64> {
-    if values.is_empty() {
-        return None;
-    }
-
-    let index = ((values.len() - 1) as f64 * percentile).floor() as usize;
-    values.get(index).copied()
-}
-
-fn default_run_id() -> String {
-    format!("small-put-bench-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))
-}
-
-fn print_size_summary(summary: &SizeSummary) {
-    println!(
-        "{}: success={} failed={} obj/s={:.3} MiB/s={:.3} avg={:.3?} p50={:.3?} p90={:.3?} p99={:.3?}",
-        summary.label,
-        summary.succeeded,
-        summary.failed,
-        summary.object_rate,
-        summary.throughput_mib_per_sec,
-        summary.avg_ms,
-        summary.p50_ms,
-        summary.p90_ms,
-        summary.p99_ms,
-    );
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn parse_size_spec_supports_binary_units() {
-        let four_kib = parse_size_spec("4KiB").expect("4KiB should parse");
-        assert_eq!(four_kib.bytes, 4 * 1024);
-
-        let one_mib = parse_size_spec("1MiB").expect("1MiB should parse");
-        assert_eq!(one_mib.bytes, 1024 * 1024);
-    }
-
-    #[test]
-    fn percentile_returns_expected_bucket() {
-        let values = vec![10.0, 20.0, 30.0, 40.0, 50.0];
-        assert_eq!(percentile(&values, 0.50), Some(30.0));
-        assert_eq!(percentile(&values, 0.90), Some(40.0));
-    }
-}

crates/e2e_test/src/checksum_upload_test.rs

@@ -19,13 +19,9 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
-    use aws_sdk_s3::primitives::{ByteStream, SdkBody};
+    use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
     use base64::Engine;
-    use bytes::Bytes;
-    use futures::StreamExt;
-    use http_body::Frame;
-    use http_body_util::StreamBody;
     use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
@@ -68,53 +64,6 @@ mod tests {
             .encoded
     }
 
-    fn streamed_body_70kib_of_a() -> ByteStream {
-        let bytes = Bytes::from_static(&[b'a'; 1024]);
-        let stream = futures::stream::repeat_with(move || {
-            let frame = Frame::data(bytes.clone());
-            Ok::<_, std::io::Error>(frame)
-        });
-        let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
-        ByteStream::new(SdkBody::from_body_1_x(body))
-    }
-
-    struct WithSizeHint<T> {
-        inner: T,
-        size_hint: usize,
-    }
-
-    impl<T> WithSizeHint<T> {
-        fn new(inner: T, size_hint: usize) -> Self {
-            Self { inner, size_hint }
-        }
-    }
-
-    impl<T> http_body::Body for WithSizeHint<T>
-    where
-        T: http_body::Body + Unpin,
-    {
-        type Data = T::Data;
-        type Error = T::Error;
-
-        fn poll_frame(
-            self: std::pin::Pin<&mut Self>,
-            cx: &mut std::task::Context<'_>,
-        ) -> std::task::Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
-            let this = self.get_mut();
-            std::pin::Pin::new(&mut this.inner).poll_frame(cx)
-        }
-
-        fn is_end_stream(&self) -> bool {
-            self.inner.is_end_stream()
-        }
-
-        fn size_hint(&self) -> http_body::SizeHint {
-            let mut hint = self.inner.size_hint();
-            hint.set_exact(self.size_hint as u64);
-            hint
-        }
-    }
-
     /// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
     #[tokio::test]
     #[serial]
@@ -187,121 +136,6 @@ mod tests {
         info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
     }
 
-    /// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect
-    /// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum.
-    #[tokio::test]
-    #[serial]
-    async fn test_put_object_with_checksum_algorithm_only() {
-        init_logging();
-        info!("TEST: PutObject with checksum_algorithm only");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-checksum-algorithm-only";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let key = "obj-with-checksum-algorithm-only.txt";
-        let content = vec![b'a'; 70 * 1024];
-
-        let put_resp = client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .body(ByteStream::from(content.clone()))
-            .send()
-            .await
-            .expect("PutObject with checksum_algorithm should succeed");
-
-        let put_checksum = put_resp
-            .checksum_crc32()
-            .expect("PutObject should return checksum_crc32 when checksum_algorithm is used")
-            .to_string();
-
-        let mut get_resp = client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_mode(ChecksumMode::Enabled)
-            .send()
-            .await
-            .expect("GetObject should succeed");
-
-        let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty()))
-            .collect()
-            .await
-            .expect("collect body")
-            .into_bytes();
-
-        assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content");
-        assert_eq!(
-            get_resp.checksum_crc32().map(str::to_string),
-            Some(put_checksum),
-            "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum"
-        );
-    }
-
-    /// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec<u8>)`.
-    #[tokio::test]
-    #[serial]
-    async fn test_put_object_with_checksum_algorithm_only_streaming_body() {
-        init_logging();
-        info!("TEST: PutObject with checksum_algorithm only using streaming body");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-checksum-algorithm-streaming";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let key = "obj-with-checksum-algorithm-streaming.txt";
-        let expected_content = vec![b'a'; 70 * 1024];
-
-        let put_resp = client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .body(streamed_body_70kib_of_a())
-            .send()
-            .await
-            .expect("PutObject with streaming checksum_algorithm should succeed");
-
-        let put_checksum = put_resp
-            .checksum_crc32()
-            .expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads")
-            .to_string();
-
-        let mut get_resp = client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .checksum_mode(ChecksumMode::Enabled)
-            .send()
-            .await
-            .expect("GetObject should succeed");
-
-        let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty()))
-            .collect()
-            .await
-            .expect("collect body")
-            .into_bytes();
-
-        assert_eq!(
-            body_bytes.as_ref(),
-            expected_content.as_slice(),
-            "GetObject body must match uploaded content"
-        );
-        assert_eq!(
-            get_resp.checksum_crc32().map(str::to_string),
-            Some(put_checksum),
-            "GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads"
-        );
-    }
-
     /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
     /// Uses part size >= 5MB (server minimum) for two parts.
     #[tokio::test]
@@ -400,114 +234,6 @@ mod tests {
         info!("PASSED: MultipartUpload with checksum and GetObject content match");
     }
 
-    /// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation,
-    /// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed.
-    #[tokio::test]
-    #[serial]
-    async fn test_multipart_upload_with_crc32_algorithm_only() {
-        init_logging();
-        info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-multipart-checksum-crc32-auto";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let key = "multipart-with-crc32-auto.bin";
-        let part1_content = "a".repeat(5 * 1024 * 1024 + 1);
-        let part2_content = "b".repeat(1024);
-
-        let create_resp = client
-            .create_multipart_upload()
-            .bucket(bucket)
-            .key(key)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .send()
-            .await
-            .expect("CreateMultipartUpload should succeed");
-
-        let upload_id = create_resp.upload_id().expect("upload_id should be present");
-
-        let part1_resp = client
-            .upload_part()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(upload_id)
-            .part_number(1)
-            .body(ByteStream::from(part1_content.clone().into_bytes()))
-            .send()
-            .await
-            .expect("UploadPart 1 should succeed");
-        let part1_checksum = part1_resp
-            .checksum_crc32()
-            .expect("UploadPart 1 should return checksum_crc32")
-            .to_string();
-
-        let part2_resp = client
-            .upload_part()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(upload_id)
-            .part_number(2)
-            .body(ByteStream::from(part2_content.clone().into_bytes()))
-            .send()
-            .await
-            .expect("UploadPart 2 should succeed");
-        let part2_checksum = part2_resp
-            .checksum_crc32()
-            .expect("UploadPart 2 should return checksum_crc32")
-            .to_string();
-
-        let completed_upload = CompletedMultipartUpload::builder()
-            .parts(
-                CompletedPart::builder()
-                    .part_number(1)
-                    .e_tag(part1_resp.e_tag().expect("etag part 1"))
-                    .checksum_crc32(part1_checksum)
-                    .build(),
-            )
-            .parts(
-                CompletedPart::builder()
-                    .part_number(2)
-                    .e_tag(part2_resp.e_tag().expect("etag part 2"))
-                    .checksum_crc32(part2_checksum)
-                    .build(),
-            )
-            .build();
-
-        client
-            .complete_multipart_upload()
-            .bucket(bucket)
-            .key(key)
-            .upload_id(upload_id)
-            .multipart_upload(completed_upload)
-            .send()
-            .await
-            .expect("CompleteMultipartUpload should succeed");
-
-        let body_bytes = client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .send()
-            .await
-            .expect("GetObject should succeed")
-            .body
-            .collect()
-            .await
-            .expect("collect body")
-            .into_bytes();
-
-        let expected_content = format!("{part1_content}{part2_content}");
-        assert_eq!(
-            body_bytes.as_ref(),
-            expected_content.as_bytes(),
-            "completed multipart object must match concatenated parts"
-        );
-    }
-
     /// Regression test for issue #2282:
     /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
     #[tokio::test]

crates/e2e_test/src/content_encoding_test.rs

@@ -138,4 +138,63 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding,
+    /// only the effective encoding should roundtrip through GET/HEAD.
+    #[tokio::test]
+    #[serial]
+    async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() {
+        init_logging();
+        info!("aws-chunked,gzip should persist only gzip");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "content-encoding-aws-chunked-gzip-test";
+        let key = "streamed/object.txt";
+        let content = b"streaming upload body with effective gzip encoding";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .content_type("text/plain")
+            .content_encoding("aws-chunked,gzip")
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PUT failed");
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
+        assert_eq!(
+            get_resp.content_encoding(),
+            Some("gzip"),
+            "GET must return only the effective content encoding after aws-chunked is stripped"
+        );
+        let body = get_resp.body.collect().await.unwrap().into_bytes();
+        assert_eq!(body.as_ref(), content, "Body content mismatch");
+
+        let head_resp = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("HEAD failed");
+        assert_eq!(
+            head_resp.content_encoding(),
+            Some("gzip"),
+            "HEAD must return only the effective content encoding after aws-chunked is stripped"
+        );
+
+        env.stop_server();
+    }
 }

crates/e2e_test/src/kms/kms_local_test.rs

@@ -344,9 +344,6 @@ async fn test_local_kms_multipart_upload() {
     test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET)
         .await
         .expect("SSE-C multipart upload test failed");
-    test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET)
-        .await
-        .expect("SSE-C multipart wrong-key download test failed");
 
     // Test 4: Large multipart upload (test streaming encryption with multiple blocks)
     // TODO: Re-enable after fixing streaming encryption issues with large files
@@ -651,99 +648,6 @@ async fn test_multipart_upload_with_sse_c(
     Ok(())
 }
 
-async fn test_multipart_download_with_wrong_sse_c_key_fails(
-    s3_client: &aws_sdk_s3::Client,
-    bucket: &str,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let object_key = "multipart-sse-c-bad-download-test";
-    let part_size = 5 * 1024 * 1024;
-    let total_parts = 2;
-    let total_size = part_size * total_parts;
-
-    let encryption_key = "01234567890123456789012345678901";
-    let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
-    let key_md5 = sse_customer_key_md5_base64(encryption_key);
-
-    let wrong_key = "abcdefghijklmnopqrstuvwxyz012345";
-    let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
-    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
-
-    let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect();
-
-    let create_multipart_output = s3_client
-        .create_multipart_upload()
-        .bucket(bucket)
-        .key(object_key)
-        .sse_customer_algorithm("AES256")
-        .sse_customer_key(&key_b64)
-        .sse_customer_key_md5(&key_md5)
-        .send()
-        .await?;
-
-    let upload_id = create_multipart_output.upload_id().unwrap();
-    let mut completed_parts = Vec::new();
-
-    for part_number in 1..=total_parts {
-        let start = (part_number - 1) * part_size;
-        let end = std::cmp::min(start + part_size, total_size);
-        let part_data = &test_data[start..end];
-
-        let upload_part_output = s3_client
-            .upload_part()
-            .bucket(bucket)
-            .key(object_key)
-            .upload_id(upload_id)
-            .part_number(part_number as i32)
-            .body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()))
-            .sse_customer_algorithm("AES256")
-            .sse_customer_key(&key_b64)
-            .sse_customer_key_md5(&key_md5)
-            .send()
-            .await?;
-
-        completed_parts.push(
-            aws_sdk_s3::types::CompletedPart::builder()
-                .part_number(part_number as i32)
-                .e_tag(upload_part_output.e_tag().unwrap())
-                .build(),
-        );
-    }
-
-    let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
-        .set_parts(Some(completed_parts))
-        .build();
-
-    s3_client
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(object_key)
-        .upload_id(upload_id)
-        .multipart_upload(completed_multipart_upload)
-        .send()
-        .await?;
-
-    let err = s3_client
-        .get_object()
-        .bucket(bucket)
-        .key(object_key)
-        .sse_customer_algorithm("AES256")
-        .sse_customer_key(&wrong_key_b64)
-        .sse_customer_key_md5(&wrong_key_md5)
-        .send()
-        .await
-        .expect_err("multipart SSE-C download with the wrong key should fail");
-
-    let service_err = err.into_service_error();
-    assert_eq!(
-        service_err.meta().code(),
-        Some("InvalidRequest"),
-        "wrong-key multipart SSE-C download should return InvalidRequest, got {:?}",
-        service_err.meta().code()
-    );
-
-    Ok(())
-}
-
 /// Test large multipart upload to verify streaming encryption works correctly
 #[allow(dead_code)]
 async fn test_large_multipart_upload(

crates/e2e_test/src/kms/test_runner.rs

@@ -450,11 +450,7 @@ impl KMSTestSuite {
         if failed > 0 {
             warn!("❌ Failing tests:");
             for result in results.iter().filter(|r| !r.success) {
-                warn!(
-                    "  - {}: {}",
-                    result.test_name,
-                    result.error_message.as_ref().unwrap_or(&"Unknown error".to_string())
-                );
+                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
             }
         }
     }

crates/e2e_test/src/lib.rs

@@ -100,10 +100,6 @@ mod cluster_concurrency_test;
 #[cfg(test)]
 mod checksum_upload_test;
 
-// Range request regression tests
-#[cfg(test)]
-mod range_request_test;
-
 // Group deletion tests
 #[cfg(test)]
 mod group_delete_test;

crates/e2e_test/src/range_request_test.rs

@@ -1,80 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! End-to-end regression test for invalid GET object ranges.
-
-#[cfg(test)]
-mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
-    use aws_sdk_s3::Client;
-    use aws_sdk_s3::error::SdkError;
-    use aws_sdk_s3::primitives::ByteStream;
-    use serial_test::serial;
-    use tracing::info;
-
-    fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
-        env.create_s3_client()
-    }
-
-    async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        match client.create_bucket().bucket(bucket).send().await {
-            Ok(_) => Ok(()),
-            Err(err) => {
-                if err.to_string().contains("BucketAlreadyOwnedByYou") || err.to_string().contains("BucketAlreadyExists") {
-                    Ok(())
-                } else {
-                    Err(Box::new(err))
-                }
-            }
-        }
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn test_get_object_invalid_range_returns_416_issue_s3_implemented_tests() {
-        init_logging();
-        info!("TEST: GetObject invalid range should return InvalidRange/416");
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-invalid-range";
-        let key = "range.txt";
-        let content = b"testcontent";
-
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-        client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(ByteStream::from_static(content))
-            .send()
-            .await
-            .expect("PutObject should succeed");
-
-        let result = client.get_object().bucket(bucket).key(key).range("bytes=40-50").send().await;
-
-        let err = result.expect_err("GetObject with an unsatisfiable range should fail");
-        match err {
-            SdkError::ServiceError(service_err) => {
-                assert_eq!(service_err.raw().status().as_u16(), 416, "invalid range should return HTTP 416");
-
-                let s3_err = service_err.into_err();
-                assert_eq!(s3_err.meta().code(), Some("InvalidRange"), "invalid range should map to InvalidRange");
-            }
-            other_err => panic!("Expected S3 service error, got: {other_err:?}"),
-        }
-    }
-}

crates/e2e_test/src/reliant/conditional_writes.rs

@@ -259,7 +259,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
     let upload_id = initiate_response
         .upload_id()
-        .ok_or(std::io::Error::other("No upload ID returned"))?;
+        .ok_or_else(|| std::io::Error::other("No upload ID returned"))?;
 
     // Upload parts
     for part_number in 1..=num_parts {
@@ -277,7 +277,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
         let part_etag = upload_part_response
             .e_tag()
-            .ok_or(std::io::Error::other("Do not have etag"))?
+            .ok_or_else(|| std::io::Error::other("Do not have etag"))?
             .to_string();
 
         let completed_part = CompletedPart::builder().part_number(part_number).e_tag(part_etag).build();

crates/e2e_test/src/reliant/lifecycle.rs

@@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 #[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
+async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
     use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
 
     let client = create_aws_s3_client().await?;
@@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::er
         .build()?;
     let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
 
-    let err = client
+    client
         .put_bucket_lifecycle_configuration()
         .bucket(BUCKET)
         .lifecycle_configuration(lifecycle)
         .send()
-        .await
-        .expect_err("zero-day lifecycle expiration should be rejected");
-
-    let err_msg = format!("{err:?}");
-    assert!(
-        err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"),
-        "unexpected error: {err_msg}"
-    );
+        .await?;
 
     Ok(())
 }

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -154,7 +154,7 @@ async fn walk_dir() -> Result<(), Box<dyn Error>> {
             match response.next().await {
                 Some(Ok(resp)) => {
                     if !resp.success {
-                        println!("{}", resp.error_info.unwrap_or("".to_string()));
+                        println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
                     }
                     let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
                         .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))

crates/e2e_test/src/reliant/sql.rs

@@ -281,8 +281,11 @@ async fn test_select_object_content_csv_limit() -> Result<(), Box<dyn Error>> {
     println!("CSV Limit result: {result_str}");
 
     // Verify only first 2 records are returned
-    let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
-    assert_eq!(lines.len(), 2, "Should return exactly 2 records");
+    assert_eq!(
+        result_str.lines().filter(|line| !line.trim().is_empty()).count(),
+        2,
+        "Should return exactly 2 records"
+    );
 
     Ok(())
 }
@@ -321,8 +324,10 @@ async fn test_select_object_content_csv_order_by() -> Result<(), Box<dyn Error>>
     println!("CSV Order By result: {result_str}");
 
     // Verify ordered by age descending
-    let lines: Vec<&str> = result_str.lines().filter(|line| !line.trim().is_empty()).collect();
-    assert!(lines.len() >= 2, "Should return at least 2 records");
+    assert!(
+        result_str.lines().filter(|line| !line.trim().is_empty()).count() >= 2,
+        "Should return at least 2 records"
+    );
 
     // Check if contains highest age records
     assert!(result_str.contains("Charlie,35"));

crates/e2e_test/src/replication_extension_test.rs

@@ -190,6 +190,14 @@ async fn put_bucket_replication(
     Ok(())
 }
 
+async fn delete_bucket_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication", env.url);
+    signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
 async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
     let client = env.create_s3_client();
     client
@@ -1084,6 +1092,54 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
     Ok(())
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-delete-config-src";
+    let target_bucket = "replication-delete-config-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    let delete_response = delete_bucket_replication(&source_env, source_bucket).await?;
+    assert!(
+        delete_response.status().is_success(),
+        "unexpected delete status: {}",
+        delete_response.status()
+    );
+
+    let targets_response = list_replication_targets_request(&source_env, Some(source_bucket)).await?;
+    assert_eq!(targets_response.status(), StatusCode::OK);
+    let targets: Vec<serde_json::Value> = targets_response.json().await?;
+    assert!(
+        targets
+            .iter()
+            .all(|target| target.get("arn").and_then(|arn| arn.as_str()) != Some(target_arn.as_str())),
+        "deleted replication config left stale target {target_arn}: {targets:?}"
+    );
+
+    let recreated_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &recreated_arn).await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {

crates/e2e_test/src/special_chars_test.rs

@@ -26,10 +26,16 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
     use aws_sdk_s3::Client;
     use aws_sdk_s3::primitives::ByteStream;
+    use http::StatusCode;
+    use http::header::HOST;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
     use serial_test::serial;
+    use std::error::Error;
     use tracing::{debug, info};
 
     /// Helper function to create an S3 client for testing
@@ -56,6 +62,30 @@ mod tests {
         }
     }
 
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
     /// Test PUT and GET with space character in path
     ///
     /// This reproduces Part A of the issue:
@@ -274,6 +304,73 @@ mod tests {
         info!("Test completed successfully");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_missing_object_with_trailing_equals_returns_no_such_key() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-missing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let url = format!("{}/{}/path/sitemap.xmlage=", env.url, bucket);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::NOT_FOUND,
+            "missing object key ending with '=' should pass signature validation before object lookup"
+        );
+
+        let body = response.text().await?;
+        assert!(body.contains("<Code>NoSuchKey</Code>"), "expected NoSuchKey XML response, got: {body}");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_existing_object_with_trailing_equals_returns_content() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-existing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let key = "path/sitemap.xmlage=";
+        let content = b"object content for raw signed URL with trailing equals";
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}/{}", env.url, bucket, key);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::OK,
+            "existing object key ending with '=' should pass signature validation and return content"
+        );
+
+        let body = response.bytes().await?;
+        assert_eq!(body.as_ref(), content);
+
+        env.stop_server();
+        Ok(())
+    }
+
     /// Test DELETE operation with special characters
     #[tokio::test]
     #[serial]

crates/ecstore/Cargo.toml

@@ -70,7 +70,6 @@ reed-solomon-erasure = { workspace = true }
 reed-solomon-simd = { workspace = true }
 lazy_static.workspace = true
 rustfs-lock.workspace = true
-rustfs-io-core.workspace = true
 rustfs-io-metrics.workspace = true
 regex = { workspace = true }
 path-absolutize = { workspace = true }
@@ -87,7 +86,7 @@ hyper.workspace = true
 hyper-util.workspace = true
 hyper-rustls.workspace = true
 rustls.workspace = true
-tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
+tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower.workspace = true
@@ -140,17 +139,5 @@ harness = false
 name = "comparison_benchmark"
 harness = false
 
-[[bench]]
-name = "direct_chunk_benchmark"
-harness = false
-
-[[bench]]
-name = "reconstructed_chunk_benchmark"
-harness = false
-
-[[bench]]
-name = "bitrot_chunk_benchmark"
-harness = false
-
 [lib]
 doctest = false

crates/ecstore/README.md

@@ -32,43 +32,6 @@
 
 For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
 
-## 📈 Benchmarks
-
-ECStore ships several Criterion benchmarks under [`crates/ecstore/benches/`](./benches/).
-
-### Direct Chunk Path
-
-Use the direct chunk benchmark to compare the current slice-forwarding path against the previous assembled-copy path:
-
-```bash
-cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark
-```
-
-To run only the end-to-end ECStore range-read benchmark:
-
-```bash
-cargo bench -p rustfs-ecstore --bench direct_chunk_benchmark ecstore_get_object_chunks
-```
-
-To run the reconstructed multi-disk range-read benchmark:
-
-```bash
-cargo bench -p rustfs-ecstore --bench reconstructed_chunk_benchmark
-```
-
-### Saved Comparison Points
-
-Latest local measurements on this branch:
-
-- `direct_chunk_path/slice_forwarding/single_block_aligned`: about `477 ns`
-- `direct_chunk_path/assembled_copy/single_block_aligned`: about `3.25 us`
-- `direct_chunk_path/slice_forwarding/multi_block_unaligned`: about `963 ns`
-- `direct_chunk_path/assembled_copy/multi_block_unaligned`: about `7.25 us`
-- `ecstore_get_object_chunks/drain/multi_disk_range`: about `644-654 us`, throughput about `2.86-2.90 GiB/s`
-- `reconstructed_chunk_path/drain/multi_disk_missing_shard`: about `1.292-1.304 ms`, throughput about `1.43-1.45 GiB/s`
-
-These numbers are intended as branch-local reference points. Re-run the benchmark on your target machine before treating them as a regression baseline.
-
 ## 📄 License
 
 This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.

crates/ecstore/benches/bitrot_chunk_benchmark.rs

@@ -1,106 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use bytes::Bytes;
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use rustfs_ecstore::bitrot::decode_bitrot_chunk_source_for_bench;
-use rustfs_io_core::IoChunk;
-use rustfs_utils::HashAlgorithm;
-use std::hint::black_box;
-
-struct BitrotChunkBenchCase {
-    name: &'static str,
-    source_chunks: Vec<IoChunk>,
-    shard_size: usize,
-    expected_decoded_len: usize,
-    expected_copied: bool,
-}
-
-fn encode_shard(checksum_algo: HashAlgorithm, shard: &[u8]) -> Vec<u8> {
-    let mut encoded = Vec::with_capacity(checksum_algo.size() + shard.len());
-    encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
-    encoded.extend_from_slice(shard);
-    encoded
-}
-
-fn bitrot_chunk_bench_cases() -> [BitrotChunkBenchCase; 2] {
-    let checksum_algo = HashAlgorithm::Md5;
-    let shard_one = b"abcd";
-    let shard_two = b"efgh";
-    let encoded_one = encode_shard(checksum_algo.clone(), shard_one);
-    let encoded_two = encode_shard(checksum_algo.clone(), shard_two);
-
-    let mut cross_chunk = Vec::with_capacity(encoded_one.len() + encoded_two.len());
-    cross_chunk.extend_from_slice(&encoded_one);
-    cross_chunk.extend_from_slice(&encoded_two);
-    let split = checksum_algo.size() + 2;
-
-    [
-        BitrotChunkBenchCase {
-            name: "aligned_multi_chunk_no_copy",
-            source_chunks: vec![
-                IoChunk::Shared(Bytes::from(encoded_one)),
-                IoChunk::Shared(Bytes::from(encoded_two)),
-            ],
-            shard_size: shard_one.len(),
-            expected_decoded_len: shard_one.len() + shard_two.len(),
-            expected_copied: false,
-        },
-        BitrotChunkBenchCase {
-            name: "cross_chunk_frame_copy",
-            source_chunks: vec![
-                IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[..split])),
-                IoChunk::Shared(Bytes::copy_from_slice(&cross_chunk[split..])),
-            ],
-            shard_size: shard_one.len(),
-            expected_decoded_len: shard_one.len() + shard_two.len(),
-            expected_copied: true,
-        },
-    ]
-}
-
-fn bench_bitrot_chunk_decode(c: &mut Criterion) {
-    let checksum_algo = HashAlgorithm::Md5;
-    let mut group = c.benchmark_group("bitrot_chunk_decode");
-    group.sample_size(20);
-
-    for case in bitrot_chunk_bench_cases() {
-        let (decoded, copied) =
-            decode_bitrot_chunk_source_for_bench(&case.source_chunks, case.shard_size, checksum_algo.clone(), false)
-                .expect("decode bitrot source");
-        let decoded_len: usize = decoded.iter().map(IoChunk::len).sum();
-
-        assert_eq!(decoded_len, case.expected_decoded_len);
-        assert_eq!(copied, case.expected_copied);
-
-        group.throughput(Throughput::Bytes(case.expected_decoded_len as u64));
-        group.bench_with_input(BenchmarkId::new("decode", case.name), &case, |b, case| {
-            b.iter(|| {
-                let result = decode_bitrot_chunk_source_for_bench(
-                    black_box(&case.source_chunks),
-                    black_box(case.shard_size),
-                    checksum_algo.clone(),
-                    false,
-                )
-                .expect("decode bitrot source");
-                black_box(result);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-criterion_group!(benches, bench_bitrot_chunk_decode);
-criterion_main!(benches);

crates/ecstore/benches/direct_chunk_benchmark.rs

@@ -1,390 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use bytes::{Bytes, BytesMut};
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use futures_util::StreamExt;
-use futures_util::stream;
-use http::HeaderMap;
-use rustfs_ecstore::bucket::metadata_sys;
-use rustfs_ecstore::disk::endpoint::Endpoint;
-use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
-use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
-use rustfs_ecstore::set_disk::collect_direct_data_shard_chunks_for_benchmark;
-use rustfs_ecstore::store::{ECStore, init_local_disks};
-use rustfs_ecstore::store_api::{
-    BucketOperations, BucketOptions, ChunkNativePutData, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions, ObjectIO,
-    ObjectOptions,
-};
-use rustfs_io_core::{BoxChunkStream, IoChunk, MappedChunk};
-use std::hint::black_box;
-use std::net::SocketAddr;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU16, Ordering};
-use tempfile::TempDir;
-use tokio::runtime::Runtime;
-use tokio_util::sync::CancellationToken;
-
-#[derive(Clone)]
-struct BenchCase {
-    name: &'static str,
-    data_shards: usize,
-    block_size: usize,
-    blocks: usize,
-    offset: usize,
-    length: usize,
-}
-
-fn bench_cases() -> [BenchCase; 2] {
-    [
-        BenchCase {
-            name: "single_block_aligned",
-            data_shards: 4,
-            block_size: 256 * 1024,
-            blocks: 1,
-            offset: 0,
-            length: 256 * 1024,
-        },
-        BenchCase {
-            name: "multi_block_unaligned",
-            data_shards: 4,
-            block_size: 256 * 1024,
-            blocks: 8,
-            offset: 123_457,
-            length: 2 * 256 * 1024 + 33_333,
-        },
-    ]
-}
-
-#[derive(Clone)]
-struct EcstoreBenchCase {
-    name: &'static str,
-    disk_count: usize,
-    payload_len: usize,
-    range: HTTPRangeSpec,
-    expected_copy_mode: GetObjectChunkCopyMode,
-}
-
-struct EcstoreBenchEnv {
-    _temp_dir: TempDir,
-    store: Arc<ECStore>,
-    bucket: String,
-    key: String,
-    range: HTTPRangeSpec,
-    opts: ObjectOptions,
-    expected_len: usize,
-    expected_copy_mode: GetObjectChunkCopyMode,
-}
-
-fn ecstore_bench_cases() -> [EcstoreBenchCase; 1] {
-    [EcstoreBenchCase {
-        name: "multi_disk_range",
-        disk_count: 4,
-        payload_len: 3 * 1024 * 1024 + 137,
-        range: HTTPRangeSpec {
-            is_suffix_length: false,
-            start: 123_457,
-            end: 2 * 1024 * 1024 + 33_333,
-        },
-        expected_copy_mode: expected_direct_copy_mode(),
-    }]
-}
-
-#[cfg(unix)]
-const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
-    GetObjectChunkCopyMode::TrueZeroCopy
-}
-
-#[cfg(not(unix))]
-const fn expected_direct_copy_mode() -> GetObjectChunkCopyMode {
-    GetObjectChunkCopyMode::SharedBytes
-}
-
-fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
-    HTTPRangeSpec {
-        is_suffix_length: range.is_suffix_length,
-        start: range.start,
-        end: range.end,
-    }
-}
-
-fn next_loopback_addr() -> SocketAddr {
-    static NEXT_PORT: AtomicU16 = AtomicU16::new(39013);
-    let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
-    SocketAddr::from(([127, 0, 0, 1], port))
-}
-
-fn build_endpoint_pools(paths: &[std::path::PathBuf]) -> EndpointServerPools {
-    let mut endpoints = Vec::with_capacity(paths.len());
-    for (idx, disk_path) in paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(idx);
-        endpoints.push(endpoint);
-    }
-
-    EndpointServerPools(vec![PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: paths.len(),
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "bench".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    }])
-}
-
-async fn build_ecstore_bench_env(case: &EcstoreBenchCase) -> EcstoreBenchEnv {
-    let temp_dir = tempfile::tempdir().expect("tempdir");
-    let mut disk_paths = Vec::with_capacity(case.disk_count);
-    for idx in 0..case.disk_count {
-        let path = temp_dir.path().join(format!("disk{}", idx + 1));
-        tokio::fs::create_dir_all(&path).await.expect("create disk dir");
-        disk_paths.push(path);
-    }
-
-    let endpoint_pools = build_endpoint_pools(&disk_paths);
-    GLOBAL_LOCAL_DISK_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
-    init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
-
-    let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
-        .await
-        .expect("create ecstore");
-
-    let buckets = store
-        .list_bucket(&BucketOptions {
-            no_metadata: true,
-            ..Default::default()
-        })
-        .await
-        .expect("list buckets")
-        .into_iter()
-        .map(|bucket| bucket.name)
-        .collect();
-    metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
-
-    let object_id = case.name.replace('_', "-");
-    let bucket = format!("bench-direct-{object_id}");
-    let key = format!("objects/{object_id}.bin");
-    store
-        .make_bucket(
-            &bucket,
-            &MakeBucketOptions {
-                versioning_enabled: true,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("make bucket");
-
-    let payload: Vec<u8> = (0..case.payload_len).map(|idx| (idx % 251) as u8).collect();
-    let mut reader = ChunkNativePutData::from_vec(payload);
-    let put_info = store
-        .put_object(&bucket, &key, &mut reader, &ObjectOptions::default())
-        .await
-        .expect("put object");
-    if case.disk_count > 1 {
-        assert!(put_info.data_blocks > 1, "expected multi-data-shard object");
-    }
-
-    let (_, expected_len) = case.range.get_offset_length(case.payload_len as i64).expect("range length");
-
-    EcstoreBenchEnv {
-        _temp_dir: temp_dir,
-        store,
-        bucket,
-        key,
-        range: clone_range_spec(&case.range),
-        opts: ObjectOptions::default(),
-        expected_len: expected_len as usize,
-        expected_copy_mode: case.expected_copy_mode,
-    }
-}
-
-async fn run_ecstore_get_object_chunks_bench(env: &EcstoreBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
-    let mut result = env
-        .store
-        .get_object_chunks(&env.bucket, &env.key, Some(clone_range_spec(&env.range)), HeaderMap::new(), &env.opts)
-        .await
-        .expect("get object chunks");
-    let copy_mode = result.copy_mode;
-    let mut total_len = 0usize;
-    let mut chunk_count = 0usize;
-    while let Some(chunk) = result.stream.next().await {
-        let chunk = chunk.expect("chunk");
-        total_len += chunk.len();
-        chunk_count += 1;
-    }
-
-    (copy_mode, total_len, chunk_count)
-}
-
-fn build_shard_bytes(case: &BenchCase) -> Vec<Vec<Bytes>> {
-    let total_len = case.block_size * case.blocks;
-    let payload: Vec<u8> = (0..total_len).map(|idx| (idx % 251) as u8).collect();
-    let mut shards = vec![Vec::with_capacity(case.blocks); case.data_shards];
-
-    for block in 0..case.blocks {
-        let block_start = block * case.block_size;
-        let block_slice = &payload[block_start..block_start + case.block_size];
-        let shard_width = case.block_size / case.data_shards;
-        for (shard_idx, shard) in shards.iter_mut().enumerate().take(case.data_shards) {
-            let shard_start = shard_idx * shard_width;
-            let shard_end = shard_start + shard_width;
-            shard.push(Bytes::copy_from_slice(&block_slice[shard_start..shard_end]));
-        }
-    }
-
-    shards
-}
-
-fn build_mapped_streams(shards: &[Vec<Bytes>]) -> Vec<BoxChunkStream> {
-    shards
-        .iter()
-        .map(|shard_chunks| {
-            let chunks: Vec<_> = shard_chunks
-                .iter()
-                .map(|chunk| {
-                    let mapped = MappedChunk::new(chunk.clone(), 0, chunk.len()).expect("mapped chunk");
-                    Ok::<IoChunk, std::io::Error>(IoChunk::Mapped(mapped))
-                })
-                .collect();
-            Box::pin(stream::iter(chunks)) as BoxChunkStream
-        })
-        .collect()
-}
-
-fn collect_old_assembly(
-    shards: &[Vec<Bytes>],
-    data_shards: usize,
-    block_size: usize,
-    offset: usize,
-    length: usize,
-) -> Vec<IoChunk> {
-    if length == 0 {
-        return Vec::new();
-    }
-
-    let start_block = offset / block_size;
-    let end_block = offset.saturating_add(length - 1) / block_size;
-    let mut result = Vec::with_capacity(end_block - start_block + 1);
-
-    for block_index in start_block..=end_block {
-        let block_offset = if block_index == start_block { offset % block_size } else { 0 };
-        let block_length = if start_block == end_block {
-            length
-        } else if block_index == start_block {
-            block_size - (offset % block_size)
-        } else if block_index == end_block {
-            (offset + length) % block_size
-        } else {
-            block_size
-        };
-
-        if block_length == 0 {
-            break;
-        }
-
-        let mut block = BytesMut::with_capacity(block_length);
-        let mut write_left = block_length;
-        let mut skip = block_offset;
-
-        for shard in shards.iter().take(data_shards) {
-            let shard_chunk = &shard[block_index];
-            if skip >= shard_chunk.len() {
-                skip -= shard_chunk.len();
-                continue;
-            }
-
-            let available = &shard_chunk[skip..];
-            skip = 0;
-            let take = available.len().min(write_left);
-            block.extend_from_slice(&available[..take]);
-            write_left -= take;
-
-            if write_left == 0 {
-                break;
-            }
-        }
-
-        result.push(IoChunk::Shared(block.freeze()));
-    }
-
-    result
-}
-
-fn bench_direct_chunk_path(c: &mut Criterion) {
-    let runtime = Runtime::new().expect("tokio runtime");
-    let mut group = c.benchmark_group("direct_chunk_path");
-
-    for case in bench_cases() {
-        let shard_bytes = build_shard_bytes(&case);
-        group.throughput(Throughput::Bytes(case.length as u64));
-
-        group.bench_with_input(BenchmarkId::new("slice_forwarding", case.name), &case, |b, case| {
-            b.iter(|| {
-                let streams = build_mapped_streams(&shard_bytes);
-                let chunks = runtime
-                    .block_on(collect_direct_data_shard_chunks_for_benchmark(
-                        streams,
-                        case.data_shards,
-                        case.block_size,
-                        case.blocks * case.block_size,
-                        false,
-                        case.offset,
-                        case.length,
-                    ))
-                    .expect("collect direct chunks");
-                black_box(chunks);
-            });
-        });
-
-        group.bench_with_input(BenchmarkId::new("assembled_copy", case.name), &case, |b, case| {
-            b.iter(|| {
-                let chunks = collect_old_assembly(&shard_bytes, case.data_shards, case.block_size, case.offset, case.length);
-                black_box(chunks);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-fn bench_ecstore_get_object_chunks(c: &mut Criterion) {
-    let runtime = Runtime::new().expect("tokio runtime");
-    let mut group = c.benchmark_group("ecstore_get_object_chunks");
-    group.sample_size(10);
-
-    for case in ecstore_bench_cases() {
-        let env = runtime.block_on(build_ecstore_bench_env(&case));
-        let (copy_mode, total_len, _) = runtime.block_on(run_ecstore_get_object_chunks_bench(&env));
-        assert_eq!(copy_mode, env.expected_copy_mode);
-        assert_eq!(total_len, env.expected_len);
-
-        group.throughput(Throughput::Bytes(env.expected_len as u64));
-        group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
-            b.iter(|| {
-                let result = runtime.block_on(run_ecstore_get_object_chunks_bench(env));
-                black_box(result);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-criterion_group!(benches, bench_direct_chunk_path, bench_ecstore_get_object_chunks);
-criterion_main!(benches);

crates/ecstore/benches/reconstructed_chunk_benchmark.rs

@@ -1,393 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use futures_util::StreamExt;
-use http::HeaderMap;
-use rustfs_ecstore::bucket::metadata_sys;
-use rustfs_ecstore::disk::endpoint::Endpoint;
-use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
-use rustfs_ecstore::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
-use rustfs_ecstore::store::{ECStore, init_local_disks};
-use rustfs_ecstore::store_api::{
-    BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, GetObjectChunkCopyMode, HTTPRangeSpec, MakeBucketOptions,
-    MultipartOperations, ObjectOperations, ObjectOptions,
-};
-use std::hint::black_box;
-use std::net::SocketAddr;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU16, Ordering};
-use tempfile::TempDir;
-use tokio::runtime::Runtime;
-use tokio_util::sync::CancellationToken;
-
-#[derive(Clone)]
-enum ReconstructedReadSpec {
-    PartNumber {
-        part_number: usize,
-        missing_part_name: &'static str,
-    },
-    Range {
-        start: u64,
-        end: u64,
-        missing_part_name: &'static str,
-    },
-}
-
-#[derive(Clone)]
-struct ReconstructedBenchCase {
-    object_id: &'static str,
-    name: &'static str,
-    read_spec: ReconstructedReadSpec,
-}
-
-struct ReconstructedBenchEnv {
-    store: Arc<ECStore>,
-    bucket: String,
-    key: String,
-    range: HTTPRangeSpec,
-    opts: ObjectOptions,
-    expected_len: usize,
-}
-
-struct ReconstructedBenchSuite {
-    _temp_dir: TempDir,
-    store: Arc<ECStore>,
-    disk_paths: Vec<PathBuf>,
-}
-
-const MULTIPART_PART_ONE_LEN: usize = 5 * 1024 * 1024;
-const MULTIPART_PART_TWO_LEN: usize = 5 * 1024 * 1024 + 137;
-const MULTIPART_PART_THREE_LEN: usize = 1024 * 1024 + 77;
-
-fn reconstructed_bench_cases() -> [ReconstructedBenchCase; 4] {
-    [
-        ReconstructedBenchCase {
-            object_id: "mp-part2",
-            name: "multi_disk_missing_shard_multipart_part2",
-            read_spec: ReconstructedReadSpec::PartNumber {
-                part_number: 2,
-                missing_part_name: "part.2",
-            },
-        },
-        ReconstructedBenchCase {
-            object_id: "mp-cross-range",
-            name: "multi_disk_missing_shard_multipart_cross_part_range",
-            read_spec: ReconstructedReadSpec::Range {
-                start: (MULTIPART_PART_ONE_LEN - 32 * 1024) as u64,
-                end: (MULTIPART_PART_ONE_LEN + 96 * 1024) as u64,
-                missing_part_name: "part.2",
-            },
-        },
-        ReconstructedBenchCase {
-            object_id: "mp-part3",
-            name: "multi_disk_missing_shard_multipart_part3",
-            read_spec: ReconstructedReadSpec::PartNumber {
-                part_number: 3,
-                missing_part_name: "part.3",
-            },
-        },
-        ReconstructedBenchCase {
-            object_id: "mp-cross-final-range",
-            name: "multi_disk_missing_shard_multipart_cross_final_part_range",
-            read_spec: ReconstructedReadSpec::Range {
-                start: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN - 32 * 1024) as u64,
-                end: (MULTIPART_PART_ONE_LEN + MULTIPART_PART_TWO_LEN + 96 * 1024) as u64,
-                missing_part_name: "part.3",
-            },
-        },
-    ]
-}
-
-fn next_loopback_addr() -> SocketAddr {
-    static NEXT_PORT: AtomicU16 = AtomicU16::new(39113);
-    let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
-    SocketAddr::from(([127, 0, 0, 1], port))
-}
-
-fn clone_range_spec(range: &HTTPRangeSpec) -> HTTPRangeSpec {
-    HTTPRangeSpec {
-        is_suffix_length: range.is_suffix_length,
-        start: range.start,
-        end: range.end,
-    }
-}
-
-fn build_endpoint_pools(paths: &[PathBuf]) -> EndpointServerPools {
-    let mut endpoints = Vec::with_capacity(paths.len());
-    for (idx, disk_path) in paths.iter().enumerate() {
-        let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("utf8 path")).expect("endpoint");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(idx);
-        endpoints.push(endpoint);
-    }
-
-    EndpointServerPools(vec![PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: paths.len(),
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "bench".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    }])
-}
-
-fn find_part_files(root: &Path, part_name: &str, out: &mut Vec<PathBuf>) {
-    let Ok(entries) = std::fs::read_dir(root) else {
-        return;
-    };
-    for entry in entries.flatten() {
-        let path = entry.path();
-        if path.is_dir() {
-            find_part_files(&path, part_name, out);
-            continue;
-        }
-
-        if path.file_name().and_then(|name| name.to_str()) == Some(part_name) {
-            out.push(path);
-        }
-    }
-}
-
-async fn remove_part_files(root: &Path, part_name: &str) -> Vec<(PathBuf, Vec<u8>)> {
-    let mut paths = Vec::new();
-    find_part_files(root, part_name, &mut paths);
-
-    let mut removed = Vec::with_capacity(paths.len());
-    for path in paths {
-        let content = tokio::fs::read(&path).await.expect("read part file before removal");
-        tokio::fs::remove_file(&path).await.expect("remove part file");
-        removed.push((path, content));
-    }
-
-    removed
-}
-
-async fn restore_part_files(files: Vec<(PathBuf, Vec<u8>)>) {
-    for (path, content) in files {
-        if let Some(parent) = path.parent() {
-            tokio::fs::create_dir_all(parent)
-                .await
-                .expect("ensure parent for part restore");
-        }
-        tokio::fs::write(&path, content).await.expect("restore part file");
-    }
-}
-
-async fn run_reconstructed_get_object_chunks(
-    store: &Arc<ECStore>,
-    bucket: &str,
-    key: &str,
-    range: &HTTPRangeSpec,
-    opts: &ObjectOptions,
-) -> (GetObjectChunkCopyMode, usize, usize) {
-    let mut result = store
-        .get_object_chunks(bucket, key, Some(clone_range_spec(range)), HeaderMap::new(), opts)
-        .await
-        .expect("get object chunks");
-
-    let copy_mode = result.copy_mode;
-    let mut total_len = 0usize;
-    let mut chunk_count = 0usize;
-    while let Some(chunk) = result.stream.next().await {
-        let chunk = chunk.expect("chunk");
-        total_len += chunk.len();
-        chunk_count += 1;
-    }
-
-    (copy_mode, total_len, chunk_count)
-}
-
-async fn create_multipart_object(store: &Arc<ECStore>, bucket: &str, key: &str, parts: &[Vec<u8>]) -> usize {
-    let upload = store
-        .new_multipart_upload(bucket, key, &ObjectOptions::default())
-        .await
-        .expect("new multipart upload");
-
-    let mut completed_parts = Vec::with_capacity(parts.len());
-    for (idx, part) in parts.iter().enumerate() {
-        let mut reader = ChunkNativePutData::from_vec(part.clone());
-        let part_info = store
-            .put_object_part(bucket, key, &upload.upload_id, idx + 1, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("put object part");
-        completed_parts.push(CompletePart {
-            part_num: idx + 1,
-            etag: part_info.etag,
-            ..Default::default()
-        });
-    }
-
-    store
-        .clone()
-        .complete_multipart_upload(bucket, key, &upload.upload_id, completed_parts, &ObjectOptions::default())
-        .await
-        .expect("complete multipart upload");
-
-    parts.iter().map(Vec::len).sum()
-}
-
-async fn build_reconstructed_bench_suite() -> ReconstructedBenchSuite {
-    let temp_dir = tempfile::tempdir().expect("tempdir");
-    let disk_paths: Vec<_> = (1..=4).map(|idx| temp_dir.path().join(format!("disk{idx}"))).collect();
-    for disk_path in &disk_paths {
-        tokio::fs::create_dir_all(disk_path).await.expect("create disk dir");
-    }
-
-    let endpoint_pools = build_endpoint_pools(&disk_paths);
-    GLOBAL_LOCAL_DISK_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
-    GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
-    init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
-
-    let store = ECStore::new(next_loopback_addr(), endpoint_pools, CancellationToken::new())
-        .await
-        .expect("create ecstore");
-
-    let buckets = store
-        .list_bucket(&BucketOptions {
-            no_metadata: true,
-            ..Default::default()
-        })
-        .await
-        .expect("list buckets")
-        .into_iter()
-        .map(|bucket| bucket.name)
-        .collect();
-    metadata_sys::init_bucket_metadata_sys(store.clone(), buckets).await;
-
-    ReconstructedBenchSuite {
-        _temp_dir: temp_dir,
-        store,
-        disk_paths,
-    }
-}
-
-async fn build_reconstructed_bench_env(suite: &ReconstructedBenchSuite, case: &ReconstructedBenchCase) -> ReconstructedBenchEnv {
-    let bucket = format!("bench-r-{}", case.object_id);
-    let key = format!("objects/{}.bin", case.object_id);
-    suite
-        .store
-        .make_bucket(
-            &bucket,
-            &MakeBucketOptions {
-                versioning_enabled: true,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("make bucket");
-
-    let part_one: Vec<u8> = (0..MULTIPART_PART_ONE_LEN).map(|idx| (idx % 251) as u8).collect();
-    let part_two: Vec<u8> = (0..MULTIPART_PART_TWO_LEN).map(|idx| ((idx + 11) % 251) as u8).collect();
-    let part_three: Vec<u8> = (0..MULTIPART_PART_THREE_LEN).map(|idx| ((idx + 29) % 251) as u8).collect();
-    let payload_len = create_multipart_object(&suite.store, &bucket, &key, &[part_one, part_two, part_three]).await;
-    let info = suite
-        .store
-        .get_object_info(&bucket, &key, &ObjectOptions::default())
-        .await
-        .expect("get object info");
-    let (range, opts, missing_part_name) = match case.read_spec.clone() {
-        ReconstructedReadSpec::PartNumber {
-            part_number,
-            missing_part_name,
-        } => {
-            let range = HTTPRangeSpec::from_object_info(&info, part_number).expect("part_number range");
-            let opts = ObjectOptions {
-                part_number: Some(part_number),
-                ..Default::default()
-            };
-            (range, opts, missing_part_name)
-        }
-        ReconstructedReadSpec::Range {
-            start,
-            end,
-            missing_part_name,
-        } => (
-            HTTPRangeSpec {
-                is_suffix_length: false,
-                start: start as i64,
-                end: end as i64,
-            },
-            ObjectOptions::default(),
-            missing_part_name,
-        ),
-    };
-
-    let (_, expected_len) = range.get_offset_length(payload_len as i64).expect("range length");
-    let mut selected_reconstructed = false;
-    for disk_path in &suite.disk_paths {
-        let object_root = disk_path
-            .join(&bucket)
-            .join("objects")
-            .join(format!("{}.bin", case.object_id));
-        let removed_parts = remove_part_files(&object_root, missing_part_name).await;
-        if removed_parts.is_empty() {
-            continue;
-        }
-
-        let (copy_mode, total_len, _) = run_reconstructed_get_object_chunks(&suite.store, &bucket, &key, &range, &opts).await;
-        if copy_mode == GetObjectChunkCopyMode::Reconstructed && total_len == expected_len as usize {
-            selected_reconstructed = true;
-            break;
-        }
-
-        restore_part_files(removed_parts).await;
-    }
-    assert!(
-        selected_reconstructed,
-        "failed to select a missing shard placement that triggers reconstructed copy mode for benchmark case {}",
-        case.name
-    );
-
-    ReconstructedBenchEnv {
-        store: suite.store.clone(),
-        bucket,
-        key,
-        range: clone_range_spec(&range),
-        opts,
-        expected_len: expected_len as usize,
-    }
-}
-
-async fn run_reconstructed_get_object_chunks_bench(env: &ReconstructedBenchEnv) -> (GetObjectChunkCopyMode, usize, usize) {
-    run_reconstructed_get_object_chunks(&env.store, &env.bucket, &env.key, &env.range, &env.opts).await
-}
-
-fn bench_reconstructed_chunk_path(c: &mut Criterion) {
-    let runtime = Runtime::new().expect("tokio runtime");
-    let suite = runtime.block_on(build_reconstructed_bench_suite());
-    let mut group = c.benchmark_group("reconstructed_chunk_path");
-    group.sample_size(10);
-    for case in reconstructed_bench_cases() {
-        let env = runtime.block_on(build_reconstructed_bench_env(&suite, &case));
-        let (copy_mode, total_len, _) = runtime.block_on(run_reconstructed_get_object_chunks_bench(&env));
-        assert_eq!(copy_mode, GetObjectChunkCopyMode::Reconstructed);
-        assert_eq!(total_len, env.expected_len);
-
-        group.throughput(Throughput::Bytes(env.expected_len as u64));
-        group.bench_with_input(BenchmarkId::new("drain", case.name), &env, |b, env| {
-            b.iter(|| {
-                let result = runtime.block_on(run_reconstructed_get_object_chunks_bench(env));
-                black_box(result);
-            });
-        });
-    }
-    group.finish();
-}
-
-criterion_group!(benches, bench_reconstructed_chunk_path);
-criterion_main!(benches);

crates/ecstore/run_benchmarks.sh

@@ -119,16 +119,6 @@ run_large_data_test() {
     print_success "Large-dataset tests completed"
 }
 
-# Run direct chunk path benchmarks
-run_direct_chunk_benchmark() {
-    print_info "📦 Starting direct chunk path benchmarks..."
-    echo "================================================"
-
-    cargo bench --bench direct_chunk_benchmark
-
-    print_success "Direct chunk path benchmarks completed"
-}
-
 # Generate comparison report
 generate_comparison_report() {
     print_info "📊 Generating performance report..."
@@ -178,7 +168,6 @@ show_help() {
     echo "  full         Run the full benchmark suite"
     echo "  performance  Run detailed performance tests"
     echo "  simd         Run the SIMD-only tests"
-    echo "  direct       Run the direct chunk path benchmarks"
     echo "  large        Run large-dataset tests"
     echo "  clean        Remove previous results"
     echo "  help         Show this help message"
@@ -188,7 +177,6 @@ show_help() {
     echo "  $0 performance        # Detailed performance test"
     echo "  $0 full              # Full benchmark suite"
     echo "  $0 simd              # SIMD-only benchmark"
-    echo "  $0 direct            # Direct chunk path benchmark"
     echo "  $0 large             # Large-dataset benchmark"
     echo ""
     echo "Features:"
@@ -252,11 +240,6 @@ main() {
             run_simd_benchmark
             generate_comparison_report
             ;;
-        "direct")
-            cleanup
-            run_direct_chunk_benchmark
-            generate_comparison_report
-            ;;
         "large")
             cleanup
             run_large_data_test

crates/ecstore/src/admin_server_info.rs

@@ -162,8 +162,9 @@ pub async fn get_local_server_property() -> ServerProperties {
 
     let mut props = ServerProperties {
         endpoint: addr,
-        uptime: SystemTime::now()
-            .duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
+        uptime: GLOBAL_BOOT_TIME
+            .get()
+            .and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok())
             .unwrap_or_default()
             .as_secs(),
         network,

crates/ecstore/src/bitrot.rs

@@ -14,328 +14,13 @@
 
 use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError};
 use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
-use crate::store_api::{GetObjectChunkCopyMode, GetObjectChunkPath, GetObjectChunkResult};
-use bytes::{Bytes, BytesMut};
-use futures_util::{StreamExt, stream};
-use rustfs_io_core::{BoxChunkStream, IoChunk};
+use bytes::Bytes;
 use rustfs_utils::HashAlgorithm;
-use std::collections::VecDeque;
 use std::io::Cursor;
 use std::time::Instant;
 use tokio::io::AsyncRead;
 use tracing::debug;
 
-const BITROT_READ_OPERATION: &str = "bitrot_read";
-
-fn classify_chunk_copy_mode(source_direct: bool, copied: bool) -> GetObjectChunkCopyMode {
-    if copied {
-        GetObjectChunkCopyMode::SingleCopy
-    } else if source_direct {
-        GetObjectChunkCopyMode::TrueZeroCopy
-    } else {
-        GetObjectChunkCopyMode::SharedBytes
-    }
-}
-
-struct ChunkSpan {
-    bytes: Bytes,
-    chunk: IoChunk,
-    copied: bool,
-}
-
-fn take_contiguous_chunk_span(chunk: &IoChunk, offset: usize, len: usize) -> std::io::Result<ChunkSpan> {
-    match chunk {
-        IoChunk::Shared(bytes) => {
-            let bytes = bytes.slice(offset..offset + len);
-            Ok(ChunkSpan {
-                bytes: bytes.clone(),
-                chunk: IoChunk::Shared(bytes),
-                copied: false,
-            })
-        }
-        IoChunk::Mapped(mapped) => {
-            let chunk = IoChunk::Mapped(mapped.slice(offset, len)?);
-            let bytes = chunk.as_bytes();
-            Ok(ChunkSpan {
-                bytes,
-                chunk,
-                copied: false,
-            })
-        }
-        IoChunk::Pooled(pooled) => {
-            let chunk = IoChunk::Pooled(pooled.slice(offset, len)?);
-            let bytes = chunk.as_bytes();
-            Ok(ChunkSpan {
-                bytes,
-                chunk,
-                copied: false,
-            })
-        }
-    }
-}
-
-struct BitrotChunkSource {
-    source_stream: BoxChunkStream,
-    source_chunks: VecDeque<IoChunk>,
-    source_chunk_offset: usize,
-    source_buffered_bytes: usize,
-    source_done: bool,
-}
-
-struct BitrotChunkStreamState {
-    source: BitrotChunkSource,
-    decoded_remaining: usize,
-    trim_prefix: usize,
-    output_remaining: usize,
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-}
-
-struct ChunkCursor<'a> {
-    chunks: &'a [IoChunk],
-    chunk_index: usize,
-    chunk_offset: usize,
-    consumed: usize,
-    total_len: usize,
-}
-
-impl<'a> ChunkCursor<'a> {
-    fn new(chunks: &'a [IoChunk]) -> Self {
-        Self {
-            chunks,
-            chunk_index: 0,
-            chunk_offset: 0,
-            consumed: 0,
-            total_len: chunks.iter().map(IoChunk::len).sum(),
-        }
-    }
-
-    fn remaining(&self) -> usize {
-        self.total_len.saturating_sub(self.consumed)
-    }
-
-    fn skip_empty_chunks(&mut self) {
-        while let Some(chunk) = self.chunks.get(self.chunk_index) {
-            if self.chunk_offset < chunk.len() {
-                break;
-            }
-            self.chunk_index += 1;
-            self.chunk_offset = 0;
-        }
-    }
-
-    fn advance(&mut self, len: usize) {
-        self.consumed += len;
-        self.chunk_offset += len;
-        self.skip_empty_chunks();
-    }
-
-    fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
-        self.skip_empty_chunks();
-        if self.remaining() < len {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-        }
-
-        let Some(chunk) = self.chunks.get(self.chunk_index) else {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
-        };
-        let available = chunk.len().saturating_sub(self.chunk_offset);
-
-        if len <= available {
-            let span = take_contiguous_chunk_span(chunk, self.chunk_offset, len)?;
-            self.advance(len);
-            return Ok(span);
-        }
-
-        let mut aggregate = BytesMut::with_capacity(len);
-        let mut remaining = len;
-        while remaining > 0 {
-            self.skip_empty_chunks();
-            let Some(chunk) = self.chunks.get(self.chunk_index) else {
-                return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-            };
-            let available = chunk.len().saturating_sub(self.chunk_offset);
-            let take = available.min(remaining);
-            aggregate.extend_from_slice(&chunk.as_bytes()[self.chunk_offset..self.chunk_offset + take]);
-            self.advance(take);
-            remaining -= take;
-        }
-
-        let bytes = aggregate.freeze();
-        Ok(ChunkSpan {
-            bytes: bytes.clone(),
-            chunk: IoChunk::Shared(bytes),
-            copied: true,
-        })
-    }
-}
-
-impl BitrotChunkSource {
-    fn new(source_stream: BoxChunkStream, source_chunks: VecDeque<IoChunk>, source_done: bool) -> Self {
-        let source_buffered_bytes = source_chunks.iter().map(IoChunk::len).sum();
-        Self {
-            source_stream,
-            source_chunks,
-            source_chunk_offset: 0,
-            source_buffered_bytes,
-            source_done,
-        }
-    }
-
-    fn skip_empty_chunks(&mut self) {
-        while let Some(chunk) = self.source_chunks.front() {
-            if self.source_chunk_offset < chunk.len() {
-                break;
-            }
-            self.source_chunks.pop_front();
-            self.source_chunk_offset = 0;
-        }
-    }
-
-    async fn fill(&mut self, min_bytes: usize) -> std::io::Result<()> {
-        while self.source_buffered_bytes < min_bytes && !self.source_done {
-            match self.source_stream.next().await {
-                Some(Ok(chunk)) => {
-                    self.source_buffered_bytes += chunk.len();
-                    self.source_chunks.push_back(chunk);
-                }
-                Some(Err(err)) => return Err(err),
-                None => self.source_done = true,
-            }
-        }
-
-        if self.source_buffered_bytes < min_bytes {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-        }
-
-        Ok(())
-    }
-
-    fn advance(&mut self, len: usize) {
-        self.source_buffered_bytes = self.source_buffered_bytes.saturating_sub(len);
-        self.source_chunk_offset += len;
-        self.skip_empty_chunks();
-    }
-
-    fn take_span(&mut self, len: usize) -> std::io::Result<ChunkSpan> {
-        self.skip_empty_chunks();
-        if self.source_buffered_bytes < len {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-        }
-
-        let Some(chunk) = self.source_chunks.front() else {
-            return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "missing bitrot chunk source"));
-        };
-        let available = chunk.len().saturating_sub(self.source_chunk_offset);
-
-        if len <= available {
-            let span = take_contiguous_chunk_span(chunk, self.source_chunk_offset, len)?;
-            self.advance(len);
-            return Ok(span);
-        }
-
-        let mut aggregate = BytesMut::with_capacity(len);
-        let mut remaining = len;
-        while remaining > 0 {
-            self.skip_empty_chunks();
-            let Some(chunk) = self.source_chunks.front() else {
-                return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated bitrot chunk source"));
-            };
-            let available = chunk.len().saturating_sub(self.source_chunk_offset);
-            let take = available.min(remaining);
-            aggregate.extend_from_slice(&chunk.as_bytes()[self.source_chunk_offset..self.source_chunk_offset + take]);
-            self.advance(take);
-            remaining -= take;
-        }
-
-        let bytes = aggregate.freeze();
-        Ok(ChunkSpan {
-            bytes: bytes.clone(),
-            chunk: IoChunk::Shared(bytes),
-            copied: true,
-        })
-    }
-}
-
-impl BitrotChunkStreamState {
-    #[allow(clippy::too_many_arguments)]
-    fn new(
-        source_stream: BoxChunkStream,
-        source_chunks: VecDeque<IoChunk>,
-        source_done: bool,
-        decoded_remaining: usize,
-        trim_prefix: usize,
-        output_remaining: usize,
-        shard_size: usize,
-        checksum_algo: HashAlgorithm,
-        skip_verify: bool,
-    ) -> Self {
-        Self {
-            source: BitrotChunkSource::new(source_stream, source_chunks, source_done),
-            decoded_remaining,
-            trim_prefix,
-            output_remaining,
-            shard_size,
-            checksum_algo,
-            skip_verify,
-        }
-    }
-
-    fn hash_size(&self) -> usize {
-        self.checksum_algo.size()
-    }
-
-    async fn next_verified_chunk(&mut self) -> std::io::Result<Option<IoChunk>> {
-        let hash_size = self.hash_size();
-
-        while self.output_remaining > 0 && self.decoded_remaining > 0 {
-            let data_len = self.shard_size.min(self.decoded_remaining);
-
-            let expected_hash = if hash_size > 0 {
-                self.source.fill(hash_size).await?;
-                Some(self.source.take_span(hash_size)?)
-            } else {
-                None
-            };
-
-            self.source.fill(data_len).await?;
-            let data_span = self.source.take_span(data_len)?;
-
-            if let Some(expected_hash) = expected_hash
-                && !self.skip_verify
-                && self.checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref()
-            {
-                return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
-            }
-
-            self.decoded_remaining -= data_len;
-
-            if self.trim_prefix >= data_len {
-                self.trim_prefix -= data_len;
-                continue;
-            }
-
-            let start = self.trim_prefix;
-            self.trim_prefix = 0;
-            let take = (data_len - start).min(self.output_remaining);
-            self.output_remaining -= take;
-
-            let chunk = if start == 0 && take == data_len {
-                data_span.chunk
-            } else {
-                data_span.chunk.slice(start, take)?
-            };
-
-            if !chunk.is_empty() {
-                return Ok(Some(chunk));
-            }
-        }
-
-        Ok(None)
-    }
-}
-
 /// Create a BitrotReader from either inline data or disk file stream
 ///
 /// # Parameters
@@ -379,40 +64,21 @@ pub async fn create_bitrot_reader(
         Ok(Some(reader))
     } else if let Some(disk) = disk {
         // Read from disk
-        if use_zero_copy {
-            if !disk.is_local() {
-                rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
-                rustfs_io_metrics::record_io_fallback(
-                    rustfs_io_metrics::IoStage::ReadSetup,
-                    rustfs_io_metrics::FallbackReason::NonLocalBackend,
-                );
-
-                let rd = disk.read_file_stream(bucket, path, offset, length).await?;
-                let reader = BitrotReader::new(rd, shard_size, checksum_algo, skip_verify);
-                return Ok(Some(reader));
-            }
-
+        if use_zero_copy && disk.is_local() {
             // Try zero-copy read first (uses mmap on Unix)
             let start = Instant::now();
             match disk.read_file_zero_copy(bucket, path, offset, length).await {
                 Ok(bytes) => {
                     let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
 
-                    rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Fast);
-                    // `read_file_zero_copy()` returns a shared `Bytes` view, but it may still
-                    // internally aggregate multiple chunk windows. The exact chunk-native copy
-                    // mode is only preserved by `create_bitrot_chunk_stream()`.
-                    rustfs_io_metrics::record_io_copy_mode(
-                        BITROT_READ_OPERATION,
-                        rustfs_io_metrics::CopyMode::SharedBytes,
-                        bytes.len(),
-                    );
+                    // Record zero-copy metrics
+                    rustfs_io_metrics::record_zero_copy_read(bytes.len(), duration_ms);
 
+                    // Log successful zero-copy read
                     debug!(
                         size = bytes.len(),
-                        duration_ms,
                         path = %path,
-                        "bitrot_fast_read_success"
+                        "zero_copy_read_success"
                     );
 
                     // Wrap Bytes in Cursor for AsyncRead
@@ -427,16 +93,14 @@ pub async fn create_bitrot_reader(
                     Ok(Some(reader))
                 }
                 Err(e) => {
-                    rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
-                    rustfs_io_metrics::record_io_fallback(
-                        rustfs_io_metrics::IoStage::ReadSetup,
-                        rustfs_io_metrics::FallbackReason::Unknown,
-                    );
+                    // Record zero-copy fallback
+                    rustfs_io_metrics::record_zero_copy_fallback(&format!("{:?}", e));
 
+                    // Log zero-copy fallback
                     debug!(
-                        reason = %e,
+                        reason = %format!("{:?}", e),
                         path = %path,
-                        "bitrot_fast_read_fallback"
+                        "zero_copy_fallback"
                     );
 
                     // Fall back to regular stream read on error
@@ -453,7 +117,6 @@ pub async fn create_bitrot_reader(
                 }
             }
         } else {
-            rustfs_io_metrics::record_io_path_selected(BITROT_READ_OPERATION, rustfs_io_metrics::IoPath::Legacy);
             // Use regular stream read
             match disk.read_file_stream(bucket, path, offset, length).await {
                 Ok(rd) => {
@@ -469,198 +132,6 @@ pub async fn create_bitrot_reader(
     }
 }
 
-/// Create a chunk stream from bitrot-encoded data, preserving source chunk provenance when possible.
-#[allow(clippy::too_many_arguments)]
-pub async fn create_bitrot_chunk_stream(
-    inline_data: Option<&[u8]>,
-    disk: Option<&DiskStore>,
-    bucket: &str,
-    path: &str,
-    offset: usize,
-    length: usize,
-    total_data_size: usize,
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-    use_zero_copy: bool,
-) -> disk::error::Result<Option<GetObjectChunkResult>> {
-    let fetch_start = (offset / shard_size) * shard_size;
-    let fetch_end = (offset + length).div_ceil(shard_size) * shard_size;
-    let fetch_end = fetch_end.min(total_data_size);
-    let fetch_length = fetch_end.saturating_sub(fetch_start);
-    let trim_prefix = offset.saturating_sub(fetch_start);
-    let hash_size = checksum_algo.size();
-    let encoded_length = fetch_length.div_ceil(shard_size) * hash_size + fetch_length;
-    let encoded_offset = fetch_start.div_ceil(shard_size) * hash_size + fetch_start;
-
-    let mut source_done = false;
-    let (source_stream, mut prefetched_chunks, source_direct) = if let Some(data) = inline_data {
-        source_done = true;
-        let mut chunks = VecDeque::new();
-        chunks.push_back(IoChunk::Shared(
-            Bytes::copy_from_slice(data).slice(encoded_offset..encoded_offset + encoded_length),
-        ));
-        let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
-        (source_stream, chunks, false)
-    } else if let Some(disk) = disk {
-        if use_zero_copy {
-            let mut source_stream = disk.read_file_chunks(bucket, path, encoded_offset, encoded_length).await?;
-            let mut prefetched_chunks = VecDeque::new();
-            let mut direct = true;
-            while prefetched_chunks.len() < 2 {
-                let Some(chunk) = source_stream.next().await else {
-                    source_done = true;
-                    break;
-                };
-                let chunk = chunk?;
-                direct &= matches!(chunk, IoChunk::Mapped(_));
-                prefetched_chunks.push_back(chunk);
-            }
-            (source_stream, prefetched_chunks, direct)
-        } else {
-            source_done = true;
-            let bytes = disk.read_file_zero_copy(bucket, path, encoded_offset, encoded_length).await?;
-            let mut chunks = VecDeque::new();
-            chunks.push_back(IoChunk::Shared(bytes));
-            let source_stream: BoxChunkStream = Box::pin(stream::empty::<std::io::Result<IoChunk>>());
-            (source_stream, chunks, false)
-        }
-    } else {
-        return Ok(None);
-    };
-
-    let copied = predicted_stream_copy(encoded_length, shard_size, checksum_algo.size(), &prefetched_chunks, source_done);
-    let state = BitrotChunkStreamState::new(
-        source_stream,
-        std::mem::take(&mut prefetched_chunks),
-        source_done,
-        fetch_length,
-        trim_prefix,
-        length,
-        shard_size,
-        checksum_algo,
-        skip_verify,
-    );
-    let stream = stream::unfold(Some(state), |state| async move {
-        let mut state = match state {
-            Some(state) => state,
-            None => return None,
-        };
-
-        match state.next_verified_chunk().await {
-            Ok(Some(chunk)) => {
-                let next_state = if state.output_remaining == 0 { None } else { Some(state) };
-                Some((Ok::<IoChunk, std::io::Error>(chunk), next_state))
-            }
-            Ok(None) => None,
-            Err(err) => Some((Err(err), None)),
-        }
-    });
-    Ok(Some(GetObjectChunkResult {
-        stream: Box::pin(stream),
-        path: GetObjectChunkPath::Direct,
-        copy_mode: classify_chunk_copy_mode(source_direct, copied),
-    }))
-}
-
-fn predicted_stream_copy(
-    encoded_length: usize,
-    shard_size: usize,
-    hash_size: usize,
-    prefetched_chunks: &VecDeque<IoChunk>,
-    source_done: bool,
-) -> bool {
-    if prefetched_chunks.is_empty() {
-        return false;
-    }
-
-    if source_done && prefetched_chunks.len() == 1 {
-        return false;
-    }
-
-    let full_frame_len = hash_size + shard_size;
-    if full_frame_len == 0 {
-        return false;
-    }
-
-    let first_window_len = prefetched_chunks.front().map(IoChunk::len).unwrap_or(encoded_length);
-    encoded_length > first_window_len && !first_window_len.is_multiple_of(full_frame_len)
-}
-
-fn trim_chunk_vec(chunks: Vec<IoChunk>, offset: usize, length: usize) -> std::io::Result<Vec<IoChunk>> {
-    let mut skip = offset;
-    let mut remaining = length;
-    let mut result = Vec::new();
-
-    for chunk in chunks {
-        if remaining == 0 {
-            break;
-        }
-
-        let chunk_len = chunk.len();
-        if skip >= chunk_len {
-            skip -= chunk_len;
-            continue;
-        }
-
-        let start = skip;
-        let take = (chunk_len - start).min(remaining);
-        result.push(chunk.slice(start, take)?);
-        remaining -= take;
-        skip = 0;
-    }
-
-    Ok(result)
-}
-
-fn decode_bitrot_chunk_source(
-    source_chunks: &[IoChunk],
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-) -> std::io::Result<(Vec<IoChunk>, bool)> {
-    let hash_size = checksum_algo.size();
-    let mut cursor = ChunkCursor::new(source_chunks);
-    let mut result = Vec::new();
-    let mut copied = false;
-
-    while cursor.remaining() > 0 {
-        let expected_hash = if hash_size > 0 {
-            Some(cursor.take_span(hash_size)?)
-        } else {
-            None
-        };
-
-        let data_len = shard_size.min(cursor.remaining());
-        if data_len == 0 {
-            break;
-        }
-
-        let data_span = cursor.take_span(data_len)?;
-        copied |= data_span.copied;
-        if let Some(expected_hash) = expected_hash {
-            copied |= expected_hash.copied;
-            if !skip_verify && checksum_algo.hash_encode(data_span.bytes.as_ref()).as_ref() != expected_hash.bytes.as_ref() {
-                return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
-            }
-        }
-
-        result.push(data_span.chunk);
-    }
-
-    Ok((result, copied))
-}
-
-#[doc(hidden)]
-pub fn decode_bitrot_chunk_source_for_bench(
-    source_chunks: &[IoChunk],
-    shard_size: usize,
-    checksum_algo: HashAlgorithm,
-    skip_verify: bool,
-) -> std::io::Result<(Vec<IoChunk>, bool)> {
-    decode_bitrot_chunk_source(source_chunks, shard_size, checksum_algo, skip_verify)
-}
-
 /// Create a new BitrotWriterWrapper based on the provided parameters
 ///
 /// # Parameters
@@ -705,7 +176,6 @@ pub async fn create_bitrot_writer(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use futures_util::StreamExt;
 
     #[tokio::test]
     async fn test_create_bitrot_reader_with_inline_data() {
@@ -756,246 +226,6 @@ mod tests {
         assert!(result.unwrap().is_some());
     }
 
-    #[tokio::test]
-    async fn test_create_bitrot_chunk_stream_with_inline_data() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::HighwayHash256S;
-        let shard1 = b"abcd";
-        let shard2 = b"ef";
-
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        encoded.extend_from_slice(shard1);
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        encoded.extend_from_slice(shard2);
-
-        let mut stream = create_bitrot_chunk_stream(
-            Some(&encoded),
-            None,
-            "test-bucket",
-            "test-path",
-            0,
-            shard1.len() + shard2.len(),
-            shard1.len() + shard2.len(),
-            shard_size,
-            checksum_algo,
-            false,
-            false,
-        )
-        .await
-        .unwrap()
-        .unwrap()
-        .stream;
-
-        let mut collected = Vec::new();
-        while let Some(chunk) = stream.next().await {
-            collected.extend_from_slice(&chunk.unwrap().as_bytes());
-        }
-
-        assert_eq!(collected, b"abcdef");
-    }
-
-    #[tokio::test]
-    async fn test_create_bitrot_chunk_stream_detects_hash_mismatch() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::HighwayHash256S;
-        let shard = b"abcd";
-
-        let mut encoded = Vec::new();
-        let mut bad_hash = checksum_algo.hash_encode(shard).as_ref().to_vec();
-        bad_hash[0] ^= 0xFF;
-        encoded.extend_from_slice(&bad_hash);
-        encoded.extend_from_slice(shard);
-
-        let result = create_bitrot_chunk_stream(
-            Some(&encoded),
-            None,
-            "test-bucket",
-            "test-path",
-            0,
-            shard.len(),
-            shard.len(),
-            shard_size,
-            checksum_algo,
-            false,
-            false,
-        )
-        .await;
-
-        let mut stream = result.unwrap().unwrap().stream;
-        let err = stream.next().await.unwrap().unwrap_err();
-        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
-        assert!(err.to_string().contains("bitrot hash mismatch"));
-    }
-
-    #[tokio::test]
-    async fn test_create_bitrot_chunk_stream_trims_range_after_decode() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::HighwayHash256S;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        encoded.extend_from_slice(shard1);
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        encoded.extend_from_slice(shard2);
-
-        let mut stream = create_bitrot_chunk_stream(
-            Some(&encoded),
-            None,
-            "test-bucket",
-            "test-path",
-            1,
-            5,
-            shard1.len() + shard2.len(),
-            shard_size,
-            checksum_algo,
-            false,
-            false,
-        )
-        .await
-        .unwrap()
-        .unwrap()
-        .stream;
-
-        let mut collected = Vec::new();
-        while let Some(chunk) = stream.next().await {
-            collected.extend_from_slice(&chunk.unwrap().as_bytes());
-        }
-
-        assert_eq!(collected, b"bcdef");
-    }
-
-    #[test]
-    fn test_decode_bitrot_chunk_source_preserves_aligned_multi_chunk_slices() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let mut encoded_chunk_one = Vec::new();
-        encoded_chunk_one.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        encoded_chunk_one.extend_from_slice(shard1);
-
-        let mut encoded_chunk_two = Vec::new();
-        encoded_chunk_two.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        encoded_chunk_two.extend_from_slice(shard2);
-
-        let source_chunks = vec![
-            IoChunk::Shared(Bytes::from(encoded_chunk_one)),
-            IoChunk::Shared(Bytes::from(encoded_chunk_two)),
-        ];
-        let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
-
-        assert!(!copied, "frame-aligned multi-chunk source should not require aggregate copies");
-        assert_eq!(decoded.len(), 2);
-        assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
-        assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
-    }
-
-    #[test]
-    fn test_decode_bitrot_chunk_source_marks_cross_chunk_frame_as_copied() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let hash1 = checksum_algo.hash_encode(shard1).as_ref().to_vec();
-        let hash2 = checksum_algo.hash_encode(shard2).as_ref().to_vec();
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(&hash1);
-        encoded.extend_from_slice(shard1);
-        encoded.extend_from_slice(&hash2);
-        encoded.extend_from_slice(shard2);
-
-        let split = hash1.len() + 2;
-        let source_chunks = vec![
-            IoChunk::Shared(Bytes::copy_from_slice(&encoded[..split])),
-            IoChunk::Shared(Bytes::copy_from_slice(&encoded[split..])),
-        ];
-        let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
-
-        assert!(copied, "cross-chunk frame should be classified as requiring a copy");
-        assert_eq!(decoded.len(), 2);
-        assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
-        assert_eq!(decoded[1].as_bytes(), Bytes::from_static(b"efgh"));
-    }
-
-    #[test]
-    fn test_decode_bitrot_chunk_source_preserves_pooled_single_chunk_slice() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard = b"abcd";
-
-        let mut encoded = Vec::new();
-        encoded.extend_from_slice(checksum_algo.hash_encode(shard).as_ref());
-        encoded.extend_from_slice(shard);
-
-        let source_chunks = vec![IoChunk::Pooled(rustfs_io_core::PooledChunk::from_vec(encoded))];
-        let (decoded, copied) = decode_bitrot_chunk_source(&source_chunks, shard_size, checksum_algo, false).unwrap();
-
-        assert!(!copied, "single pooled chunk slice should preserve provenance without copy");
-        assert_eq!(decoded.len(), 1);
-        assert!(matches!(&decoded[0], IoChunk::Pooled(_)));
-        assert_eq!(decoded[0].as_bytes(), Bytes::from_static(b"abcd"));
-    }
-
-    #[tokio::test]
-    async fn test_bitrot_chunk_source_marks_cross_chunk_take_as_copied() {
-        let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
-            Ok(IoChunk::Shared(Bytes::from_static(b"ab"))),
-            Ok(IoChunk::Shared(Bytes::from_static(b"cd"))),
-        ]));
-        let mut source = BitrotChunkSource::new(source_stream, VecDeque::new(), false);
-
-        source.fill(4).await.expect("source fill should succeed");
-        let span = source.take_span(4).expect("cross-chunk take should succeed");
-
-        assert!(span.copied, "cross-chunk take should be classified as copied");
-        assert_eq!(span.bytes, Bytes::from_static(b"abcd"));
-        assert_eq!(span.chunk.as_bytes(), Bytes::from_static(b"abcd"));
-    }
-
-    #[tokio::test]
-    async fn test_bitrot_chunk_stream_state_yields_verified_prefix_before_later_truncation() {
-        let shard_size = 4;
-        let checksum_algo = HashAlgorithm::Md5;
-        let shard1 = b"abcd";
-        let shard2 = b"efgh";
-
-        let mut first_frame = Vec::new();
-        first_frame.extend_from_slice(checksum_algo.hash_encode(shard1).as_ref());
-        first_frame.extend_from_slice(shard1);
-
-        let mut second_frame_prefix = Vec::new();
-        second_frame_prefix.extend_from_slice(checksum_algo.hash_encode(shard2).as_ref());
-        second_frame_prefix.extend_from_slice(&shard2[..2]);
-
-        let source_stream: BoxChunkStream = Box::pin(stream::iter(vec![
-            Ok(IoChunk::Shared(Bytes::from(first_frame))),
-            Ok(IoChunk::Shared(Bytes::from(second_frame_prefix))),
-        ]));
-        let mut state = BitrotChunkStreamState::new(
-            source_stream,
-            VecDeque::new(),
-            false,
-            shard1.len() + shard2.len(),
-            0,
-            shard1.len() + shard2.len(),
-            shard_size,
-            checksum_algo,
-            false,
-        );
-
-        let first = state.next_verified_chunk().await.unwrap().unwrap();
-        assert_eq!(first.as_bytes(), Bytes::from_static(b"abcd"));
-
-        let err = state.next_verified_chunk().await.unwrap_err();
-        assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
-        assert!(err.to_string().contains("truncated bitrot chunk source"));
-    }
-
     #[tokio::test]
     async fn test_create_bitrot_reader_with_inline_offset_starts_at_requested_shard() {
         let shard_size = 4;

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -593,7 +593,7 @@ impl BucketTargetSys {
         };
 
         if let Some(cli) = cli {
-            return Some(cli.clone());
+            return Some(cli);
         }
 
         // TODO: spawn a task to reload the target
@@ -786,7 +786,10 @@ impl BucketTargetSys {
                         && tgt
                             .credentials
                             .as_ref()
-                            .map(|c| c.access_key == target.credentials.as_ref().unwrap_or(&Credentials::default()).access_key)
+                            .map(|c| {
+                                let default_creds = Credentials::default();
+                                c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
+                            })
                             .unwrap_or(false)
                     {
                         return (tgt.arn.clone(), true);
@@ -894,6 +897,41 @@ pub struct RemoveObjectOptions {
     pub replication_validity_check: bool,
 }
 
+fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOptions) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    if opts.force_delete {
+        insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
+    }
+    if opts.governance_bypass {
+        headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
+    }
+
+    if opts.replication_delete_marker {
+        insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
+    }
+
+    if let Some(t) = opts.replication_mtime {
+        insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
+    }
+
+    if !opts.replication_status.is_empty() {
+        headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
+    }
+
+    if let Some(version_id) = version_id {
+        insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
+    }
+
+    if opts.replication_request {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
+    }
+    if opts.replication_validity_check {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
+    }
+
+    headers
+}
+
 #[derive(Debug, Clone)]
 pub struct AdvancedPutOptions {
     pub source_version_id: String,
@@ -1424,39 +1462,15 @@ impl TargetClient {
         version_id: Option<String>,
         opts: RemoveObjectOptions,
     ) -> Result<(), S3ClientError> {
-        let mut headers = HeaderMap::new();
-        if opts.force_delete {
-            insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
-        }
-        if opts.governance_bypass {
-            headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
-        }
-
-        if opts.replication_delete_marker {
-            insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
-        }
-
-        if let Some(t) = opts.replication_mtime {
-            insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
-        }
-
-        if !opts.replication_status.is_empty() {
-            headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
-        }
-
-        if opts.replication_request {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
-        }
-        if opts.replication_validity_check {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
-        }
+        let headers = build_remove_object_headers(version_id.as_deref(), &opts);
+        let api_version_id = if opts.replication_request { None } else { version_id };
 
         match self
             .client
             .delete_object()
             .bucket(bucket)
             .key(object)
-            .set_version_id(version_id)
+            .set_version_id(api_version_id)
             .customize()
             .map_request(move |mut req| {
                 for (k, v) in headers.clone().into_iter() {
@@ -1550,3 +1564,53 @@ impl From<std::io::Error> for BucketTargetError {
 }
 
 impl Error for BucketTargetError {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: true,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert_eq!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(),
+            Some(version_id.as_str()),
+            "replication delete requests must preserve the version id in internal headers"
+        );
+    }
+
+    #[test]
+    fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: false,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(),
+            "delete-marker version purges must not masquerade as delete-marker creations"
+        );
+    }
+}

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -14,8 +14,9 @@
 
 pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
+pub mod core;
 pub mod evaluator;
-pub mod lifecycle;
+pub use self::core as lifecycle;
 pub mod rule;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;

crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs

@@ -83,7 +83,7 @@ impl LastDayTierStats {
     #[allow(dead_code)]
     fn merge(&self, m: LastDayTierStats) -> LastDayTierStats {
         let mut cl = self.clone();
-        let mut cm = m.clone();
+        let mut cm = m;
         let mut merged = LastDayTierStats::default();
 
         if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() {

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -1,4 +1,3 @@
-#![allow(unused_imports)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
@@ -109,10 +109,19 @@ impl ObjSweeper {
     }
 
     pub async fn sweep(&self) {
-        let je = self.should_remove_remote_object();
-        if !je.is_none() {
-            let mut expiry_state = GLOBAL_ExpiryState.write().await;
-            expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
+        let Some(je) = self.should_remove_remote_object() else {
+            return;
+        };
+        let hash = je.op_hash();
+        // Grab the sender under a short read lock, then release the lock so we
+        // don't hold it across the async send.
+        let wrkr = GLOBAL_ExpiryState.read().await.get_worker_ch(hash);
+        let Some(wrkr) = wrkr else {
+            GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
+            return;
+        };
+        if wrkr.send(Some(Box::new(je))).await.is_err() {
+            GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
         }
     }
 }

crates/ecstore/src/bucket/metadata.rs

@@ -678,7 +678,7 @@ impl BucketMetadata {
                 // let x = data.clone();
                 // let str = std::str::from_utf8(&x).expect("Invalid UTF-8");
                 // println!("update config:{}", str);
-                self.bucket_targets_config_json = data.clone();
+                self.bucket_targets_config_json = data;
                 self.bucket_targets_config_updated_at = updated;
             }
             BUCKET_CORS_CONFIG => {

crates/ecstore/src/bucket/migration.rs

@@ -17,7 +17,7 @@
 use crate::bucket::metadata::BUCKET_METADATA_FILE;
 use crate::bucket::replication::{decode_resync_file, encode_resync_file};
 use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
-use crate::store_api::{BucketOptions, ChunkNativePutData, ObjectOptions, StorageAPI};
+use crate::store_api::{BucketOptions, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
 use rustfs_policy::auth::UserIdentity;
 use rustfs_policy::policy::PolicyDoc;
@@ -263,7 +263,7 @@ async fn migrate_one_if_missing<S: StorageAPI>(
         }
     };
 
-    let mut put_data = ChunkNativePutData::from_vec(data);
+    let mut put_data = PutObjReader::from_vec(data);
     if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await {
         warn!("write {label}: {e}");
     } else {
@@ -341,7 +341,7 @@ pub async fn try_migrate_iam_config<S: StorageAPI>(store: Arc<S>) {
                     continue;
                 }
             };
-            let mut put_data = ChunkNativePutData::from_vec(data);
+            let mut put_data = PutObjReader::from_vec(data);
             if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await {
                 warn!("write IAM config {path}: {e}");
             } else {

crates/ecstore/src/bucket/replication/config.rs

@@ -152,7 +152,16 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
             }
 
             if obj.op_type == ReplicationType::Delete {
+                if !rule.metadata_replicate(obj) {
+                    return false;
+                }
+
                 if obj.version_id.is_some() {
+                    if obj.delete_marker {
+                        return rule.delete_marker_replication.clone().is_some_and(|d| {
+                            d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
+                        });
+                    }
                     return rule
                         .delete_replication
                         .clone()

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -754,7 +754,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
         buckets: Vec<String>,
     ) -> Result<(), EcstoreError> {
         // Load bucket metadata system in background
-        let pool_clone = self.clone();
+        let pool_clone = self;
 
         tokio::spawn(async move {
             pool_clone.start_resync_routine(buckets, cancellation_token).await;
@@ -824,10 +824,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
         let cancel_token = CancellationToken::new();
         resyncer.register_cancel_token(&opts, cancel_token.clone()).await;
         tokio::spawn(async move {
-            resyncer
-                .clone()
-                .resync_bucket(cancel_token, storage, false, opts.clone())
-                .await;
+            Box::pin(resyncer.clone().resync_bucket(cancel_token, storage, false, opts.clone())).await;
             resyncer.clear_cancel_token(&opts).await;
         });
 
@@ -914,7 +911,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
                         };
                         tokio::spawn(async move {
                             resync.register_cancel_token(&opts, ctx.clone()).await;
-                            resync.clone().resync_bucket(ctx, storage, true, opts.clone()).await;
+                            Box::pin(resync.clone().resync_bucket(ctx, storage, true, opts.clone())).await;
                             resync.clear_cancel_token(&opts).await;
                         });
                     }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -34,7 +34,7 @@ use crate::global::get_global_bucket_monitor;
 use crate::set_disk::get_lock_acquire_timeout;
 use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
-use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
@@ -85,7 +85,8 @@ use tokio::task::JoinSet;
 use tokio::time::Duration as TokioDuration;
 use tokio_util::io::ReaderStream;
 use tokio_util::sync::CancellationToken;
-use tracing::{error, info, instrument, warn};
+use tracing::{debug, error, info, instrument, warn};
+use uuid::Uuid;
 
 pub(crate) const REPLICATION_DIR: &str = ".replication";
 pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -1271,15 +1272,7 @@ pub async fn check_replicate_delete(
         return ReplicateDecision::default();
     }
 
-    let opts = ObjectOpts {
-        name: dobj.object_name.clone(),
-        ssec: is_ssec_encrypted(&oi.user_defined),
-        user_tags: oi.user_tags.clone(),
-        delete_marker: oi.delete_marker,
-        version_id: dobj.version_id,
-        op_type: ReplicationType::Delete,
-        ..Default::default()
-    };
+    let opts = delete_replication_object_opts(dobj, oi);
 
     let tgt_arns = rcfg.filter_target_arns(&opts);
     let mut dsc = ReplicateDecision::new();
@@ -1331,6 +1324,19 @@ pub async fn check_replicate_delete(
     dsc
 }
 
+fn delete_replication_object_opts(dobj: &ObjectToDelete, oi: &ObjectInfo) -> ObjectOpts {
+    ObjectOpts {
+        name: dobj.object_name.clone(),
+        ssec: is_ssec_encrypted(&oi.user_defined),
+        user_tags: oi.user_tags.clone(),
+        delete_marker: oi.delete_marker,
+        version_id: dobj.version_id,
+        op_type: ReplicationType::Delete,
+        replica: oi.replication_status == ReplicationStatusType::Replica,
+        ..Default::default()
+    }
+}
+
 /// Check if the user-defined metadata contains SSEC encryption headers
 fn is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
     user_defined.contains_key(SSEC_ALGORITHM_HEADER)
@@ -1497,6 +1503,54 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
         }
     };
 
+    if dobj.delete_object.delete_marker
+        && let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
+    {
+        let source_marker_state = storage
+            .get_object_info(
+                &bucket,
+                &dobj.delete_object.object_name,
+                &ObjectOptions {
+                    version_id: Some(delete_marker_version_id.to_string()),
+                    versioned: BucketVersioningSys::prefix_enabled(&bucket, &dobj.delete_object.object_name).await,
+                    version_suspended: BucketVersioningSys::prefix_suspended(&bucket, &dobj.delete_object.object_name).await,
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        match source_marker_state {
+            Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
+            Ok(_) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    "skipping stale delete-marker replication because source version is no longer a delete marker"
+                );
+                return;
+            }
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    "skipping stale delete-marker replication because source version no longer exists"
+                );
+                return;
+            }
+            Err(err) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    error = %err,
+                    "failed to verify source delete-marker state before replication"
+                );
+            }
+        }
+    }
+
     let dsc = match parse_replicate_decision(
         &bucket,
         &dobj
@@ -1529,7 +1583,6 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
             return;
         }
     };
-
     let ns_lock = match storage
         .new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
         .await
@@ -1653,7 +1706,33 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
         }
     }
 
-    let (replication_status, prev_status) = if dobj.delete_object.version_id.is_none() {
+    let is_version_purge = is_version_delete_replication(&dobj.delete_object);
+
+    if should_retry_delete_marker_purge(&dobj.delete_object) {
+        let bucket_clone = bucket.clone();
+        let dobj_clone = dobj.clone();
+        let dsc_clone = dsc.clone();
+        let storage_clone = storage.clone();
+        tokio::spawn(async move {
+            for _ in 0..5 {
+                if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
+                    && source_delete_marker_missing(
+                        &*storage_clone,
+                        &bucket_clone,
+                        &dobj_clone.delete_object.object_name,
+                        delete_marker_version_id,
+                    )
+                    .await
+                {
+                    replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
+                    break;
+                }
+                tokio::time::sleep(TokioDuration::from_secs(1)).await;
+            }
+        });
+    }
+
+    let (replication_status, prev_status) = if !is_version_purge {
         (
             rinfos.replication_status(),
             dobj.delete_object
@@ -1741,6 +1820,65 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
     }
 }
 
+async fn source_delete_marker_missing<S: StorageAPI>(
+    storage: &S,
+    bucket: &str,
+    object_name: &str,
+    delete_marker_version_id: Uuid,
+) -> bool {
+    match storage
+        .get_object_info(
+            bucket,
+            object_name,
+            &ObjectOptions {
+                version_id: Some(delete_marker_version_id.to_string()),
+                versioned: BucketVersioningSys::prefix_enabled(bucket, object_name).await,
+                version_suspended: BucketVersioningSys::prefix_suspended(bucket, object_name).await,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(info) => !info.delete_marker || info.version_id != Some(delete_marker_version_id),
+        Err(err) => is_err_object_not_found(&err) || is_err_version_not_found(&err),
+    }
+}
+
+async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
+    let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
+        return;
+    };
+
+    for tgt_entry in dsc.targets_map.values() {
+        if !tgt_entry.replicate {
+            continue;
+        }
+        if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
+            continue;
+        }
+        let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &tgt_entry.arn).await else {
+            continue;
+        };
+
+        let _ = tgt_client
+            .remove_object(
+                &tgt_client.bucket,
+                &dobj.delete_object.object_name,
+                Some(delete_marker_version_id.to_string()),
+                RemoveObjectOptions {
+                    force_delete: false,
+                    governance_bypass: false,
+                    replication_delete_marker: false,
+                    replication_mtime: dobj.delete_object.delete_marker_mtime,
+                    replication_status: ReplicationStatusType::Replica,
+                    replication_request: true,
+                    replication_validity_check: false,
+                },
+            )
+            .await;
+    }
+}
+
 async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
     let bucket = &dobj.bucket;
     let object_name = &dobj.delete_object.object_name;
@@ -1924,6 +2062,18 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
     }
 }
 
+fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
+    dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
+}
+
+fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool {
+    dobj.delete_marker_version_id.is_some()
+}
+
+fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
+    !is_not_found && !matches!(code, Some("MethodNotAllowed" | "405"))
+}
+
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
     let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
         version_id.to_owned()
@@ -1941,7 +2091,8 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
     rinfo.endpoint = tgt_client.endpoint.clone();
     rinfo.secure = tgt_client.secure;
 
-    if dobj.delete_object.version_id.is_none()
+    let is_version_purge = is_version_delete_replication(&dobj.delete_object);
+    if !is_version_purge
         && rinfo.prev_replication_status == ReplicationStatusType::Completed
         && dobj.op_type != ReplicationType::ExistingObject
     {
@@ -1949,12 +2100,12 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         return rinfo;
     }
 
-    if dobj.delete_object.version_id.is_some() && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
+    if is_version_purge && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
         return rinfo;
     }
 
     if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
-        if dobj.delete_object.version_id.is_none() {
+        if !is_version_purge {
             rinfo.replication_status = ReplicationStatusType::Failed;
         } else {
             rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -1968,18 +2119,29 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         Some(version_id.to_string())
     };
 
-    if dobj.delete_object.delete_marker_version_id.is_some()
-        && let Err(e) = tgt_client
+    if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
+        match tgt_client
             .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
             .await
-        && let SdkError::ServiceError(service_err) = &e
-        && !service_err.err().is_not_found()
-    {
-        rinfo.replication_status = ReplicationStatusType::Failed;
-        rinfo.error = Some(e.to_string());
-
-        return rinfo;
-    };
+        {
+            Ok(_) => {}
+            Err(e) => {
+                let non_retryable = matches!(
+                    &e,
+                    SdkError::ServiceError(service_err)
+                        if is_retryable_delete_replication_head_error(
+                            service_err.err().is_not_found(),
+                            service_err.err().code(),
+                        )
+                );
+                if non_retryable {
+                    rinfo.replication_status = ReplicationStatusType::Failed;
+                    rinfo.error = Some(e.to_string());
+                    return rinfo;
+                }
+            }
+        }
+    }
 
     match tgt_client
         .remove_object(
@@ -1989,7 +2151,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
             RemoveObjectOptions {
                 force_delete: false,
                 governance_bypass: false,
-                replication_delete_marker: dobj.delete_object.delete_marker_version_id.is_some(),
+                replication_delete_marker: dobj.delete_object.delete_marker,
                 replication_mtime: dobj.delete_object.delete_marker_mtime,
                 replication_status: ReplicationStatusType::Replica,
                 replication_request: true,
@@ -1999,15 +2161,32 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         .await
     {
         Ok(_) => {
-            if dobj.delete_object.version_id.is_none() {
+            debug!(
+                bucket = tgt_client.bucket,
+                object = dobj.delete_object.object_name,
+                version_id = ?version_id,
+                delete_marker = dobj.delete_object.delete_marker,
+                is_version_purge,
+                "replicate_delete_to_target succeeded"
+            );
+            if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Completed;
             } else {
                 rinfo.version_purge_status = VersionPurgeStatusType::Complete;
             }
         }
         Err(e) => {
+            warn!(
+                bucket = tgt_client.bucket,
+                object = dobj.delete_object.object_name,
+                version_id = ?version_id,
+                delete_marker = dobj.delete_object.delete_marker,
+                is_version_purge,
+                error = %e,
+                "replicate_delete_to_target failed"
+            );
             rinfo.error = Some(e.to_string());
-            if dobj.delete_object.version_id.is_none() {
+            if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Failed;
             } else {
                 rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -3450,6 +3629,125 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_delete_replication_object_opts_marks_replica_deletes() {
+        let dobj = ObjectToDelete {
+            object_name: "obj".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            replication_status: ReplicationStatusType::Replica,
+            ..Default::default()
+        };
+
+        let opts = delete_replication_object_opts(&dobj, &oi);
+
+        assert!(
+            opts.replica,
+            "replica deletes must preserve replica status for downstream ReplicaModifications rules"
+        );
+        assert_eq!(opts.version_id, dobj.version_id);
+        assert_eq!(opts.name, dobj.object_name);
+        assert_eq!(opts.op_type, ReplicationType::Delete);
+    }
+
+    #[test]
+    fn test_delete_replication_object_opts_keeps_non_replica_deletes_local() {
+        let dobj = ObjectToDelete {
+            object_name: "obj".to_string(),
+            ..Default::default()
+        };
+        let oi = ObjectInfo {
+            bucket: "b".to_string(),
+            name: "obj".to_string(),
+            replication_status: ReplicationStatusType::Completed,
+            ..Default::default()
+        };
+
+        let opts = delete_replication_object_opts(&dobj, &oi);
+
+        assert!(!opts.replica, "source-originated deletes should not be treated as replica modifications");
+    }
+
+    #[test]
+    fn test_is_version_delete_replication_for_delete_marker_version_purge() {
+        let dobj = DeletedObject {
+            delete_marker: false,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            is_version_delete_replication(&dobj),
+            "delete-marker version purges must be tracked as version purge replication, not delete-marker creation replication"
+        );
+    }
+
+    #[test]
+    fn test_is_version_delete_replication_for_delete_marker_creation() {
+        let dobj = DeletedObject {
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            !is_version_delete_replication(&dobj),
+            "delete-marker creation should remain on the delete-marker replication path"
+        );
+    }
+
+    #[test]
+    fn test_should_retry_delete_marker_purge_for_version_purge() {
+        let dobj = DeletedObject {
+            delete_marker: false,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            should_retry_delete_marker_purge(&dobj),
+            "delete-marker version purge should schedule delayed target cleanup in case the target marker arrives late"
+        );
+    }
+
+    #[test]
+    fn test_should_retry_delete_marker_purge_for_delete_marker_creation() {
+        let dobj = DeletedObject {
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            should_retry_delete_marker_purge(&dobj),
+            "delete-marker creation should keep the late-arrival cleanup path so downstream purges can catch up"
+        );
+    }
+
+    #[test]
+    fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
+        assert!(
+            !is_retryable_delete_replication_head_error(false, Some("405")),
+            "numeric 405 responses should not block delete-marker purge replication"
+        );
+        assert!(
+            !is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")),
+            "MethodNotAllowed responses should not block delete-marker purge replication"
+        );
+        assert!(
+            !is_retryable_delete_replication_head_error(true, Some("NoSuchKey")),
+            "not-found responses should not block delete-marker purge replication"
+        );
+        assert!(
+            is_retryable_delete_replication_head_error(false, Some("AccessDenied")),
+            "unexpected head errors should still fail fast"
+        );
+    }
+
     #[tokio::test]
     async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
         let oi = ObjectInfo {

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -832,61 +832,55 @@ impl ReplicationStats {
         let mut rs = ReplStat::new();
 
         match status {
-            ReplicationStatusType::Pending => {
-                if ri.op_type.is_data_replication() && prev_status != status {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        Duration::default(),
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Pending if ri.op_type.is_data_replication() && prev_status != status => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    Duration::default(),
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Completed => {
-                if ri.op_type.is_data_replication() {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        ri.duration,
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Completed if ri.op_type.is_data_replication() => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    ri.duration,
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Failed => {
-                if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        ri.duration,
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Failed
+                if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending =>
+            {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    ri.duration,
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Replica => {
-                if ri.op_type == ReplicationType::Object {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        Duration::default(),
-                        status,
-                        ri.op_type,
-                        String::new(),
-                        false,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Replica if ri.op_type == ReplicationType::Object => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    Duration::default(),
+                    status,
+                    ri.op_type,
+                    String::new(),
+                    false,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
             _ => {}
         }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -12,12 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::disk_store::get_drive_walkdir_stall_timeout;
 use crate::disk::error::DiskError;
 use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
 use futures::future::join_all;
+use metrics::counter;
 use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
-use std::{future::Future, pin::Pin};
+use std::{future::Future, pin::Pin, time::Duration};
+use tokio::io::AsyncRead;
 use tokio::spawn;
+use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -26,6 +30,21 @@ pub type PartialFn =
     Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 
+#[derive(Debug)]
+enum PeekOutcome {
+    Ready(Option<MetaCacheEntry>),
+    Error(rustfs_filemeta::Error),
+    TimedOut,
+}
+
+async fn peek_with_timeout<R: AsyncRead + Unpin>(reader: &mut MetacacheReader<R>, timeout_duration: Duration) -> PeekOutcome {
+    match timeout(timeout_duration, reader.peek()).await {
+        Ok(Ok(entry)) => PeekOutcome::Ready(entry),
+        Ok(Err(err)) => PeekOutcome::Error(err),
+        Err(_) => PeekOutcome::TimedOut,
+    }
+}
+
 #[derive(Default)]
 pub struct ListPathRawOptions {
     pub disks: Vec<Option<DiskStore>>,
@@ -160,6 +179,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     }
 
     let revjob = spawn(async move {
+        let peek_timeout = get_drive_walkdir_stall_timeout();
         let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
         for _ in 0..readers.len() {
             errs.push(None);
@@ -191,8 +211,8 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                     continue;
                 }
 
-                let entry = match r.peek().await {
-                    Ok(res) => {
+                let entry = match peek_with_timeout(r, peek_timeout).await {
+                    PeekOutcome::Ready(res) => {
                         if let Some(entry) = res {
                             // info!("read entry disk: {}, name: {}", i, entry.name);
                             entry
@@ -203,7 +223,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
-                    Err(err) => {
+                    PeekOutcome::Error(err) => {
                         if err == rustfs_filemeta::Error::Unexpected {
                             at_eof += 1;
                             // warn!("list_path_raw: peek err eof, disk: {}", i);
@@ -236,6 +256,31 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
+                    PeekOutcome::TimedOut => {
+                        has_err += 1;
+                        errs[i] = Some(DiskError::Timeout);
+                        let endpoint = opts
+                            .disks
+                            .get(i)
+                            .and_then(|disk| disk.as_ref().map(|disk| disk.endpoint().to_string()))
+                            .unwrap_or_else(|| "missing".to_string());
+                        counter!(
+                            "rustfs_list_path_raw_stall_total",
+                            "drive" => endpoint.clone()
+                        )
+                        .increment(1);
+                        warn!(
+                            drive = %endpoint,
+                            bucket = %opts.bucket,
+                            path = %opts.path,
+                            timeout_ms = peek_timeout.as_millis(),
+                            "list_path_raw reader peek timed out; excluding drive from current merge"
+                        );
+                        let (detached_rd, write_half) = tokio::io::duplex(1);
+                        drop(write_half);
+                        *r = MetacacheReader::new(detached_rd);
+                        continue;
+                    }
                 };
 
                 // warn!("list_path_raw: loop entry: {:?}, disk: {}", &entry.name, i);
@@ -364,3 +409,42 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     // warn!("list_path_raw: done");
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_filemeta::MetacacheWriter;
+
+    #[tokio::test]
+    async fn peek_with_timeout_times_out_on_silent_reader() {
+        let (_writer, reader) = tokio::io::duplex(64);
+        let mut reader = MetacacheReader::new(reader);
+
+        let outcome = peek_with_timeout(&mut reader, Duration::from_millis(20)).await;
+        assert!(matches!(outcome, PeekOutcome::TimedOut));
+    }
+
+    #[tokio::test]
+    async fn peek_with_timeout_reads_entry_before_deadline() {
+        let (reader, writer) = tokio::io::duplex(256);
+        let mut metacache_reader = MetacacheReader::new(reader);
+
+        tokio::spawn(async move {
+            let mut writer = MetacacheWriter::new(writer);
+            let entry = MetaCacheEntry {
+                name: "bucket/object".to_string(),
+                metadata: vec![1, 2, 3],
+                cached: None,
+                reusable: false,
+            };
+            writer.write(&[entry]).await.expect("entry should be written");
+            writer.close().await.expect("writer should close");
+        });
+
+        let outcome = peek_with_timeout(&mut metacache_reader, Duration::from_secs(1)).await;
+        match outcome {
+            PeekOutcome::Ready(Some(entry)) => assert_eq!(entry.name, "bucket/object"),
+            other => panic!("expected ready entry, got {other:?}"),
+        }
+    }
+}

crates/ecstore/src/client/api_error_response.rs

@@ -100,7 +100,7 @@ pub fn http_resp_to_error_response(
     bucket_name: &str,
     object_name: &str,
 ) -> ErrorResponse {
-    let err_body = String::from_utf8(b).unwrap();
+    let err_body = String::from_utf8_lossy(&b).to_string();
     if h.is_empty() || resp_status.is_client_error() || resp_status.is_server_error() {
         return ErrorResponse {
             status_code: resp_status,
@@ -178,36 +178,46 @@ pub fn http_resp_to_error_response(
                 };
             }
         }
-    } else {
-        err_resp = err_resp_.unwrap();
+    } else if let Ok(parsed_resp) = err_resp_ {
+        err_resp = parsed_resp;
     }
     err_resp.status_code = resp_status;
     if let Some(server_name) = h.get("Server") {
-        err_resp.server = server_name.to_str().expect("err").to_string();
+        if let Ok(server_str) = server_name.to_str() {
+            err_resp.server = server_str.to_string();
+        }
     }
 
-    let code = h.get("x-minio-error-code");
-    if code.is_some() {
-        err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
+    if let Some(code) = h.get("x-minio-error-code") {
+        if let Ok(code_str) = code.to_str() {
+            err_resp.code = S3ErrorCode::Custom(code_str.into());
+        }
     }
-    let desc = h.get("x-minio-error-desc");
-    if desc.is_some() {
-        err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
+    if let Some(desc) = h.get("x-minio-error-desc") {
+        if let Ok(desc_str) = desc.to_str() {
+            err_resp.message = desc_str.trim_matches('"').to_string();
+        }
     }
 
     if err_resp.request_id == "" {
         if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
-            err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
+            if let Ok(request_id_str) = x_amz_request_id.to_str() {
+                err_resp.request_id = request_id_str.to_string();
+            }
         }
     }
     if err_resp.host_id == "" {
         if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
-            err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
+            if let Ok(host_id_str) = x_amz_id_2.to_str() {
+                err_resp.host_id = host_id_str.to_string();
+            }
         }
     }
     if err_resp.region == "" {
         if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
-            err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
+            if let Ok(region_str) = x_amz_bucket_region.to_str() {
+                err_resp.region = region_str.to_string();
+            }
         }
     }
     if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {

crates/ecstore/src/client/api_get_object.rs

@@ -1,4 +1,3 @@
-#![allow(clippy::map_entry)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(clippy::map_entry)]
 #![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]

crates/ecstore/src/client/api_put_object.rs

@@ -137,21 +137,21 @@ impl Default for PutObjectOptions {
 impl PutObjectOptions {
     fn set_match_etag(&mut self, etag: &str) {
         if etag == "*" {
-            self.custom_header
-                .insert("If-Match", HeaderValue::from_str("*").expect("err"));
+            self.custom_header.insert("If-Match", HeaderValue::from_static("*"));
         } else {
-            self.custom_header
-                .insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{}\"", etag)) {
+                self.custom_header.insert("If-Match", etag_value);
+            }
         }
     }
 
     fn set_match_etag_except(&mut self, etag: &str) {
         if etag == "*" {
-            self.custom_header
-                .insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
+            self.custom_header.insert("If-None-Match", HeaderValue::from_static("*"));
         } else {
-            self.custom_header
-                .insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
+            if let Ok(etag_value) = HeaderValue::from_str(&format!("\"{etag}\"")) {
+                self.custom_header.insert("If-None-Match", etag_value);
+            }
         }
     }
 
@@ -162,59 +162,75 @@ impl PutObjectOptions {
         if content_type == "" {
             content_type = "application/octet-stream".to_string();
         }
-        header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
+        if let Ok(content_type_value) = HeaderValue::from_str(&content_type) {
+            header.insert("Content-Type", content_type_value);
+        }
 
         if self.content_encoding != "" {
-            header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
+            if let Ok(encoding_value) = HeaderValue::from_str(&self.content_encoding) {
+                header.insert("Content-Encoding", encoding_value);
+            }
         }
         if self.content_disposition != "" {
-            header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
+            if let Ok(disposition_value) = HeaderValue::from_str(&self.content_disposition) {
+                header.insert("Content-Disposition", disposition_value);
+            }
         }
         if self.content_language != "" {
-            header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
+            if let Ok(language_value) = HeaderValue::from_str(&self.content_language) {
+                header.insert("Content-Language", language_value);
+            }
         }
         if self.cache_control != "" {
-            header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
+            if let Ok(cache_value) = HeaderValue::from_str(&self.cache_control) {
+                header.insert("Cache-Control", cache_value);
+            }
         }
 
         if self.expires.unix_timestamp() != 0 {
-            header.insert(
-                "Expires",
-                HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
-            ); //rustfs invalid header
+            if let Ok(expires_str) = self.expires.format(ISO8601_DATEFORMAT) {
+                if let Ok(expires_value) = HeaderValue::from_str(&expires_str) {
+                    header.insert("Expires", expires_value);
+                }
+            }
         }
 
         if self.mode.as_str() != "" {
-            header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
+            if let Ok(mode_value) = HeaderValue::from_str(self.mode.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_MODE, mode_value);
+            }
         }
 
         if self.retain_until_date.unix_timestamp() != 0 {
-            header.insert(
-                X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
-                HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
-            );
+            if let Ok(retain_str) = self.retain_until_date.format(ISO8601_DATEFORMAT) {
+                if let Ok(retain_value) = HeaderValue::from_str(&retain_str) {
+                    header.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, retain_value);
+                }
+            }
         }
 
         if self.legalhold.as_str() != "" {
-            header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
+            if let Ok(legalhold_value) = HeaderValue::from_str(self.legalhold.as_str()) {
+                header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, legalhold_value);
+            }
         }
 
         if self.storage_class != "" {
-            header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
+            if let Ok(storage_class_value) = HeaderValue::from_str(&self.storage_class) {
+                header.insert(X_AMZ_STORAGE_CLASS, storage_class_value);
+            }
         }
 
         if self.website_redirect_location != "" {
-            header.insert(
-                X_AMZ_WEBSITE_REDIRECT_LOCATION,
-                HeaderValue::from_str(&self.website_redirect_location).expect("err"),
-            );
+            if let Ok(redirect_value) = HeaderValue::from_str(&self.website_redirect_location) {
+                header.insert(X_AMZ_WEBSITE_REDIRECT_LOCATION, redirect_value);
+            }
         }
 
         if !self.internal.replication_status.as_str().is_empty() {
-            header.insert(
-                X_AMZ_REPLICATION_STATUS,
-                HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
-            );
+            if let Ok(replication_status_value) = HeaderValue::from_str(self.internal.replication_status.as_str()) {
+                header.insert(X_AMZ_REPLICATION_STATUS, replication_status_value);
+            }
         }
 
         for (k, v) in &self.user_metadata {
@@ -360,17 +376,21 @@ impl TransitionClient {
 
             let mut md5_base64: String = "".to_string();
             if opts.send_content_md5 {
-                let mut md5_hasher = self.md5_hasher.lock().unwrap();
-                let hash = md5_hasher.as_mut().expect("err");
-                let hash = hash.hash_encode(&buf[..length]);
-                md5_base64 = base64_encode(hash.as_ref());
+                if let Some(mut md5_hasher) = self.md5_hasher.lock().unwrap().as_mut() {
+                    let hash = md5_hasher.hash_encode(&buf[..length]);
+                    md5_base64 = base64_encode(hash.as_ref());
+                }
             } else {
                 let mut crc = opts.auto_checksum.hasher()?;
                 crc.update(&buf[..length]);
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }

crates/ecstore/src/client/api_put_object_multipart.rs

@@ -127,7 +127,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -309,27 +313,27 @@ impl TransitionClient {
         let h = resp.headers();
         let mut obj_part = ObjectPart {
             checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
-                h_checksum_crc32.to_str().expect("err").to_string()
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
-                h_checksum_crc32c.to_str().expect("err").to_string()
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
-                h_checksum_sha1.to_str().expect("err").to_string()
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
-                h_checksum_sha256.to_str().expect("err").to_string()
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
-                h_checksum_crc64nvme.to_str().expect("err").to_string()
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
@@ -338,7 +342,7 @@ impl TransitionClient {
         obj_part.size = p.size;
         obj_part.part_num = p.part_number;
         obj_part.etag = if let Some(h_etag) = h.get("ETag") {
-            h_etag.to_str().expect("err").trim_matches('"').to_string()
+            h_etag.to_str().unwrap_or("").trim_matches('"').to_string()
         } else {
             "".to_string()
         };
@@ -398,7 +402,7 @@ impl TransitionClient {
             key: complete_multipart_upload_result.key,
             etag: trim_etag(&complete_multipart_upload_result.etag),
             version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
-                h_x_amz_version_id.to_str().expect("err").to_string()
+                h_x_amz_version_id.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },

crates/ecstore/src/client/api_put_object_streaming.rs

@@ -21,6 +21,7 @@
 use bytes::Bytes;
 use futures::future::join_all;
 use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use std::io::Error;
 use std::sync::RwLock;
 use std::{collections::HashMap, sync::Arc};
 use time::{OffsetDateTime, format_description};
@@ -152,7 +153,10 @@ impl TransitionClient {
 
             if opts.send_content_md5 {
                 let mut md5_hasher = self.md5_hasher.lock().unwrap();
-                let md5_hash = md5_hasher.as_mut().expect("err");
+                let md5_hash = match md5_hasher.as_mut() {
+                    Some(hasher) => hasher,
+                    None => return Err(std::io::Error::other("MD5 hasher not initialized")),
+                };
                 let hash = md5_hash.hash_encode(&buf[..length]);
                 md5_base64 = base64_encode(hash.as_ref());
             } else {
@@ -161,7 +165,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -275,11 +283,14 @@ impl TransitionClient {
         for part_number in 1..=total_parts_count {
             let mut buf = Vec::<u8>::new();
             select! {
-                buf = bufs_rx.recv() => {}
+                buf1 = bufs_rx.recv() => {
+                    if let Some(buf1) = buf1 {
+                        buf = buf1;
+                    }
+                }
                 err = err_rx.recv() => {
                     //cancel_token.cancel();
-                    //wg.Wait()
-                    return Err(err.expect("err"));
+                    return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
                 }
                 else => (),
             }
@@ -309,7 +320,11 @@ impl TransitionClient {
                 let csum = crc.finalize();
 
                 if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
-                    custom_header.insert(header_name, base64_encode(csum.as_ref()).parse().expect("err"));
+                    if let Ok(header_value) = base64_encode(csum.as_ref()).parse() {
+                        custom_header.insert(header_name, header_value);
+                    } else {
+                        warn!("Failed to parse checksum value");
+                    }
                 } else {
                     warn!("Invalid header name: {}", opts.auto_checksum.key());
                 }
@@ -319,12 +334,19 @@ impl TransitionClient {
             let clone_parts_info = parts_info.clone();
             let clone_upload_id = upload_id.clone();
             let clone_self = self.clone();
+            let err_tx_clone = err_tx.clone();
             futures.push(async move {
                 let mut md5_base64: String = "".to_string();
 
                 if opts.send_content_md5 {
                     let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
-                    let md5_hash = md5_hasher.as_mut().expect("err");
+                    let md5_hash = match md5_hasher.as_mut() {
+                        Some(hasher) => hasher,
+                        None => {
+                            //let _ = err_tx_clone.send(std::io::Error::other("MD5 hasher not initialized")).await;
+                            return Ok::<(), Error>(());
+                        }
+                    };
                     let hash = md5_hash.hash_encode(&buf[..length]);
                     md5_base64 = base64_encode(hash.as_ref());
                 }
@@ -344,12 +366,21 @@ impl TransitionClient {
                     sha256_hex: "".to_string(),
                     trailer: HeaderMap::new(),
                 };
-                let obj_part = clone_self.upload_part(&mut p).await.expect("err");
+                let obj_part = match clone_self.upload_part(&mut p).await {
+                    Ok(part) => part,
+                    Err(err) => {
+                        let _ = err_tx_clone.send(std::io::Error::other(err.to_string())).await;
+                        return Err::<(), Error>(err);
+                    }
+                };
 
-                let mut clone_parts_info = clone_parts_info.write().unwrap();
-                clone_parts_info.entry(part_number).or_insert(obj_part);
+                {
+                    let mut clone_parts_info = clone_parts_info.write().unwrap();
+                    clone_parts_info.entry(part_number).or_insert(obj_part);
+                }
 
-                clone_bufs_tx.send(buf);
+                let _ = clone_bufs_tx.send(buf).await;
+                Ok::<(), Error>(())
             });
 
             total_uploaded_size += length as i64;
@@ -359,7 +390,7 @@ impl TransitionClient {
 
         select! {
             err = err_rx.recv() => {
-                return Err(err.expect("err"));
+                return Err(err.unwrap_or_else(|| std::io::Error::other("Unknown error received from channel")));
             }
             else => (),
         }
@@ -504,9 +535,10 @@ impl TransitionClient {
         Ok(UploadInfo {
             bucket: bucket_name.to_string(),
             key: object_name.to_string(),
-            etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
+            etag: trim_etag(h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("")),
+
             version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
-                h_x_amz_version_id.to_str().expect("err").to_string()
+                h_x_amz_version_id.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
@@ -514,27 +546,27 @@ impl TransitionClient {
             expiration: exp_time,
             expiration_rule_id: rule_id,
             checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
-                h_checksum_crc32.to_str().expect("err").to_string()
+                h_checksum_crc32.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
-                h_checksum_crc32c.to_str().expect("err").to_string()
+                h_checksum_crc32c.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
-                h_checksum_sha1.to_str().expect("err").to_string()
+                h_checksum_sha1.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
-                h_checksum_sha256.to_str().expect("err").to_string()
+                h_checksum_sha256.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },
             checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
-                h_checksum_crc64nvme.to_str().expect("err").to_string()
+                h_checksum_crc64nvme.to_str().unwrap_or("").to_string()
             } else {
                 "".to_string()
             },

crates/ecstore/src/client/api_remove.rs

@@ -25,7 +25,7 @@ use hyper::body::Bytes;
 use rustfs_utils::HashAlgorithm;
 use s3s::S3ErrorCode;
 use s3s::dto::ReplicationStatus;
-use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
+use s3s::header::{X_AMZ_BYPASS_GOVERNANCE_RETENTION, X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
 use serde::Deserialize;
 use std::fmt::Display;
 use std::{
@@ -111,8 +111,9 @@ impl TransitionClient {
             .await?;
 
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.delete(bucket_name);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
         }
         Ok(())
     }
@@ -142,8 +143,9 @@ impl TransitionClient {
             .await?;
 
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.delete(bucket_name);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.delete(bucket_name);
+            }
         }
 
         Ok(())
@@ -168,7 +170,7 @@ impl TransitionClient {
         let mut headers = HeaderMap::new();
 
         if opts.governance_bypass {
-            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
+            headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true")); //amzBypassGovernance
         }
 
         let resp = self
@@ -197,13 +199,12 @@ impl TransitionClient {
         Ok(RemoveObjectResult {
             object_name: object_name.to_string(),
             object_version_id: opts.version_id,
-            delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
+            delete_marker: resp.headers().get(X_AMZ_DELETE_MARKER).map_or(false, |v| v == "true"),
             delete_marker_version_id: resp
                 .headers()
-                .get("x-amz-version-id")
-                .expect("err")
-                .to_str()
-                .expect("err")
+                .get(X_AMZ_VERSION_ID)
+                .and_then(|v| v.to_str().ok())
+                .unwrap_or_default()
                 .to_string(),
             ..Default::default()
         })
@@ -290,15 +291,15 @@ impl TransitionClient {
                             bucket_name,
                             &object.name,
                             RemoveObjectOptions {
-                                version_id: object.version_id.expect("err").to_string(),
+                                version_id: object.version_id.map(|id| id.to_string()).unwrap_or_default(),
                                 governance_bypass: opts.governance_bypass,
                                 ..Default::default()
                             },
                         )
                         .await?;
                     let remove_result_clone = remove_result.clone();
-                    if !remove_result.err.is_none() {
-                        match to_error_response(&remove_result.err.expect("err")).code {
+                    if let Some(err) = &remove_result.err {
+                        match to_error_response(err).code {
                             S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
                                 continue;
                             }
@@ -326,7 +327,7 @@ impl TransitionClient {
 
             let mut headers = HeaderMap::new();
             if opts.governance_bypass {
-                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
+                headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, HeaderValue::from_static("true"));
             }
 
             let remove_bytes = generate_remove_multi_objects_request(&batch);
@@ -423,23 +424,20 @@ impl TransitionClient {
                         request_id: resp
                             .headers()
                             .get("x-amz-request-id")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         host_id: resp
                             .headers()
                             .get("x-amz-id-2")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         region: resp
                             .headers()
                             .get("x-amz-bucket-region")
-                            .expect("err")
-                            .to_str()
-                            .expect("err")
+                            .and_then(|v| v.to_str().ok())
+                            .unwrap_or_default()
                             .to_string(),
                         ..Default::default()
                     };
@@ -472,10 +470,11 @@ pub struct RemoveObjectError {
 
 impl Display for RemoveObjectError {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        if self.err.is_none() {
-            return write!(f, "unexpected remove object error result");
+        if let Some(err) = &self.err {
+            write!(f, "{}", err.to_string())
+        } else {
+            write!(f, "unexpected remove object error result")
         }
-        write!(f, "{}", self.err.as_ref().expect("err").to_string())
     }
 }
 

crates/ecstore/src/client/bucket_cache.rs

@@ -70,10 +70,10 @@ impl TransitionClient {
 
         let mut location;
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            let ret = bucket_loc_cache.get(bucket_name);
-            if let Some(location) = ret {
-                return Ok(location);
+            if let Ok(bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                if let Some(location) = bucket_loc_cache.get(bucket_name) {
+                    return Ok(location);
+                }
             }
             //location = ret?;
         }
@@ -83,8 +83,9 @@ impl TransitionClient {
         let mut resp = self.doit(req).await?;
         location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
         {
-            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-            bucket_loc_cache.set(bucket_name, &location);
+            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                bucket_loc_cache.set(bucket_name, &location);
+            }
         }
         Ok(location)
     }
@@ -108,7 +109,11 @@ impl TransitionClient {
             url_str.push_str("://");
             url_str.push_str(bucket_name);
             url_str.push_str(".");
-            url_str.push_str(target_url.host_str().expect("err"));
+            url_str.push_str(
+                target_url
+                    .host_str()
+                    .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "host is none"))?,
+            );
             url_str.push_str("/?location");
         } else {
             let mut path = bucket_name.to_string();
@@ -135,13 +140,16 @@ impl TransitionClient {
 
         let value;
         {
-            let mut creds_provider = self.creds_provider.lock().unwrap();
-            value = match creds_provider.get_with_context(Some(self.cred_context())) {
-                Ok(v) => v,
-                Err(err) => {
-                    return Err(std::io::Error::other(err));
-                }
-            };
+            if let Ok(mut creds_provider) = self.creds_provider.lock() {
+                value = match creds_provider.get_with_context(Some(self.cred_context())) {
+                    Ok(v) => v,
+                    Err(err) => {
+                        return Err(std::io::Error::other(err));
+                    }
+                };
+            } else {
+                return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
+            }
         }
 
         let mut signer_type = value.signer_type.clone();
@@ -171,8 +179,9 @@ impl TransitionClient {
             content_sha256 = UNSIGNED_PAYLOAD.to_string();
         }
 
-        req.headers_mut()
-            .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
+        if let Ok(content_sha256_value) = content_sha256.parse() {
+            req.headers_mut().insert("X-Amz-Content-Sha256", content_sha256_value);
+        }
         let req = rustfs_signer::sign_v4(req, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
         Ok(req)
     }
@@ -228,13 +237,16 @@ async fn process_bucket_location_response(
     }
     let mut location = "".to_string();
     if tier_type == "huaweicloud" {
-        let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(body_vec).unwrap()).unwrap();
-        location = d.location_constraint;
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(d) = quick_xml::de::from_str::<CreateBucketConfiguration>(&body_str) {
+                location = d.location_constraint;
+            }
+        }
     } else {
-        if let Ok(LocationConstraint { field }) =
-            quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(body_vec).unwrap())
-        {
-            location = field;
+        if let Ok(body_str) = String::from_utf8(body_vec) {
+            if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&body_str) {
+                location = field;
+            }
         }
     }
     //debug!("location: {}", location);

crates/ecstore/src/client/credentials.rs

@@ -1,4 +1,3 @@
-#![allow(unused_imports)]
 // Copyright 2024 RustFS Team
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![allow(unused_imports)]
 #![allow(unused_variables)]
 #![allow(unused_mut)]
 #![allow(unused_assignments)]
@@ -57,7 +57,9 @@ impl<P: Provider + Default> Credentials<P> {
 
     pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
         if self.is_expired() {
-            let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
+            let creds = self.provider.retrieve_with_cred_context(cc.unwrap_or(CredContext {
+                endpoint: "".to_string(),
+            }));
             self.creds = creds;
             self.force_refresh = false;
         }

crates/ecstore/src/client/object_handlers_common.rs

@@ -12,14 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+use tracing::warn;
+
 use crate::bucket::lifecycle::lifecycle;
+use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::store::ECStore;
 use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
+use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
 use rustfs_lock::MAX_DELETE_LIST;
 
-pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
+fn lifecycle_version_delete_replication_state(
+    replicate_decision_str: String,
+    pending_status: Option<String>,
+) -> ReplicationState {
+    ReplicationState {
+        replicate_decision_str,
+        version_purge_status_internal: pending_status.clone(),
+        purge_targets: version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
+        ..Default::default()
+    }
+}
+
+pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
+    let version_suspended = match BucketVersioningSys::get(bucket).await {
+        Ok(vc) => vc.suspended(),
+        Err(err) => {
+            warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup");
+            return;
+        }
+    };
     let mut remaining = to_del;
     loop {
         let mut to_del = remaining;
@@ -29,15 +53,94 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
         } else {
             remaining = &[];
         }
-        let vc = BucketVersioningSys::get(bucket).await.expect("err!");
-        let _deleted_objs = api.delete_objects(
-            bucket,
-            to_del.to_vec(),
-            ObjectOptions {
-                //prefix_enabled_fn:  vc.prefix_enabled(""),
-                version_suspended: vc.suspended(),
+
+        let mut replication_candidates: Vec<Option<ReplicationState>> = Vec::with_capacity(to_del.len());
+        for object in to_del.iter() {
+            let version_id = object.version_id.map(|vid| vid.to_string());
+            let opts = ObjectOptions {
+                version_id: version_id.clone(),
+                versioned: true,
+                version_suspended,
                 ..Default::default()
-            },
-        );
+            };
+            let candidate = match api.get_object_info(bucket, &object.object_name, &opts).await {
+                Ok(info) => {
+                    let dsc = check_replicate_delete(bucket, object, &info, &opts, None).await;
+                    dsc.replicate_any()
+                        .then(|| lifecycle_version_delete_replication_state(dsc.to_string(), dsc.pending_status()))
+                }
+                Err(err) => {
+                    warn!(
+                        bucket,
+                        object = %object.object_name,
+                        version_id = ?version_id,
+                        error = ?err,
+                        "failed to get object info during lifecycle noncurrent version cleanup; skipping delete replication scheduling"
+                    );
+                    None
+                }
+            };
+            replication_candidates.push(candidate);
+        }
+
+        let (mut deleted_objs, errors) = api
+            .delete_objects(
+                bucket,
+                to_del.to_vec(),
+                ObjectOptions {
+                    version_suspended,
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        for (i, deleted_obj) in deleted_objs.iter_mut().enumerate() {
+            if errors.get(i).and_then(|err| err.as_ref()).is_some() {
+                continue;
+            }
+            let Some(replication_state) = replication_candidates.get(i).and_then(|c| c.clone()) else {
+                continue;
+            };
+            deleted_obj.replication_state = Some(replication_state);
+            schedule_replication_delete(DeletedObjectReplicationInfo {
+                delete_object: deleted_obj.clone(),
+                bucket: bucket.to_string(),
+                event_type: REPLICATE_INCOMING_DELETE.to_string(),
+                ..Default::default()
+            })
+            .await;
+        }
+
+        for (i, err) in errors.iter().enumerate() {
+            if let Some(e) = err {
+                let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or("<unknown>");
+                let vid = to_del
+                    .get(i)
+                    .and_then(|o| o.version_id)
+                    .map(|v| v.to_string())
+                    .unwrap_or_default();
+                warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup");
+            }
+        }
+        if remaining.is_empty() {
+            break;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::lifecycle_version_delete_replication_state;
+
+    #[test]
+    fn lifecycle_version_delete_replication_state_tracks_pending_purge_targets() {
+        let state = lifecycle_version_delete_replication_state(
+            "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
+            Some("arn:aws:s3:::target=PENDING;".to_string()),
+        );
+
+        assert_eq!(state.version_purge_status_internal.as_deref(), Some("arn:aws:s3:::target=PENDING;"));
+        assert!(state.purge_targets.contains_key("arn:aws:s3:::target"));
+        assert_eq!(state.replicate_decision_str, "arn:aws:s3:::target=true;false;arn:aws:s3:::target;");
     }
 }

crates/ecstore/src/client/transition_api.rs

@@ -251,9 +251,10 @@ impl TransitionClient {
         };
 
         {
-            let mut md5_hasher = client.md5_hasher.lock().unwrap();
-            if md5_hasher.is_none() {
-                *md5_hasher = Some(HashAlgorithm::Md5);
+            if let Ok(mut md5_hasher) = client.md5_hasher.lock() {
+                if md5_hasher.is_none() {
+                    *md5_hasher = Some(HashAlgorithm::Md5);
+                }
             }
         }
         if client.sha256_hasher.is_none() {
@@ -275,25 +276,30 @@ impl TransitionClient {
     }
 
     fn trace_errors_only_off(&self) {
-        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
-        *trace_errors_only = false;
+        if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
+            *trace_errors_only = false;
+        }
     }
 
     fn trace_off(&self) {
-        let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap();
-        *is_trace_enabled = false;
-        let mut trace_errors_only = self.trace_errors_only.lock().unwrap();
-        *trace_errors_only = false;
+        if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
+            *is_trace_enabled = false;
+        }
+        if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
+            *trace_errors_only = false;
+        }
     }
 
     fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
-        let mut endpoint = self.s3_accelerate_endpoint.lock().unwrap();
-        *endpoint = accelerate_endpoint.to_string();
+        if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
+            *endpoint = accelerate_endpoint.to_string();
+        }
     }
 
     fn set_s3_enable_dual_stack(&self, enabled: bool) {
-        let mut dual_stack = self.s3_dual_stack_enabled.lock().unwrap();
-        *dual_stack = enabled;
+        if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
+            *dual_stack = enabled;
+        }
     }
 
     pub fn hash_materials(
@@ -352,7 +358,6 @@ impl TransitionClient {
         let resp;
         let http_client = self.http_client.clone();
         {
-            //let mut http_client = http_client.lock().unwrap();
             req_method = req.method().clone();
             req_uri = req.uri().clone();
             req_headers = req.headers().clone();
@@ -368,7 +373,10 @@ impl TransitionClient {
             return Err(std::io::Error::other(err));
         }
 
-        let resp = resp.unwrap();
+        let resp = match resp {
+            Ok(r) => r,
+            Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
+        };
         debug!("http_resp: {:?}", resp);
 
         //let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
@@ -455,11 +463,13 @@ impl TransitionClient {
                             return Err(std::io::Error::other(err_response));
                         }
                         if metadata.bucket_name != "" {
-                            let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
-                            let location = bucket_loc_cache.get(&metadata.bucket_name);
-                            if location.is_some() && location.unwrap() != err_response.region {
-                                bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
-                                //continue;
+                            if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
+                                if let Some(location) = bucket_loc_cache.get(&metadata.bucket_name) {
+                                    if location != err_response.region {
+                                        bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
+                                        //continue;
+                                    }
+                                }
                             }
                         } else if err_response.region != metadata.bucket_location {
                             metadata.bucket_location = err_response.region.clone();
@@ -518,8 +528,11 @@ impl TransitionClient {
 
         let value;
         {
-            let mut creds_provider = self.creds_provider.lock().unwrap();
-            value = creds_provider.get_with_context(Some(self.cred_context()))?;
+            if let Ok(mut creds_provider) = self.creds_provider.lock() {
+                value = creds_provider.get_with_context(Some(self.cred_context()))?;
+            } else {
+                return Err(std::io::Error::other("Failed to acquire credentials provider lock"));
+            }
         }
 
         let mut signer_type = value.signer_type.clone();
@@ -548,8 +561,10 @@ impl TransitionClient {
                     )));
                 }
                 let headers = req.headers_mut();
-                for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() {
-                    headers.insert(k, v.clone());
+                if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
+                    for (k, v) in extra_headers {
+                        headers.insert(k, v.clone());
+                    }
                 }
             }
             if signer_type == SignatureType::SignatureV2 {
@@ -571,18 +586,22 @@ impl TransitionClient {
         self.set_user_agent(&mut req);
 
         for (k, v) in metadata.custom_header.clone() {
-            req.headers_mut().insert(k.expect("err"), v);
+            if let Some(key) = k {
+                req.headers_mut().insert(key, v);
+            }
         }
 
         //req.content_length = metadata.content_length;
         if metadata.content_length <= -1 {
-            let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err");
-            req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
+            if let Ok(chunked_value) = HeaderValue::from_str(&vec!["chunked"].join(",")) {
+                req.headers_mut().insert(http::header::TRANSFER_ENCODING, chunked_value);
+            }
         }
 
         if metadata.content_md5_base64.len() > 0 {
-            let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err");
-            req.headers_mut().insert("Content-Md5", md5_value);
+            if let Ok(md5_value) = HeaderValue::from_str(&metadata.content_md5_base64) {
+                req.headers_mut().insert("Content-Md5", md5_value);
+            }
         }
 
         if signer_type == SignatureType::SignatureAnonymous {
@@ -607,8 +626,13 @@ impl TransitionClient {
             } else if metadata.trailer.len() > 0 {
                 sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string();
             }
-            req.headers_mut()
-                .insert("X-Amz-Content-Sha256".parse::<HeaderName>().unwrap(), sha_header.parse().expect("err"));
+            let header_name = "X-Amz-Content-Sha256"
+                .parse::<HeaderName>()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+            let header_value = sha_header
+                .parse()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+            req.headers_mut().insert(header_name, header_value);
 
             req = rustfs_signer::sign_v4_trailer(
                 req,
@@ -636,7 +660,7 @@ impl TransitionClient {
 
     pub fn set_user_agent(&self, req: &mut Request<s3s::Body>) {
         let headers = req.headers_mut();
-        headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
+        headers.insert("User-Agent", HeaderValue::from_static(C_USER_AGENT));
     }
 
     fn make_target_url(
@@ -648,7 +672,10 @@ impl TransitionClient {
         query_values: &HashMap<String, String>,
     ) -> Result<Url, std::io::Error> {
         let scheme = self.endpoint_url.scheme();
-        let host = self.endpoint_url.host().unwrap();
+        let host = self
+            .endpoint_url
+            .host()
+            .ok_or_else(|| std::io::Error::other("Endpoint URL has no host"))?;
         let default_port = if scheme == "https" { 443 } else { 80 };
         let port = self.endpoint_url.port().unwrap_or(default_port);
 
@@ -1155,9 +1182,10 @@ pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Re
         for (name, value) in h.iter() {
             let header_name = name.as_str().to_lowercase();
             if header_name.starts_with("x-amz-meta-") {
-                let key = header_name.strip_prefix("x-amz-meta-").unwrap().to_string();
-                if let Ok(value_str) = value.to_str() {
-                    meta.insert(key, value_str.to_string());
+                if let Some(key) = header_name.strip_prefix("x-amz-meta-") {
+                    if let Ok(value_str) = value.to_str() {
+                        meta.insert(key.to_string(), value_str.to_string());
+                    }
                 }
             }
         }

crates/ecstore/src/config/audit.rs

@@ -16,9 +16,13 @@ use crate::config::{KV, KVS};
 use rustfs_config::{
     COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
     MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
-    WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
+    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
+    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
+    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
+    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
+    PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
+    WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+    WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -192,3 +196,144 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_ADDRESS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_SUBJECT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_CREDENTIALS_FILE.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_REQUIRED.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+#[allow(dead_code)]
+pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_AUTH_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
+            value: EnableState::On.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/com.rs

@@ -16,10 +16,16 @@ use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, audit, notify, oidc, stor
 use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::global::is_first_cluster_node_local;
-use crate::store_api::{ChunkNativePutData, ObjectInfo, ObjectOptions, StorageAPI};
+use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
-use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{
+    AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
+    AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS,
+    NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
 use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -34,6 +40,8 @@ const CONFIG_FILE: &str = "config.json";
 
 pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
 
+pub const COMMA_SEPARATED_LISTS: &[&str] = &[rustfs_config::oidc::OIDC_SCOPES, rustfs_config::oidc::OIDC_OTHER_AUDIENCES];
+
 static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
 
 static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
@@ -42,6 +50,72 @@ static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
     h
 });
 
+#[derive(Clone, Copy)]
+struct TargetConfigDescriptor {
+    external_key: &'static str,
+    subsystem_key: &'static str,
+    default_kvs: &'static LazyLock<KVS>,
+    valid_keys: &'static [&'static str],
+}
+
+fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
+    [
+        TargetConfigDescriptor {
+            external_key: "webhook",
+            subsystem_key: NOTIFY_WEBHOOK_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
+            valid_keys: NOTIFY_WEBHOOK_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "mqtt",
+            subsystem_key: NOTIFY_MQTT_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_MQTT_KVS,
+            valid_keys: NOTIFY_MQTT_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "nats",
+            subsystem_key: NOTIFY_NATS_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_NATS_KVS,
+            valid_keys: NOTIFY_NATS_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "pulsar",
+            subsystem_key: NOTIFY_PULSAR_SUB_SYS,
+            default_kvs: &notify::DEFAULT_NOTIFY_PULSAR_KVS,
+            valid_keys: NOTIFY_PULSAR_KEYS,
+        },
+    ]
+}
+
+fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
+    [
+        TargetConfigDescriptor {
+            external_key: "webhook",
+            subsystem_key: AUDIT_WEBHOOK_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
+            valid_keys: AUDIT_WEBHOOK_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "mqtt",
+            subsystem_key: AUDIT_MQTT_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_MQTT_KVS,
+            valid_keys: AUDIT_MQTT_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "nats",
+            subsystem_key: AUDIT_NATS_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_NATS_KVS,
+            valid_keys: AUDIT_NATS_KEYS,
+        },
+        TargetConfigDescriptor {
+            external_key: "pulsar",
+            subsystem_key: AUDIT_PULSAR_SUB_SYS,
+            default_kvs: &audit::DEFAULT_AUDIT_PULSAR_KVS,
+            valid_keys: AUDIT_PULSAR_KEYS,
+        },
+    ]
+}
+
 #[instrument(skip(api))]
 pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
     let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
@@ -128,7 +202,7 @@ pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()>
 }
 
 pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
-    let mut put_data = ChunkNativePutData::from_vec(data);
+    let mut put_data = PutObjReader::from_vec(data);
     if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
         error!("save_config_with_opts: err: {:?}, file: {}", err, file);
         return Err(err);
@@ -192,15 +266,15 @@ fn parse_oidc_scalar_value(key: &str, value: &Value) -> Option<String> {
         }),
         Value::Bool(v) => Some(v.to_string()),
         Value::Number(v) => Some(v.to_string()),
-        Value::Array(values) if key == rustfs_config::oidc::OIDC_SCOPES => {
-            let scopes = values
+        Value::Array(values) if COMMA_SEPARATED_LISTS.contains(&key) => {
+            let values_str = values
                 .iter()
                 .filter_map(Value::as_str)
                 .map(str::trim)
-                .filter(|scope| !scope.is_empty())
+                .filter(|val| !val.is_empty())
                 .collect::<Vec<_>>()
                 .join(",");
-            Some(scopes)
+            Some(values_str)
         }
         Value::Null => None,
         _ => None,
@@ -363,29 +437,31 @@ fn apply_external_notify_section(
     applied
 }
 
+fn apply_external_target_descriptors(
+    cfg: &mut Config,
+    section_obj: &Map<String, Value>,
+    descriptors: &[TargetConfigDescriptor],
+) -> bool {
+    let mut applied = false;
+    for descriptor in descriptors {
+        applied |= apply_external_notify_section(
+            cfg,
+            section_obj,
+            descriptor.external_key,
+            descriptor.subsystem_key,
+            descriptor.default_kvs,
+            descriptor.valid_keys,
+        );
+    }
+    applied
+}
+
 fn apply_external_notify_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
     let Some(Value::Object(notify_obj)) = root.get("notify") else {
         return false;
     };
 
-    let mut applied = false;
-    applied |= apply_external_notify_section(
-        cfg,
-        notify_obj,
-        "webhook",
-        NOTIFY_WEBHOOK_SUB_SYS,
-        &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
-        NOTIFY_WEBHOOK_KEYS,
-    );
-    applied |= apply_external_notify_section(
-        cfg,
-        notify_obj,
-        "mqtt",
-        NOTIFY_MQTT_SUB_SYS,
-        &notify::DEFAULT_NOTIFY_MQTT_KVS,
-        NOTIFY_MQTT_KEYS,
-    );
-    applied
+    apply_external_target_descriptors(cfg, notify_obj, &notify_target_descriptors())
 }
 
 fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
@@ -394,24 +470,7 @@ fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
         return false;
     };
 
-    let mut applied = false;
-    applied |= apply_external_notify_section(
-        cfg,
-        audit_obj,
-        "webhook",
-        AUDIT_WEBHOOK_SUB_SYS,
-        &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
-        AUDIT_WEBHOOK_KEYS,
-    );
-    applied |= apply_external_notify_section(
-        cfg,
-        audit_obj,
-        "mqtt",
-        AUDIT_MQTT_SUB_SYS,
-        &audit::DEFAULT_AUDIT_MQTT_KVS,
-        AUDIT_MQTT_KEYS,
-    );
-    applied
+    apply_external_target_descriptors(cfg, audit_obj, &audit_target_descriptors())
 }
 
 fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
@@ -513,15 +572,15 @@ fn build_oidc_provider_object(kvs: &KVS) -> Map<String, Value> {
             continue;
         }
 
-        if kv.key == rustfs_config::oidc::OIDC_SCOPES {
-            let scopes = kv
+        if COMMA_SEPARATED_LISTS.contains(&kv.key.as_str()) {
+            let values = kv
                 .value
                 .split(',')
                 .map(str::trim)
-                .filter(|scope| !scope.is_empty())
-                .map(|scope| Value::String(scope.to_string()))
+                .filter(|val| !val.is_empty())
+                .map(|val| Value::String(val.to_string()))
                 .collect::<Vec<_>>();
-            provider.insert(kv.key.clone(), Value::Array(scopes));
+            provider.insert(kv.key.clone(), Value::Array(values));
             continue;
         }
 
@@ -537,7 +596,7 @@ fn build_oidc_object(cfg: &Config) -> Map<String, Value> {
     };
 
     let mut providers = subsystem.iter().collect::<Vec<_>>();
-    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    providers.sort_by_key(|(lhs, _)| *lhs);
 
     let mut oidc_obj = Map::new();
     for (instance_key, kvs) in providers {
@@ -572,7 +631,7 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     };
 
     let mut providers = subsystem.iter().collect::<Vec<_>>();
-    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    providers.sort_by_key(|(lhs, _)| *lhs);
 
     let mut oidc_obj = Map::new();
     for (instance_key, kvs) in providers {
@@ -605,7 +664,15 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
 }
 
 fn is_notify_bool_key(key: &str) -> bool {
-    key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+    matches!(
+        key,
+        ENABLE_KEY
+            | rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+            | rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
+            | rustfs_config::NATS_TLS_REQUIRED
+            | rustfs_config::PULSAR_TLS_ALLOW_INSECURE
+            | rustfs_config::PULSAR_TLS_HOSTNAME_VERIFICATION
+    )
 }
 
 fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
@@ -691,7 +758,7 @@ fn build_notify_subsystem_object(
         .iter()
         .filter(|(instance_key, _)| instance_key.as_str() != DEFAULT_DELIMITER)
         .collect::<Vec<_>>();
-    instances.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    instances.sort_by_key(|(lhs, _)| *lhs);
 
     for (instance_key, kvs) in instances {
         let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
@@ -703,38 +770,43 @@ fn build_notify_subsystem_object(
     subsystem_obj
 }
 
+fn build_target_object(cfg: &Config, descriptors: &[TargetConfigDescriptor]) -> Map<String, Value> {
+    let mut target_obj = Map::new();
+    for descriptor in descriptors {
+        let subsystem_obj =
+            build_notify_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
+        if !subsystem_obj.is_empty() {
+            target_obj.insert(descriptor.external_key.to_string(), Value::Object(subsystem_obj));
+        }
+    }
+    target_obj
+}
+
 fn build_notify_object(cfg: &Config) -> Map<String, Value> {
-    let mut notify_obj = Map::new();
-
-    let webhook_obj =
-        build_notify_subsystem_object(cfg, NOTIFY_WEBHOOK_SUB_SYS, &notify::DEFAULT_NOTIFY_WEBHOOK_KVS, NOTIFY_WEBHOOK_KEYS);
-    if !webhook_obj.is_empty() {
-        notify_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
-    }
-
-    let mqtt_obj = build_notify_subsystem_object(cfg, NOTIFY_MQTT_SUB_SYS, &notify::DEFAULT_NOTIFY_MQTT_KVS, NOTIFY_MQTT_KEYS);
-    if !mqtt_obj.is_empty() {
-        notify_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
-    }
-
-    notify_obj
+    build_target_object(cfg, &notify_target_descriptors())
 }
 
 fn build_audit_object(cfg: &Config) -> Map<String, Value> {
-    let mut audit_obj = Map::new();
+    build_target_object(cfg, &audit_target_descriptors())
+}
 
-    let webhook_obj =
-        build_notify_subsystem_object(cfg, AUDIT_WEBHOOK_SUB_SYS, &audit::DEFAULT_AUDIT_WEBHOOK_KVS, AUDIT_WEBHOOK_KEYS);
-    if !webhook_obj.is_empty() {
-        audit_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
+fn sync_rendered_target_object(
+    target_obj: &mut Map<String, Value>,
+    rendered_target: &Map<String, Value>,
+    descriptors: &[TargetConfigDescriptor],
+) {
+    for descriptor in descriptors {
+        match rendered_target.get(descriptor.external_key) {
+            Some(Value::Object(v)) => {
+                target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone()));
+                target_obj.remove(descriptor.subsystem_key);
+            }
+            _ => {
+                target_obj.remove(descriptor.external_key);
+                target_obj.remove(descriptor.subsystem_key);
+            }
+        }
     }
-
-    let mqtt_obj = build_notify_subsystem_object(cfg, AUDIT_MQTT_SUB_SYS, &audit::DEFAULT_AUDIT_MQTT_KVS, AUDIT_MQTT_KEYS);
-    if !mqtt_obj.is_empty() {
-        audit_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
-    }
-
-    audit_obj
 }
 
 fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
@@ -771,67 +843,31 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
         _ => Map::new(),
     };
     let rendered_notify = build_notify_object(cfg);
-    match rendered_notify.get("webhook") {
-        Some(Value::Object(v)) => {
-            notify_obj.insert("webhook".to_string(), Value::Object(v.clone()));
-            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
-        }
-        _ => {
-            notify_obj.remove("webhook");
-            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
-        }
-    }
-    match rendered_notify.get("mqtt") {
-        Some(Value::Object(v)) => {
-            notify_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
-            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
-        }
-        _ => {
-            notify_obj.remove("mqtt");
-            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
-        }
-    }
+    sync_rendered_target_object(&mut notify_obj, &rendered_notify, &notify_target_descriptors());
     if notify_obj.is_empty() {
         root.remove("notify");
     } else {
         root.insert("notify".to_string(), Value::Object(notify_obj));
     }
-    root.remove(NOTIFY_WEBHOOK_SUB_SYS);
-    root.remove(NOTIFY_MQTT_SUB_SYS);
+    for descriptor in notify_target_descriptors() {
+        root.remove(descriptor.subsystem_key);
+    }
 
     let mut logger_obj = match root.remove("logger") {
         Some(Value::Object(v)) => v,
         _ => Map::new(),
     };
     let rendered_audit = build_audit_object(cfg);
-    match rendered_audit.get("webhook") {
-        Some(Value::Object(v)) => {
-            logger_obj.insert("webhook".to_string(), Value::Object(v.clone()));
-            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
-        }
-        _ => {
-            logger_obj.remove("webhook");
-            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
-        }
-    }
-    match rendered_audit.get("mqtt") {
-        Some(Value::Object(v)) => {
-            logger_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
-            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
-        }
-        _ => {
-            logger_obj.remove("mqtt");
-            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
-        }
-    }
+    sync_rendered_target_object(&mut logger_obj, &rendered_audit, &audit_target_descriptors());
     if logger_obj.is_empty() {
         root.remove("logger");
     } else {
         root.insert("logger".to_string(), Value::Object(logger_obj));
     }
     root.remove("audit");
-    root.remove(AUDIT_WEBHOOK_SUB_SYS);
-    root.remove(AUDIT_MQTT_SUB_SYS);
+    for descriptor in audit_target_descriptors() {
+        root.remove(descriptor.subsystem_key);
+    }
 
     Ok(serde_json::to_vec(&Value::Object(root))?)
 }
@@ -1076,10 +1112,10 @@ mod tests {
     use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
     use crate::set_disk::SetDisks;
     use crate::store_api::{
-        BucketInfo, BucketOperations, BucketOptions, ChunkNativePutData, CompletePart, DeleteBucketOptions, DeletedObject,
-        GetObjectReader, HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info,
-        ListOperations, MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo,
-        ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, StorageAPI, WalkOptions,
+        BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader,
+        HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
+        MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations,
+        ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
     };
     use http::HeaderMap;
     use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
@@ -1302,7 +1338,7 @@ mod tests {
             &self,
             _bucket: &str,
             _object: &str,
-            _data: &mut ChunkNativePutData,
+            _data: &mut PutObjReader,
             _opts: &ObjectOptions,
         ) -> Result<ObjectInfo> {
             panic!("unused in test")
@@ -1489,7 +1525,7 @@ mod tests {
             _object: &str,
             _upload_id: &str,
             _part_id: usize,
-            _data: &mut ChunkNativePutData,
+            _data: &mut PutObjReader,
             _opts: &ObjectOptions,
         ) -> Result<PartInfo> {
             panic!("unused in test")
@@ -1655,6 +1691,7 @@ mod tests {
               "client_id":"console",
               "client_secret":"secret-value",
               "scopes":["openid","profile","email"],
+              "other_audiences":["aud1", "aud2"],
               "redirect_uri_dynamic":true,
               "display_name":"Default Provider"
             },
@@ -1679,6 +1716,7 @@ mod tests {
         );
         assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_CLIENT_ID), "console");
         assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_SCOPES), "openid,profile,email");
+        assert_eq!(default_kvs.get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES), "aud1,aud2");
         assert_eq!(default_kvs.get(ENABLE_KEY), EnableState::On.to_string());
 
         let smoke_kvs = cfg
@@ -1863,6 +1901,7 @@ mod tests {
         );
         default_provider.insert(rustfs_config::oidc::OIDC_CLIENT_ID.to_string(), "console".to_string());
         default_provider.insert(rustfs_config::oidc::OIDC_SCOPES.to_string(), "openid,profile,email".to_string());
+        default_provider.insert(rustfs_config::oidc::OIDC_OTHER_AUDIENCES.to_string(), "aud1,aud2".to_string());
         oidc_section.insert(DEFAULT_DELIMITER.to_string(), default_provider);
         cfg.0.insert(IDENTITY_OPENID_SUB_SYS.to_string(), oidc_section);
 
@@ -1890,6 +1929,13 @@ mod tests {
                 .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
             Some(vec!["openid", "profile", "email"])
         );
+        assert_eq!(
+            default_provider
+                .get(rustfs_config::oidc::OIDC_OTHER_AUDIENCES)
+                .and_then(Value::as_array)
+                .map(|values| values.iter().filter_map(Value::as_str).collect::<Vec<_>>()),
+            Some(vec!["aud1", "aud2"])
+        );
         assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
     }
 

crates/ecstore/src/config/mod.rs

@@ -25,8 +25,8 @@ use crate::store::ECStore;
 use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
 use rustfs_config::COMMENT_KEY;
 use rustfs_config::DEFAULT_DELIMITER;
-use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
+use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
 use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -144,7 +144,7 @@ impl KVS {
     pub fn insert(&mut self, key: String, value: String) {
         for kv in self.0.iter_mut() {
             if kv.key == key {
-                kv.value = value.clone();
+                kv.value = value;
                 return;
             }
         }
@@ -241,6 +241,10 @@ pub fn init() {
     kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
     kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone());
     kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());
+    kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone());
+    kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
+    kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
+    kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
     kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
 
     // Register all default configurations

crates/ecstore/src/config/notify.rs

@@ -16,9 +16,12 @@ use crate::config::{KV, KVS};
 use rustfs_config::{
     COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
     MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
-    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
-    WEBHOOK_SKIP_TLS_VERIFY,
+    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
+    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
+    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
+    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
+    PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
+    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -171,3 +174,143 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
         },
     ])
 });
+
+pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_ADDRESS.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_SUBJECT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_CREDENTIALS_FILE.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_CERT.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_CLIENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: NATS_TLS_REQUIRED.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: NATS_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});
+
+pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
+    KVS(vec![
+        KV {
+            key: ENABLE_KEY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_BROKER.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TOPIC.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_AUTH_TOKEN.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_USERNAME.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_PASSWORD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        },
+        KV {
+            key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
+            value: EnableState::On.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_DIR.to_owned(),
+            value: EVENT_DEFAULT_DIR.to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: PULSAR_QUEUE_LIMIT.to_owned(),
+            value: DEFAULT_LIMIT.to_string(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+    ])
+});

crates/ecstore/src/config/oidc.rs

@@ -17,9 +17,9 @@ use rustfs_config::{
     ENABLE_KEY, EnableState,
     oidc::{
         OIDC_CLAIM_NAME, OIDC_CLAIM_PREFIX, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_CONFIG_URL, OIDC_DEFAULT_CLAIM_NAME,
-        OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_SCOPES, OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME,
-        OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_SCOPES,
-        OIDC_USERNAME_CLAIM,
+        OIDC_DEFAULT_EMAIL_CLAIM, OIDC_DEFAULT_GROUPS_CLAIM, OIDC_DEFAULT_ROLES_CLAIM, OIDC_DEFAULT_SCOPES,
+        OIDC_DEFAULT_USERNAME_CLAIM, OIDC_DISPLAY_NAME, OIDC_EMAIL_CLAIM, OIDC_GROUPS_CLAIM, OIDC_OTHER_AUDIENCES,
+        OIDC_REDIRECT_URI, OIDC_REDIRECT_URI_DYNAMIC, OIDC_ROLE_POLICY, OIDC_ROLES_CLAIM, OIDC_SCOPES, OIDC_USERNAME_CLAIM,
     },
 };
 use std::sync::LazyLock;
@@ -52,6 +52,11 @@ pub static DEFAULT_IDENTITY_OPENID_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: OIDC_DEFAULT_SCOPES.to_owned(),
             hidden_if_empty: false,
         },
+        KV {
+            key: OIDC_OTHER_AUDIENCES.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
         KV {
             key: OIDC_REDIRECT_URI.to_owned(),
             value: "".to_owned(),
@@ -87,6 +92,11 @@ pub static DEFAULT_IDENTITY_OPENID_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: OIDC_DEFAULT_GROUPS_CLAIM.to_owned(),
             hidden_if_empty: false,
         },
+        KV {
+            key: OIDC_ROLES_CLAIM.to_owned(),
+            value: OIDC_DEFAULT_ROLES_CLAIM.to_owned(),
+            hidden_if_empty: false,
+        },
         KV {
             key: OIDC_EMAIL_CLAIM.to_owned(),
             value: OIDC_DEFAULT_EMAIL_CLAIM.to_owned(),

crates/ecstore/src/config/storageclass.rs

@@ -150,7 +150,18 @@ impl Config {
             return false;
         }
 
-        shard_size as usize <= self.inline_shard_limit_bytes(versioned)
+        let shard_size = shard_size as usize;
+
+        let mut inline_block = DEFAULT_INLINE_BLOCK;
+        if self.initialized {
+            inline_block = self.inline_block;
+        }
+
+        if versioned {
+            shard_size <= inline_block / 8
+        } else {
+            shard_size <= inline_block
+        }
     }
 
     pub fn inline_block(&self) -> usize {
@@ -161,15 +172,6 @@ impl Config {
         }
     }
 
-    pub fn inline_shard_limit_bytes(&self, versioned: bool) -> usize {
-        let inline_block = self.inline_block();
-        if versioned { inline_block / 8 } else { inline_block }
-    }
-
-    pub fn inline_object_limit_bytes(&self, data_shards: usize, versioned: bool) -> usize {
-        self.inline_shard_limit_bytes(versioned).saturating_mul(data_shards.max(1))
-    }
-
     pub fn capacity_optimized(&self) -> bool {
         if !self.initialized {
             false
@@ -334,32 +336,3 @@ pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_coun
     }
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn inline_object_limit_matches_default_non_versioned_budget() {
-        let cfg = Config {
-            initialized: true,
-            inline_block: DEFAULT_INLINE_BLOCK,
-            ..Default::default()
-        };
-
-        assert_eq!(cfg.inline_shard_limit_bytes(false), DEFAULT_INLINE_BLOCK);
-        assert_eq!(cfg.inline_object_limit_bytes(8, false), DEFAULT_INLINE_BLOCK * 8);
-    }
-
-    #[test]
-    fn inline_object_limit_scales_down_for_versioned_objects() {
-        let cfg = Config {
-            initialized: true,
-            inline_block: DEFAULT_INLINE_BLOCK,
-            ..Default::default()
-        };
-
-        assert_eq!(cfg.inline_shard_limit_bytes(true), DEFAULT_INLINE_BLOCK / 8);
-        assert_eq!(cfg.inline_object_limit_bytes(8, true), DEFAULT_INLINE_BLOCK);
-    }
-}

crates/ecstore/src/data_movement.rs

@@ -14,11 +14,9 @@
 
 use crate::error::{Error, Result};
 use crate::store::ECStore;
-use crate::store_api::{
-    ChunkNativePutData, CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions,
-};
+use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
 use bytes::Bytes;
-use rustfs_rio::{BlockReadable, BoxReadBlockFuture, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
+use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
 use std::io::Cursor;
 use std::pin::Pin;
 use std::sync::{
@@ -56,11 +54,6 @@ impl<R: AsyncRead + Unpin + Send + Sync> TryGetIndex for IndexedDataMovementRead
     }
 }
 
-impl<R: AsyncRead + Unpin + Send + Sync> BlockReadable for IndexedDataMovementReader<R> {
-    fn read_block<'a>(&'a mut self, buf: &'a mut [u8]) -> BoxReadBlockFuture<'a> {
-        Box::pin(rustfs_utils::read_full(self, buf))
-    }
-}
 pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
     let bytes = index?;
     let mut decoded = Index::new();
@@ -71,7 +64,7 @@ pub fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
     }
 }
 
-pub fn put_data_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<ChunkNativePutData> {
+pub fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<PutObjReader> {
     use sha2::{Digest, Sha256};
 
     let sha256hex = if !chunk.is_empty() {
@@ -81,8 +74,8 @@ pub fn put_data_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: O
     };
 
     let reader = IndexedDataMovementReader::new(Cursor::new(chunk), index);
-    let hash_reader = HashReader::from_reader(reader, size, actual_size, None, sha256hex, false)?;
-    Ok(ChunkNativePutData::new(hash_reader))
+    let hash_reader = HashReader::from_stream(reader, size, actual_size, None, sha256hex, false)?;
+    Ok(PutObjReader::new(hash_reader))
 }
 
 pub fn new_multipart_abort_flag() -> Arc<AtomicBool> {
@@ -179,7 +172,7 @@ pub(crate) async fn migrate_object(
                 let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
                 let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
                 let index = decode_part_index(part.index.as_ref());
-                let mut data = put_data_from_chunk(chunk, part_size, part_actual_size, index)?;
+                let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
 
                 let pi = match store
                     .put_object_part(
@@ -261,8 +254,8 @@ pub(crate) async fn migrate_object(
         .first()
         .and_then(|part| decode_part_index(part.index.as_ref()));
     let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index);
-    let hrd = HashReader::from_reader(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
-    let mut data = ChunkNativePutData::new(hrd);
+    let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
+    let mut data = PutObjReader::new(hrd);
 
     if let Err(err) = store
         .put_object(

crates/ecstore/src/disk/disk_store.rs

@@ -16,12 +16,17 @@ use crate::disk::{
     CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
     FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo,
     WalkDirOptions,
+    health_state::{
+        RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
+        get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
+        record_drive_recovery_class, record_drive_runtime_state, record_drive_state_transition,
+    },
     local::{LocalDisk, ScanGuard},
 };
 use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
+use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
-use rustfs_io_core::BoxChunkStream;
 use std::{
     path::PathBuf,
     sync::{
@@ -41,7 +46,6 @@ const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
 pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
 pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
@@ -52,9 +56,52 @@ lazy_static::lazy_static! {
 }
 
 pub fn get_max_timeout_duration() -> Duration {
-    std::env::var(ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION)
-        .map(|v| Duration::from_secs(v.parse::<u64>().unwrap_or(30)))
-        .unwrap_or(Duration::from_secs(30))
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION,
+        rustfs_config::DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS,
+    ))
+}
+
+fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
+    Duration::from_secs(
+        rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
+            .unwrap_or(default_secs),
+    )
+}
+
+pub fn get_drive_metadata_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_disk_info_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_list_dir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_stall_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+    )
 }
 
 /// DiskHealthTracker tracks the health status of a disk.
@@ -69,6 +116,16 @@ pub struct DiskHealthTracker {
     pub status: AtomicU32,
     /// Atomic number of waiting operations
     pub waiting: AtomicU32,
+    /// Runtime drive health state
+    pub runtime_state: AtomicU32,
+    /// Consecutive failures while transitioning away from online
+    pub consecutive_failures: AtomicU32,
+    /// Consecutive successes while returning online
+    pub consecutive_successes: AtomicU32,
+    /// When the drive first left the online state
+    pub offline_since_unix_secs: AtomicI64,
+    /// Last runtime state transition timestamp
+    pub last_transition_unix_secs: AtomicI64,
 }
 
 impl DiskHealthTracker {
@@ -84,6 +141,11 @@ impl DiskHealthTracker {
             last_started: AtomicI64::new(now),
             status: AtomicU32::new(DISK_HEALTH_OK),
             waiting: AtomicU32::new(0),
+            runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
+            consecutive_failures: AtomicU32::new(0),
+            consecutive_successes: AtomicU32::new(0),
+            offline_since_unix_secs: AtomicI64::new(0),
+            last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
         }
     }
 
@@ -111,12 +173,139 @@ impl DiskHealthTracker {
         self.status.store(DISK_HEALTH_OK, Ordering::Release);
     }
 
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.runtime_state.store(state as u32, Ordering::Release);
+        match state {
+            RuntimeDriveHealthState::Offline => self.set_faulty(),
+            RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => {
+                self.set_ok();
+            }
+        }
+    }
+
     pub fn swap_ok_to_faulty(&self) -> bool {
         self.status
             .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed)
             .is_ok()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire))
+    }
+
+    pub fn offline_duration(&self) -> Option<Duration> {
+        let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
+        if offline_since <= 0 {
+            return None;
+        }
+        let now = current_unix_secs();
+        Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
+    }
+
+    pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let now = current_unix_secs();
+        let next = match current {
+            RuntimeDriveHealthState::Online => {
+                self.consecutive_failures.store(1, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                self.offline_since_unix_secs
+                    .compare_exchange(0, now as i64, Ordering::AcqRel, Ordering::Relaxed)
+                    .ok();
+                RuntimeDriveHealthState::Suspect
+            }
+            RuntimeDriveHealthState::Suspect => {
+                let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
+                if failures >= get_drive_suspect_failure_threshold() {
+                    RuntimeDriveHealthState::Offline
+                } else {
+                    RuntimeDriveHealthState::Suspect
+                }
+            }
+            RuntimeDriveHealthState::Returning => {
+                self.consecutive_failures.store(0, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                RuntimeDriveHealthState::Offline
+            }
+            RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
+        };
+
+        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        self.transition_state(endpoint, current, next, reason);
+        current == RuntimeDriveHealthState::Online
+    }
+
+    pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let next = match current {
+            RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Suspect => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Offline => {
+                self.consecutive_successes.store(1, Ordering::Release);
+                RuntimeDriveHealthState::Returning
+            }
+            RuntimeDriveHealthState::Returning => {
+                let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1;
+                if successes >= get_drive_returning_success_threshold() {
+                    RuntimeDriveHealthState::Online
+                } else {
+                    RuntimeDriveHealthState::Returning
+                }
+            }
+        };
+
+        let became_online = next == RuntimeDriveHealthState::Online;
+        if became_online {
+            self.status.store(DISK_HEALTH_OK, Ordering::Release);
+            self.consecutive_failures.store(0, Ordering::Release);
+            self.consecutive_successes.store(0, Ordering::Release);
+        }
+        self.transition_state(endpoint, current, next, reason);
+        if became_online {
+            self.log_success();
+        }
+        became_online
+    }
+
+    fn transition_state(
+        &self,
+        endpoint: &Endpoint,
+        current: RuntimeDriveHealthState,
+        next: RuntimeDriveHealthState,
+        reason: &'static str,
+    ) {
+        if current == next {
+            return;
+        }
+
+        self.runtime_state.store(next as u32, Ordering::Release);
+        self.last_transition_unix_secs
+            .store(current_unix_secs() as i64, Ordering::Release);
+
+        if matches!(
+            next,
+            RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Offline | RuntimeDriveHealthState::Returning
+        ) && self.offline_since_unix_secs.load(Ordering::Acquire) == 0
+        {
+            self.offline_since_unix_secs
+                .store(current_unix_secs() as i64, Ordering::Release);
+        }
+
+        if next == RuntimeDriveHealthState::Online {
+            if let Some(duration) = self.offline_duration() {
+                record_drive_offline_duration(endpoint, duration);
+                record_drive_recovery_class(classify_drive_recovery(duration));
+            }
+            self.offline_since_unix_secs.store(0, Ordering::Release);
+        } else if let Some(duration) = self.offline_duration() {
+            record_drive_offline_duration(endpoint, duration);
+        }
+
+        record_drive_state_transition(endpoint, current, next, reason);
+        record_drive_runtime_state(endpoint, next);
+    }
+
     /// Increment waiting operations counter
     pub fn increment_waiting(&self) {
         self.waiting.fetch_add(1, Ordering::Relaxed);
@@ -138,6 +327,13 @@ impl DiskHealthTracker {
     }
 }
 
+fn current_unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_secs()
+}
+
 impl Default for DiskHealthTracker {
     fn default() -> Self {
         Self::new()
@@ -187,19 +383,34 @@ impl LocalDiskWrapper {
         let env_health_check =
             rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
 
-        Self {
+        let wrapper = Self {
             disk,
             health: Arc::new(DiskHealthTracker::new()),
             health_check: health_check && env_health_check,
             cancel_token: CancellationToken::new(),
             disk_id: Arc::new(RwLock::new(None)),
-        }
+        };
+        record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
+        wrapper
     }
 
     pub fn get_disk(&self) -> Arc<LocalDisk> {
         self.disk.clone()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        self.health.runtime_state()
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        self.health.offline_duration().map(|duration| duration.as_secs())
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.health.force_runtime_state_for_test(state);
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -220,6 +431,20 @@ impl LocalDiskWrapper {
         self.cancel_token.cancel();
     }
 
+    fn spawn_recovery_monitor_if_needed(&self) {
+        if !self.health_check {
+            return;
+        }
+
+        self.health.increment_waiting();
+        let health = Arc::clone(&self.health);
+        let disk = Arc::clone(&self.disk);
+        let cancel_token = self.cancel_token.clone();
+        tokio::spawn(async move {
+            Self::monitor_disk_status(disk, health, cancel_token).await;
+        });
+    }
+
     /// Monitor disk writability periodically
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         // TODO: config interval
@@ -257,7 +482,9 @@ impl LocalDiskWrapper {
 
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err() && health.swap_ok_to_faulty() {
+                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                        && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
+                    {
                         // Health check failed, disk is considered faulty
                         warn!("health check: failed, disk is considered faulty");
 
@@ -346,9 +573,9 @@ impl LocalDiskWrapper {
 
     /// Monitor disk status and try to bring it back online
     async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        const CHECK_EVERY: Duration = Duration::from_secs(5);
+        let check_every = get_drive_returning_probe_interval();
 
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(check_every);
 
         loop {
             tokio::select! {
@@ -363,12 +590,18 @@ impl LocalDiskWrapper {
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
                     match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
                         Ok(_) => {
+                            let state_before = health.runtime_state();
+                            let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
+                            info!("Disk {} recovery probe succeeded; state={:?}", disk.to_string(), state_before);
+                            if !is_online {
+                                continue;
+                            }
                             info!("Disk {} is back online", disk.to_string());
-                            health.set_ok();
                             health.decrement_waiting();
                             return;
                         }
                         Err(e) => {
+                            health.mark_failure(&disk.endpoint(), "recovery_probe_failed");
                             warn!("Disk {} still faulty: {:?}", disk.to_string(), e);
                         }
                     }
@@ -436,6 +669,19 @@ impl LocalDiskWrapper {
     /// Track disk health for an operation.
     /// This method should wrap disk operations to ensure health checking.
     pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        self.track_disk_health_with_op("unknown", operation, timeout_duration).await
+    }
+
+    pub async fn track_disk_health_with_op<T, F, Fut>(
+        &self,
+        op: &'static str,
+        operation: F,
+        timeout_duration: Duration,
+    ) -> Result<T>
     where
         F: FnOnce() -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
@@ -480,7 +726,21 @@ impl LocalDiskWrapper {
             Err(_) => {
                 // Timeout occurred, mark disk as potentially faulty and decrement waiting counter
                 self.health.decrement_waiting();
-                warn!("disk operation timeout after {:?}", timeout_duration);
+                if self.health.mark_failure(&self.endpoint(), "operation_timeout") {
+                    self.spawn_recovery_monitor_if_needed();
+                }
+                counter!(
+                    "rustfs_drive_op_timeout_total",
+                    "endpoint" => self.endpoint().to_string(),
+                    "op" => op.to_string()
+                )
+                .increment(1);
+                warn!(
+                    endpoint = %self.endpoint(),
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    "Local disk operation timed out"
+                );
                 Err(DiskError::other(format!("disk operation timeout after {timeout_duration:?}")))
             }
         }
@@ -490,8 +750,12 @@ impl LocalDiskWrapper {
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        self.track_disk_health(|| async { self.disk.read_metadata(volume, path).await }, Duration::ZERO)
-            .await
+        self.track_disk_health_with_op(
+            "read_metadata",
+            || async { self.disk.read_metadata(volume, path).await },
+            get_drive_metadata_timeout(),
+        )
+        .await
     }
 
     fn start_scan(&self) -> ScanGuard {
@@ -566,15 +830,22 @@ impl DiskAPI for LocalDiskWrapper {
             return Err(DiskError::FaultyDisk);
         }
 
-        let result = self.disk.disk_info(opts).await?;
+        self.track_disk_health_with_op(
+            "disk_info",
+            || async {
+                let result = self.disk.disk_info(opts).await?;
 
-        if let Some(current_disk_id) = *self.disk_id.read().await
-            && Some(current_disk_id) != result.id
-        {
-            return Err(DiskError::DiskNotFound);
-        };
+                if let Some(current_disk_id) = *self.disk_id.read().await
+                    && Some(current_disk_id) != result.id
+                {
+                    return Err(DiskError::DiskNotFound);
+                };
 
-        Ok(result)
+                Ok(result)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
     }
 
     async fn make_volume(&self, volume: &str) -> Result<()> {
@@ -588,7 +859,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
-        self.track_disk_health(|| async { self.disk.list_volumes().await }, Duration::ZERO)
+        self.track_disk_health_with_op("list_volumes", || async { self.disk.list_volumes().await }, Duration::ZERO)
             .await
     }
 
@@ -603,7 +874,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
-        self.track_disk_health(|| async { self.disk.walk_dir(opts, wr).await }, Duration::ZERO)
+        self.track_disk_health_with_op("walk_dir", || async { self.disk.walk_dir(opts, wr).await }, get_drive_walkdir_timeout())
             .await
     }
 
@@ -711,9 +982,10 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
-        self.track_disk_health(
+        self.track_disk_health_with_op(
+            "list_dir",
             || async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
-            get_max_timeout_duration(),
+            get_drive_list_dir_timeout(),
         )
         .await
     }
@@ -739,14 +1011,6 @@ impl DiskAPI for LocalDiskWrapper {
         .await
     }
 
-    async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream> {
-        self.track_disk_health(
-            || async { self.disk.read_file_chunks(volume, path, offset, length).await },
-            get_max_timeout_duration(),
-        )
-        .await
-    }
-
     async fn append_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileWriter> {
         self.track_disk_health(|| async { self.disk.append_file(volume, path).await }, Duration::ZERO)
             .await
@@ -811,3 +1075,74 @@ impl DiskAPI for LocalDiskWrapper {
             .await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::endpoint::Endpoint;
+    use crate::disk::health_state::RuntimeDriveHealthState;
+
+    #[test]
+    fn drive_metadata_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
+                assert_eq!(
+                    get_drive_metadata_timeout(),
+                    Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(17));
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_prefers_canonical_over_legacy() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("7"), || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(7));
+            });
+        });
+    }
+
+    #[test]
+    fn runtime_state_transitions_from_online_to_suspect_then_offline() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+
+        assert!(!health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+        assert!(health.offline_duration().is_some());
+    }
+
+    #[test]
+    fn runtime_state_transitions_back_online_after_recovery_threshold() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        health.mark_failure(&endpoint, "timeout");
+        health.mark_failure(&endpoint, "timeout");
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.offline_duration().is_none());
+    }
+}

crates/ecstore/src/disk/error.rs

@@ -724,10 +724,7 @@ mod tests {
         let path = PathBuf::from("/test/path");
         let io_error = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "permission denied");
 
-        let context_error = FileAccessDeniedWithContext {
-            path: path.clone(),
-            source: io_error,
-        };
+        let context_error = FileAccessDeniedWithContext { path, source: io_error };
 
         let display_str = format!("{context_error}");
         assert!(display_str.contains("/test/path"));

crates/ecstore/src/disk/error_reduce.rs

@@ -67,7 +67,7 @@ pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize,
     let (best_err, best_count) = err_counts
         .into_iter()
         .max_by(|(_, c1), (_, c2)| c1.cmp(c2))
-        .unwrap_or((nil_error.clone(), 0));
+        .unwrap_or((nil_error, 0));
 
     // Compare nil errors with the top non-nil error and prefer the nil error
     if nil_count > best_count || (nil_count == best_count && nil_count > 0) {
@@ -112,7 +112,7 @@ mod tests {
     fn test_reduce_errs_basic() {
         let e1 = err_io("a");
         let e2 = err_io("b");
-        let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
+        let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2), None];
         let ignored = vec![];
         let (count, err) = reduce_errs(&errors, &ignored);
         assert_eq!(count, 2);
@@ -124,7 +124,7 @@ mod tests {
         let e1 = err_io("a");
         let e2 = err_io("b");
         let errors = vec![Some(e1.clone()), Some(e2.clone()), Some(e1.clone()), Some(e2.clone()), None];
-        let ignored = vec![e2.clone()];
+        let ignored = vec![e2];
         let (count, err) = reduce_errs(&errors, &ignored);
         assert_eq!(count, 2);
         assert_eq!(err, Some(e1));
@@ -134,7 +134,7 @@ mod tests {
     fn test_reduce_quorum_errs() {
         let e1 = err_io("a");
         let e2 = err_io("b");
-        let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2.clone()), None];
+        let errors = vec![Some(e1.clone()), Some(e1.clone()), Some(e2), None];
         let ignored = vec![];
         let quorum_err = Error::FaultyDisk;
         // quorum = 2, should return e1
@@ -167,7 +167,7 @@ mod tests {
     fn test_reduce_errs_nil_tiebreak() {
         // Error::Nil and another error have the same count, should prefer Nil
         let e1 = err_io("a");
-        let errors = vec![Some(e1.clone()), None, Some(e1.clone()), None]; // e1:2, Nil:2
+        let errors = vec![Some(e1.clone()), None, Some(e1), None]; // e1:2, Nil:2
         let ignored = vec![];
         let (count, err) = reduce_errs(&errors, &ignored);
         assert_eq!(count, 2);

crates/ecstore/src/disk/health_state.rs

@@ -0,0 +1,237 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{DiskAPI, DiskStore};
+use crate::disk::endpoint::Endpoint;
+use metrics::{counter, gauge};
+use std::time::Duration;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u32)]
+pub enum RuntimeDriveHealthState {
+    Online = 0,
+    Suspect = 1,
+    Offline = 2,
+    Returning = 3,
+}
+
+impl RuntimeDriveHealthState {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Online => "online",
+            Self::Suspect => "suspect",
+            Self::Offline => "offline",
+            Self::Returning => "returning",
+        }
+    }
+
+    pub fn from_u32(value: u32) -> Self {
+        match value {
+            1 => Self::Suspect,
+            2 => Self::Offline,
+            3 => Self::Returning,
+            _ => Self::Online,
+        }
+    }
+
+    pub fn is_snapshot_eligible(self) -> bool {
+        matches!(self, Self::Online | Self::Suspect | Self::Returning)
+    }
+
+    pub fn is_strictly_online(self) -> bool {
+        matches!(self, Self::Online)
+    }
+
+    pub fn should_probe_for_admin(self) -> bool {
+        matches!(self, Self::Online | Self::Returning)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DriveRecoveryClass {
+    ShortOffline,
+    MediumOffline,
+    LongOffline,
+}
+
+impl DriveRecoveryClass {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::ShortOffline => "short_offline",
+            Self::MediumOffline => "medium_offline",
+            Self::LongOffline => "long_offline",
+        }
+    }
+}
+
+pub fn get_drive_suspect_failure_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_success_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_probe_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_offline_grace_period() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+        rustfs_config::DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+    ))
+}
+
+pub fn get_drive_long_offline_threshold() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+        rustfs_config::DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+    ))
+}
+
+pub fn classify_drive_recovery(duration: Duration) -> DriveRecoveryClass {
+    if duration <= get_drive_offline_grace_period() {
+        DriveRecoveryClass::ShortOffline
+    } else if duration >= get_drive_long_offline_threshold() {
+        DriveRecoveryClass::LongOffline
+    } else {
+        DriveRecoveryClass::MediumOffline
+    }
+}
+
+pub fn record_drive_runtime_state(endpoint: &Endpoint, state: RuntimeDriveHealthState) {
+    let endpoint_label = endpoint.to_string();
+    let pool_label = endpoint.pool_idx.to_string();
+    let set_label = endpoint.set_idx.to_string();
+    let disk_label = endpoint.disk_idx.to_string();
+
+    for candidate in [
+        RuntimeDriveHealthState::Online,
+        RuntimeDriveHealthState::Suspect,
+        RuntimeDriveHealthState::Offline,
+        RuntimeDriveHealthState::Returning,
+    ] {
+        gauge!(
+            "rustfs_drive_runtime_state",
+            "endpoint" => endpoint_label.clone(),
+            "pool" => pool_label.clone(),
+            "set" => set_label.clone(),
+            "disk" => disk_label.clone(),
+            "state" => candidate.as_str().to_string()
+        )
+        .set(if candidate == state { 1.0 } else { 0.0 });
+    }
+}
+
+pub fn record_drive_state_transition(
+    endpoint: &Endpoint,
+    from: RuntimeDriveHealthState,
+    to: RuntimeDriveHealthState,
+    reason: &'static str,
+) {
+    counter!(
+        "rustfs_drive_state_transition_total",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string(),
+        "from" => from.as_str().to_string(),
+        "to" => to.as_str().to_string(),
+        "reason" => reason.to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_recovery_class(class: DriveRecoveryClass) {
+    counter!(
+        "rustfs_drive_recovery_class_total",
+        "class" => class.as_str().to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_offline_duration(endpoint: &Endpoint, duration: Duration) {
+    gauge!(
+        "rustfs_drive_offline_duration_seconds",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string()
+    )
+    .set(duration.as_secs_f64());
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DriveMembershipSnapshot {
+    pub online: Vec<DiskStore>,
+    pub suspect: Vec<DiskStore>,
+    pub returning: Vec<DiskStore>,
+    pub offline: Vec<DiskStore>,
+}
+
+impl DriveMembershipSnapshot {
+    pub fn from_optional_disks(disks: &[Option<DiskStore>]) -> Self {
+        let mut snapshot = Self::default();
+
+        for disk in disks.iter().flatten() {
+            match disk.runtime_state() {
+                RuntimeDriveHealthState::Online => snapshot.online.push(disk.clone()),
+                RuntimeDriveHealthState::Suspect => snapshot.suspect.push(disk.clone()),
+                RuntimeDriveHealthState::Returning => snapshot.returning.push(disk.clone()),
+                RuntimeDriveHealthState::Offline => snapshot.offline.push(disk.clone()),
+            }
+        }
+
+        snapshot
+    }
+
+    pub fn scanner_heal_candidates(&self) -> Vec<DiskStore> {
+        let mut disks = Vec::with_capacity(self.online.len() + self.suspect.len() + self.returning.len());
+        disks.extend(self.online.iter().cloned());
+        disks.extend(self.suspect.iter().cloned());
+        disks.extend(self.returning.iter().cloned());
+        disks
+    }
+
+    pub fn strict_online_candidates(&self) -> Vec<DiskStore> {
+        self.online.clone()
+    }
+
+    pub fn strict_online_local_candidates(&self) -> Vec<DiskStore> {
+        self.online.iter().filter(|disk| disk.is_local()).cloned().collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn runtime_drive_health_state_snapshot_eligibility_matches_membership_policy() {
+        assert!(RuntimeDriveHealthState::Online.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Suspect.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Returning.is_snapshot_eligible());
+        assert!(!RuntimeDriveHealthState::Offline.is_snapshot_eligible());
+    }
+}

crates/ecstore/src/disk/mod.rs

@@ -19,6 +19,7 @@ pub mod error_conv;
 pub mod error_reduce;
 pub mod format;
 pub mod fs;
+pub mod health_state;
 pub mod local;
 pub mod os;
 
@@ -33,6 +34,7 @@ pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
 use crate::disk::disk_store::LocalDiskWrapper;
+use crate::disk::health_state::RuntimeDriveHealthState;
 use crate::disk::local::ScanGuard;
 use crate::rpc::RemoteDisk;
 use bytes::Bytes;
@@ -41,7 +43,6 @@ use error::DiskError;
 use error::{Error, Result};
 use local::LocalDisk;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
-use rustfs_io_core::BoxChunkStream;
 use rustfs_madmin::info_commands::DiskMetrics;
 use serde::{Deserialize, Serialize};
 use std::{fmt::Debug, path::PathBuf, sync::Arc};
@@ -296,14 +297,6 @@ impl DiskAPI for Disk {
         }
     }
 
-    #[tracing::instrument(skip(self))]
-    async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream> {
-        match self {
-            Disk::Local(local_disk) => local_disk.read_file_chunks(volume, path, offset, length).await,
-            Disk::Remote(remote_disk) => remote_disk.read_file_chunks(volume, path, offset, length).await,
-        }
-    }
-
     #[tracing::instrument(skip(self))]
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
         match self {
@@ -419,6 +412,30 @@ impl DiskAPI for Disk {
     }
 }
 
+impl Disk {
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        match self {
+            Disk::Local(local_disk) => local_disk.runtime_state(),
+            Disk::Remote(remote_disk) => remote_disk.runtime_state(),
+        }
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        match self {
+            Disk::Local(local_disk) => local_disk.offline_duration_secs(),
+            Disk::Remote(remote_disk) => remote_disk.offline_duration_secs(),
+        }
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        match self {
+            Disk::Local(local_disk) => local_disk.force_runtime_state_for_test(state),
+            Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
+        }
+    }
+}
+
 impl Disk {
     /// Enable health monitoring on this disk.
     /// Called after startup format loading completes so that remote peers
@@ -514,9 +531,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     /// On other platforms, falls back to efficient read operations.
     async fn read_file_zero_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Bytes>;
 
-    /// Chunk-based file read compatibility layer for the zero-copy data plane.
-    async fn read_file_chunks(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<BoxChunkStream>;
-
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
     async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<FileWriter>;
     // ReadFileStream

crates/ecstore/src/endpoints.rs

@@ -554,7 +554,7 @@ impl EndpointServerPools {
 
         for pool in self.0.iter() {
             for ep in pool.endpoints.as_ref() {
-                let n = node_map.entry(ep.host_port()).or_insert(Node {
+                let n = node_map.entry(ep.host_port()).or_insert_with(|| Node {
                     url: ep.url.clone(),
                     pools: vec![],
                     is_local: ep.is_local,

crates/ecstore/src/erasure_coding/bitrot.rs

@@ -172,52 +172,6 @@ where
     }
 }
 
-impl BitrotWriter<CustomWriter> {
-    fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        if buf.is_empty() {
-            return Ok(0);
-        }
-
-        if self.finished {
-            return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished"));
-        }
-
-        if buf.len() > self.shard_size {
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::InvalidInput,
-                format!("data size {} exceeds shard size {}", buf.len(), self.shard_size),
-            ));
-        }
-
-        if buf.len() < self.shard_size {
-            self.finished = true;
-        }
-
-        match &mut self.inner {
-            CustomWriter::InlineBuffer(data) => {
-                if self.hash_algo.size() > 0 {
-                    let hash = self.hash_algo.hash_encode(buf);
-                    if hash.as_ref().is_empty() {
-                        error!("bitrot writer write hash error: hash is empty");
-                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty"));
-                    }
-                    data.extend_from_slice(hash.as_ref());
-                }
-                data.extend_from_slice(buf);
-                Ok(buf.len())
-            }
-            CustomWriter::Other(_) => Err(std::io::Error::other("inline sync write requires inline buffer writer")),
-        }
-    }
-
-    fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
-        match self.inner {
-            CustomWriter::InlineBuffer(_) => Ok(()),
-            CustomWriter::Other(_) => Err(std::io::Error::other("inline sync shutdown requires inline buffer writer")),
-        }
-    }
-}
-
 async fn write_all_vectored<W>(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()>
 where
     W: AsyncWrite + Unpin,
@@ -326,10 +280,6 @@ impl CustomWriter {
             Self::Other(_) => None,
         }
     }
-
-    pub fn is_inline_buffer(&self) -> bool {
-        matches!(self, Self::InlineBuffer(_))
-    }
 }
 
 impl AsyncWrite for CustomWriter {
@@ -447,24 +397,6 @@ impl BitrotWriterWrapper {
         self.bitrot_writer.shutdown().await
     }
 
-    pub fn is_inline_buffer(&self) -> bool {
-        matches!(self.writer_type, WriterType::InlineBuffer)
-    }
-
-    pub fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        if !self.is_inline_buffer() {
-            return Err(std::io::Error::other("inline sync write requires inline buffer writer"));
-        }
-        self.bitrot_writer.write_inline_sync(buf)
-    }
-
-    pub fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
-        if !self.is_inline_buffer() {
-            return Err(std::io::Error::other("inline sync shutdown requires inline buffer writer"));
-        }
-        self.bitrot_writer.shutdown_inline_sync()
-    }
-
     /// Extract the inline buffer data, consuming the wrapper
     pub fn into_inline_data(self) -> Option<Vec<u8>> {
         match self.writer_type {

crates/ecstore/src/erasure_coding/decode.rs

@@ -17,7 +17,6 @@ use crate::disk::error_reduce::reduce_errs;
 use crate::erasure_coding::{BitrotReader, Erasure};
 use futures::stream::{FuturesUnordered, StreamExt};
 use pin_project_lite::pin_project;
-use rustfs_io_core::{IoChunk, PooledChunk};
 use std::io;
 use std::io::ErrorKind;
 use tokio::io::AsyncRead;
@@ -156,68 +155,6 @@ fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
     size
 }
 
-fn block_window(
-    offset: usize,
-    length: usize,
-    block_size: usize,
-    block_index: usize,
-    start_block: usize,
-    end_block: usize,
-) -> (usize, usize) {
-    let end_remainder = offset.saturating_add(length) % block_size;
-    if start_block == end_block {
-        (offset % block_size, length)
-    } else if block_index == start_block {
-        (offset % block_size, block_size - (offset % block_size))
-    } else if block_index == end_block {
-        (0, if end_remainder == 0 { block_size } else { end_remainder })
-    } else {
-        (0, block_size)
-    }
-}
-
-fn take_data_blocks_as_chunks(
-    shards: &mut [Option<Vec<u8>>],
-    data_blocks: usize,
-    mut offset: usize,
-    length: usize,
-) -> io::Result<Vec<IoChunk>> {
-    if get_data_block_len(shards, data_blocks) < length {
-        error!("take_data_blocks_as_chunks get_data_block_len < length");
-        return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
-    }
-
-    let mut chunks = Vec::new();
-    let mut remaining = length;
-    for block_op in shards.iter_mut().take(data_blocks) {
-        let Some(block) = block_op.take() else {
-            error!("take_data_blocks_as_chunks block_op.is_none()");
-            return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block"));
-        };
-
-        if offset >= block.len() {
-            offset -= block.len();
-            continue;
-        }
-
-        let start = offset;
-        offset = 0;
-        let take = (block.len() - start).min(remaining);
-        let chunk = if start == 0 && take == block.len() {
-            IoChunk::Pooled(PooledChunk::from_vec(block))
-        } else {
-            IoChunk::Pooled(PooledChunk::from_vec(block).slice(start, take)?)
-        };
-        chunks.push(chunk);
-        remaining -= take;
-        if remaining == 0 {
-            break;
-        }
-    }
-
-    Ok(chunks)
-}
-
 /// Write data blocks from encoded blocks to target, supporting offset and length
 async fn write_data_blocks<W>(
     writer: &mut W,
@@ -276,134 +213,6 @@ where
     Ok(total_written)
 }
 
-pub(crate) struct ErasureChunkDecoder<R> {
-    erasure: Erasure,
-    reader: ParallelReader<R>,
-    offset: usize,
-    length: usize,
-    start_block: usize,
-    end_block: usize,
-    current_block: usize,
-    written: usize,
-    healable_error: Option<Error>,
-    finished: bool,
-}
-
-impl<R> ErasureChunkDecoder<R>
-where
-    R: AsyncRead + Unpin + Send + Sync,
-{
-    pub(crate) fn new(
-        erasure: Erasure,
-        readers: Vec<Option<BitrotReader<R>>>,
-        offset: usize,
-        length: usize,
-        total_length: usize,
-    ) -> io::Result<Self> {
-        if readers.len() != erasure.data_shards + erasure.parity_shards {
-            return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
-        }
-
-        let end_offset = offset
-            .checked_add(length)
-            .ok_or_else(|| io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))?;
-        if end_offset > total_length {
-            return Err(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"));
-        }
-
-        let start_block = offset / erasure.block_size;
-        let end_block = if length == 0 {
-            start_block
-        } else {
-            end_offset.saturating_sub(1) / erasure.block_size
-        };
-        let reader = ParallelReader::new(readers, erasure.clone(), offset, total_length);
-
-        Ok(Self {
-            erasure,
-            reader,
-            offset,
-            length,
-            start_block,
-            end_block,
-            current_block: start_block,
-            written: 0,
-            healable_error: None,
-            finished: length == 0,
-        })
-    }
-
-    pub(crate) async fn next_chunks(&mut self) -> io::Result<Option<Vec<IoChunk>>> {
-        if self.finished {
-            return Ok(None);
-        }
-
-        if self.current_block > self.end_block {
-            self.finished = true;
-            return Ok(None);
-        }
-
-        let block_index = self.current_block;
-        self.current_block += 1;
-
-        let (block_offset, block_length) = block_window(
-            self.offset,
-            self.length,
-            self.erasure.block_size,
-            block_index,
-            self.start_block,
-            self.end_block,
-        );
-        if block_length == 0 {
-            self.finished = true;
-            return Ok(None);
-        }
-
-        let (mut shards, errs) = self.reader.read().await;
-
-        if self.healable_error.is_none()
-            && let (_, Some(err)) = reduce_errs(&errs, &[])
-            && (err == Error::FileNotFound || err == Error::FileCorrupt)
-        {
-            self.healable_error = Some(err);
-        }
-
-        if !self.reader.can_decode(&shards) {
-            self.finished = true;
-            error!("reconstructed chunk decoder can_decode errs: {:?}", &errs);
-            return Err(Error::ErasureReadQuorum.into());
-        }
-
-        if let Err(err) = self.erasure.decode_data(&mut shards) {
-            self.finished = true;
-            error!("reconstructed chunk decoder decode_data err: {:?}", err);
-            return Err(err);
-        }
-
-        let chunks = take_data_blocks_as_chunks(&mut shards, self.erasure.data_shards, block_offset, block_length)?;
-        self.written += chunks.iter().map(IoChunk::len).sum::<usize>();
-        Ok(Some(chunks))
-    }
-
-    pub(crate) fn written(&self) -> usize {
-        self.written
-    }
-
-    pub(crate) fn finish_error(&self) -> Option<io::Error> {
-        if self.written < self.length {
-            Some(Error::LessData.into())
-        } else {
-            None
-        }
-    }
-
-    pub(crate) fn take_healable_error(&mut self) -> Option<Error> {
-        self.healable_error.take()
-    }
-}
-
-pub(crate) type ReconstructedChunkDecoder<R> = ErasureChunkDecoder<R>;
-
 impl Erasure {
     pub async fn decode<W, R>(
         &self,
@@ -511,7 +320,6 @@ mod tests {
         disk::error::DiskError,
         erasure_coding::{BitrotReader, BitrotWriter},
     };
-    use bytes::Bytes;
     use rustfs_utils::HashAlgorithm;
     use std::io::Cursor;
 
@@ -652,47 +460,4 @@ mod tests {
         let reader_cursor = Cursor::new(buf);
         BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false)
     }
-
-    async fn create_bitrot_reader_from_shard(
-        shard: Bytes,
-        shard_size: usize,
-        hash_algo: &HashAlgorithm,
-    ) -> BitrotReader<Cursor<Vec<u8>>> {
-        let writer = Cursor::new(Vec::new());
-        let mut writer = BitrotWriter::new(writer, shard_size, hash_algo.clone());
-        writer.write(shard.as_ref()).await.unwrap();
-        let reader_cursor = Cursor::new(writer.into_inner().into_inner());
-        BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false)
-    }
-
-    #[tokio::test]
-    async fn test_erasure_chunk_decoder_reconstructs_missing_data_shard_as_pooled_chunks() {
-        let erasure = Erasure::new(2, 1, 4);
-        let original = b"abcd";
-        let encoded = erasure.encode_data(original).unwrap();
-        let shard_size = erasure.shard_size();
-        let hash_algo = HashAlgorithm::None;
-
-        let readers = vec![
-            None,
-            Some(create_bitrot_reader_from_shard(encoded[1].clone(), shard_size, &hash_algo).await),
-            Some(create_bitrot_reader_from_shard(encoded[2].clone(), shard_size, &hash_algo).await),
-        ];
-
-        let mut decoder = ErasureChunkDecoder::new(erasure, readers, 0, original.len(), original.len()).unwrap();
-        let first_batch = decoder.next_chunks().await.unwrap().unwrap();
-        assert!(
-            first_batch.iter().all(|chunk| matches!(chunk, IoChunk::Pooled(_))),
-            "reconstructed decoder should produce pooled chunks"
-        );
-        let collected = first_batch
-            .into_iter()
-            .flat_map(|chunk| chunk.as_bytes().to_vec())
-            .collect::<Vec<_>>();
-
-        assert_eq!(collected, original);
-        assert!(decoder.next_chunks().await.unwrap().is_none());
-        assert_eq!(decoder.written(), original.len());
-        assert!(decoder.finish_error().is_none());
-    }
 }