test: cover S3 error XML body compatibility (#2606)

Co-authored-by: houseme <housemecn@gmail.com>

.docker/observability/grafana/dashboards/rustfs.json

@@ -2641,7 +2641,7 @@
       },
       "gridPos": {
         "h": 7,
-        "w": 24,
+        "w": 12,
         "x": 0,
         "y": 73
       },
@@ -2670,7 +2670,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_network_host_network_io{job=~\"$job\", direction=\"received\"}[5m]))",
           "legendFormat": "RX - {{job}}",
           "range": true,
           "refId": "C"
@@ -2681,7 +2681,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (job) (rate(rustfs_system_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "expr": "sum by (job) (rate(rustfs_system_network_host_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))",
           "legendFormat": "TX - {{job}}",
           "range": true,
           "refId": "D"
@@ -2690,6 +2690,115 @@
       "title": "Network",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 73
+      },
+      "id": 500,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job, interface) (rate(rustfs_system_network_host_network_io_per_interface{job=~\"$job\", direction=\"received\"}[5m]))",
+          "legendFormat": "RX {{interface}} - {{job}}",
+          "range": true,
+          "refId": "E"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (job, interface) (rate(rustfs_system_network_host_network_io_per_interface{job=~\"$job\", direction=\"transmitted\"}[5m]))",
+          "legendFormat": "TX {{interface}} - {{job}}",
+          "range": true,
+          "refId": "F"
+        }
+      ],
+      "title": "Network by Interface",
+      "type": "timeseries"
+    },
     {
       "datasource": {
         "type": "prometheus",

AGENTS.md

@@ -6,24 +6,39 @@ Use the nearest subdirectory `AGENTS.md` for path-specific guidance.
 ## Rule Precedence
 
 1. System/developer instructions.
-2. This file (global defaults).
-3. The nearest `AGENTS.md` in the current path (more specific scope wins).
+2. Current user/task instructions.
+3. The nearest `AGENTS.md` in the current path.
+4. This file (global defaults).
 
 If repo-level instructions conflict, follow the nearest file and keep behavior aligned with CI.
 
+## Execution Discipline
+
+- Read the relevant existing code, tests, and local guidance before changing behavior.
+- State assumptions when they affect the implementation or verification path.
+- If a task has multiple plausible interpretations, list the options briefly and choose the narrowest reasonable path; ask when the ambiguity would make the change risky.
+- For multi-step work, keep the plan minimal and tied to verifiable outcomes.
+- Avoid redundant file reads, repeated commands, and unnecessary exploratory work once enough context is available.
+- A good result is a minimal diff with clear assumptions, no over-engineering, and independent verification.
+
 ## Communication and Language
 
 - Respond in the same language used by the requester.
 - Keep source code, comments, commit messages, and PR title/body in English.
+- Be concise. Avoid sycophantic openers, closing fluff, and verbose status reporting.
 
 ## Change Style for Existing Logic
 
 - Prefer direct, local code over extracting one-off helpers.
 - Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow.
+- Solve only the requested problem; do not add speculative features, configurability, or adjacent improvements.
+- Prefer editing existing code over rewriting files or reshaping unrelated logic.
+- Modify only what is required and remove only artifacts introduced by your own changes.
 - Preserve the existing control-flow and logic shape when fixing bugs or addressing review comments, especially in init, distributed coordination, locking, metadata, and concurrency paths.
 - Do not refactor existing code only to make it easier to unit test.
 - Keep fixes narrowly aligned with the requested behavior; avoid semantic-adjacent rewrites while touching sensitive paths.
 - Keep code elegant, concise, and direct. Prefer minimal, readable implementations over over-engineering and excessive abstraction. Use comments to clarify non-obvious intent and invariants, not to compensate for unclear code.
+- Mention unrelated issues when useful, but do not fix them as part of a narrow task.
 
 ## Constant and String Usage
 
@@ -44,6 +59,8 @@ Reference the source files above instead.
 
 ## Verification Before PR
 
+Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion.
+
 For code changes, run and pass the following before opening a PR:
 
 ```bash
@@ -69,6 +86,7 @@ Do not open a PR with code changes when the required checks fail.
 - Use `N/A` for non-applicable template sections.
 - Include verification commands in the PR description.
 - When using `gh pr create`/`gh pr edit`, use `--body-file` instead of inline `--body` for multiline markdown.
+- Do not include the literal sequence `\n` in any GitHub issue, pull request, or discussion comment.
 - After fixing code review comments or CI findings, always mark corresponding review
   comments/threads as resolved before returning to the user.
 - In handling review comments, confirm the underlying issue before changing code.

ARCHITECTURE.md

@@ -86,7 +86,7 @@ Crates are organized in a dependency DAG with 9 depth levels (0 = leaf, 8 = top)
 ```
 Depth 0 — LEAF (no internal deps):
   appauth, checksums, config, credentials, crypto, io-metrics,
-  madmin, mcp, s3-common, workers, zip
+  madmin, s3-common, workers, zip
 
 Depth 1:
   io-core (→ io-metrics)
@@ -195,7 +195,6 @@ Depth 8 — TOP:
 | `trusted-proxies` | 4.0K | Trusted proxy / IP forwarding |
 | `zip` | 986 | ZIP archive support for bulk downloads |
 | `workers` | 136 | Simple worker abstraction |
-| `mcp` | 2.0K | Model Context Protocol server (AI tooling) |
 
 ## Architecture Invariants
 
@@ -224,7 +223,7 @@ Depth 8 — TOP:
 6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
    not bare `Error`).
    - ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
-     `mcp` and `heal` use `anyhow` in library code.
+      `heal` use `anyhow` in library code.
 
 ## Known Structural Issues
 
@@ -285,9 +284,9 @@ anyhow::Result<T>             // in library code (OK in tests/CLI)
 
 ### Metrics
 
-- Prometheus-style metrics via `rustfs-metrics` crate
+- Prometheus-style metrics via `rustfs-obs` runtime and schema
 - I/O-specific counters via `rustfs-io-metrics`
-- Registration happens at crate level, collection in `metrics` crate
+- Registration happens at crate level, collection/reporting in `rustfs-obs`
 
 ### Testing
 
@@ -369,7 +368,7 @@ The binary (`main.rs`) boots in this order:
   Add handler in `admin/handlers/`, register in `admin/router.rs`
 
 - **"Where do I add a new metric?"**
-  Define in `crates/metrics/`, register collector, expose via `/minio/v2/metrics`
+  Define descriptor/collector in `crates/obs/src/metrics/`, expose via `/minio/v2/metrics`
 
 ---
 

Cargo.toml

@@ -31,8 +31,6 @@ members = [
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
     "crates/madmin", # Management dashboard and admin API interface
-    "crates/mcp", # MCP server for S3 operations
-    "crates/metrics", # Metrics collection and reporting
     "crates/notify", # Notification system for events
     "crates/obs", # Observability utilities
     "crates/object-capacity", # Capacity scan and refresh core
@@ -95,8 +93,6 @@ rustfs-keystone = { path = "crates/keystone", version = "0.0.5" }
 rustfs-kms = { path = "crates/kms", version = "0.0.5" }
 rustfs-lock = { path = "crates/lock", version = "0.0.5" }
 rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
-rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
-rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
 rustfs-notify = { path = "crates/notify", version = "0.0.5" }
 rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
 rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
@@ -104,6 +100,7 @@ rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
 rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
 rustfs-protos = { path = "crates/protos", version = "0.0.5" }
+rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 rustfs-rio = { path = "crates/rio", version = "0.0.5" }
 rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" }
 rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
@@ -115,7 +112,6 @@ rustfs-targets = { path = "crates/targets", version = "0.0.5" }
 rustfs-utils = { path = "crates/utils", version = "0.0.5" }
 rustfs-workers = { path = "crates/workers", version = "0.0.5" }
 rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
-rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
@@ -135,7 +131,7 @@ http-body = "1.0.1"
 http-body-util = "0.1.3"
 reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 socket2 = { version = "0.6.3", features = ["all"] }
-tokio = { version = "1.52.0", features = ["fs", "rt-multi-thread", "io-uring"] }
+tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
 tokio-stream = { version = "0.1.18" }
 tokio-test = "0.4.5"
@@ -154,18 +150,16 @@ flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
 quick-xml = "0.39.2"
-rmcp = { version = "1.4.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
-schemars = "1.2.1"
 
 # Cryptography and Security
 aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
 argon2 = { version = "0.6.0-rc.8" }
-blake2 = "0.11.0-rc.5"
+blake2 = "0.11.0-rc.6"
 chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
 hmac = { version = "0.13.0" }
@@ -202,7 +196,7 @@ base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.6.0", features = ["derive", "env"] }
+clap = { version = "4.6.1", features = ["derive", "env"] }
 const-str = { version = "1.1.0", features = ["std", "proc"] }
 convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
@@ -210,7 +204,7 @@ crossbeam-queue = "0.3.12"
 crossbeam-channel = "0.5.15"
 crossbeam-deque = "0.8.6"
 crossbeam-utils = "0.8.21"
-datafusion = "53.0.0"
+datafusion = "53.1.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
@@ -250,11 +244,11 @@ rayon = "1.12.0"
 reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.12.3" }
-rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] }
+rumqttc = { package = "rumqttc-next", version = "0.30.0", features = ["websocket"] }
 rustix = { version = "1.1.4", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.2" }
-s3s = { git = "https://github.com/rustfs/s3s", rev = "0dd8fcaaa72eda68fafd49c38daea43bb8697558", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s", rev = "a3b16608df35aaeed8fff08b4988d03f4ca9445b", features = ["minio"] }
 serial_test = "3.4.0"
 shadow-rs = { version = "1.7.1", default-features = false }
 siphasher = "1.0.2"
@@ -270,14 +264,14 @@ tempfile = "3.27.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
-tracing-appender = "0.2.4"
+tracing-appender = "0.2.5"
 tracing-error = "0.2.1"
 tracing-opentelemetry = "0.32.1"
 tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.23.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 vaultrs = { version = "0.8.0" }
 walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
@@ -288,7 +282,7 @@ zstd = "0.13.3"
 
 # Observability and Metrics
 metrics = "0.24.3"
-dial9-tokio-telemetry = "0.2"
+dial9-tokio-telemetry = "0.3"
 opentelemetry = { version = "0.31.0" }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes", "spec_unstable_logs_enabled"] }
 opentelemetry-otlp = { version = "0.31.1", features = ["gzip-http", "reqwest-rustls"] }
@@ -320,7 +314,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
 pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] }
 
 [workspace.metadata.cargo-shear]
-ignored = ["rustfs", "rustfs-mcp"]
+ignored = ["rustfs"]
 
 [profile.release]
 opt-level = 3

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM alpine:3.23 AS build
+FROM alpine:3.23.4 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
@@ -54,7 +54,7 @@ RUN set -eux; \
     rm -rf rustfs.zip /build/.tmp || true
 
 
-FROM alpine:3.23
+FROM alpine:3.23.4
 
 ARG RELEASE=latest
 ARG BUILD_DATE
@@ -73,7 +73,7 @@ LABEL name="RustFS" \
       license="Apache-2.0"
 
 RUN apk update && \
-    apk add --no-cache ca-certificates coreutils curl "zlib>=1.3.2-r0"
+    apk add --no-cache ca-certificates coreutils curl
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs

crates/audit/src/factory.rs

@@ -73,7 +73,9 @@ impl TargetFactory for WebhookTargetFactory {
             enable: true, // If we are here, it's already enabled.
             endpoint: endpoint_url,
             auth_token: config.lookup(WEBHOOK_AUTH_TOKEN).unwrap_or_default(),
-            queue_dir: config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
+            queue_dir: config
+                .lookup(WEBHOOK_QUEUE_DIR)
+                .unwrap_or_else(|| AUDIT_DEFAULT_DIR.to_string()),
             queue_limit: config
                 .lookup(WEBHOOK_QUEUE_LIMIT)
                 .and_then(|v| v.parse::<u64>().ok())
@@ -111,7 +113,9 @@ impl TargetFactory for WebhookTargetFactory {
             ));
         }
 
-        let queue_dir = config.lookup(WEBHOOK_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string());
+        let queue_dir = config
+            .lookup(WEBHOOK_QUEUE_DIR)
+            .unwrap_or_else(|| AUDIT_DEFAULT_DIR.to_string());
         if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
             return Err(TargetError::Configuration("Webhook queue directory must be an absolute path".to_string()));
         }
@@ -178,7 +182,7 @@ impl TargetFactory for MQTTTargetFactory {
                 config.lookup(MQTT_TLS_TRUST_LEAF_AS_CA).as_deref(),
                 config.lookup(MQTT_WS_PATH_ALLOWLIST).as_deref(),
             )?,
-            queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or(AUDIT_DEFAULT_DIR.to_string()),
+            queue_dir: config.lookup(MQTT_QUEUE_DIR).unwrap_or_else(|| AUDIT_DEFAULT_DIR.to_string()),
             queue_limit: config
                 .lookup(MQTT_QUEUE_LIMIT)
                 .and_then(|v| v.parse::<u64>().ok())

crates/audit/src/global.rs

@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{AuditEntry, AuditResult, AuditSystem};
+use crate::{AuditEntry, AuditResult, AuditSystem, system::AuditTargetMetricSnapshot};
 use rustfs_ecstore::config::Config;
 use std::sync::{Arc, OnceLock};
 use tracing::{debug, error, trace, warn};
@@ -89,6 +89,15 @@ pub async fn reload_audit_config(config: Config) -> AuditResult<()> {
     with_audit_system!(|system: Arc<AuditSystem>| async move { system.reload_config(config).await })
 }
 
+/// Returns per-target audit delivery metrics for Prometheus collection.
+pub async fn audit_target_metrics() -> Vec<AuditTargetMetricSnapshot> {
+    if let Some(system) = audit_system() {
+        system.snapshot_target_metrics().await
+    } else {
+        Vec::new()
+    }
+}
+
 /// Check if the global audit system is running
 pub async fn is_audit_system_running() -> bool {
     if let Some(system) = audit_system() {

crates/audit/src/lib.rs

@@ -31,4 +31,4 @@ pub use error::{AuditError, AuditResult};
 pub use global::*;
 pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation};
 pub use registry::AuditRegistry;
-pub use system::AuditSystem;
+pub use system::{AuditSystem, AuditTargetMetricSnapshot};

crates/audit/src/system.rs

@@ -24,6 +24,14 @@ use std::sync::Arc;
 use tokio::sync::{Mutex, RwLock, mpsc};
 use tracing::{error, info, warn};
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct AuditTargetMetricSnapshot {
+    pub failed_messages: u64,
+    pub queue_length: u64,
+    pub target_id: String,
+    pub total_messages: u64,
+}
+
 /// State of the audit system
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum AuditSystemState {
@@ -537,8 +545,14 @@ impl AuditSystem {
                                     TargetError::Timeout(_) => {
                                         warn!("Timeout sending to target {}, retrying...", target.id());
                                     }
+                                    TargetError::Dropped(reason) => {
+                                        warn!("Dropped queued payload for target {}: {}", target.id(), reason);
+                                        observability::record_target_failure();
+                                        break;
+                                    }
                                     _ => {
                                         error!("Permanent error for target {}: {}", target.id(), e);
+                                        target.record_final_failure();
                                         observability::record_target_failure();
                                         break;
                                     }
@@ -552,6 +566,7 @@ impl AuditSystem {
 
                     if retries >= MAX_RETRIES && !success {
                         warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id());
+                        target.record_final_failure();
                         observability::record_target_failure();
                     }
                 }
@@ -664,6 +679,25 @@ impl AuditSystem {
         registry.list_target_values()
     }
 
+    /// Returns per-target delivery metrics for Prometheus collection.
+    pub async fn snapshot_target_metrics(&self) -> Vec<AuditTargetMetricSnapshot> {
+        let targets = self.get_target_values().await;
+        let mut snapshots = Vec::with_capacity(targets.len());
+
+        for target in targets {
+            let delivery = target.delivery_snapshot();
+            snapshots.push(AuditTargetMetricSnapshot {
+                failed_messages: delivery.failed_messages,
+                queue_length: delivery.queue_length,
+                target_id: target.id().to_string(),
+                total_messages: delivery.total_messages,
+            });
+        }
+
+        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
+        snapshots
+    }
+
     /// Gets information about a specific target
     ///
     /// # Arguments

crates/common/src/data_usage.rs

@@ -51,14 +51,14 @@ impl AllTierStats {
     pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
         for (tier, st) in tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
     pub fn merge(&mut self, other: AllTierStats) {
         for (tier, st) in other.tiers {
             self.tiers
-                .insert(tier.clone(), self.tiers.get(&tier).unwrap_or(&TierStats::default()).add(&st));
+                .insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
         }
     }
 
@@ -673,7 +673,7 @@ impl DataUsageCache {
         let mut leaves = Vec::new();
         let mut remove = total - limit;
         add(self, path, &mut leaves);
-        leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
+        leaves.sort_by_key(|a| a.objects);
 
         while remove > 0 && !leaves.is_empty() {
             let Some(e) = leaves.first() else {

crates/common/src/heal_channel.rs

@@ -18,7 +18,7 @@ use std::{
     fmt::{self, Display},
     sync::OnceLock,
 };
-use tokio::sync::{broadcast, mpsc};
+use tokio::sync::{broadcast, mpsc, oneshot};
 use uuid::Uuid;
 
 pub const HEAL_DELETE_DANGLING: bool = true;
@@ -206,11 +206,59 @@ pub struct HealOpts {
     pub set: Option<usize>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionDropReason {
+    QueueFull,
+    PolicyDropped,
+}
+
+impl HealAdmissionDropReason {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::QueueFull => "queue_full",
+            Self::PolicyDropped => "policy_dropped",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HealAdmissionResult {
+    Accepted,
+    Merged,
+    Full,
+    Dropped(HealAdmissionDropReason),
+}
+
+impl HealAdmissionResult {
+    pub fn result_label(self) -> &'static str {
+        match self {
+            Self::Accepted => "accepted",
+            Self::Merged => "merged",
+            Self::Full => "full",
+            Self::Dropped(_) => "dropped",
+        }
+    }
+
+    pub fn reason_label(self) -> &'static str {
+        match self {
+            Self::Dropped(reason) => reason.as_str(),
+            _ => "none",
+        }
+    }
+
+    pub fn is_admitted(self) -> bool {
+        matches!(self, Self::Accepted | Self::Merged)
+    }
+}
+
 /// Heal channel command type
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub enum HealChannelCommand {
     /// Start a new heal task
-    Start(HealChannelRequest),
+    Start {
+        request: HealChannelRequest,
+        response_tx: oneshot::Sender<Result<HealAdmissionResult, String>>,
+    },
     /// Query heal task status
     Query { heal_path: String, client_token: String },
     /// Cancel heal task
@@ -339,9 +387,22 @@ pub fn subscribe_heal_responses() -> broadcast::Receiver<HealChannelResponse> {
     heal_response_sender().subscribe()
 }
 
+/// Send heal start request and wait for structured admission feedback.
+pub async fn send_heal_request_with_admission(request: HealChannelRequest) -> Result<HealAdmissionResult, String> {
+    let (response_tx, response_rx) = oneshot::channel();
+    send_heal_command(HealChannelCommand::Start { request, response_tx }).await?;
+    response_rx
+        .await
+        .map_err(|e| format!("Failed to receive heal admission response: {e}"))?
+}
+
 /// Send heal start request
 pub async fn send_heal_request(request: HealChannelRequest) -> Result<(), String> {
-    send_heal_command(HealChannelCommand::Start(request)).await
+    match send_heal_request_with_admission(request).await? {
+        HealAdmissionResult::Accepted | HealAdmissionResult::Merged => Ok(()),
+        HealAdmissionResult::Full => Err("Heal request queue is full".to_string()),
+        HealAdmissionResult::Dropped(reason) => Err(format!("Heal request dropped: {}", reason.as_str())),
+    }
 }
 
 /// Send heal query request
@@ -450,7 +511,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str)
         }
 
         if let Some(e) = &rule.noncurrent_version_expiration {
-            if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
+            if e.noncurrent_days.is_some() {
                 return true;
             }
             if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
@@ -542,6 +603,19 @@ pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPri
 mod tests {
     use super::*;
 
+    #[test]
+    fn heal_admission_result_labels_are_stable() {
+        assert_eq!(HealAdmissionResult::Accepted.result_label(), "accepted");
+        assert_eq!(HealAdmissionResult::Merged.result_label(), "merged");
+        assert_eq!(HealAdmissionResult::Full.result_label(), "full");
+        assert_eq!(
+            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull).reason_label(),
+            "queue_full"
+        );
+        assert!(HealAdmissionResult::Merged.is_admitted());
+        assert!(!HealAdmissionResult::Full.is_admitted());
+    }
+
     #[tokio::test]
     async fn heal_response_broadcast_reaches_subscriber() {
         let mut receiver = subscribe_heal_responses();

crates/common/src/internode_metrics.rs

@@ -101,11 +101,7 @@ impl InternodeMetrics {
     pub fn snapshot(&self) -> InternodeMetricsSnapshot {
         let dial_samples_total = self.dial_samples_total.load(Ordering::Relaxed);
         let dial_total_time_nanos = self.dial_total_time_nanos.load(Ordering::Relaxed);
-        let dial_avg_time_nanos = if dial_samples_total == 0 {
-            0
-        } else {
-            dial_total_time_nanos / dial_samples_total
-        };
+        let dial_avg_time_nanos = dial_total_time_nanos.checked_div(dial_samples_total).unwrap_or(0);
 
         InternodeMetricsSnapshot {
             sent_bytes_total: self.sent_bytes_total.load(Ordering::Relaxed),

crates/concurrency/Cargo.toml

@@ -26,7 +26,7 @@ thiserror = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]
-tokio = { workspace = true, features = ["full"] }
+tokio = { workspace = true, features = ["test-util","macros","rt-multi-thread"] }
 
 [features]
 default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"]

crates/config/README.md

@@ -53,8 +53,24 @@ Current guidance:
 
 ## Scanner environment aliases
 
+- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
+- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
 - `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
 - `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
+- `RUSTFS_SCANNER_IDLE_MODE` (canonical)
+- `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
+
+## Drive timeout environment variables
+
+- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS`
+- `RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS`
+
+Legacy compatibility fallback:
+- `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
+  This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
 
 ## 📄 License
 

crates/config/src/constants/drive.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Legacy global drive timeout fallback.
+/// Deprecated in favor of per-operation drive timeout knobs.
+pub const ENV_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
+
+/// Default timeout in seconds for the legacy global drive timeout fallback.
+pub const DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS: u64 = 30;
+
+/// Timeout for metadata-oriented drive operations such as `read_metadata`.
+pub const ENV_DRIVE_METADATA_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_METADATA_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_METADATA_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `disk_info()` calls on local and remote drives.
+pub const ENV_DRIVE_DISK_INFO_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS: u64 = 5;
+
+/// Timeout for `list_dir()` style metadata listing operations.
+pub const ENV_DRIVE_LIST_DIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS: u64 = 5;
+
+/// Total timeout for `walk_dir()` operations.
+pub const ENV_DRIVE_WALKDIR_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
+
+/// Maximum time without forward progress while consuming a `walk_dir()` stream.
+pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
+pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
+
+/// Number of consecutive failures before a suspect drive is classified as offline.
+pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
+pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
+
+/// Number of consecutive successful recovery probes before a returning drive is considered online again.
+pub const ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD: &str = "RUSTFS_DRIVE_RETURNING_SUCCESS_THRESHOLD";
+pub const DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD: u64 = 3;
+
+/// Probe interval in seconds while a drive is in the recovery path.
+pub const ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS: &str = "RUSTFS_DRIVE_RETURNING_PROBE_INTERVAL_SECS";
+pub const DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS: u64 = 2;
+
+/// Duration in seconds for classifying a recovered drive as a short offline event.
+pub const ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS: &str = "RUSTFS_DRIVE_OFFLINE_GRACE_PERIOD_SECS";
+pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
+
+/// Duration in seconds after which a recovered drive is classified as long offline.
+pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
+pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;

crates/config/src/constants/heal.rs

@@ -56,6 +56,15 @@ pub const ENV_HEAL_TASK_TIMEOUT_SECS: &str = "RUSTFS_HEAL_TASK_TIMEOUT_SECS";
 /// - Note: A higher concurrency limit can speed up healing but may lead to resource contention.
 pub const ENV_HEAL_MAX_CONCURRENT_HEALS: &str = "RUSTFS_HEAL_MAX_CONCURRENT_HEALS";
 
+/// Environment variable name that specifies the maximum number of concurrent heal operations
+/// allowed for a single erasure set.
+///
+/// - Purpose: Prevent one degraded set from consuming all global heal slots.
+/// - Unit: number of operations (usize).
+/// - Valid values: any positive integer.
+/// - Example: `export RUSTFS_HEAL_MAX_CONCURRENT_PER_SET=1`
+pub const ENV_HEAL_MAX_CONCURRENT_PER_SET: &str = "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET";
+
 /// Default value for enabling authentication for heal operations if not specified in the environment variable.
 /// - Value: true (authentication enabled).
 /// - Rationale: Enabling authentication by default enhances security for heal operations.
@@ -86,3 +95,46 @@ pub const DEFAULT_HEAL_TASK_TIMEOUT_SECS: u64 = 300; // 5 minutes
 /// - Rationale: This default concurrency limit helps balance healing speed with resource usage, preventing system overload.
 /// - Adjustments: Users may modify this value via the `RUSTFS_HEAL_MAX_CONCURRENT_HEALS` environment variable based on their system capacity and expected heal workload.
 pub const DEFAULT_HEAL_MAX_CONCURRENT_HEALS: usize = 4;
+
+/// Default maximum number of concurrent heal operations per erasure set.
+///
+/// - Value: 1 concurrent heal operation per set.
+/// - Rationale: Keeps a degraded set from monopolizing the global heal scheduler.
+pub const DEFAULT_HEAL_MAX_CONCURRENT_PER_SET: usize = 1;
+
+/// Environment variable that controls whether low-priority heal requests should merge into
+/// an existing queued request with the same deduplication key.
+pub const ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE: &str = "RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE";
+
+/// Environment variable that allows low-priority heal requests to be dropped when the queue is full.
+pub const ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: &str = "RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL";
+
+/// Environment variable that controls concurrent object heals within a single erasure-set page.
+pub const ENV_HEAL_PAGE_OBJECT_CONCURRENCY: &str = "RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY";
+
+/// Environment variable that toggles notify-driven scheduler wakeups.
+pub const ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: &str = "RUSTFS_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE";
+
+/// Environment variable that toggles per-set bulkhead scheduling.
+pub const ENV_HEAL_SET_BULKHEAD_ENABLE: &str = "RUSTFS_HEAL_SET_BULKHEAD_ENABLE";
+
+/// Environment variable that toggles page-level parallel object healing for erasure-set repair.
+pub const ENV_HEAL_PAGE_PARALLEL_ENABLE: &str = "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE";
+
+/// Default behavior is to merge duplicate low-priority requests.
+pub const DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE: bool = true;
+
+/// Default behavior is to drop low-priority requests instead of blocking when the queue is full.
+pub const DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL: bool = true;
+
+/// Default per-page object heal concurrency for erasure-set healing.
+pub const DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY: usize = 8;
+
+/// Default behavior is to keep notify-driven scheduler wakeups enabled.
+pub const DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE: bool = true;
+
+/// Default behavior is to keep per-set bulkhead scheduling enabled.
+pub const DEFAULT_HEAL_SET_BULKHEAD_ENABLE: bool = true;
+
+/// Default behavior is to keep erasure-set page parallelism enabled.
+pub const DEFAULT_HEAL_PAGE_PARALLEL_ENABLE: bool = true;

crates/config/src/constants/mod.rs

@@ -17,6 +17,7 @@ pub(crate) mod body_limits;
 pub(crate) mod capacity;
 pub(crate) mod compress;
 pub(crate) mod console;
+pub(crate) mod drive;
 pub(crate) mod env;
 pub(crate) mod heal;
 pub(crate) mod object;

crates/config/src/constants/scanner.rs

@@ -25,6 +25,12 @@ pub const ENV_SCANNER_START_DELAY_SECS: &str = "RUSTFS_SCANNER_START_DELAY_SECS"
 #[deprecated(note = "Use RUSTFS_SCANNER_START_DELAY_SECS instead")]
 pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
 
+/// Environment variable that specifies the scanner cycle interval in seconds.
+/// If set, this overrides the cycle interval derived from `RUSTFS_SCANNER_SPEED`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CYCLE=3600` (1 hour)
+pub const ENV_SCANNER_CYCLE: &str = "RUSTFS_SCANNER_CYCLE";
+
 /// Environment variable that selects the scanner speed preset.
 /// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
 /// Controls the sleep factor, maximum sleep duration, and cycle interval.
@@ -39,9 +45,24 @@ pub const DEFAULT_SCANNER_SPEED: &str = "default";
 /// - Example: `export RUSTFS_SCANNER_IDLE_MODE=false`
 pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
 
+/// Environment variable that controls scanner cache save timeout in seconds.
+/// The scanner enforces a minimum value of `1`.
+/// - Unit: seconds (u64).
+/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
+pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
+
 /// Default scanner idle mode.
 pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 
+/// Compatibility flag kept for Patch 3 rollback windows.
+///
+/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
+/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
+pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
+
+/// Default inline scanner heal compatibility mode.
+pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
+
 /// Scanner speed preset controlling throttling behavior.
 ///
 /// Each preset defines three parameters:
@@ -56,6 +77,8 @@ pub const DEFAULT_SCANNER_IDLE_MODE: bool = true;
 /// | `default` | 2x     | 1 second  | 1 minute       |
 /// | `slow`    | 10x    | 15 seconds| 1 minute       |
 /// | `slowest` | 100x   | 15 seconds| 30 minutes     |
+///
+/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum ScannerSpeed {
     Fastest,

crates/config/src/lib.rs

@@ -25,6 +25,8 @@ pub use constants::compress::*;
 #[cfg(feature = "constants")]
 pub use constants::console::*;
 #[cfg(feature = "constants")]
+pub use constants::drive::*;
+#[cfg(feature = "constants")]
 pub use constants::env::*;
 #[cfg(feature = "constants")]
 pub use constants::heal::*;

crates/credentials/src/credentials.rs

@@ -20,7 +20,7 @@ use std::collections::HashMap;
 use std::env;
 use std::fmt;
 use std::io::Error;
-use std::sync::OnceLock;
+use std::sync::{LazyLock, OnceLock};
 use time::OffsetDateTime;
 
 /// Global active credentials
@@ -314,6 +314,17 @@ impl fmt::Debug for Credentials {
 }
 
 impl Credentials {
+    /// Returns a reference to this credential's claims, or a shared empty map
+    /// when the credential has no claims attached. Avoids per-call allocation
+    /// at call sites that need an `&HashMap<String, Value>`.
+    pub fn claims_or_empty(&self) -> &HashMap<String, Value> {
+        static EMPTY: LazyLock<HashMap<String, Value>> = LazyLock::new(HashMap::new);
+        match &self.claims {
+            Some(c) => c,
+            None => &EMPTY,
+        }
+    }
+
     pub fn is_expired(&self) -> bool {
         if self.expiration.is_none() {
             return false;

crates/e2e_test/Cargo.toml

@@ -41,7 +41,6 @@ serde_json.workspace = true
 tonic = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
-clap = { workspace = true }
 rustfs-madmin.workspace = true
 rustfs-filemeta.workspace = true
 bytes.workspace = true
@@ -53,8 +52,6 @@ async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
 flate2.workspace = true
 http.workspace = true
-http-body.workspace = true
-http-body-util.workspace = true
 reqwest = { workspace = true }
 rustfs-signer.workspace = true
 tracing = { workspace = true }

crates/e2e_test/src/archive_download_integrity_test.rs

@@ -218,6 +218,34 @@ mod tests {
         Ok(builder.send().await?)
     }
 
+    async fn signed_get_request_with_headers(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        extra_headers: &[(&str, &str)],
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let mut request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+        for (name, value) in extra_headers {
+            request = request.header(*name, *value);
+        }
+
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut builder = client.get(url);
+        for (name, value) in signed.headers() {
+            builder = builder.header(name, value);
+        }
+
+        Ok(builder.send().await?)
+    }
+
     async fn assert_archive_object_content_encoding(
         client: &S3Client,
         bucket: &str,
@@ -655,6 +683,42 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+        env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
+
+        let client = env.create_s3_client();
+        let key = "multipart-empty-conditional-headers.zip";
+        let zip_bytes =
+            complete_archive_multipart_upload_with_content_encoding(&client, MULTIPART_ARCHIVE_TEST_BUCKET, key, None).await?;
+        let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, key);
+
+        let response = signed_get_request_with_headers(
+            &object_url,
+            &env.access_key,
+            &env.secret_key,
+            &[("if-match", ""), ("if-none-match", "")],
+        )
+        .await?;
+        let status = response.status();
+        let body = response.bytes().await?;
+
+        assert_eq!(
+            status,
+            StatusCode::OK,
+            "unexpected multipart GET status {status}, body: {}",
+            String::from_utf8_lossy(body.as_ref())
+        );
+        assert_eq!(body.as_ref(), zip_bytes.as_slice());
+
+        env.stop_server();
+        Ok(())
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()

crates/e2e_test/src/kms/test_runner.rs

@@ -450,11 +450,7 @@ impl KMSTestSuite {
         if failed > 0 {
             warn!("❌ Failing tests:");
             for result in results.iter().filter(|r| !r.success) {
-                warn!(
-                    "  - {}: {}",
-                    result.test_name,
-                    result.error_message.as_ref().unwrap_or(&"Unknown error".to_string())
-                );
+                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
             }
         }
     }

crates/e2e_test/src/reliant/conditional_writes.rs

@@ -259,7 +259,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
     let upload_id = initiate_response
         .upload_id()
-        .ok_or(std::io::Error::other("No upload ID returned"))?;
+        .ok_or_else(|| std::io::Error::other("No upload ID returned"))?;
 
     // Upload parts
     for part_number in 1..=num_parts {
@@ -277,7 +277,7 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
 
         let part_etag = upload_part_response
             .e_tag()
-            .ok_or(std::io::Error::other("Do not have etag"))?
+            .ok_or_else(|| std::io::Error::other("Do not have etag"))?
             .to_string();
 
         let completed_part = CompletedPart::builder().part_number(part_number).e_tag(part_etag).build();

crates/e2e_test/src/reliant/lifecycle.rs

@@ -160,7 +160,7 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 #[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
+async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
     use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
 
     let client = create_aws_s3_client().await?;
@@ -176,19 +176,12 @@ async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::er
         .build()?;
     let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
 
-    let err = client
+    client
         .put_bucket_lifecycle_configuration()
         .bucket(BUCKET)
         .lifecycle_configuration(lifecycle)
         .send()
-        .await
-        .expect_err("zero-day lifecycle expiration should be rejected");
-
-    let err_msg = format!("{err:?}");
-    assert!(
-        err_msg.contains("InvalidArgument") && err_msg.contains("greater than 0"),
-        "unexpected error: {err_msg}"
-    );
+        .await?;
 
     Ok(())
 }

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -154,7 +154,7 @@ async fn walk_dir() -> Result<(), Box<dyn Error>> {
             match response.next().await {
                 Some(Ok(resp)) => {
                     if !resp.success {
-                        println!("{}", resp.error_info.unwrap_or("".to_string()));
+                        println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
                     }
                     let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
                         .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))

crates/e2e_test/src/replication_extension_test.rs

@@ -190,6 +190,14 @@ async fn put_bucket_replication(
     Ok(())
 }
 
+async fn delete_bucket_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication", env.url);
+    signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
 async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
     let client = env.create_s3_client();
     client
@@ -1084,6 +1092,54 @@ async fn test_remove_remote_target_rejects_target_used_by_replication() -> Resul
     Ok(())
 }
 
+#[tokio::test]
+#[serial]
+async fn test_delete_bucket_replication_removes_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-delete-config-src";
+    let target_bucket = "replication-delete-config-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    let delete_response = delete_bucket_replication(&source_env, source_bucket).await?;
+    assert!(
+        delete_response.status().is_success(),
+        "unexpected delete status: {}",
+        delete_response.status()
+    );
+
+    let targets_response = list_replication_targets_request(&source_env, Some(source_bucket)).await?;
+    assert_eq!(targets_response.status(), StatusCode::OK);
+    let targets: Vec<serde_json::Value> = targets_response.json().await?;
+    assert!(
+        targets
+            .iter()
+            .all(|target| target.get("arn").and_then(|arn| arn.as_str()) != Some(target_arn.as_str())),
+        "deleted replication config left stale target {target_arn}: {targets:?}"
+    );
+
+    let recreated_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &recreated_arn).await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {

crates/e2e_test/src/special_chars_test.rs

@@ -26,10 +26,16 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestEnvironment, init_logging};
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
     use aws_sdk_s3::Client;
     use aws_sdk_s3::primitives::ByteStream;
+    use http::StatusCode;
+    use http::header::HOST;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
     use serial_test::serial;
+    use std::error::Error;
     use tracing::{debug, info};
 
     /// Helper function to create an S3 client for testing
@@ -56,6 +62,30 @@ mod tests {
         }
     }
 
+    async fn signed_get(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .body(Body::empty())?;
+
+        let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+        let client = local_http_client();
+        let mut request_builder = client.get(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+
+        Ok(request_builder.send().await?)
+    }
+
     /// Test PUT and GET with space character in path
     ///
     /// This reproduces Part A of the issue:
@@ -274,6 +304,73 @@ mod tests {
         info!("Test completed successfully");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_missing_object_with_trailing_equals_returns_no_such_key() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-missing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let url = format!("{}/{}/path/sitemap.xmlage=", env.url, bucket);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::NOT_FOUND,
+            "missing object key ending with '=' should pass signature validation before object lookup"
+        );
+
+        let body = response.text().await?;
+        assert!(body.contains("<Code>NoSuchKey</Code>"), "expected NoSuchKey XML response, got: {body}");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_signed_get_existing_object_with_trailing_equals_returns_content() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = create_s3_client(&env);
+        let bucket = "test-existing-equals-key";
+        create_bucket(&client, bucket).await?;
+
+        let key = "path/sitemap.xmlage=";
+        let content = b"object content for raw signed URL with trailing equals";
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await?;
+
+        let url = format!("{}/{}/{}", env.url, bucket, key);
+        let response = signed_get(&url, &env.access_key, &env.secret_key).await?;
+
+        assert_eq!(
+            response.status(),
+            StatusCode::OK,
+            "existing object key ending with '=' should pass signature validation and return content"
+        );
+
+        let body = response.bytes().await?;
+        assert_eq!(body.as_ref(), content);
+
+        env.stop_server();
+        Ok(())
+    }
+
     /// Test DELETE operation with special characters
     #[tokio::test]
     #[serial]

crates/ecstore/Cargo.toml

@@ -86,7 +86,7 @@ hyper.workspace = true
 hyper-util.workspace = true
 hyper-rustls.workspace = true
 rustls.workspace = true
-tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
+tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] }
 tonic.workspace = true
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower.workspace = true

crates/ecstore/src/admin_server_info.rs

@@ -162,8 +162,9 @@ pub async fn get_local_server_property() -> ServerProperties {
 
     let mut props = ServerProperties {
         endpoint: addr,
-        uptime: SystemTime::now()
-            .duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
+        uptime: GLOBAL_BOOT_TIME
+            .get()
+            .and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok())
             .unwrap_or_default()
             .as_secs(),
         network,

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -786,7 +786,10 @@ impl BucketTargetSys {
                         && tgt
                             .credentials
                             .as_ref()
-                            .map(|c| c.access_key == target.credentials.as_ref().unwrap_or(&Credentials::default()).access_key)
+                            .map(|c| {
+                                let default_creds = Credentials::default();
+                                c.access_key == target.credentials.as_ref().unwrap_or(&default_creds).access_key
+                            })
                             .unwrap_or(false)
                     {
                         return (tgt.arn.clone(), true);
@@ -894,6 +897,41 @@ pub struct RemoveObjectOptions {
     pub replication_validity_check: bool,
 }
 
+fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOptions) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    if opts.force_delete {
+        insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
+    }
+    if opts.governance_bypass {
+        headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
+    }
+
+    if opts.replication_delete_marker {
+        insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
+    }
+
+    if let Some(t) = opts.replication_mtime {
+        insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
+    }
+
+    if !opts.replication_status.is_empty() {
+        headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
+    }
+
+    if let Some(version_id) = version_id {
+        insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
+    }
+
+    if opts.replication_request {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
+    }
+    if opts.replication_validity_check {
+        insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
+    }
+
+    headers
+}
+
 #[derive(Debug, Clone)]
 pub struct AdvancedPutOptions {
     pub source_version_id: String,
@@ -1424,39 +1462,15 @@ impl TargetClient {
         version_id: Option<String>,
         opts: RemoveObjectOptions,
     ) -> Result<(), S3ClientError> {
-        let mut headers = HeaderMap::new();
-        if opts.force_delete {
-            insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
-        }
-        if opts.governance_bypass {
-            headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
-        }
-
-        if opts.replication_delete_marker {
-            insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
-        }
-
-        if let Some(t) = opts.replication_mtime {
-            insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
-        }
-
-        if !opts.replication_status.is_empty() {
-            headers.insert(AMZ_BUCKET_REPLICATION_STATUS, opts.replication_status.as_str().parse().unwrap());
-        }
-
-        if opts.replication_request {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
-        }
-        if opts.replication_validity_check {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
-        }
+        let headers = build_remove_object_headers(version_id.as_deref(), &opts);
+        let api_version_id = if opts.replication_request { None } else { version_id };
 
         match self
             .client
             .delete_object()
             .bucket(bucket)
             .key(object)
-            .set_version_id(version_id)
+            .set_version_id(api_version_id)
             .customize()
             .map_request(move |mut req| {
                 for (k, v) in headers.clone().into_iter() {
@@ -1550,3 +1564,53 @@ impl From<std::io::Error> for BucketTargetError {
 }
 
 impl Error for BucketTargetError {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn build_remove_object_headers_includes_internal_version_id_for_replication_delete() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: true,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert_eq!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_VERSION_ID).as_deref(),
+            Some(version_id.as_str()),
+            "replication delete requests must preserve the version id in internal headers"
+        );
+    }
+
+    #[test]
+    fn build_remove_object_headers_omits_delete_marker_flag_for_marker_version_purge() {
+        let version_id = Uuid::new_v4().to_string();
+        let headers = build_remove_object_headers(
+            Some(version_id.as_str()),
+            &RemoveObjectOptions {
+                force_delete: false,
+                governance_bypass: false,
+                replication_delete_marker: false,
+                replication_mtime: None,
+                replication_status: ReplicationStatusType::Replica,
+                replication_request: true,
+                replication_validity_check: false,
+            },
+        );
+
+        assert!(
+            rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_DELETEMARKER).is_none(),
+            "delete-marker version purges must not masquerade as delete-marker creations"
+        );
+    }
+}

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -11,20 +11,18 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#![allow(unused_imports)]
-#![allow(unused_variables)]
-#![allow(unused_mut)]
-#![allow(unused_assignments)]
-#![allow(unused_must_use)]
-#![allow(clippy::all)]
 
 use crate::bucket::lifecycle::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc};
+use crate::bucket::lifecycle::evaluator::Evaluator;
 use crate::bucket::lifecycle::lifecycle::{
     self, ExpirationOptions, Lifecycle, ObjectOpts, TransitionOptions, abort_incomplete_multipart_upload_due,
 };
 use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
 use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier};
 use crate::bucket::object_lock::objectlock_sys::check_object_lock_for_deletion;
+use crate::bucket::replication::{
+    DeletedObjectReplicationInfo, ReplicationConfig, check_replicate_delete, schedule_replication_delete,
+};
 use crate::bucket::{metadata_sys, metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys};
 use crate::client::object_api_utils::new_getobjectreader;
 use crate::disk::error::DiskError;
@@ -37,34 +35,33 @@ use crate::global::GLOBAL_LocalNodeName;
 use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
 use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
 use crate::store::ECStore;
-use crate::store_api::StorageAPI;
 use crate::store_api::{
     GetObjectReader, HTTPRangeSpec, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions,
     ObjectToDelete,
 };
 use crate::tier::warm_backend::WarmBackendGetOpts;
 use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
-use bytes::BytesMut;
 use futures::Future;
 use http::HeaderMap;
 use lazy_static::lazy_static;
 use rustfs_common::data_usage::TierStats;
 use rustfs_common::heal_channel::rep_has_active_rules;
 use rustfs_common::metrics::{IlmAction, Metrics};
-use rustfs_filemeta::{FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, get_file_info, is_restored_object_on_disk};
+use rustfs_filemeta::{
+    FileInfo, FileInfoOpts, NULL_VERSION_ID, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicationState, RestoreStatusOps,
+    VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
+};
 use rustfs_s3_common::EventName;
 use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
-use s3s::Body;
 use s3s::dto::{
     BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
-    ServerSideEncryption, Timestamp,
+    Timestamp,
 };
-use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_STORAGE_CLASS};
+use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
 use sha2::{Digest, Sha256};
 use std::any::Any;
 use std::collections::HashMap;
 use std::env;
-use std::io::Write;
 use std::pin::Pin;
 use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::{Arc, Mutex, Weak};
@@ -73,7 +70,7 @@ use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{RwLock, mpsc};
-use tracing::{debug, error, info, warn};
+use tracing::{error, info, warn};
 use uuid::Uuid;
 use xxhash_rust::xxh64;
 
@@ -111,7 +108,7 @@ impl LifecycleSys {
     pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
         match get_lifecycle_config(bucket).await {
             Ok((lc, _)) => Some(lc),
-            Err(err) if err == Error::ConfigNotFound => None,
+            Err(Error::ConfigNotFound) => None,
             Err(err) => {
                 warn!(bucket, error = ?err, "failed to load lifecycle config");
                 None
@@ -119,8 +116,25 @@ impl LifecycleSys {
         }
     }
 
-    pub fn trace(_oi: &ObjectInfo) -> TraceFn {
-        Arc::new(|_oi, _ctx| Box::pin(async move {}))
+    pub fn trace(oi: &ObjectInfo) -> TraceFn {
+        let bucket = oi.bucket.clone();
+        let name = oi.name.clone();
+        let version_id = oi.version_id.map(|v| v.to_string()).unwrap_or_default();
+        Arc::new(move |_action: String, _ctx: HashMap<String, String>| {
+            let bucket = bucket.clone();
+            let name = name.clone();
+            let version_id = version_id.clone();
+            Box::pin(async move {
+                info!(
+                    bucket = %bucket,
+                    object = %name,
+                    version_id = %version_id,
+                    action = %_action,
+                    "ILM lifecycle trace: {} on {}/{} (version: {})",
+                    _action, bucket, name, version_id
+                );
+            })
+        })
     }
 }
 
@@ -133,8 +147,8 @@ struct ExpiryTask {
 impl ExpiryOp for ExpiryTask {
     fn op_hash(&self) -> u64 {
         let mut hasher = Sha256::new();
-        hasher.update(format!("{}", self.obj_info.bucket).as_bytes());
-        hasher.update(format!("{}", self.obj_info.name).as_bytes());
+        hasher.update(self.obj_info.bucket.as_bytes());
+        hasher.update(self.obj_info.name.as_bytes());
         xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
     }
 
@@ -188,8 +202,8 @@ struct FreeVersionTask(ObjectInfo);
 impl ExpiryOp for FreeVersionTask {
     fn op_hash(&self) -> u64 {
         let mut hasher = Sha256::new();
-        hasher.update(format!("{}", self.0.transitioned_object.tier).as_bytes());
-        hasher.update(format!("{}", self.0.transitioned_object.name).as_bytes());
+        hasher.update(self.0.transitioned_object.tier.as_bytes());
+        hasher.update(self.0.transitioned_object.name.as_bytes());
         xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
     }
 
@@ -207,8 +221,8 @@ struct NewerNoncurrentTask {
 impl ExpiryOp for NewerNoncurrentTask {
     fn op_hash(&self) -> u64 {
         let mut hasher = Sha256::new();
-        hasher.update(format!("{}", self.bucket).as_bytes());
-        hasher.update(format!("{}", self.versions[0].object_name).as_bytes());
+        hasher.update(self.bucket.as_bytes());
+        hasher.update(self.versions[0].object_name.as_bytes());
         xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
     }
 
@@ -240,7 +254,7 @@ impl ExpiryState {
 
     pub async fn pending_tasks(&self) -> usize {
         let rxs = &self.tasks_rx;
-        if rxs.len() == 0 {
+        if rxs.is_empty() {
             return 0;
         }
         let mut tasks = 0;
@@ -253,32 +267,33 @@ impl ExpiryState {
     pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> {
         let wrkr = self.get_worker_ch(je.op_hash());
         if wrkr.is_none() {
-            *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
+            *self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
+            return Ok(());
         }
-        let wrkr = wrkr.expect("err");
+        let wrkr = wrkr.expect("worker channel should exist after None check");
         select! {
             //_ -> GlobalContext.Done() => ()
             _ = wrkr.send(Some(Box::new(je.clone()))) => (),
             else => {
-                *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1;
+                *self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
             }
         }
-        return Ok(());
+        Ok(())
     }
 
     pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) {
         let task = FreeVersionTask(oi);
         let wrkr = self.get_worker_ch(task.op_hash());
         if wrkr.is_none() {
-            *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
+            *self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1;
             return;
         }
-        let wrkr = wrkr.expect("err!");
+        let wrkr = wrkr.expect("worker channel should exist after None check");
         select! {
             //_ -> GlobalContext.Done() => {}
             _ = wrkr.send(Some(Box::new(task))) => (),
             else => {
-                *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1;
+                *self.stats.as_mut().expect("stats lock").missed_freevers_tasks.get_mut() += 1;
             }
         }
     }
@@ -291,21 +306,21 @@ impl ExpiryState {
         };
         let wrkr = self.get_worker_ch(task.op_hash());
         if wrkr.is_none() {
-            *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
             return;
         }
-        let wrkr = wrkr.expect("err!");
+        let wrkr = wrkr.expect("worker channel should exist after None check");
         select! {
             //_ -> GlobalContext.Done() => {}
             _ = wrkr.send(Some(Box::new(task))) => (),
             else => {
-                *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+                *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
             }
         }
     }
 
     pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
-        if versions.len() == 0 {
+        if versions.is_empty() {
             return;
         }
 
@@ -316,26 +331,30 @@ impl ExpiryState {
         };
         let wrkr = self.get_worker_ch(task.op_hash());
         if wrkr.is_none() {
-            *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+            *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
             return;
         }
-        let wrkr = wrkr.expect("err!");
+        let wrkr = wrkr.expect("worker channel should exist after None check");
         select! {
             //_ -> GlobalContext.Done() => {}
             _ = wrkr.send(Some(Box::new(task))) => (),
             else => {
-                *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1;
+                *self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
             }
         }
     }
 
     pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
-        if self.tasks_tx.len() == 0 {
+        if self.tasks_tx.is_empty() {
             return None;
         }
         Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone())
     }
 
+    pub fn increment_missed_tier_journal_tasks(&mut self) {
+        *self.stats.as_mut().expect("stats lock").missed_tier_journal_tasks.get_mut() += 1;
+    }
+
     pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
         if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 {
             return;
@@ -349,11 +368,11 @@ impl ExpiryState {
             let rx = Arc::new(tokio::sync::Mutex::new(rx));
             state.tasks_tx.push(tx);
             state.tasks_rx.push(rx.clone());
-            *state.stats.as_mut().expect("err").workers.get_mut() += 1;
+            *state.stats.as_mut().expect("stats lock").workers.get_mut() += 1;
             tokio::spawn(async move {
                 let mut rx = rx.lock().await;
                 //let mut expiry_state = GLOBAL_ExpiryState.read().await;
-                ExpiryState::worker(&mut *rx, api).await;
+                ExpiryState::worker(&mut rx, api).await;
             });
         }
 
@@ -363,48 +382,49 @@ impl ExpiryState {
             worker.send(None).await.unwrap_or(());
             state.tasks_tx.remove(l - 1);
             state.tasks_rx.remove(l - 1);
-            *state.stats.as_mut().expect("err").workers.get_mut() -= 1;
+            *state.stats.as_mut().expect("stats lock").workers.get_mut() -= 1;
             l -= 1;
         }
     }
 
     pub async fn worker(rx: &mut Receiver<Option<ExpiryOpType>>, api: Arc<ECStore>) {
-        //let cancel_token =
-        //    get_background_services_cancel_token().ok_or_else(|| Error::other("Background services not initialized"))?;
+        let cancel_token = crate::global::get_background_services_cancel_token().unwrap_or_else(|| {
+            static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
+            FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new)
+        });
 
         loop {
             select! {
-                //_ = cancel_token.cancelled() => {
-                _ = tokio::signal::ctrl_c() => {
-                    info!("got ctrl+c, exits");
+                _ = cancel_token.cancelled() => {
+                    info!("lifecycle expiry worker received shutdown signal, exiting");
                     break;
                 }
                 v = rx.recv() => {
                     if v.is_none() {
                         break;
                     }
-                    let v = v.expect("err!");
+                    let v = v.expect("channel closed unexpectedly");
                     if v.is_none() {
                         //rx.close();
                         //drop(rx);
                         let _ = rx;
                         return;
                     }
-                    let v = v.expect("err!");
+                    let v = v.expect("received None after None check");
                     if v.as_any().is::<ExpiryTask>() {
-                        let v = v.as_any().downcast_ref::<ExpiryTask>().expect("err!");
-                        if v.obj_info.transitioned_object.status != "" {
+                        let v = v.as_any().downcast_ref::<ExpiryTask>().expect("ExpiryTask downcast failed");
+                        if !v.obj_info.transitioned_object.status.is_empty() {
                             apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await;
                         } else {
                             apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await;
                         }
                     }
                     else if v.as_any().is::<NewerNoncurrentTask>() {
-                        let _v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("err!");
-                        //delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
+                        let v = v.as_any().downcast_ref::<NewerNoncurrentTask>().expect("NewerNoncurrentTask downcast failed");
+                        crate::client::object_handlers_common::delete_object_versions(&api, &v.bucket, &v.versions, v.event.clone()).await;
                     }
                     else if v.as_any().is::<Jentry>() {
-                        let v = v.as_any().downcast_ref::<Jentry>().expect("err!");
+                        let v = v.as_any().downcast_ref::<Jentry>().expect("Jentry downcast failed");
                         if let Err(err) = delete_object_from_remote_tier(&v.obj_name, &v.version_id, &v.tier_name).await {
                             warn!(
                                 object = %v.obj_name,
@@ -416,7 +436,7 @@ impl ExpiryState {
                         }
                     }
                     else if v.as_any().is::<FreeVersionTask>() {
-                        let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
+                        let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
                         let oi = v.0.clone();
                         if let Err(err) = delete_object_from_remote_tier(
                             &oi.transitioned_object.name,
@@ -499,7 +519,7 @@ struct TransitionTask {
 impl ExpiryOp for TransitionTask {
     fn op_hash(&self) -> u64 {
         let mut hasher = Sha256::new();
-        hasher.update(format!("{}", self.obj_info.bucket).as_bytes());
+        hasher.update(self.obj_info.bucket.as_bytes());
         // hasher.update(format!("{}", self.obj_info.versions[0].object_name).as_bytes());
         xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
     }
@@ -594,22 +614,40 @@ impl TransitionState {
                     if task.is_err() {
                         break;
                     }
-                    let task = task.expect("err!");
+                    let task = task.expect("channel recv should succeed after error check");
                     if task.is_none() {
                         //self.transition_rx.close();
                         //drop(self.transition_rx);
                         return;
                     }
-                    let task = task.expect("err!");
+                    let task = task.expect("received None after None check");
                     if task.as_any().is::<TransitionTask>() {
-                        let task = task.as_any().downcast_ref::<TransitionTask>().expect("err!");
+                        let task = task.as_any().downcast_ref::<TransitionTask>().expect("TransitionTask downcast failed");
 
                         GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst);
+
+                        let obj_info_for_event = ObjectInfo {
+                            bucket: task.obj_info.bucket.clone(),
+                            name: task.obj_info.name.clone(),
+                            size: task.obj_info.size,
+                            version_id: task.obj_info.version_id,
+                            ..Default::default()
+                        };
+
                         if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await {
                             if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") {
                                 error!("Transition to {} failed for {}/{} version:{} with {}",
                                     task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.map(|v| v.to_string()).unwrap_or_default(), err.to_string());
                             }
+                            // Send s3:ObjectTransition:Failed event
+                            send_event(EventArgs {
+                                event_name: EventName::ObjectTransitionFailed.to_string(),
+                                bucket_name: obj_info_for_event.bucket.clone(),
+                                object: obj_info_for_event,
+                                user_agent: "Internal: [ILM-Transition]".to_string(),
+                                host: GLOBAL_LocalNodeName.to_string(),
+                                ..Default::default()
+                            });
                         } else {
                             let mut ts = TierStats {
                                 total_size: task.obj_info.size as u64,
@@ -620,6 +658,16 @@ impl TransitionState {
                                 ts.num_objects = 1;
                             }
                             GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts);
+
+                            // Send s3:ObjectTransition:Complete event
+                            send_event(EventArgs {
+                                event_name: EventName::ObjectTransitionComplete.to_string(),
+                                bucket_name: obj_info_for_event.bucket.clone(),
+                                object: obj_info_for_event,
+                                user_agent: "Internal: [ILM-Transition]".to_string(),
+                                host: GLOBAL_LocalNodeName.to_string(),
+                                ..Default::default()
+                            });
                         }
                         GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst);
                     }
@@ -634,7 +682,7 @@ impl TransitionState {
         tier_stats
             .entry(tier.to_string())
             .and_modify(|e| e.add_stats(ts))
-            .or_insert(LastDayTierStats::default());
+            .or_default();
     }
 
     pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
@@ -666,15 +714,15 @@ impl TransitionState {
             tokio::spawn(async move {
                 TransitionState::worker(clone_api).await;
             });
-            num_workers = num_workers + 1;
+            num_workers += 1;
             GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst);
         }
 
         let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
         while num_workers > n {
             let worker = GLOBAL_TransitionState.kill_tx.clone();
-            worker.send(()).await;
-            num_workers = num_workers - 1;
+            let _ = worker.send(()).await;
+            num_workers -= 1;
             GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst);
         }
     }
@@ -683,10 +731,10 @@ impl TransitionState {
 pub async fn init_background_expiry(api: Arc<ECStore>) {
     let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
     //globalILMConfig.getExpirationWorkers()
-    if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") {
-        if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
-            workers = num_expirations;
-        }
+    if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS")
+        && let Ok(num_expirations) = env_expiration_workers.parse::<usize>()
+    {
+        workers = num_expirations;
     }
 
     if workers == 0 {
@@ -1029,24 +1077,24 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
     for rule in &lc.rules {
         if let Some(transitions) = &rule.transitions {
             for transition in transitions {
-                if let Some(storage_class) = &transition.storage_class {
-                    if storage_class.as_str() != "" {
-                        let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
-                        if !valid {
-                            return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
-                        }
+                if let Some(storage_class) = &transition.storage_class
+                    && storage_class.as_str() != ""
+                {
+                    let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
+                    if !valid {
+                        return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
                     }
                 }
             }
         }
         if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions {
             for noncurrent_version_transition in noncurrent_version_transitions {
-                if let Some(storage_class) = &noncurrent_version_transition.storage_class {
-                    if storage_class.as_str() != "" {
-                        let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
-                        if !valid {
-                            return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
-                        }
+                if let Some(storage_class) = &noncurrent_version_transition.storage_class
+                    && storage_class.as_str() != ""
+                {
+                    let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str());
+                    if !valid {
+                        return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS));
                     }
                 }
             }
@@ -1067,6 +1115,104 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
     }
 }
 
+pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
+    let Some(lifecycle) = GLOBAL_LifecycleSys.get(&oi.bucket).await else {
+        return;
+    };
+    let Some(api) = crate::new_object_layer_fn() else {
+        return;
+    };
+
+    let mut marker = None;
+    let mut version_marker = None;
+    let mut object_infos = Vec::new();
+
+    loop {
+        let Ok(page) = api
+            .clone()
+            .list_object_versions(&oi.bucket, &oi.name, marker.clone(), version_marker.clone(), None, 1000)
+            .await
+        else {
+            return;
+        };
+
+        object_infos.extend(page.objects.into_iter().filter(|object| object.name == oi.name));
+
+        if !page.is_truncated {
+            break;
+        }
+
+        marker = page.next_marker;
+        version_marker = page.next_version_idmarker;
+    }
+
+    if object_infos.is_empty() {
+        object_infos.push(oi.clone());
+    }
+
+    let lock_config = match metadata_sys::get_object_lock_config(&oi.bucket).await {
+        Ok((cfg, _)) => Some(Arc::new(cfg)),
+        Err(_) => None,
+    };
+    let replication = match metadata_sys::get_replication_config(&oi.bucket).await {
+        Ok((cfg, _)) if !cfg.rules.is_empty() => Some(Arc::new(ReplicationConfig::new(Some(cfg), None))),
+        _ => None,
+    };
+
+    let object_opts = object_infos
+        .iter()
+        .map(|object| object.to_lifecycle_opts())
+        .collect::<Vec<ObjectOpts>>();
+    let Ok(events) = Evaluator::new(Arc::new(lifecycle))
+        .with_lock_retention(lock_config)
+        .with_replication_config(replication)
+        .eval(&object_opts)
+        .await
+    else {
+        return;
+    };
+
+    let mut to_delete_objs = Vec::new();
+    let mut noncurrent_event = None;
+
+    for (object, event) in object_infos.iter().zip(events.iter()) {
+        if event.due != Some(OffsetDateTime::UNIX_EPOCH) {
+            continue;
+        }
+
+        match event.action {
+            IlmAction::DeleteAction
+            | IlmAction::DeleteRestoredAction
+            | IlmAction::DeleteRestoredVersionAction
+            | IlmAction::DeleteAllVersionsAction
+            | IlmAction::DelMarkerDeleteAllVersionsAction => {
+                apply_expiry_rule(event, &src, object).await;
+            }
+            IlmAction::DeleteVersionAction => {
+                to_delete_objs.push(ObjectToDelete {
+                    object_name: object.name.clone(),
+                    version_id: object.version_id,
+                    ..Default::default()
+                });
+                if noncurrent_event.is_none() {
+                    noncurrent_event = Some(event.clone());
+                }
+            }
+            _ => {}
+        }
+    }
+
+    if !to_delete_objs.is_empty()
+        && let Some(event) = noncurrent_event
+    {
+        GLOBAL_ExpiryState
+            .write()
+            .await
+            .enqueue_by_newer_noncurrent(&oi.bucket, to_delete_objs, event)
+            .await;
+    }
+}
+
 pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
     let Some(lc) = GLOBAL_LifecycleSys.get(bucket).await else {
         return Ok(());
@@ -1094,6 +1240,60 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket:
     }
 }
 
+pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
+    let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
+        return Ok(());
+    };
+    let lock_retention = metadata_sys::get_object_lock_config(bucket)
+        .await
+        .ok()
+        .and_then(|(cfg, _)| cfg.rule.and_then(|rule| rule.default_retention));
+    let replication_config = metadata_sys::get_replication_config(bucket).await.ok();
+    let mut marker = None;
+    let mut version_marker = None;
+    let src = LcEventSrc::Scanner;
+
+    loop {
+        let page = api
+            .clone()
+            .list_object_versions(bucket, "", marker.clone(), version_marker.clone(), None, 1000)
+            .await?;
+
+        for object in &page.objects {
+            let event = eval_action_from_lifecycle(&lc, lock_retention.clone(), replication_config.clone(), object).await;
+            match event.action {
+                IlmAction::DeleteAction
+                | IlmAction::DeleteVersionAction
+                | IlmAction::DeleteRestoredAction
+                | IlmAction::DeleteRestoredVersionAction
+                | IlmAction::DeleteAllVersionsAction
+                | IlmAction::DelMarkerDeleteAllVersionsAction => {
+                    if event
+                        .due
+                        .is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp())
+                    {
+                        if object.is_remote() {
+                            apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await;
+                        } else {
+                            apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await;
+                        }
+                    } else {
+                        apply_expiry_rule(&event, &src, object).await;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        if !page.is_truncated {
+            return Ok(());
+        }
+
+        marker = page.next_marker;
+        version_marker = page.next_version_idmarker;
+    }
+}
+
 async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) {
     let event = lc.eval(&oi.to_lifecycle_opts()).await;
     match event.action {
@@ -1154,6 +1354,8 @@ pub async fn expire_transitioned_object(
         }
     };
 
+    schedule_lifecycle_replication_delete_if_needed(oi).await;
+
     //defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
 
     let event_name = if oi.delete_marker {
@@ -1227,8 +1429,19 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
     result
 }
 
-pub fn audit_tier_actions(_api: ECStore, _tier: &str, _bytes: i64) -> TimeFn {
-    Arc::new(|| Box::pin(async move {}))
+pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
+    let tier = _tier.to_string();
+    Arc::new(move || {
+        let tier = tier.clone();
+        Box::pin(async move {
+            info!(
+                tier = %tier,
+                bytes = bytes,
+                "ILM tier transition audit: completed transition of {} bytes to tier '{}'",
+                bytes, tier
+            );
+        })
+    })
 }
 
 pub async fn get_transitioned_object_reader(
@@ -1245,11 +1458,11 @@ pub async fn get_transitioned_object_reader(
         Err(err) => return Err(std::io::Error::other(err)),
     };
 
-    let ret = new_getobjectreader(rs, &oi, opts, &h);
+    let ret = new_getobjectreader(rs, oi, opts, h);
     if let Err(err) = ret {
         return Err(error_resp_to_object_err(err, vec![bucket, object]));
     }
-    let (get_fn, off, length) = ret.expect("err");
+    let (get_fn, off, length) = ret.expect("get_transitioned_object_reader should succeed after error check");
     let mut gopts = WarmBackendGetOpts::default();
 
     if off >= 0 && length >= 0 {
@@ -1269,8 +1482,8 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
     let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
     let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
     let vid = version_id.trim();
-    if vid != "" && vid != NULL_VERSION_ID {
-        if let Err(err) = Uuid::parse_str(vid) {
+    if !vid.is_empty() && vid != NULL_VERSION_ID {
+        if let Err(_err) = Uuid::parse_str(vid) {
             return Err(std::io::Error::other(
                 StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
             ));
@@ -1326,30 +1539,27 @@ pub async fn put_restore_opts(
             if !strings_has_prefix_fold(&v.name.clone().unwrap(), "x-amz-meta") {
                 meta.insert(
                     format!("x-amz-meta-{}", v.name.as_ref().unwrap()),
-                    v.value.clone().unwrap_or("".to_string()),
+                    v.value.clone().unwrap_or_else(|| "".to_string()),
                 );
                 continue;
             }
-            meta.insert(v.name.clone().unwrap(), v.value.clone().unwrap_or("".to_string()));
+            meta.insert(v.name.clone().unwrap(), v.value.clone().unwrap_or_else(|| "".to_string()));
         }
-        if let Some(output_location) = rreq.output_location.as_ref() {
-            if let Some(s3) = &output_location.s3 {
-                if let Some(tags) = &s3.tagging {
-                    meta.insert(
-                        AMZ_OBJECT_TAGGING.to_string(),
-                        serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or("".to_string()),
-                    );
-                }
-            }
+        if let Some(output_location) = rreq.output_location.as_ref()
+            && let Some(s3) = &output_location.s3
+            && let Some(tags) = &s3.tagging
+        {
+            meta.insert(
+                AMZ_OBJECT_TAGGING.to_string(),
+                serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
+            );
         }
-        if let Some(output_location) = rreq.output_location.as_ref() {
-            if let Some(s3) = &output_location.s3 {
-                if let Some(encryption) = &s3.encryption {
-                    if encryption.encryption_type.as_str() != "" {
-                        meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
-                    }
-                }
-            }
+        if let Some(output_location) = rreq.output_location.as_ref()
+            && let Some(s3) = &output_location.s3
+            && let Some(encryption) = &s3.encryption
+            && encryption.encryption_type.as_str() != ""
+        {
+            meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
         }
         return Ok(ObjectOptions {
             versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
@@ -1361,7 +1571,7 @@ pub async fn put_restore_opts(
     for (k, v) in &oi.user_defined {
         meta.insert(k.to_string(), v.clone());
     }
-    if oi.user_tags.len() != 0 {
+    if !oi.user_tags.is_empty() {
         meta.insert(AMZ_OBJECT_TAGGING.to_string(), oi.user_tags.clone());
     }
     let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
@@ -1394,7 +1604,7 @@ impl LifecycleOps for ObjectInfo {
         lifecycle::ObjectOpts {
             name: self.name.clone(),
             user_tags: self.user_tags.clone(),
-            version_id: self.version_id.clone(),
+            version_id: self.version_id,
             mod_time: self.mod_time,
             size: self.size as usize,
             is_latest: self.is_latest,
@@ -1421,38 +1631,42 @@ pub trait RestoreRequestOps {
 }
 
 impl RestoreRequestOps for RestoreRequest {
-    fn validate(&self, api: Arc<ECStore>) -> Result<(), std::io::Error> {
-        /*if self.type_.is_none() && self.select_parameters.is_some() {
+    fn validate(&self, _api: Arc<ECStore>) -> Result<(), std::io::Error> {
+        // SELECT type requires select_parameters, and vice versa
+        if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() {
             return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
         }
-        if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.select_parameters.is_none() {
+        if let Some(type_) = &self.type_
+            && type_.as_str() == RestoreRequestType::SELECT
+            && self.select_parameters.is_none()
+        {
             return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
         }
 
-        if self.type_.is_none() && self.output_location.is_some() {
-            return Err(std::io::Error::other("OutputLocation required only for SELECT request type"));
+        // OutputLocation is only valid for SELECT requests
+        if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() {
+            return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type"));
         }
-        if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.output_location.is_none() {
+        if let Some(type_) = &self.type_
+            && type_.as_str() == RestoreRequestType::SELECT
+            && self.output_location.is_none()
+        {
             return Err(std::io::Error::other("OutputLocation required for SELECT requests"));
         }
 
-        if let Some(type_) = self.type_ && type_ == RestoreRequestType::SELECT && self.days != 0 {
+        // Days must not be specified with SELECT requests
+        if let Some(type_) = &self.type_
+            && type_.as_str() == RestoreRequestType::SELECT
+            && self.days.is_some_and(|d| d > 0)
+        {
             return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
         }
-        if self.days == 0 && self.type_.is_none() {
+
+        // For non-SELECT requests, days must be at least 1
+        if self.type_.is_none() && self.days.is_none_or(|d| d <= 0) {
             return Err(std::io::Error::other("restoration days should be at least 1"));
         }
-        if self.output_location.is_some() {
-            if _, err := api.get_bucket_info(self.output_location.s3.bucket_name, BucketOptions{}); err != nil {
-                return err
-            }
-            if self.output_location.s3.prefix == "" {
-                return Err(std::io::Error::other("Prefix is a required parameter in OutputLocation"));
-            }
-            if self.output_location.s3.encryption.encryption_type.as_str() != ServerSideEncryption::AES256 {
-                return NotImplemented{}
-            }
-        }*/
+
         Ok(())
     }
 }
@@ -1473,10 +1687,8 @@ pub async fn eval_action_from_lifecycle(
     let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false };
 
     match event.action {
-        IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
-            if lock_enabled {
-                return lifecycle::Event::default();
-            }
+        IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction if lock_enabled => {
+            return lifecycle::Event::default();
         }
         IlmAction::DeleteVersionAction | IlmAction::DeleteRestoredVersionAction => {
             if oi.version_id.is_none() {
@@ -1497,10 +1709,10 @@ pub async fn eval_action_from_lifecycle(
                 //}
                 return lifecycle::Event::default();
             }
-            if let Some(rcfg) = rcfg {
-                if rep_has_active_rules(&rcfg.0, &oi.name, true) {
-                    return lifecycle::Event::default();
-                }
+            if let Some(rcfg) = rcfg
+                && rep_has_active_rules(&rcfg.0, &oi.name, true)
+            {
+                return lifecycle::Event::default();
             }
         }
         _ => (),
@@ -1566,6 +1778,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
             return false;
         }
     };
+    schedule_lifecycle_replication_delete_if_needed(oi).await;
     //debug!("dobj: {:?}", dobj);
     if dobj.name.is_empty() {
         dobj = oi.clone();
@@ -1606,6 +1819,70 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
     true
 }
 
+async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo) {
+    if !oi.delete_marker || oi.version_id.is_none() {
+        return;
+    }
+
+    let replication_state = lifecycle_delete_replication_state(oi).await;
+    if replication_state.is_none() {
+        return;
+    }
+
+    schedule_replication_delete(DeletedObjectReplicationInfo {
+        delete_object: crate::store_api::DeletedObject {
+            object_name: oi.name.clone(),
+            delete_marker_version_id: oi.version_id,
+            delete_marker: false,
+            delete_marker_mtime: oi.mod_time,
+            replication_state,
+            ..Default::default()
+        },
+        bucket: oi.bucket.clone(),
+        event_type: REPLICATE_INCOMING_DELETE.to_string(),
+        ..Default::default()
+    })
+    .await;
+}
+
+async fn lifecycle_delete_replication_state(oi: &ObjectInfo) -> Option<ReplicationState> {
+    if !oi.replication_decision.is_empty() || oi.version_purge_status == VersionPurgeStatusType::Pending {
+        return Some(oi.replication_state());
+    }
+
+    let dsc = check_replicate_delete(
+        &oi.bucket,
+        &ObjectToDelete {
+            object_name: oi.name.clone(),
+            version_id: oi.version_id,
+            ..Default::default()
+        },
+        oi,
+        &ObjectOptions {
+            version_id: oi.version_id.map(|v| v.to_string()),
+            versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
+            ..Default::default()
+        },
+        None,
+    )
+    .await;
+    if !dsc.replicate_any() {
+        return None;
+    }
+
+    Some(replication_state_for_version_delete(dsc))
+}
+
+fn replication_state_for_version_delete(dsc: ReplicateDecision) -> ReplicationState {
+    let pending_status = dsc.pending_status();
+    ReplicationState {
+        replicate_decision_str: dsc.to_string(),
+        version_purge_status_internal: pending_status.clone(),
+        purge_targets: rustfs_filemeta::version_purge_statuses_map(pending_status.as_deref().unwrap_or_default()),
+        ..Default::default()
+    }
+}
+
 pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
     let mut success = false;
     match event.action {

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -14,8 +14,9 @@
 
 pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
+pub mod core;
 pub mod evaluator;
-pub mod lifecycle;
+pub use self::core as lifecycle;
 pub mod rule;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -109,10 +109,19 @@ impl ObjSweeper {
     }
 
     pub async fn sweep(&self) {
-        let je = self.should_remove_remote_object();
-        if !je.is_none() {
-            let mut expiry_state = GLOBAL_ExpiryState.write().await;
-            expiry_state.enqueue_tier_journal_entry(&je.expect("err!"));
+        let Some(je) = self.should_remove_remote_object() else {
+            return;
+        };
+        let hash = je.op_hash();
+        // Grab the sender under a short read lock, then release the lock so we
+        // don't hold it across the async send.
+        let wrkr = GLOBAL_ExpiryState.read().await.get_worker_ch(hash);
+        let Some(wrkr) = wrkr else {
+            GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
+            return;
+        };
+        if wrkr.send(Some(Box::new(je))).await.is_err() {
+            GLOBAL_ExpiryState.write().await.increment_missed_tier_journal_tasks();
         }
     }
 }

crates/ecstore/src/bucket/replication/config.rs

@@ -153,6 +153,11 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
 
             if obj.op_type == ReplicationType::Delete {
                 if obj.version_id.is_some() {
+                    if obj.delete_marker {
+                        return rule.delete_marker_replication.clone().is_some_and(|d| {
+                            d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
+                        });
+                    }
                     return rule
                         .delete_replication
                         .clone()

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -824,10 +824,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
         let cancel_token = CancellationToken::new();
         resyncer.register_cancel_token(&opts, cancel_token.clone()).await;
         tokio::spawn(async move {
-            resyncer
-                .clone()
-                .resync_bucket(cancel_token, storage, false, opts.clone())
-                .await;
+            Box::pin(resyncer.clone().resync_bucket(cancel_token, storage, false, opts.clone())).await;
             resyncer.clear_cancel_token(&opts).await;
         });
 
@@ -914,7 +911,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
                         };
                         tokio::spawn(async move {
                             resync.register_cancel_token(&opts, ctx.clone()).await;
-                            resync.clone().resync_bucket(ctx, storage, true, opts.clone()).await;
+                            Box::pin(resync.clone().resync_bucket(ctx, storage, true, opts.clone())).await;
                             resync.clear_cancel_token(&opts).await;
                         });
                     }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -34,7 +34,7 @@ use crate::global::get_global_bucket_monitor;
 use crate::set_disk::get_lock_acquire_timeout;
 use crate::store_api::{DeletedObject, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
-use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedPart, ObjectLockLegalHoldStatus};
@@ -86,6 +86,7 @@ use tokio::time::Duration as TokioDuration;
 use tokio_util::io::ReaderStream;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, instrument, warn};
+use uuid::Uuid;
 
 pub(crate) const REPLICATION_DIR: &str = ".replication";
 pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -1497,6 +1498,54 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
         }
     };
 
+    if dobj.delete_object.delete_marker
+        && let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
+    {
+        let source_marker_state = storage
+            .get_object_info(
+                &bucket,
+                &dobj.delete_object.object_name,
+                &ObjectOptions {
+                    version_id: Some(delete_marker_version_id.to_string()),
+                    versioned: BucketVersioningSys::prefix_enabled(&bucket, &dobj.delete_object.object_name).await,
+                    version_suspended: BucketVersioningSys::prefix_suspended(&bucket, &dobj.delete_object.object_name).await,
+                    ..Default::default()
+                },
+            )
+            .await;
+
+        match source_marker_state {
+            Ok(info) if info.delete_marker && info.version_id == Some(delete_marker_version_id) => {}
+            Ok(_) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    "skipping stale delete-marker replication because source version is no longer a delete marker"
+                );
+                return;
+            }
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    "skipping stale delete-marker replication because source version no longer exists"
+                );
+                return;
+            }
+            Err(err) => {
+                warn!(
+                    bucket,
+                    object = dobj.delete_object.object_name,
+                    version_id = %delete_marker_version_id,
+                    error = %err,
+                    "failed to verify source delete-marker state before replication"
+                );
+            }
+        }
+    }
+
     let dsc = match parse_replicate_decision(
         &bucket,
         &dobj
@@ -1529,7 +1578,6 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
             return;
         }
     };
-
     let ns_lock = match storage
         .new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
         .await
@@ -1653,7 +1701,33 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
         }
     }
 
-    let (replication_status, prev_status) = if dobj.delete_object.version_id.is_none() {
+    let is_version_purge = is_version_delete_replication(&dobj.delete_object);
+
+    if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
+        let bucket_clone = bucket.clone();
+        let dobj_clone = dobj.clone();
+        let dsc_clone = dsc.clone();
+        let storage_clone = storage.clone();
+        tokio::spawn(async move {
+            for _ in 0..5 {
+                if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
+                    && source_delete_marker_missing(
+                        &*storage_clone,
+                        &bucket_clone,
+                        &dobj_clone.delete_object.object_name,
+                        delete_marker_version_id,
+                    )
+                    .await
+                {
+                    replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
+                    break;
+                }
+                tokio::time::sleep(TokioDuration::from_secs(1)).await;
+            }
+        });
+    }
+
+    let (replication_status, prev_status) = if !is_version_purge {
         (
             rinfos.replication_status(),
             dobj.delete_object
@@ -1741,6 +1815,65 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
     }
 }
 
+async fn source_delete_marker_missing<S: StorageAPI>(
+    storage: &S,
+    bucket: &str,
+    object_name: &str,
+    delete_marker_version_id: Uuid,
+) -> bool {
+    match storage
+        .get_object_info(
+            bucket,
+            object_name,
+            &ObjectOptions {
+                version_id: Some(delete_marker_version_id.to_string()),
+                versioned: BucketVersioningSys::prefix_enabled(bucket, object_name).await,
+                version_suspended: BucketVersioningSys::prefix_suspended(bucket, object_name).await,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(info) => !info.delete_marker || info.version_id != Some(delete_marker_version_id),
+        Err(err) => is_err_object_not_found(&err) || is_err_version_not_found(&err),
+    }
+}
+
+async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
+    let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
+        return;
+    };
+
+    for tgt_entry in dsc.targets_map.values() {
+        if !tgt_entry.replicate {
+            continue;
+        }
+        if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
+            continue;
+        }
+        let Some(tgt_client) = BucketTargetSys::get().get_remote_target_client(bucket, &tgt_entry.arn).await else {
+            continue;
+        };
+
+        let _ = tgt_client
+            .remove_object(
+                &tgt_client.bucket,
+                &dobj.delete_object.object_name,
+                Some(delete_marker_version_id.to_string()),
+                RemoveObjectOptions {
+                    force_delete: false,
+                    governance_bypass: false,
+                    replication_delete_marker: false,
+                    replication_mtime: dobj.delete_object.delete_marker_mtime,
+                    replication_status: ReplicationStatusType::Replica,
+                    replication_request: true,
+                    replication_validity_check: false,
+                },
+            )
+            .await;
+    }
+}
+
 async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
     let bucket = &dobj.bucket;
     let object_name = &dobj.delete_object.object_name;
@@ -1924,6 +2057,14 @@ async fn replicate_force_delete_to_targets<S: StorageAPI>(dobj: &DeletedObjectRe
     }
 }
 
+fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
+    dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
+}
+
+fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
+    !is_not_found && !matches!(code, Some("MethodNotAllowed" | "405"))
+}
+
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
     let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
         version_id.to_owned()
@@ -1941,7 +2082,8 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
     rinfo.endpoint = tgt_client.endpoint.clone();
     rinfo.secure = tgt_client.secure;
 
-    if dobj.delete_object.version_id.is_none()
+    let is_version_purge = is_version_delete_replication(&dobj.delete_object);
+    if !is_version_purge
         && rinfo.prev_replication_status == ReplicationStatusType::Completed
         && dobj.op_type != ReplicationType::ExistingObject
     {
@@ -1949,12 +2091,12 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         return rinfo;
     }
 
-    if dobj.delete_object.version_id.is_some() && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
+    if is_version_purge && rinfo.version_purge_status == VersionPurgeStatusType::Complete {
         return rinfo;
     }
 
     if BucketTargetSys::get().is_offline(&tgt_client.to_url()).await {
-        if dobj.delete_object.version_id.is_none() {
+        if !is_version_purge {
             rinfo.replication_status = ReplicationStatusType::Failed;
         } else {
             rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -1968,18 +2110,29 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         Some(version_id.to_string())
     };
 
-    if dobj.delete_object.delete_marker_version_id.is_some()
-        && let Err(e) = tgt_client
+    if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
+        match tgt_client
             .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
             .await
-        && let SdkError::ServiceError(service_err) = &e
-        && !service_err.err().is_not_found()
-    {
-        rinfo.replication_status = ReplicationStatusType::Failed;
-        rinfo.error = Some(e.to_string());
-
-        return rinfo;
-    };
+        {
+            Ok(_) => {}
+            Err(e) => {
+                let non_retryable = matches!(
+                    &e,
+                    SdkError::ServiceError(service_err)
+                        if is_retryable_delete_replication_head_error(
+                            service_err.err().is_not_found(),
+                            service_err.err().code(),
+                        )
+                );
+                if non_retryable {
+                    rinfo.replication_status = ReplicationStatusType::Failed;
+                    rinfo.error = Some(e.to_string());
+                    return rinfo;
+                }
+            }
+        }
+    }
 
     match tgt_client
         .remove_object(
@@ -1989,7 +2142,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
             RemoveObjectOptions {
                 force_delete: false,
                 governance_bypass: false,
-                replication_delete_marker: dobj.delete_object.delete_marker_version_id.is_some(),
+                replication_delete_marker: dobj.delete_object.delete_marker,
                 replication_mtime: dobj.delete_object.delete_marker_mtime,
                 replication_status: ReplicationStatusType::Replica,
                 replication_request: true,
@@ -1999,7 +2152,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         .await
     {
         Ok(_) => {
-            if dobj.delete_object.version_id.is_none() {
+            if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Completed;
             } else {
                 rinfo.version_purge_status = VersionPurgeStatusType::Complete;
@@ -2007,7 +2160,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         }
         Err(e) => {
             rinfo.error = Some(e.to_string());
-            if dobj.delete_object.version_id.is_none() {
+            if !is_version_purge {
                 rinfo.replication_status = ReplicationStatusType::Failed;
             } else {
                 rinfo.version_purge_status = VersionPurgeStatusType::Failed;
@@ -3450,6 +3603,54 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_is_version_delete_replication_for_delete_marker_version_purge() {
+        let dobj = DeletedObject {
+            delete_marker: false,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            is_version_delete_replication(&dobj),
+            "delete-marker version purges must be tracked as version purge replication, not delete-marker creation replication"
+        );
+    }
+
+    #[test]
+    fn test_is_version_delete_replication_for_delete_marker_creation() {
+        let dobj = DeletedObject {
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+
+        assert!(
+            !is_version_delete_replication(&dobj),
+            "delete-marker creation should remain on the delete-marker replication path"
+        );
+    }
+
+    #[test]
+    fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
+        assert!(
+            !is_retryable_delete_replication_head_error(false, Some("405")),
+            "numeric 405 responses should not block delete-marker purge replication"
+        );
+        assert!(
+            !is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")),
+            "MethodNotAllowed responses should not block delete-marker purge replication"
+        );
+        assert!(
+            !is_retryable_delete_replication_head_error(true, Some("NoSuchKey")),
+            "not-found responses should not block delete-marker purge replication"
+        );
+        assert!(
+            is_retryable_delete_replication_head_error(false, Some("AccessDenied")),
+            "unexpected head errors should still fail fast"
+        );
+    }
+
     #[tokio::test]
     async fn test_get_heal_replicate_object_info_failed_object_returns_heal_roi() {
         let oi = ObjectInfo {

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -832,61 +832,55 @@ impl ReplicationStats {
         let mut rs = ReplStat::new();
 
         match status {
-            ReplicationStatusType::Pending => {
-                if ri.op_type.is_data_replication() && prev_status != status {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        Duration::default(),
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Pending if ri.op_type.is_data_replication() && prev_status != status => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    Duration::default(),
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Completed => {
-                if ri.op_type.is_data_replication() {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        ri.duration,
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Completed if ri.op_type.is_data_replication() => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    ri.duration,
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Failed => {
-                if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        ri.duration,
-                        status,
-                        ri.op_type,
-                        ri.endpoint.clone(),
-                        ri.secure,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Failed
+                if ri.op_type.is_data_replication() && prev_status == ReplicationStatusType::Pending =>
+            {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    ri.duration,
+                    status,
+                    ri.op_type,
+                    ri.endpoint.clone(),
+                    ri.secure,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
-            ReplicationStatusType::Replica => {
-                if ri.op_type == ReplicationType::Object {
-                    rs.set(
-                        ri.arn.clone(),
-                        ri.size,
-                        Duration::default(),
-                        status,
-                        ri.op_type,
-                        String::new(),
-                        false,
-                        ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
-                    );
-                }
+            ReplicationStatusType::Replica if ri.op_type == ReplicationType::Object => {
+                rs.set(
+                    ri.arn.clone(),
+                    ri.size,
+                    Duration::default(),
+                    status,
+                    ri.op_type,
+                    String::new(),
+                    false,
+                    ri.error.as_ref().map(|e| crate::error::Error::other(e.clone())),
+                );
             }
             _ => {}
         }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -12,12 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::disk_store::get_drive_walkdir_stall_timeout;
 use crate::disk::error::DiskError;
 use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
 use futures::future::join_all;
+use metrics::counter;
 use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
-use std::{future::Future, pin::Pin};
+use std::{future::Future, pin::Pin, time::Duration};
+use tokio::io::AsyncRead;
 use tokio::spawn;
+use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -26,6 +30,21 @@ pub type PartialFn =
     Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 
+#[derive(Debug)]
+enum PeekOutcome {
+    Ready(Option<MetaCacheEntry>),
+    Error(rustfs_filemeta::Error),
+    TimedOut,
+}
+
+async fn peek_with_timeout<R: AsyncRead + Unpin>(reader: &mut MetacacheReader<R>, timeout_duration: Duration) -> PeekOutcome {
+    match timeout(timeout_duration, reader.peek()).await {
+        Ok(Ok(entry)) => PeekOutcome::Ready(entry),
+        Ok(Err(err)) => PeekOutcome::Error(err),
+        Err(_) => PeekOutcome::TimedOut,
+    }
+}
+
 #[derive(Default)]
 pub struct ListPathRawOptions {
     pub disks: Vec<Option<DiskStore>>,
@@ -160,6 +179,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     }
 
     let revjob = spawn(async move {
+        let peek_timeout = get_drive_walkdir_stall_timeout();
         let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
         for _ in 0..readers.len() {
             errs.push(None);
@@ -191,8 +211,8 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                     continue;
                 }
 
-                let entry = match r.peek().await {
-                    Ok(res) => {
+                let entry = match peek_with_timeout(r, peek_timeout).await {
+                    PeekOutcome::Ready(res) => {
                         if let Some(entry) = res {
                             // info!("read entry disk: {}, name: {}", i, entry.name);
                             entry
@@ -203,7 +223,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
-                    Err(err) => {
+                    PeekOutcome::Error(err) => {
                         if err == rustfs_filemeta::Error::Unexpected {
                             at_eof += 1;
                             // warn!("list_path_raw: peek err eof, disk: {}", i);
@@ -236,6 +256,31 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
+                    PeekOutcome::TimedOut => {
+                        has_err += 1;
+                        errs[i] = Some(DiskError::Timeout);
+                        let endpoint = opts
+                            .disks
+                            .get(i)
+                            .and_then(|disk| disk.as_ref().map(|disk| disk.endpoint().to_string()))
+                            .unwrap_or_else(|| "missing".to_string());
+                        counter!(
+                            "rustfs_list_path_raw_stall_total",
+                            "drive" => endpoint.clone()
+                        )
+                        .increment(1);
+                        warn!(
+                            drive = %endpoint,
+                            bucket = %opts.bucket,
+                            path = %opts.path,
+                            timeout_ms = peek_timeout.as_millis(),
+                            "list_path_raw reader peek timed out; excluding drive from current merge"
+                        );
+                        let (detached_rd, write_half) = tokio::io::duplex(1);
+                        drop(write_half);
+                        *r = MetacacheReader::new(detached_rd);
+                        continue;
+                    }
                 };
 
                 // warn!("list_path_raw: loop entry: {:?}, disk: {}", &entry.name, i);
@@ -364,3 +409,42 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     // warn!("list_path_raw: done");
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_filemeta::MetacacheWriter;
+
+    #[tokio::test]
+    async fn peek_with_timeout_times_out_on_silent_reader() {
+        let (_writer, reader) = tokio::io::duplex(64);
+        let mut reader = MetacacheReader::new(reader);
+
+        let outcome = peek_with_timeout(&mut reader, Duration::from_millis(20)).await;
+        assert!(matches!(outcome, PeekOutcome::TimedOut));
+    }
+
+    #[tokio::test]
+    async fn peek_with_timeout_reads_entry_before_deadline() {
+        let (reader, writer) = tokio::io::duplex(256);
+        let mut metacache_reader = MetacacheReader::new(reader);
+
+        tokio::spawn(async move {
+            let mut writer = MetacacheWriter::new(writer);
+            let entry = MetaCacheEntry {
+                name: "bucket/object".to_string(),
+                metadata: vec![1, 2, 3],
+                cached: None,
+                reusable: false,
+            };
+            writer.write(&[entry]).await.expect("entry should be written");
+            writer.close().await.expect("writer should close");
+        });
+
+        let outcome = peek_with_timeout(&mut metacache_reader, Duration::from_secs(1)).await;
+        match outcome {
+            PeekOutcome::Ready(Some(entry)) => assert_eq!(entry.name, "bucket/object"),
+            other => panic!("expected ready entry, got {other:?}"),
+        }
+    }
+}

crates/ecstore/src/client/object_handlers_common.rs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+use tracing::warn;
+
 use crate::bucket::lifecycle::lifecycle;
 use crate::bucket::versioning::VersioningApi;
 use crate::bucket::versioning_sys::BucketVersioningSys;
@@ -19,7 +22,14 @@ use crate::store::ECStore;
 use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
 use rustfs_lock::MAX_DELETE_LIST;
 
-pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
+pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
+    let version_suspended = match BucketVersioningSys::get(bucket).await {
+        Ok(vc) => vc.suspended(),
+        Err(err) => {
+            warn!(bucket, error = ?err, "failed to get versioning config during lifecycle noncurrent version cleanup");
+            return;
+        }
+    };
     let mut remaining = to_del;
     loop {
         let mut to_del = remaining;
@@ -29,15 +39,29 @@ pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[Object
         } else {
             remaining = &[];
         }
-        let vc = BucketVersioningSys::get(bucket).await.expect("err!");
-        let _deleted_objs = api.delete_objects(
-            bucket,
-            to_del.to_vec(),
-            ObjectOptions {
-                //prefix_enabled_fn:  vc.prefix_enabled(""),
-                version_suspended: vc.suspended(),
-                ..Default::default()
-            },
-        );
+        let (_deleted_objs, errors) = api
+            .delete_objects(
+                bucket,
+                to_del.to_vec(),
+                ObjectOptions {
+                    version_suspended,
+                    ..Default::default()
+                },
+            )
+            .await;
+        for (i, err) in errors.iter().enumerate() {
+            if let Some(e) = err {
+                let obj_name = to_del.get(i).map(|o| o.object_name.as_str()).unwrap_or("<unknown>");
+                let vid = to_del
+                    .get(i)
+                    .and_then(|o| o.version_id)
+                    .map(|v| v.to_string())
+                    .unwrap_or_default();
+                warn!(bucket, object = obj_name, version_id = %vid, error = ?e, "failed to delete noncurrent version during lifecycle cleanup");
+            }
+        }
+        if remaining.is_empty() {
+            break;
+        }
     }
 }

crates/ecstore/src/config/com.rs

@@ -537,7 +537,7 @@ fn build_oidc_object(cfg: &Config) -> Map<String, Value> {
     };
 
     let mut providers = subsystem.iter().collect::<Vec<_>>();
-    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    providers.sort_by_key(|(lhs, _)| *lhs);
 
     let mut oidc_obj = Map::new();
     for (instance_key, kvs) in providers {
@@ -572,7 +572,7 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     };
 
     let mut providers = subsystem.iter().collect::<Vec<_>>();
-    providers.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    providers.sort_by_key(|(lhs, _)| *lhs);
 
     let mut oidc_obj = Map::new();
     for (instance_key, kvs) in providers {
@@ -691,7 +691,7 @@ fn build_notify_subsystem_object(
         .iter()
         .filter(|(instance_key, _)| instance_key.as_str() != DEFAULT_DELIMITER)
         .collect::<Vec<_>>();
-    instances.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+    instances.sort_by_key(|(lhs, _)| *lhs);
 
     for (instance_key, kvs) in instances {
         let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);

crates/ecstore/src/disk/disk_store.rs

@@ -16,10 +16,16 @@ use crate::disk::{
     CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
     FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo,
     WalkDirOptions,
+    health_state::{
+        RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
+        get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
+        record_drive_recovery_class, record_drive_runtime_state, record_drive_state_transition,
+    },
     local::{LocalDisk, ScanGuard},
 };
 use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
+use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use std::{
     path::PathBuf,
@@ -40,7 +46,6 @@ const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
 pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
 pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
@@ -51,9 +56,52 @@ lazy_static::lazy_static! {
 }
 
 pub fn get_max_timeout_duration() -> Duration {
-    std::env::var(ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION)
-        .map(|v| Duration::from_secs(v.parse::<u64>().unwrap_or(30)))
-        .unwrap_or(Duration::from_secs(30))
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION,
+        rustfs_config::DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS,
+    ))
+}
+
+fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
+    Duration::from_secs(
+        rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
+            .unwrap_or(default_secs),
+    )
+}
+
+pub fn get_drive_metadata_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_disk_info_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_list_dir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_stall_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+    )
 }
 
 /// DiskHealthTracker tracks the health status of a disk.
@@ -68,6 +116,16 @@ pub struct DiskHealthTracker {
     pub status: AtomicU32,
     /// Atomic number of waiting operations
     pub waiting: AtomicU32,
+    /// Runtime drive health state
+    pub runtime_state: AtomicU32,
+    /// Consecutive failures while transitioning away from online
+    pub consecutive_failures: AtomicU32,
+    /// Consecutive successes while returning online
+    pub consecutive_successes: AtomicU32,
+    /// When the drive first left the online state
+    pub offline_since_unix_secs: AtomicI64,
+    /// Last runtime state transition timestamp
+    pub last_transition_unix_secs: AtomicI64,
 }
 
 impl DiskHealthTracker {
@@ -83,6 +141,11 @@ impl DiskHealthTracker {
             last_started: AtomicI64::new(now),
             status: AtomicU32::new(DISK_HEALTH_OK),
             waiting: AtomicU32::new(0),
+            runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
+            consecutive_failures: AtomicU32::new(0),
+            consecutive_successes: AtomicU32::new(0),
+            offline_since_unix_secs: AtomicI64::new(0),
+            last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
         }
     }
 
@@ -110,12 +173,139 @@ impl DiskHealthTracker {
         self.status.store(DISK_HEALTH_OK, Ordering::Release);
     }
 
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.runtime_state.store(state as u32, Ordering::Release);
+        match state {
+            RuntimeDriveHealthState::Offline => self.set_faulty(),
+            RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => {
+                self.set_ok();
+            }
+        }
+    }
+
     pub fn swap_ok_to_faulty(&self) -> bool {
         self.status
             .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed)
             .is_ok()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire))
+    }
+
+    pub fn offline_duration(&self) -> Option<Duration> {
+        let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
+        if offline_since <= 0 {
+            return None;
+        }
+        let now = current_unix_secs();
+        Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
+    }
+
+    pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let now = current_unix_secs();
+        let next = match current {
+            RuntimeDriveHealthState::Online => {
+                self.consecutive_failures.store(1, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                self.offline_since_unix_secs
+                    .compare_exchange(0, now as i64, Ordering::AcqRel, Ordering::Relaxed)
+                    .ok();
+                RuntimeDriveHealthState::Suspect
+            }
+            RuntimeDriveHealthState::Suspect => {
+                let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
+                if failures >= get_drive_suspect_failure_threshold() {
+                    RuntimeDriveHealthState::Offline
+                } else {
+                    RuntimeDriveHealthState::Suspect
+                }
+            }
+            RuntimeDriveHealthState::Returning => {
+                self.consecutive_failures.store(0, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                RuntimeDriveHealthState::Offline
+            }
+            RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
+        };
+
+        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        self.transition_state(endpoint, current, next, reason);
+        current == RuntimeDriveHealthState::Online
+    }
+
+    pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let next = match current {
+            RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Suspect => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Offline => {
+                self.consecutive_successes.store(1, Ordering::Release);
+                RuntimeDriveHealthState::Returning
+            }
+            RuntimeDriveHealthState::Returning => {
+                let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1;
+                if successes >= get_drive_returning_success_threshold() {
+                    RuntimeDriveHealthState::Online
+                } else {
+                    RuntimeDriveHealthState::Returning
+                }
+            }
+        };
+
+        let became_online = next == RuntimeDriveHealthState::Online;
+        if became_online {
+            self.status.store(DISK_HEALTH_OK, Ordering::Release);
+            self.consecutive_failures.store(0, Ordering::Release);
+            self.consecutive_successes.store(0, Ordering::Release);
+        }
+        self.transition_state(endpoint, current, next, reason);
+        if became_online {
+            self.log_success();
+        }
+        became_online
+    }
+
+    fn transition_state(
+        &self,
+        endpoint: &Endpoint,
+        current: RuntimeDriveHealthState,
+        next: RuntimeDriveHealthState,
+        reason: &'static str,
+    ) {
+        if current == next {
+            return;
+        }
+
+        self.runtime_state.store(next as u32, Ordering::Release);
+        self.last_transition_unix_secs
+            .store(current_unix_secs() as i64, Ordering::Release);
+
+        if matches!(
+            next,
+            RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Offline | RuntimeDriveHealthState::Returning
+        ) && self.offline_since_unix_secs.load(Ordering::Acquire) == 0
+        {
+            self.offline_since_unix_secs
+                .store(current_unix_secs() as i64, Ordering::Release);
+        }
+
+        if next == RuntimeDriveHealthState::Online {
+            if let Some(duration) = self.offline_duration() {
+                record_drive_offline_duration(endpoint, duration);
+                record_drive_recovery_class(classify_drive_recovery(duration));
+            }
+            self.offline_since_unix_secs.store(0, Ordering::Release);
+        } else if let Some(duration) = self.offline_duration() {
+            record_drive_offline_duration(endpoint, duration);
+        }
+
+        record_drive_state_transition(endpoint, current, next, reason);
+        record_drive_runtime_state(endpoint, next);
+    }
+
     /// Increment waiting operations counter
     pub fn increment_waiting(&self) {
         self.waiting.fetch_add(1, Ordering::Relaxed);
@@ -137,6 +327,13 @@ impl DiskHealthTracker {
     }
 }
 
+fn current_unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_secs()
+}
+
 impl Default for DiskHealthTracker {
     fn default() -> Self {
         Self::new()
@@ -186,19 +383,34 @@ impl LocalDiskWrapper {
         let env_health_check =
             rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
 
-        Self {
+        let wrapper = Self {
             disk,
             health: Arc::new(DiskHealthTracker::new()),
             health_check: health_check && env_health_check,
             cancel_token: CancellationToken::new(),
             disk_id: Arc::new(RwLock::new(None)),
-        }
+        };
+        record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
+        wrapper
     }
 
     pub fn get_disk(&self) -> Arc<LocalDisk> {
         self.disk.clone()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        self.health.runtime_state()
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        self.health.offline_duration().map(|duration| duration.as_secs())
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.health.force_runtime_state_for_test(state);
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -219,6 +431,20 @@ impl LocalDiskWrapper {
         self.cancel_token.cancel();
     }
 
+    fn spawn_recovery_monitor_if_needed(&self) {
+        if !self.health_check {
+            return;
+        }
+
+        self.health.increment_waiting();
+        let health = Arc::clone(&self.health);
+        let disk = Arc::clone(&self.disk);
+        let cancel_token = self.cancel_token.clone();
+        tokio::spawn(async move {
+            Self::monitor_disk_status(disk, health, cancel_token).await;
+        });
+    }
+
     /// Monitor disk writability periodically
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         // TODO: config interval
@@ -256,7 +482,9 @@ impl LocalDiskWrapper {
 
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err() && health.swap_ok_to_faulty() {
+                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                        && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
+                    {
                         // Health check failed, disk is considered faulty
                         warn!("health check: failed, disk is considered faulty");
 
@@ -345,9 +573,9 @@ impl LocalDiskWrapper {
 
     /// Monitor disk status and try to bring it back online
     async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        const CHECK_EVERY: Duration = Duration::from_secs(5);
+        let check_every = get_drive_returning_probe_interval();
 
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(check_every);
 
         loop {
             tokio::select! {
@@ -362,12 +590,18 @@ impl LocalDiskWrapper {
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
                     match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
                         Ok(_) => {
+                            let state_before = health.runtime_state();
+                            let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
+                            info!("Disk {} recovery probe succeeded; state={:?}", disk.to_string(), state_before);
+                            if !is_online {
+                                continue;
+                            }
                             info!("Disk {} is back online", disk.to_string());
-                            health.set_ok();
                             health.decrement_waiting();
                             return;
                         }
                         Err(e) => {
+                            health.mark_failure(&disk.endpoint(), "recovery_probe_failed");
                             warn!("Disk {} still faulty: {:?}", disk.to_string(), e);
                         }
                     }
@@ -435,6 +669,19 @@ impl LocalDiskWrapper {
     /// Track disk health for an operation.
     /// This method should wrap disk operations to ensure health checking.
     pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        self.track_disk_health_with_op("unknown", operation, timeout_duration).await
+    }
+
+    pub async fn track_disk_health_with_op<T, F, Fut>(
+        &self,
+        op: &'static str,
+        operation: F,
+        timeout_duration: Duration,
+    ) -> Result<T>
     where
         F: FnOnce() -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
@@ -479,7 +726,21 @@ impl LocalDiskWrapper {
             Err(_) => {
                 // Timeout occurred, mark disk as potentially faulty and decrement waiting counter
                 self.health.decrement_waiting();
-                warn!("disk operation timeout after {:?}", timeout_duration);
+                if self.health.mark_failure(&self.endpoint(), "operation_timeout") {
+                    self.spawn_recovery_monitor_if_needed();
+                }
+                counter!(
+                    "rustfs_drive_op_timeout_total",
+                    "endpoint" => self.endpoint().to_string(),
+                    "op" => op.to_string()
+                )
+                .increment(1);
+                warn!(
+                    endpoint = %self.endpoint(),
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    "Local disk operation timed out"
+                );
                 Err(DiskError::other(format!("disk operation timeout after {timeout_duration:?}")))
             }
         }
@@ -489,8 +750,12 @@ impl LocalDiskWrapper {
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        self.track_disk_health(|| async { self.disk.read_metadata(volume, path).await }, Duration::ZERO)
-            .await
+        self.track_disk_health_with_op(
+            "read_metadata",
+            || async { self.disk.read_metadata(volume, path).await },
+            get_drive_metadata_timeout(),
+        )
+        .await
     }
 
     fn start_scan(&self) -> ScanGuard {
@@ -565,15 +830,22 @@ impl DiskAPI for LocalDiskWrapper {
             return Err(DiskError::FaultyDisk);
         }
 
-        let result = self.disk.disk_info(opts).await?;
+        self.track_disk_health_with_op(
+            "disk_info",
+            || async {
+                let result = self.disk.disk_info(opts).await?;
 
-        if let Some(current_disk_id) = *self.disk_id.read().await
-            && Some(current_disk_id) != result.id
-        {
-            return Err(DiskError::DiskNotFound);
-        };
+                if let Some(current_disk_id) = *self.disk_id.read().await
+                    && Some(current_disk_id) != result.id
+                {
+                    return Err(DiskError::DiskNotFound);
+                };
 
-        Ok(result)
+                Ok(result)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
     }
 
     async fn make_volume(&self, volume: &str) -> Result<()> {
@@ -587,7 +859,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
-        self.track_disk_health(|| async { self.disk.list_volumes().await }, Duration::ZERO)
+        self.track_disk_health_with_op("list_volumes", || async { self.disk.list_volumes().await }, Duration::ZERO)
             .await
     }
 
@@ -602,7 +874,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
-        self.track_disk_health(|| async { self.disk.walk_dir(opts, wr).await }, Duration::ZERO)
+        self.track_disk_health_with_op("walk_dir", || async { self.disk.walk_dir(opts, wr).await }, get_drive_walkdir_timeout())
             .await
     }
 
@@ -710,9 +982,10 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
-        self.track_disk_health(
+        self.track_disk_health_with_op(
+            "list_dir",
             || async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
-            get_max_timeout_duration(),
+            get_drive_list_dir_timeout(),
         )
         .await
     }
@@ -802,3 +1075,74 @@ impl DiskAPI for LocalDiskWrapper {
             .await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::endpoint::Endpoint;
+    use crate::disk::health_state::RuntimeDriveHealthState;
+
+    #[test]
+    fn drive_metadata_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
+                assert_eq!(
+                    get_drive_metadata_timeout(),
+                    Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(17));
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_prefers_canonical_over_legacy() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("7"), || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(7));
+            });
+        });
+    }
+
+    #[test]
+    fn runtime_state_transitions_from_online_to_suspect_then_offline() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+
+        assert!(!health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+        assert!(health.offline_duration().is_some());
+    }
+
+    #[test]
+    fn runtime_state_transitions_back_online_after_recovery_threshold() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        health.mark_failure(&endpoint, "timeout");
+        health.mark_failure(&endpoint, "timeout");
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.offline_duration().is_none());
+    }
+}

crates/ecstore/src/disk/health_state.rs

@@ -0,0 +1,237 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{DiskAPI, DiskStore};
+use crate::disk::endpoint::Endpoint;
+use metrics::{counter, gauge};
+use std::time::Duration;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u32)]
+pub enum RuntimeDriveHealthState {
+    Online = 0,
+    Suspect = 1,
+    Offline = 2,
+    Returning = 3,
+}
+
+impl RuntimeDriveHealthState {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Online => "online",
+            Self::Suspect => "suspect",
+            Self::Offline => "offline",
+            Self::Returning => "returning",
+        }
+    }
+
+    pub fn from_u32(value: u32) -> Self {
+        match value {
+            1 => Self::Suspect,
+            2 => Self::Offline,
+            3 => Self::Returning,
+            _ => Self::Online,
+        }
+    }
+
+    pub fn is_snapshot_eligible(self) -> bool {
+        matches!(self, Self::Online | Self::Suspect | Self::Returning)
+    }
+
+    pub fn is_strictly_online(self) -> bool {
+        matches!(self, Self::Online)
+    }
+
+    pub fn should_probe_for_admin(self) -> bool {
+        matches!(self, Self::Online | Self::Returning)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DriveRecoveryClass {
+    ShortOffline,
+    MediumOffline,
+    LongOffline,
+}
+
+impl DriveRecoveryClass {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::ShortOffline => "short_offline",
+            Self::MediumOffline => "medium_offline",
+            Self::LongOffline => "long_offline",
+        }
+    }
+}
+
+pub fn get_drive_suspect_failure_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_success_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_probe_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_offline_grace_period() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+        rustfs_config::DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+    ))
+}
+
+pub fn get_drive_long_offline_threshold() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+        rustfs_config::DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+    ))
+}
+
+pub fn classify_drive_recovery(duration: Duration) -> DriveRecoveryClass {
+    if duration <= get_drive_offline_grace_period() {
+        DriveRecoveryClass::ShortOffline
+    } else if duration >= get_drive_long_offline_threshold() {
+        DriveRecoveryClass::LongOffline
+    } else {
+        DriveRecoveryClass::MediumOffline
+    }
+}
+
+pub fn record_drive_runtime_state(endpoint: &Endpoint, state: RuntimeDriveHealthState) {
+    let endpoint_label = endpoint.to_string();
+    let pool_label = endpoint.pool_idx.to_string();
+    let set_label = endpoint.set_idx.to_string();
+    let disk_label = endpoint.disk_idx.to_string();
+
+    for candidate in [
+        RuntimeDriveHealthState::Online,
+        RuntimeDriveHealthState::Suspect,
+        RuntimeDriveHealthState::Offline,
+        RuntimeDriveHealthState::Returning,
+    ] {
+        gauge!(
+            "rustfs_drive_runtime_state",
+            "endpoint" => endpoint_label.clone(),
+            "pool" => pool_label.clone(),
+            "set" => set_label.clone(),
+            "disk" => disk_label.clone(),
+            "state" => candidate.as_str().to_string()
+        )
+        .set(if candidate == state { 1.0 } else { 0.0 });
+    }
+}
+
+pub fn record_drive_state_transition(
+    endpoint: &Endpoint,
+    from: RuntimeDriveHealthState,
+    to: RuntimeDriveHealthState,
+    reason: &'static str,
+) {
+    counter!(
+        "rustfs_drive_state_transition_total",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string(),
+        "from" => from.as_str().to_string(),
+        "to" => to.as_str().to_string(),
+        "reason" => reason.to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_recovery_class(class: DriveRecoveryClass) {
+    counter!(
+        "rustfs_drive_recovery_class_total",
+        "class" => class.as_str().to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_offline_duration(endpoint: &Endpoint, duration: Duration) {
+    gauge!(
+        "rustfs_drive_offline_duration_seconds",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string()
+    )
+    .set(duration.as_secs_f64());
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DriveMembershipSnapshot {
+    pub online: Vec<DiskStore>,
+    pub suspect: Vec<DiskStore>,
+    pub returning: Vec<DiskStore>,
+    pub offline: Vec<DiskStore>,
+}
+
+impl DriveMembershipSnapshot {
+    pub fn from_optional_disks(disks: &[Option<DiskStore>]) -> Self {
+        let mut snapshot = Self::default();
+
+        for disk in disks.iter().flatten() {
+            match disk.runtime_state() {
+                RuntimeDriveHealthState::Online => snapshot.online.push(disk.clone()),
+                RuntimeDriveHealthState::Suspect => snapshot.suspect.push(disk.clone()),
+                RuntimeDriveHealthState::Returning => snapshot.returning.push(disk.clone()),
+                RuntimeDriveHealthState::Offline => snapshot.offline.push(disk.clone()),
+            }
+        }
+
+        snapshot
+    }
+
+    pub fn scanner_heal_candidates(&self) -> Vec<DiskStore> {
+        let mut disks = Vec::with_capacity(self.online.len() + self.suspect.len() + self.returning.len());
+        disks.extend(self.online.iter().cloned());
+        disks.extend(self.suspect.iter().cloned());
+        disks.extend(self.returning.iter().cloned());
+        disks
+    }
+
+    pub fn strict_online_candidates(&self) -> Vec<DiskStore> {
+        self.online.clone()
+    }
+
+    pub fn strict_online_local_candidates(&self) -> Vec<DiskStore> {
+        self.online.iter().filter(|disk| disk.is_local()).cloned().collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn runtime_drive_health_state_snapshot_eligibility_matches_membership_policy() {
+        assert!(RuntimeDriveHealthState::Online.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Suspect.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Returning.is_snapshot_eligible());
+        assert!(!RuntimeDriveHealthState::Offline.is_snapshot_eligible());
+    }
+}

crates/ecstore/src/disk/local.rs

@@ -1538,7 +1538,7 @@ impl DiskAPI for LocalDisk {
                 volume,
                 path_join_buf(&[
                     path,
-                    &fi.data_dir.map_or("".to_string(), |dir| dir.to_string()),
+                    &fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()),
                     &format!("part.{}", part.number),
                 ])
                 .as_str(),
@@ -1587,7 +1587,7 @@ impl DiskAPI for LocalDisk {
                 self.get_object_path(
                     bucket,
                     path_join_buf(&[
-                        path.parent().unwrap_or(Path::new("")).to_string_lossy().as_ref(),
+                        path.parent().unwrap_or_else(|| Path::new("")).to_string_lossy().as_ref(),
                         &format!("part.{num}"),
                     ])
                     .as_str(),
@@ -1648,7 +1648,7 @@ impl DiskAPI for LocalDisk {
                 volume,
                 path_join_buf(&[
                     path,
-                    &fi.data_dir.map_or("".to_string(), |dir| dir.to_string()),
+                    &fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()),
                     &format!("part.{}", part.number),
                 ])
                 .as_str(),
@@ -2497,7 +2497,7 @@ impl DiskAPI for LocalDisk {
                     let part_path = format!("part.{}", part.number);
                     let part_path = path_join_buf(&[
                         path,
-                        fi.data_dir.map_or("".to_string(), |dir| dir.to_string()).as_str(),
+                        fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(),
                         part_path.as_str(),
                     ]);
                     let part_path = self.get_object_path(volume, part_path.as_str())?;
@@ -2514,7 +2514,7 @@ impl DiskAPI for LocalDisk {
             if inline && fi.shard_file_size(fi.parts[0].actual_size) < DEFAULT_INLINE_BLOCK as i64 {
                 let part_path = path_join_buf(&[
                     path,
-                    fi.data_dir.map_or("".to_string(), |dir| dir.to_string()).as_str(),
+                    fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(),
                     format!("part.{}", fi.parts[0].number).as_str(),
                 ]);
                 let part_path = self.get_object_path(volume, part_path.as_str())?;
@@ -2838,6 +2838,32 @@ mod test {
         assert!(entries.next_entry().await.unwrap().is_some());
     }
 
+    #[tokio::test]
+    async fn cleanup_stale_tmp_objects_keeps_fresh_dirs_and_regular_files() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().unwrap();
+        let tmp = LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_BUCKET);
+        let fresh_dir = tmp.join("fresh").join("data");
+        let regular_file = tmp.join("note.txt");
+        let trash = LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_DELETED_BUCKET);
+
+        fs::create_dir_all(fresh_dir.parent().unwrap()).await.unwrap();
+        fs::create_dir_all(&trash).await.unwrap();
+        fs::write(&fresh_dir, b"temporary").await.unwrap();
+        fs::write(&regular_file, b"keep").await.unwrap();
+
+        LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), Duration::from_secs(60))
+            .await
+            .unwrap();
+
+        assert!(tmp.join("fresh").exists());
+        assert!(regular_file.exists());
+
+        let mut entries = fs::read_dir(&trash).await.unwrap();
+        assert!(entries.next_entry().await.unwrap().is_none());
+    }
+
     #[tokio::test]
     async fn test_scan_dir_includes_nested_object_dirs() {
         use rustfs_filemeta::MetacacheReader;

crates/ecstore/src/disk/mod.rs

@@ -19,6 +19,7 @@ pub mod error_conv;
 pub mod error_reduce;
 pub mod format;
 pub mod fs;
+pub mod health_state;
 pub mod local;
 pub mod os;
 
@@ -33,6 +34,7 @@ pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
 use crate::disk::disk_store::LocalDiskWrapper;
+use crate::disk::health_state::RuntimeDriveHealthState;
 use crate::disk::local::ScanGuard;
 use crate::rpc::RemoteDisk;
 use bytes::Bytes;
@@ -410,6 +412,30 @@ impl DiskAPI for Disk {
     }
 }
 
+impl Disk {
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        match self {
+            Disk::Local(local_disk) => local_disk.runtime_state(),
+            Disk::Remote(remote_disk) => remote_disk.runtime_state(),
+        }
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        match self {
+            Disk::Local(local_disk) => local_disk.offline_duration_secs(),
+            Disk::Remote(remote_disk) => remote_disk.offline_duration_secs(),
+        }
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        match self {
+            Disk::Local(local_disk) => local_disk.force_runtime_state_for_test(state),
+            Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
+        }
+    }
+}
+
 impl Disk {
     /// Enable health monitoring on this disk.
     /// Called after startup format loading completes so that remote peers

crates/ecstore/src/endpoints.rs

@@ -554,7 +554,7 @@ impl EndpointServerPools {
 
         for pool in self.0.iter() {
             for ep in pool.endpoints.as_ref() {
-                let n = node_map.entry(ep.host_port()).or_insert(Node {
+                let n = node_map.entry(ep.host_port()).or_insert_with(|| Node {
                     url: ep.url.clone(),
                     pools: vec![],
                     is_local: ep.is_local,

crates/ecstore/src/erasure_coding/encode.rs

@@ -106,7 +106,7 @@ impl<'a> MultiWriter<'a> {
             self.writers.len(),
             self.errs
                 .iter()
-                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .map(|e| e.as_ref().map_or_else(|| "<nil>".to_string(), |e| e.to_string()))
                 .collect::<Vec<_>>()
                 .join(", ")
         )))
@@ -168,7 +168,7 @@ impl<'a> MultiWriter<'a> {
             self.writers.len(),
             self.errs
                 .iter()
-                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .map(|e| e.as_ref().map_or_else(|| "<nil>".to_string(), |e| e.to_string()))
                 .collect::<Vec<_>>()
                 .join(", ")
         )))

crates/ecstore/src/notification_sys.rs

@@ -786,8 +786,9 @@ where
 
 fn offline_server_properties(host: &str, endpoints: &EndpointServerPools) -> ServerProperties {
     ServerProperties {
-        uptime: SystemTime::now()
-            .duration_since(*GLOBAL_BOOT_TIME.get().unwrap())
+        uptime: GLOBAL_BOOT_TIME
+            .get()
+            .and_then(|boot_time| SystemTime::now().duration_since(*boot_time).ok())
             .unwrap_or_default()
             .as_secs(),
         version: get_commit_id(),

crates/ecstore/src/pools.rs

@@ -1405,7 +1405,8 @@ impl ECStore {
 
         let mut fivs = load_decommission_entry_versions(&entry, &bucket, "file_info_versions")?;
 
-        fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time));
+        fivs.versions
+            .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
 
         let mut decommissioned: usize = 0;
         let mut expired: usize = 0;

crates/ecstore/src/rebalance.rs

@@ -1530,7 +1530,8 @@ impl ECStore {
         let mut fivs =
             resolve_rebalance_file_info_versions_result(entry.file_info_versions(&bucket), bucket.as_str(), entry.name.as_str())?;
 
-        fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time));
+        fivs.versions
+            .sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
 
         let mut rebalanced: usize = 0;
         let mut expired: usize = 0;

crates/ecstore/src/rpc/remote_disk.rs

@@ -17,9 +17,11 @@ use crate::disk::{
     FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
     disk_store::{
         CHECK_EVERY, CHECK_TIMEOUT_DURATION, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING,
-        SKIP_IF_SUCCESS_BEFORE, get_max_timeout_duration,
+        SKIP_IF_SUCCESS_BEFORE, get_drive_disk_info_timeout, get_drive_list_dir_timeout, get_drive_metadata_timeout,
+        get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
     },
     endpoint::Endpoint,
+    health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
 use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
@@ -31,7 +33,9 @@ use crate::{
 use bytes::Bytes;
 use futures::lock::Mutex;
 use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
+use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
+use rustfs_protos::evict_failed_connection;
 use rustfs_protos::proto_gen::node_service::RenamePartRequest;
 use rustfs_protos::proto_gen::node_service::{
     CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
@@ -43,7 +47,7 @@ use rustfs_protos::proto_gen::node_service::{
 use rustfs_rio::{HttpReader, HttpWriter};
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
-    io::Cursor,
+    io::{Cursor, ErrorKind},
     path::PathBuf,
     sync::{
         Arc,
@@ -116,10 +120,38 @@ impl RemoteDisk {
             health: Arc::new(DiskHealthTracker::new()),
             cancel_token: CancellationToken::new(),
         };
+        record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
 
         Ok(disk)
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        self.health.runtime_state()
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        self.health.offline_duration().map(|duration| duration.as_secs())
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.health.force_runtime_state_for_test(state);
+    }
+
+    fn spawn_recovery_monitor_if_needed(&self) {
+        if !self.health_check {
+            return;
+        }
+
+        let addr = self.addr.clone();
+        let endpoint = self.endpoint.clone();
+        let health = Arc::clone(&self.health);
+        let cancel_token = self.cancel_token.clone();
+        tokio::spawn(async move {
+            Self::monitor_remote_disk_recovery(addr, endpoint, health, cancel_token).await;
+        });
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes,
     /// so that remote peers have time to come online.
@@ -130,27 +162,34 @@ impl RemoteDisk {
         let health = Arc::clone(&self.health);
         let cancel_token = self.cancel_token.clone();
         let addr = self.addr.clone();
+        let endpoint = self.endpoint.clone();
 
         tokio::spawn(async move {
-            Self::monitor_remote_disk_health(addr, health, cancel_token).await;
+            Self::monitor_remote_disk_health(addr, endpoint, health, cancel_token).await;
         });
     }
 
     /// Monitor remote disk health periodically
-    async fn monitor_remote_disk_health(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
+    async fn monitor_remote_disk_health(
+        addr: String,
+        endpoint: Endpoint,
+        health: Arc<DiskHealthTracker>,
+        cancel_token: CancellationToken,
+    ) {
         let mut interval = time::interval(CHECK_EVERY);
 
         // Perform basic connectivity check
-        if Self::perform_connectivity_check(&addr).await.is_err() && health.swap_ok_to_faulty() {
+        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
             // Start recovery monitoring
             let health_clone = Arc::clone(&health);
             let addr_clone = addr.clone();
+            let endpoint_clone = endpoint.clone();
             let cancel_clone = cancel_token.clone();
 
             tokio::spawn(async move {
-                Self::monitor_remote_disk_recovery(addr_clone, health_clone, cancel_clone).await;
+                Self::monitor_remote_disk_recovery(addr_clone, endpoint_clone, health_clone, cancel_clone).await;
             });
         }
 
@@ -183,16 +222,17 @@ impl RemoteDisk {
                     }
 
                     // Perform basic connectivity check
-                    if Self::perform_connectivity_check(&addr).await.is_err() && health.swap_ok_to_faulty() {
+                    if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
                         // Start recovery monitoring
                         let health_clone = Arc::clone(&health);
                         let addr_clone = addr.clone();
+                        let endpoint_clone = endpoint.clone();
                         let cancel_clone = cancel_token.clone();
 
                         tokio::spawn(async move {
-                            Self::monitor_remote_disk_recovery(addr_clone, health_clone, cancel_clone).await;
+                            Self::monitor_remote_disk_recovery(addr_clone, endpoint_clone, health_clone, cancel_clone).await;
                         });
                     }
                 }
@@ -201,8 +241,13 @@ impl RemoteDisk {
     }
 
     /// Monitor remote disk recovery and mark as healthy when recovered
-    async fn monitor_remote_disk_recovery(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        let mut interval = time::interval(CHECK_EVERY);
+    async fn monitor_remote_disk_recovery(
+        addr: String,
+        endpoint: Endpoint,
+        health: Arc<DiskHealthTracker>,
+        cancel_token: CancellationToken,
+    ) {
+        let mut interval = time::interval(get_drive_returning_probe_interval());
 
         loop {
             tokio::select! {
@@ -211,9 +256,14 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
-                        info!("Remote disk recovered: {}", addr);
-                        health.set_ok();
-                        return;
+                        let became_online = health.mark_recovery_success(&endpoint, "connectivity_probe_success");
+                        info!("Remote disk recovery probe succeeded: {}", addr);
+                        if became_online {
+                            info!("Remote disk recovered: {}", addr);
+                            return;
+                        }
+                    } else {
+                        health.mark_failure(&endpoint, "connectivity_probe_failed");
                     }
                 }
             }
@@ -242,6 +292,19 @@ impl RemoteDisk {
 
     /// Execute operation with timeout and health tracking
     async fn execute_with_timeout<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        self.execute_with_timeout_for_op("unknown", operation, timeout_duration).await
+    }
+
+    async fn execute_with_timeout_for_op<T, F, Fut>(
+        &self,
+        op: &'static str,
+        operation: F,
+        timeout_duration: Duration,
+    ) -> Result<T>
     where
         F: FnOnce() -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
@@ -270,17 +333,81 @@ impl RemoteDisk {
                     self.health.log_success();
                 }
                 self.health.decrement_waiting();
+                if let Err(err) = &operation_result
+                    && Self::is_timeout_like_error(err)
+                {
+                    counter!(
+                        "rustfs_drive_op_timeout_total",
+                        "endpoint" => self.endpoint.to_string(),
+                        "op" => op.to_string()
+                    )
+                    .increment(1);
+                    warn!(
+                        endpoint = %self.endpoint,
+                        addr = %self.addr,
+                        op,
+                        timeout_ms = timeout_duration.as_millis(),
+                        "Remote disk operation returned a timeout-like error"
+                    );
+                    self.mark_faulty_and_evict("operation_timeout_error").await;
+                }
                 operation_result
             }
             Err(_) => {
                 // Timeout occurred, mark disk as potentially faulty
                 self.health.decrement_waiting();
-                warn!("Remote disk operation timeout after {:?}", timeout_duration);
+                counter!(
+                    "rustfs_drive_op_timeout_total",
+                    "endpoint" => self.endpoint.to_string(),
+                    "op" => op.to_string()
+                )
+                .increment(1);
+                self.mark_faulty_and_evict("operation_timeout").await;
+                warn!(
+                    endpoint = %self.endpoint,
+                    addr = %self.addr,
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    "Remote disk operation timed out"
+                );
                 Err(Error::other(format!("Remote disk operation timeout after {timeout_duration:?}")))
             }
         }
     }
 
+    fn is_timeout_like_error(err: &Error) -> bool {
+        matches!(err, DiskError::Timeout) || matches!(err, DiskError::Io(io_err) if io_err.kind() == ErrorKind::TimedOut)
+    }
+
+    async fn mark_faulty_and_evict(&self, reason: &'static str) {
+        if self.health.mark_failure(&self.endpoint, reason) {
+            self.spawn_recovery_monitor_if_needed();
+            counter!(
+                "rustfs_drive_faulty_mark_total",
+                "endpoint" => self.endpoint.to_string(),
+                "reason" => reason.to_string()
+            )
+            .increment(1);
+            warn!(
+                "Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
+                self.endpoint, self.addr, reason
+            );
+            counter!(
+                "rustfs_drive_connection_evict_total",
+                "endpoint" => self.endpoint.to_string(),
+                "reason" => reason.to_string()
+            )
+            .increment(1);
+            info!(
+                endpoint = %self.endpoint,
+                addr = %self.addr,
+                reason,
+                "Evicting cached remote disk connection after fault transition"
+            );
+            evict_failed_connection(&self.addr).await;
+        }
+    }
+
     async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
         node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
             .await
@@ -757,7 +884,8 @@ impl DiskAPI for RemoteDisk {
         let file_info = serde_json::to_string(&fi)?;
         let file_info_bin = encode_msgpack(&fi)?;
 
-        self.execute_with_timeout(
+        self.execute_with_timeout_for_op(
+            "write_metadata",
             || async {
                 let disk = self.disk_ref().await;
                 let mut client = self
@@ -786,24 +914,31 @@ impl DiskAPI for RemoteDisk {
     }
 
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        let disk = self.disk_ref().await;
-        let mut client = self
-            .get_client()
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-        let request = Request::new(ReadMetadataRequest {
-            volume: volume.to_string(),
-            path: path.to_string(),
-            disk,
-        });
+        self.execute_with_timeout_for_op(
+            "read_metadata",
+            || async {
+                let disk = self.disk_ref().await;
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let request = Request::new(ReadMetadataRequest {
+                    volume: volume.to_string(),
+                    path: path.to_string(),
+                    disk,
+                });
 
-        let response = client.read_metadata(request).await?.into_inner();
+                let response = client.read_metadata(request).await?.into_inner();
 
-        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
-        }
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
 
-        Ok(response.data)
+                Ok(response.data)
+            },
+            get_drive_metadata_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(skip(self))]
@@ -814,7 +949,8 @@ impl DiskAPI for RemoteDisk {
         let file_info_bin = encode_msgpack(&fi)?;
         let opts_bin = encode_msgpack(opts)?;
 
-        self.execute_with_timeout(
+        self.execute_with_timeout_for_op(
+            "update_metadata",
             || async {
                 let disk = self.disk_ref().await;
                 let mut client = self
@@ -967,53 +1103,66 @@ impl DiskAPI for RemoteDisk {
     async fn list_dir(&self, _origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
         debug!("list_dir {}/{}", volume, dir_path);
 
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
-        let disk = self.disk_ref().await;
+        self.execute_with_timeout(
+            || async {
+                let disk = self.disk_ref().await;
 
-        let mut client = self
-            .get_client()
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-        let request = Request::new(ListDirRequest {
-            disk,
-            volume: volume.to_string(),
-            dir_path: dir_path.to_string(),
-            count,
-        });
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let request = Request::new(ListDirRequest {
+                    disk,
+                    volume: volume.to_string(),
+                    dir_path: dir_path.to_string(),
+                    count,
+                });
 
-        let response = client.list_dir(request).await?.into_inner();
+                let response = client.list_dir(request).await?.into_inner();
 
-        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
-        }
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
 
-        Ok(response.volumes)
+                Ok(response.volumes)
+            },
+            get_drive_list_dir_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(skip(self, wr))]
     async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
         info!("walk_dir {}", self.endpoint.to_string());
 
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
-        let disk = self.disk_ref().await;
+        self.execute_with_timeout(
+            || async {
+                let disk = self.disk_ref().await;
 
-        let url = format!("{}/rustfs/rpc/walk_dir?disk={}", self.endpoint.grid_host(), urlencoding::encode(&disk),);
+                let url = format!("{}/rustfs/rpc/walk_dir?disk={}", self.endpoint.grid_host(), urlencoding::encode(&disk),);
 
-        let opts = serde_json::to_vec(&opts)?;
+                let opts = serde_json::to_vec(&opts)?;
 
-        let mut headers = HeaderMap::new();
-        headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
-        build_auth_headers(&url, &Method::GET, &mut headers);
+                let mut headers = HeaderMap::new();
+                headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+                build_auth_headers(&url, &Method::GET, &mut headers);
 
-        let mut reader = HttpReader::new(url, Method::GET, headers, Some(opts)).await?;
+                let mut reader = HttpReader::new_with_stall_timeout(
+                    url,
+                    Method::GET,
+                    headers,
+                    Some(opts),
+                    Some(get_drive_walkdir_stall_timeout()),
+                )
+                .await?;
 
-        copy_stream_with_buffer(&mut reader, wr, DEFAULT_READ_BUFFER_SIZE).await?;
+                copy_stream_with_buffer(&mut reader, wr, DEFAULT_READ_BUFFER_SIZE).await?;
 
-        Ok(())
+                Ok(())
+            },
+            get_drive_walkdir_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
@@ -1424,29 +1573,32 @@ impl DiskAPI for RemoteDisk {
 
     #[tracing::instrument(skip(self))]
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
+        self.execute_with_timeout_for_op(
+            "disk_info",
+            || async {
+                let opts = serde_json::to_string(&opts)?;
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let request = Request::new(DiskInfoRequest {
+                    disk: self.endpoint.to_string(),
+                    opts,
+                });
 
-        let opts = serde_json::to_string(&opts)?;
-        let mut client = self
-            .get_client()
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-        let request = Request::new(DiskInfoRequest {
-            disk: self.endpoint.to_string(),
-            opts,
-        });
+                let response = client.disk_info(request).await?.into_inner();
 
-        let response = client.disk_info(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
 
-        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
-        }
+                let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
 
-        let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
-
-        Ok(disk_info)
+                Ok(disk_info)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(skip(self))]
@@ -1459,9 +1611,11 @@ impl DiskAPI for RemoteDisk {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rustfs_common::GLOBAL_CONN_MAP;
     use std::sync::Once;
     use tokio::io::duplex;
     use tokio::net::TcpListener;
+    use tonic::transport::Endpoint as TonicEndpoint;
     use tracing::Level;
     use uuid::Uuid;
 
@@ -1793,6 +1947,132 @@ mod tests {
         assert!(result.is_ok());
     }
 
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_remote_disk_faulty() {
+        let url = url::Url::parse("http://remote-timeout:9000").unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let err = remote_disk
+            .execute_with_timeout(
+                || async {
+                    tokio::time::sleep(Duration::from_millis(50)).await;
+                    Ok::<(), Error>(())
+                },
+                Duration::from_millis(10),
+            )
+            .await
+            .expect_err("timeout should fail");
+
+        assert!(err.to_string().contains("timeout"));
+        assert!(!remote_disk.is_online().await, "remote disk should be marked faulty after timeout");
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_evicts_cached_connection() {
+        let addr = "http://127.0.0.1:59991".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let _ = remote_disk
+            .execute_with_timeout(
+                || async {
+                    tokio::time::sleep(Duration::from_millis(50)).await;
+                    Ok::<(), Error>(())
+                },
+                Duration::from_millis(10),
+            )
+            .await
+            .expect_err("timeout should fail");
+
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "timeout should evict cached connection"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_faulty_on_timeout_like_error() {
+        let addr = "http://127.0.0.1:59992".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(
+                || async { Err::<(), Error>(DiskError::Io(std::io::Error::new(std::io::ErrorKind::TimedOut, "stall timeout"))) },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("timeout-like operation error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io timeout error, got {other:?}"),
+            },
+            std::io::ErrorKind::TimedOut
+        );
+        assert!(!remote_disk.is_online().await, "timeout-like errors should mark remote disk faulty");
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "timeout-like errors should evict cached connection"
+        );
+    }
+
     #[test]
     fn test_remote_disk_sync_properties() {
         let url = url::Url::parse("https://secure-remote:9000/data").unwrap();

crates/ecstore/src/rpc/remote_locker.rs

@@ -18,8 +18,10 @@ use rustfs_lock::{
     LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
 };
-use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient;
 use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
+use rustfs_protos::{evict_failed_connection, proto_gen::node_service::node_service_client::NodeServiceClient};
+use std::time::Duration;
+use tokio::time::timeout;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tonic::transport::Channel;
@@ -62,6 +64,77 @@ impl RemoteClient {
             .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))
     }
 
+    async fn evict_connection(&self, op: &'static str, reason: &str) {
+        warn!(
+            addr = %self.addr,
+            op,
+            reason,
+            "Evicting cached remote lock connection after RPC failure"
+        );
+        evict_failed_connection(&self.addr).await;
+    }
+
+    fn rpc_timeout(timeout_duration: Duration) -> Duration {
+        if timeout_duration.is_zero() {
+            Duration::from_millis(1)
+        } else {
+            timeout_duration
+        }
+    }
+
+    async fn execute_rpc<T, F>(
+        &self,
+        op: &'static str,
+        timeout_duration: Duration,
+        future: F,
+    ) -> std::result::Result<T, LockError>
+    where
+        F: std::future::Future<Output = std::result::Result<T, tonic::Status>>,
+    {
+        let timeout_duration = Self::rpc_timeout(timeout_duration);
+        match timeout(timeout_duration, future).await {
+            Ok(Ok(response)) => Ok(response),
+            Ok(Err(err)) => {
+                let reason = err.to_string();
+                self.evict_connection(op, &reason).await;
+                Err(LockError::internal(format!("{op} RPC failed: {reason}")))
+            }
+            Err(_) => {
+                let reason = format!("RPC timed out after {:?}", timeout_duration);
+                self.evict_connection(op, &reason).await;
+                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), timeout_duration))
+            }
+        }
+    }
+
+    fn timeout_failure_response(request: &LockRequest) -> LockResponse {
+        LockResponse::failure("Lock acquisition timeout", request.acquire_timeout)
+    }
+
+    fn rpc_failure_response(_request: &LockRequest, err: &LockError) -> LockResponse {
+        LockResponse::failure(format!("Remote lock RPC failed: {err}"), Duration::ZERO)
+    }
+
+    fn timeout_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
+        requests.iter().map(Self::timeout_failure_response).collect()
+    }
+
+    fn rpc_failure_batch(requests: &[LockRequest], err: &LockError) -> Vec<LockResponse> {
+        requests
+            .iter()
+            .map(|request| Self::rpc_failure_response(request, err))
+            .collect()
+    }
+
+    fn batch_rpc_timeout(requests: &[LockRequest]) -> Duration {
+        requests
+            .iter()
+            .map(|request| request.acquire_timeout)
+            .max()
+            .map(Self::rpc_timeout)
+            .unwrap_or_else(|| Duration::from_millis(1))
+    }
+
     fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
         if let Some(lock_info_json) = lock_info_json {
             match serde_json::from_str::<LockInfo>(&lock_info_json) {
@@ -111,11 +184,11 @@ impl LockClient for RemoteClient {
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
 
-        let resp = client
-            .lock(req)
-            .await
-            .map_err(|e| LockError::internal(e.to_string()))?
-            .into_inner();
+        let resp = match self.execute_rpc("lock", request.acquire_timeout, client.lock(req)).await {
+            Ok(resp) => resp.into_inner(),
+            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_response(request)),
+            Err(err) => return Ok(Self::rpc_failure_response(request, &err)),
+        };
 
         // Check for explicit error first
         if let Some(error_info) = resp.error_info {
@@ -138,6 +211,10 @@ impl LockClient for RemoteClient {
     }
 
     async fn acquire_locks_batch(&self, requests: &[LockRequest]) -> Result<Vec<LockResponse>> {
+        if requests.is_empty() {
+            return Ok(Vec::new());
+        }
+
         let mut client = self.get_client().await?;
         let req = Request::new(BatchGenerallyLockRequest {
             args: requests
@@ -148,11 +225,14 @@ impl LockClient for RemoteClient {
                 .collect::<Result<Vec<_>>>()?,
         });
 
-        let resp = client
-            .lock_batch(req)
+        let resp = match self
+            .execute_rpc("lock_batch", Self::batch_rpc_timeout(requests), client.lock_batch(req))
             .await
-            .map_err(|e| LockError::internal(e.to_string()))?
-            .into_inner();
+        {
+            Ok(resp) => resp.into_inner(),
+            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_batch(requests)),
+            Err(err) => return Ok(Self::rpc_failure_batch(requests, &err)),
+        };
 
         Ok(requests
             .iter()
@@ -380,3 +460,95 @@ impl LockClient for RemoteClient {
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_common::GLOBAL_CONN_MAP;
+    use rustfs_lock::{ObjectKey, types::LockPriority};
+    use tokio::net::TcpListener;
+    use tokio::task::JoinHandle;
+    use tonic::transport::Endpoint as TonicEndpoint;
+
+    async fn spawn_hanging_listener() -> (String, JoinHandle<()>) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = format!("http://{}", listener.local_addr().unwrap());
+        let task = tokio::spawn(async move {
+            if let Ok((stream, _)) = listener.accept().await {
+                let _stream = stream;
+                tokio::time::sleep(Duration::from_secs(2)).await;
+            }
+        });
+        (addr, task)
+    }
+
+    async fn cache_lazy_channel(addr: &str) {
+        let channel = TonicEndpoint::from_shared(addr.to_string()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel);
+    }
+
+    fn test_lock_request(timeout_duration: Duration) -> LockRequest {
+        LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner-a")
+            .with_acquire_timeout(timeout_duration)
+            .with_priority(LockPriority::Normal)
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_acquire_lock_respects_request_timeout_and_evicts_connection() {
+        let (addr, accept_task) = spawn_hanging_listener().await;
+        cache_lazy_channel(&addr).await;
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let client = RemoteClient::new(addr.clone());
+        let request = test_lock_request(Duration::from_millis(50));
+        let started_at = tokio::time::Instant::now();
+
+        let response = client.acquire_lock(&request).await.unwrap();
+
+        assert!(
+            started_at.elapsed() < Duration::from_secs(1),
+            "remote lock RPC should honor request timeout"
+        );
+        assert!(!response.success, "timed out lock acquisition should fail");
+        assert_eq!(response.error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "timeout should evict cached connection"
+        );
+
+        accept_task.abort();
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_acquire_locks_batch_respects_request_timeout_and_evicts_connection() {
+        let (addr, accept_task) = spawn_hanging_listener().await;
+        cache_lazy_channel(&addr).await;
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let client = RemoteClient::new(addr.clone());
+        let requests = vec![test_lock_request(Duration::from_millis(50))];
+        let started_at = tokio::time::Instant::now();
+
+        let responses = client.acquire_locks_batch(&requests).await.unwrap();
+
+        assert!(
+            started_at.elapsed() < Duration::from_secs(1),
+            "remote batch lock RPC should honor request timeout"
+        );
+        assert_eq!(responses.len(), 1);
+        assert!(!responses[0].success, "timed out batch lock acquisition should fail");
+        assert_eq!(responses[0].error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "batch timeout should evict cached connection"
+        );
+
+        accept_task.abort();
+    }
+
+    #[test]
+    fn test_remote_client_zero_timeout_is_clamped() {
+        assert_eq!(RemoteClient::rpc_timeout(Duration::ZERO), Duration::from_millis(1));
+        assert_eq!(RemoteClient::rpc_timeout(Duration::from_millis(25)), Duration::from_millis(25));
+    }
+}

crates/ecstore/src/set_disk/lock.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::disk::health_state::DriveMembershipSnapshot;
 
 impl SetDisks {
     pub(super) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
@@ -72,41 +73,28 @@ impl SetDisks {
     }
 
     pub(super) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
-        let mut disks = self.get_disks_internal().await;
-
-        // TODO: diskinfo filter online
-
-        let mut new_disk = Vec::with_capacity(disks.len());
-
-        for disk in disks.iter() {
-            if let Some(d) = disk
-                && d.is_online().await
-            {
-                new_disk.push(disk.clone());
-            }
-        }
+        let snapshot = self.drive_membership_snapshot().await;
+        let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
 
         let mut rng = rand::rng();
-
         disks.shuffle(&mut rng);
 
-        new_disk
-        // let disks = self.get_disks_internal().await;
-        // let (filtered, _) = self.filter_online_disks(disks).await;
-        // filtered.into_iter().filter(|disk| disk.is_some()).collect()
+        disks
     }
 
     pub(super) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
-        let mut disks = self.get_online_disks().await;
+        let snapshot = self.drive_membership_snapshot().await;
+        let mut disks = snapshot
+            .strict_online_local_candidates()
+            .into_iter()
+            .map(Some)
+            .collect::<Vec<_>>();
 
         let mut rng = rand::rng();
 
         disks.shuffle(&mut rng);
 
         disks
-            .into_iter()
-            .filter(|v| v.as_ref().is_some_and(|d| d.is_local()))
-            .collect()
     }
 
     pub async fn get_online_disks_with_healing(&self, incl_healing: bool) -> (Vec<DiskStore>, bool) {
@@ -114,28 +102,41 @@ impl SetDisks {
         (disks, healing > 0)
     }
 
-    pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec<DiskStore>, Vec<DiskInfo>, usize) {
-        let mut disks = self.get_disks_internal().await;
+    pub async fn drive_membership_snapshot(&self) -> DriveMembershipSnapshot {
+        let disks = self.get_disks_internal().await;
+        DriveMembershipSnapshot::from_optional_disks(&disks)
+    }
 
-        let mut infos = Vec::with_capacity(disks.len());
+    pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec<DiskStore>, Vec<DiskInfo>, usize) {
+        let snapshot = self.drive_membership_snapshot().await;
+        let mut disks = snapshot.scanner_heal_candidates().into_iter().map(Some).collect::<Vec<_>>();
+
+        let mut infos: Vec<Option<DiskInfo>> = vec![None; disks.len()];
 
         let mut futures = Vec::with_capacity(disks.len());
-        let mut numbers: Vec<usize> = (0..disks.len()).collect();
         {
             let mut rng = rand::rng();
             disks.shuffle(&mut rng);
-
-            numbers.shuffle(&mut rng);
         }
 
-        for &i in numbers.iter() {
-            let disk = disks[i].clone();
+        for (i, disk) in disks.iter().cloned().enumerate() {
             futures.push(async move {
-                if let Some(disk) = disk {
-                    disk.disk_info(&DiskInfoOptions::default()).await
+                let info = if let Some(disk) = disk {
+                    match disk.disk_info(&DiskInfoOptions::default()).await {
+                        Ok(info) => info,
+                        Err(err) => DiskInfo {
+                            error: err.to_string(),
+                            ..Default::default()
+                        },
+                    }
                 } else {
-                    Err(DiskError::DiskNotFound)
-                }
+                    DiskInfo {
+                        error: DiskError::DiskNotFound.to_string(),
+                        ..Default::default()
+                    }
+                };
+
+                Ok((i, info))
             });
         }
 
@@ -143,13 +144,13 @@ impl SetDisks {
         let processor = get_global_processors().metadata_processor();
         let results = processor.execute_batch(futures).await;
 
-        for result in results {
+        for (submitted_idx, result) in results.into_iter().enumerate() {
             match result {
-                Ok(res) => {
-                    infos.push(res);
+                Ok((disk_idx, info)) => {
+                    infos[disk_idx] = Some(info);
                 }
                 Err(err) => {
-                    infos.push(DiskInfo {
+                    infos[submitted_idx] = Some(DiskInfo {
                         error: err.to_string(),
                         ..Default::default()
                     });
@@ -167,8 +168,11 @@ impl SetDisks {
         let mut new_disks = Vec::new();
         let mut new_infos = Vec::new();
 
-        for &i in numbers.iter() {
-            let (info, disk) = (infos[i].clone(), disks[i].clone());
+        for (disk, info) in disks.into_iter().zip(infos) {
+            let Some(info) = info else {
+                continue;
+            };
+
             if !info.error.is_empty() || disk.is_none() {
                 continue;
             }
@@ -367,3 +371,160 @@ impl SetDisks {
         Ok((new_disks, new_infos, healing))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::store_init::save_format_file;
+    use tempfile::TempDir;
+    use tokio::sync::RwLock;
+
+    async fn make_formatted_local_disk(disk_idx: usize, format: &FormatV3) -> (TempDir, Endpoint, DiskStore) {
+        let dir = tempfile::tempdir().expect("tempdir should be created");
+        let mut endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(disk_idx);
+
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("local disk should be created");
+
+        let mut disk_format = format.clone();
+        disk_format.erasure.this = format.erasure.sets[0][disk_idx];
+        save_format_file(&Some(disk.clone()), &Some(disk_format))
+            .await
+            .expect("format should be saved");
+
+        (dir, endpoint, disk)
+    }
+
+    #[tokio::test]
+    async fn get_online_disks_with_healing_and_info_keeps_disk_and_info_aligned() {
+        let disk_count = 8;
+        let format = FormatV3::new(1, disk_count);
+
+        let mut temp_dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        let mut disks = Vec::with_capacity(disk_count);
+
+        for disk_idx in 0..disk_count {
+            let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(disks)),
+            disk_count,
+            disk_count / 2,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        for _ in 0..32 {
+            let (online_disks, infos, healing) = set_disks.get_online_disks_with_healing_and_info(false).await;
+            assert_eq!(healing, 0);
+            assert_eq!(online_disks.len(), disk_count);
+            assert_eq!(infos.len(), disk_count);
+
+            for (disk, info) in online_disks.iter().zip(infos.iter()) {
+                assert!(
+                    info.error.is_empty(),
+                    "unexpected disk_info error for {}: {}",
+                    disk.endpoint(),
+                    info.error
+                );
+                assert_eq!(info.endpoint, disk.endpoint().to_string());
+                assert_eq!(
+                    info.id,
+                    disk.get_disk_id().await.expect("disk id lookup should succeed"),
+                    "disk info should stay aligned with disk {}",
+                    disk.endpoint()
+                );
+            }
+        }
+
+        drop(temp_dirs);
+    }
+
+    #[tokio::test]
+    async fn drive_membership_snapshot_filters_offline_disks_from_candidates() {
+        let disk_count = 4;
+        let format = FormatV3::new(1, disk_count);
+
+        let mut temp_dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        let mut disks = Vec::with_capacity(disk_count);
+
+        for disk_idx in 0..disk_count {
+            let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(disks)),
+            disk_count,
+            disk_count / 2,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        let all_disks = set_disks.get_disks_internal().await;
+        all_disks[1]
+            .as_ref()
+            .expect("disk 1 should exist")
+            .force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Suspect);
+        all_disks[2]
+            .as_ref()
+            .expect("disk 2 should exist")
+            .force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Returning);
+        all_disks[3]
+            .as_ref()
+            .expect("disk 3 should exist")
+            .force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
+
+        let snapshot = set_disks.drive_membership_snapshot().await;
+        assert_eq!(snapshot.online.len(), 1);
+        assert_eq!(snapshot.suspect.len(), 1);
+        assert_eq!(snapshot.returning.len(), 1);
+        assert_eq!(snapshot.offline.len(), 1);
+        assert_eq!(snapshot.scanner_heal_candidates().len(), 3);
+
+        let strict_online = set_disks.get_online_disks().await;
+        assert_eq!(strict_online.len(), 1, "strict online selection should exclude suspect/returning/offline");
+
+        let (online_disks, infos, healing) = set_disks.get_online_disks_with_healing_and_info(false).await;
+        assert_eq!(healing, 0);
+        assert_eq!(online_disks.len(), 3);
+        assert_eq!(infos.len(), 3);
+        assert!(
+            online_disks
+                .iter()
+                .all(|disk| { disk.runtime_state() != crate::disk::health_state::RuntimeDriveHealthState::Offline }),
+            "offline disks should be filtered by membership snapshot"
+        );
+
+        drop(temp_dirs);
+    }
+}

crates/ecstore/src/set_disk/metadata.rs

@@ -38,14 +38,17 @@ impl SetDisks {
         let upload_uuid = base64_simd::URL_SAFE_NO_PAD
             .decode_to_vec(upload_id.as_bytes())
             .and_then(|v| {
-                String::from_utf8(v).map_or(Ok(upload_id.to_owned()), |v| {
-                    let parts: Vec<_> = v.splitn(2, '.').collect();
-                    if parts.len() == 2 {
-                        Ok(parts[1].to_string())
-                    } else {
-                        Ok(upload_id.to_string())
-                    }
-                })
+                String::from_utf8(v).map_or_else(
+                    |_| Ok(upload_id.to_owned()),
+                    |v| {
+                        let parts: Vec<_> = v.splitn(2, '.').collect();
+                        if parts.len() == 2 {
+                            Ok(parts[1].to_string())
+                        } else {
+                            Ok(upload_id.to_string())
+                        }
+                    },
+                )
             })
             .unwrap_or_default();
 

crates/ecstore/src/set_disk/write.rs

@@ -603,7 +603,9 @@ impl SetDisks {
                 if oi.delete_marker {
                     return None;
                 }
-                if should_prevent_write(&oi, http_preconditions.if_none_match, http_preconditions.if_match) {
+                let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
+                let if_match = http_preconditions.if_match_value().map(str::to_owned);
+                if should_prevent_write(&oi, if_none_match, if_match) {
                     return Some(StorageError::PreconditionFailed);
                 }
             }
@@ -614,7 +616,7 @@ impl SetDisks {
                 // When the object is not found,
                 // - if If-Match is set, we should return 404 NotFound
                 // - if If-None-Match is set, we should be able to proceed with the request
-                if http_preconditions.if_match.is_some() {
+                if http_preconditions.if_match_value().is_some() {
                     return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string()));
                 }
             }

crates/ecstore/src/store.rs

@@ -15,7 +15,9 @@
 #![allow(clippy::map_entry)]
 
 use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
-use crate::bucket::lifecycle::bucket_lifecycle_ops::{enqueue_transition_immediate, init_background_expiry};
+use crate::bucket::lifecycle::bucket_lifecycle_ops::{
+    enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
+};
 use crate::bucket::metadata_sys::{self, set_bucket_metadata};
 use crate::bucket::utils::check_abort_multipart_args;
 use crate::bucket::utils::check_complete_multipart_args;
@@ -134,7 +136,8 @@ async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEvent
     match result {
         Ok(oi) => {
             if should_enqueue_transition_immediately(&oi) {
-                enqueue_transition_immediate(&oi, src).await;
+                enqueue_transition_immediate(&oi, src.clone()).await;
+                enqueue_immediate_expiry(&oi, src).await;
             }
             Ok(oi)
         }
@@ -160,7 +163,7 @@ mod rebalance;
 use peer::init_local_peer;
 pub use peer::{
     all_local_disk, all_local_disk_path, find_local_disk, find_local_disk_by_ref, get_disk_infos, get_disk_via_endpoint,
-    has_space_for, init_local_disks, init_lock_clients,
+    has_space_for, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
 };
 
 #[derive(Debug)]
@@ -668,6 +671,17 @@ impl ServerPoolsAvailableSpace {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::endpoints::{Endpoints, PoolEndpoints};
+    use crate::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
+    use crate::store_init::{connect_load_init_formats, init_disks};
+    use serial_test::serial;
+    use tempfile::TempDir;
+
+    async fn reset_local_disk_globals() {
+        GLOBAL_LOCAL_DISK_MAP.write().await.clear();
+        GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
+        GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
+    }
 
     #[tokio::test]
     async fn test_get_disk_infos() {
@@ -720,6 +734,73 @@ mod tests {
         assert!(result.is_none(), "Should return None for nonexistent path");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_find_local_disk_by_ref_backfills_uuid_map() {
+        reset_local_disk_globals().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for local disk ref test");
+        let disk_paths = (0..4)
+            .map(|idx| temp_dir.path().join(format!("disk{}", idx + 1)))
+            .collect::<Vec<_>>();
+        for disk_path in &disk_paths {
+            std::fs::create_dir_all(disk_path).expect("create disk path");
+        }
+
+        let mut endpoints = Vec::new();
+        for (idx, disk_path) in disk_paths.iter().enumerate() {
+            let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path to str")).expect("endpoint");
+            endpoint.set_pool_index(0);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(idx);
+            endpoints.push(endpoint);
+        }
+
+        let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 4,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: "find-local-disk-by-ref-test".to_string(),
+            platform: "test".to_string(),
+        }]);
+
+        init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
+
+        let (disks, errs) = init_disks(
+            &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
+            &DiskOption {
+                cleanup: true,
+                health_check: false,
+            },
+        )
+        .await;
+
+        assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
+        connect_load_init_formats(true, &disks, 1, 4, None)
+            .await
+            .expect("initialize format metadata");
+
+        GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
+
+        let local_disks = all_local_disk().await;
+        let first_disk = local_disks.first().expect("local disk exists");
+        let disk_id = first_disk
+            .get_disk_id()
+            .await
+            .expect("get disk id should succeed")
+            .expect("disk id should exist");
+
+        let found = find_local_disk_by_ref(&disk_id.to_string()).await;
+        assert!(found.is_some(), "disk lookup by id should backfill cache");
+        assert_eq!(
+            GLOBAL_LOCAL_DISK_ID_MAP.read().await.get(&disk_id).cloned(),
+            Some(first_disk.endpoint().to_string())
+        );
+
+        reset_local_disk_globals().await;
+    }
+
     #[tokio::test]
     async fn test_all_local_disk_path() {
         let paths = all_local_disk_path().await;

crates/ecstore/src/store/peer.rs

@@ -15,6 +15,15 @@
 use super::*;
 use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 
+async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
+    let disk_id = disk.get_disk_id().await.ok().flatten()?;
+    GLOBAL_LOCAL_DISK_ID_MAP
+        .write()
+        .await
+        .insert(disk_id, disk.endpoint().to_string());
+    Some(disk_id)
+}
+
 pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
 
@@ -27,6 +36,7 @@ pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
 
 pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
     if let Some(disk) = find_local_disk(&disk_ref.to_string()).await {
+        let _ = remember_local_disk_id(&disk).await;
         return Some(disk);
     }
 
@@ -34,8 +44,19 @@ pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
         return None;
     };
 
-    let disk_path = GLOBAL_LOCAL_DISK_ID_MAP.read().await.get(&disk_id).cloned()?;
-    find_local_disk(&disk_path).await
+    if let Some(disk_path) = GLOBAL_LOCAL_DISK_ID_MAP.read().await.get(&disk_id).cloned()
+        && let Some(disk) = find_local_disk(&disk_path).await
+    {
+        return Some(disk);
+    }
+
+    for disk in all_local_disk().await {
+        if remember_local_disk_id(&disk).await == Some(disk_id) {
+            return Some(disk);
+        }
+    }
+
+    None
 }
 
 pub async fn get_disk_via_endpoint(endpoint: &Endpoint) -> Option<DiskStore> {
@@ -70,6 +91,17 @@ pub async fn all_local_disk() -> Vec<DiskStore> {
         .collect()
 }
 
+pub async fn prewarm_local_disk_id_map() {
+    for disk in all_local_disk().await {
+        if let Err(err) = disk.get_disk_id().await {
+            warn!("prewarm_local_disk_id_map: failed to load disk id for {}: {}", disk.endpoint(), err);
+            continue;
+        }
+
+        let _ = remember_local_disk_id(&disk).await;
+    }
+}
+
 pub async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
     let opt = &DiskOption {
         cleanup: true,

crates/ecstore/src/store_api/types.rs

@@ -33,6 +33,16 @@ pub struct HTTPPreconditions {
     pub if_unmodified_since: Option<OffsetDateTime>,
 }
 
+impl HTTPPreconditions {
+    pub(crate) fn if_match_value(&self) -> Option<&str> {
+        non_empty_condition_value(self.if_match.as_deref())
+    }
+
+    pub(crate) fn if_none_match_value(&self) -> Option<&str> {
+        non_empty_condition_value(self.if_none_match.as_deref())
+    }
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
@@ -146,7 +156,10 @@ impl ObjectOptions {
         }
 
         if let Some(pre) = &self.http_preconditions {
-            if let Some(if_none_match) = &pre.if_none_match
+            let if_none_match = pre.if_none_match_value();
+            let if_match = pre.if_match_value();
+
+            if let Some(if_none_match) = if_none_match
                 && let Some(etag) = &obj_info.etag
                 && is_etag_equal(etag, if_none_match)
             {
@@ -161,7 +174,7 @@ impl ObjectOptions {
                 return Err(Error::NotModified);
             }
 
-            if let Some(if_match) = &pre.if_match {
+            if let Some(if_match) = if_match {
                 if let Some(etag) = &obj_info.etag {
                     if !is_etag_equal(etag, if_match) {
                         return Err(Error::PreconditionFailed);
@@ -171,7 +184,7 @@ impl ObjectOptions {
                 }
             }
             if has_valid_mod_time
-                && pre.if_match.is_none()
+                && if_match.is_none()
                 && let Some(if_unmodified_since) = &pre.if_unmodified_since
                 && let Some(mod_time) = &obj_info.mod_time
                 && is_modified_since(mod_time, if_unmodified_since)
@@ -184,6 +197,10 @@ impl ObjectOptions {
     }
 }
 
+fn non_empty_condition_value(value: Option<&str>) -> Option<&str> {
+    value.map(str::trim).filter(|value| !value.is_empty())
+}
+
 fn is_etag_equal(etag1: &str, etag2: &str) -> bool {
     let e1 = etag1.trim_matches('"');
     let e2 = etag2.trim_matches('"');
@@ -446,6 +463,11 @@ impl ObjectInfo {
             .replication_state_internal
             .as_ref()
             .and_then(|v| v.version_purge_status_internal.clone());
+        let replication_decision = fi
+            .replication_state_internal
+            .as_ref()
+            .map(|v| v.replicate_decision_str.clone())
+            .unwrap_or_default();
 
         let mut replication_status = fi.replication_status();
         if replication_status.is_empty()
@@ -542,6 +564,7 @@ impl ObjectInfo {
             replication_status,
             version_purge_status_internal,
             version_purge_status,
+            replication_decision,
             ..Default::default()
         }
     }
@@ -1010,6 +1033,7 @@ pub struct ObjectInfoOrErr {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rustfs_filemeta::ReplicationState;
 
     #[test]
     fn get_actual_size_prefers_actual_size_field() {
@@ -1059,6 +1083,40 @@ mod tests {
         assert_eq!(info.get_actual_size().unwrap(), 77);
     }
 
+    #[test]
+    fn precondition_check_ignores_empty_etag_conditions() {
+        let opts = ObjectOptions {
+            http_preconditions: Some(HTTPPreconditions {
+                if_match: Some(String::new()),
+                if_none_match: Some(" ".to_string()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        let info = ObjectInfo {
+            mod_time: Some(OffsetDateTime::now_utc()),
+            etag: Some("\"abc\"".to_string()),
+            ..Default::default()
+        };
+
+        assert!(opts.precondition_check(&info).is_ok());
+    }
+
+    #[test]
+    fn from_file_info_preserves_replication_decision() {
+        let fi = rustfs_filemeta::FileInfo {
+            replication_state_internal: Some(ReplicationState {
+                replicate_decision_str: "arn=true;false;arn:replication::1:dest;rule-id".to_string(),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let info = ObjectInfo::from_file_info(&fi, "bucket", "object", true);
+
+        assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
+    }
+
     #[test]
     fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() {
         let user_defined = {

crates/ecstore/src/store_list_objects.rs

@@ -162,10 +162,8 @@ impl ListPathOptions {
                     }
 
                     match kv[0] {
-                        "rustfs_cache" => {
-                            if kv[1] != MARKER_TAG_VERSION {
-                                continue;
-                            }
+                        "rustfs_cache" if kv[1] != MARKER_TAG_VERSION => {
+                            continue;
                         }
                         "id" => self.id = Some(kv[1].to_owned()),
                         "return" => {

crates/ecstore/src/tier/warm_backend_gcs.rs

@@ -107,11 +107,12 @@ impl WarmBackend for WarmBackendGCS {
             ReaderImpl::Body(content_body) => content_body.to_vec(),
             ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
         };
-        let Ok(res) = self
-            .client
-            .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
-            .send_buffered()
-            .await
+        let Ok(res) = Box::pin(
+            self.client
+                .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
+                .send_buffered(),
+        )
+        .await
         else {
             return Err(std::io::Error::other("write_object error"));
         };

crates/ecstore/tests/legacy_bitrot_read_test.rs

@@ -40,7 +40,7 @@ fn workspace_root() -> PathBuf {
     PathBuf::from(&manifest)
         .ancestors()
         .nth(2)
-        .unwrap_or(std::path::Path::new("."))
+        .unwrap_or_else(|| std::path::Path::new("."))
         .to_path_buf()
 }
 

crates/filemeta/README.md

@@ -19,6 +19,12 @@
 
 **RustFS FileMeta** provides advanced file metadata management and indexing capabilities for the [RustFS](https://rustfs.com) distributed object storage system. For the complete RustFS experience, please visit the [main RustFS repository](https://github.com/rustfs/rustfs).
 
+## Quick Use
+
+```
+cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/file/xl.meta"
+```
+
 ## ✨ Features
 
 - High-performance metadata storage and retrieval

crates/filemeta/examples/dump_fileinfo.rs

@@ -0,0 +1,50 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_filemeta::{FileInfoOpts, get_file_info};
+use std::{env, fs, path::PathBuf};
+fn main() {
+    let path = env::args()
+        .nth(1)
+        .map(PathBuf::from)
+        .expect("usage: dump_fileinfo <xl.meta path>");
+    let data = fs::read(&path).expect("read xl.meta");
+    let fi = get_file_info(
+        &data,
+        "debug-bucket",
+        "debug-object",
+        "",
+        FileInfoOpts {
+            data: false,
+            include_free_versions: true,
+        },
+    )
+    .expect("decode file info");
+    println!("path: {}", path.display());
+    println!("size: {}", fi.size);
+    println!("etag: {:?}", fi.get_etag());
+    println!("parts: {}", fi.parts.len());
+    for (idx, part) in fi.parts.iter().enumerate() {
+        println!(
+            "part#{idx}: number={} size={} actual_size={} etag={}",
+            part.number, part.size, part.actual_size, part.etag
+        );
+    }
+    println!("metadata entries: {}", fi.metadata.len());
+    let mut keys = fi.metadata.keys().cloned().collect::<Vec<_>>();
+    keys.sort();
+    for key in keys {
+        println!("meta[{key}]={}", fi.metadata.get(&key).unwrap());
+    }
+}

crates/filemeta/examples/dump_versions.rs

@@ -1,3 +1,17 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use rustfs_filemeta::FileMeta;
 use std::{env, fs, path::PathBuf};
 

crates/filemeta/src/fileinfo.rs

@@ -349,7 +349,7 @@ impl FileInfo {
 
         self.parts.push(part);
 
-        self.parts.sort_by(|a, b| a.number.cmp(&b.number));
+        self.parts.sort_by_key(|a| a.number);
     }
 
     // to_part_offset gets the part index where offset is located, returns part index and offset
@@ -429,7 +429,7 @@ impl FileInfo {
         if self.deleted {
             return "delete-marker".to_string();
         }
-        self.data_dir.map_or("".to_string(), |dir| dir.to_string())
+        self.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string())
     }
 
     /// Read quorum returns expected read quorum for this FileInfo
@@ -643,13 +643,11 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
     }
 
     match progress_tokens[1] {
-        "true" | "\"true\"" => {
-            if tokens.len() == 1 {
-                return Ok(RestoreStatus {
-                    is_restore_in_progress: Some(true),
-                    ..Default::default()
-                });
-            }
+        "true" | "\"true\"" if tokens.len() == 1 => {
+            return Ok(RestoreStatus {
+                is_restore_in_progress: Some(true),
+                ..Default::default()
+            });
         }
         "false" | "\"false\"" => {
             if tokens.len() != 2 {

crates/filemeta/src/filemeta.rs

@@ -382,6 +382,11 @@ impl FileMeta {
     #[tracing::instrument(level = "debug", skip(self))]
     pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
         let vid = Some(fi.version_id.unwrap_or(Uuid::nil()));
+        let target_is_delete_marker = self
+            .versions
+            .iter()
+            .find(|ver| ver.header.version_id == vid)
+            .is_some_and(|ver| ver.header.version_type == VersionType::Delete);
 
         let mut ventry = FileMetaVersion::default();
         if fi.deleted {
@@ -416,6 +421,10 @@ impl FileMeta {
             }
         }
 
+        if target_is_delete_marker && !fi.deleted && !fi.version_purge_status().is_empty() {
+            update_version = false;
+        }
+
         if fi.deleted {
             if !fi.delete_marker_replication_status().is_empty()
                 && let Some(delete_marker) = ventry.delete_marker.as_mut()
@@ -1659,6 +1668,45 @@ mod test {
         }
     }
 
+    #[test]
+    fn delete_version_removes_delete_marker_during_version_purge_replication() {
+        let version_id = Uuid::new_v4();
+        let mut fm = FileMeta::new();
+        fm.add_version_filemata(FileMetaVersion {
+            version_type: VersionType::Delete,
+            legacy_object: None,
+            object: None,
+            delete_marker: Some(MetaDeleteMarker {
+                version_id: Some(version_id),
+                mod_time: Some(OffsetDateTime::now_utc()),
+                meta_sys: HashMap::new(),
+            }),
+            write_version: 1,
+            uses_legacy_checksum: false,
+        })
+        .unwrap();
+
+        let fi = FileInfo {
+            deleted: false,
+            mark_deleted: false,
+            version_id: Some(version_id),
+            replication_state_internal: Some(ReplicationState {
+                version_purge_status_internal: Some("target=PENDING;".to_string()),
+                purge_targets: version_purge_statuses_map("target=PENDING;"),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let result = fm.delete_version(&fi).unwrap();
+
+        assert!(result.is_none());
+        assert!(
+            fm.versions.is_empty(),
+            "delete-marker version purge should remove the local marker instead of rewriting purge metadata onto it"
+        );
+    }
+
     #[test]
     fn test_data_integrity_validation() {
         // Test data integrity checks

crates/filemeta/src/filemeta/version.rs

@@ -517,16 +517,20 @@ impl FileMetaVersion {
                     }
                 }
             }
-            VersionType::Object => self
-                .object
-                .as_ref()
-                .unwrap_or(&MetaObject::default())
-                .into_fileinfo(volume, path, all_parts),
-            VersionType::Delete => self
-                .delete_marker
-                .as_ref()
-                .unwrap_or(&MetaDeleteMarker::default())
-                .into_fileinfo(volume, path, all_parts),
+            VersionType::Object => {
+                let default_object = MetaObject::default();
+                self.object
+                    .as_ref()
+                    .unwrap_or(&default_object)
+                    .into_fileinfo(volume, path, all_parts)
+            }
+            VersionType::Delete => {
+                let default_marker = MetaDeleteMarker::default();
+                self.delete_marker
+                    .as_ref()
+                    .unwrap_or(&default_marker)
+                    .into_fileinfo(volume, path, all_parts)
+            }
         };
         fi.uses_legacy_checksum = self.uses_legacy_checksum;
         fi

crates/filemeta/src/test_data.rs

@@ -104,7 +104,8 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
     fm.versions.push(legacy_shallow);
 
     // Sort by modification time (newest first)
-    fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
+    fm.versions
+        .sort_by_key(|v| (v.header.mod_time.is_none(), std::cmp::Reverse(v.header.mod_time)));
 
     fm.marshal_msg()
 }
@@ -263,11 +264,11 @@ pub fn create_legacy_v1_object_xlmeta() -> Result<Vec<u8>> {
     wr.extend_from_slice(&[0xc6, 0, 0, 0, 0]);
 
     let offset = wr.len();
-    rmp::encode::write_uint(&mut wr, 1).unwrap();
-    rmp::encode::write_uint(&mut wr, 1).unwrap();
-    rmp::encode::write_sint(&mut wr, 1).unwrap();
-    rmp::encode::write_bin(&mut wr, &header).unwrap();
-    rmp::encode::write_bin(&mut wr, &body).unwrap();
+    rmp::encode::write_uint(&mut wr, 1)?;
+    rmp::encode::write_uint(&mut wr, 1)?;
+    rmp::encode::write_sint(&mut wr, 1)?;
+    rmp::encode::write_bin(&mut wr, &header)?;
+    rmp::encode::write_bin(&mut wr, &body)?;
 
     let data_len = (wr.len() - offset) as u32;
     wr[offset - 4..offset].copy_from_slice(&data_len.to_be_bytes());
@@ -350,7 +351,8 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
     }
 
     // Sort by modification time (newest first)
-    fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
+    fm.versions
+        .sort_by_key(|v| (v.header.mod_time.is_none(), std::cmp::Reverse(v.header.mod_time)));
 
     fm.marshal_msg()
 }

crates/heal/Cargo.toml

@@ -32,7 +32,7 @@ rustfs-ecstore = { workspace = true }
 rustfs-common = { workspace = true }
 rustfs-madmin = { workspace = true }
 rustfs-utils = { workspace = true }
-tokio = { workspace = true, features = ["full"] }
+tokio = { workspace = true, features = ["sync","io-util","time","macros"] }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
@@ -42,6 +42,7 @@ uuid = { workspace = true, features = ["v4", "serde"] }
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 futures = { workspace = true }
+metrics = { workspace = true }
 
 [dev-dependencies]
 serde_json = { workspace = true }
@@ -50,6 +51,8 @@ tracing-subscriber = { workspace = true }
 tempfile = { workspace = true }
 walkdir = { workspace = true }
 http = { workspace = true }
+temp-env = { workspace = true }
+tokio = { workspace = true, features = ["test-util","fs"] }
 
 [lib]
 doctest = false

crates/heal/src/error.rs

@@ -68,6 +68,9 @@ pub enum Error {
     #[error("Invalid heal type: {heal_type}")]
     InvalidHealType { heal_type: String },
 
+    #[error("Transient heal skip: {message}")]
+    TransientSkip { message: String },
+
     #[error("Heal task cancelled")]
     TaskCancelled,
 
@@ -92,6 +95,11 @@ impl Error {
     {
         Error::Other(error.into().to_string())
     }
+
+    /// Create a transient skip error for retryable background heal checks.
+    pub fn transient_skip(message: impl Into<String>) -> Self {
+        Error::TransientSkip { message: message.into() }
+    }
 }
 
 impl From<Error> for std::io::Error {

crates/heal/src/heal/channel.rs

@@ -19,11 +19,11 @@ use crate::heal::{
 };
 use crate::{Error, Result};
 use rustfs_common::heal_channel::{
-    HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse, HealScanMode,
-    publish_heal_response,
+    HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse,
+    HealScanMode, publish_heal_response,
 };
 use std::sync::Arc;
-use tokio::sync::mpsc;
+use tokio::sync::{mpsc, oneshot};
 use tracing::{debug, error, info};
 
 /// Heal channel processor
@@ -82,14 +82,18 @@ impl HealChannelProcessor {
     /// Process heal command
     async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
         match command {
-            HealChannelCommand::Start(request) => self.process_start_request(request).await,
+            HealChannelCommand::Start { request, response_tx } => self.process_start_request(request, response_tx).await,
             HealChannelCommand::Query { heal_path, client_token } => self.process_query_request(heal_path, client_token).await,
             HealChannelCommand::Cancel { heal_path } => self.process_cancel_request(heal_path).await,
         }
     }
 
     /// Process start request
-    async fn process_start_request(&self, request: HealChannelRequest) -> Result<()> {
+    async fn process_start_request(
+        &self,
+        request: HealChannelRequest,
+        response_tx: oneshot::Sender<std::result::Result<HealAdmissionResult, String>>,
+    ) -> Result<()> {
         info!(
             "Processing heal start request: {} for bucket: {}/{}",
             request.id,
@@ -98,31 +102,60 @@ impl HealChannelProcessor {
         );
 
         // Convert channel request to heal request
-        let heal_request = self.convert_to_heal_request(request.clone())?;
+        let heal_request = match self.convert_to_heal_request(request.clone()) {
+            Ok(heal_request) => heal_request,
+            Err(err) => {
+                let error_text = err.to_string();
+                let _ = response_tx.send(Err(error_text.clone()));
+                self.publish_response(HealChannelResponse {
+                    request_id: request.id,
+                    success: false,
+                    data: None,
+                    error: Some(error_text),
+                });
+                return Ok(());
+            }
+        };
 
         // Submit to heal manager
         match self.heal_manager.submit_heal_request(heal_request).await {
-            Ok(task_id) => {
-                info!("Successfully submitted heal request: {} as task: {}", request.id, task_id);
+            Ok(admission) => {
+                info!(
+                    request_id = %request.id,
+                    admission = admission.result_label(),
+                    "Heal request admission decision completed"
+                );
+
+                let _ = response_tx.send(Ok(admission));
+
+                let (success, error) = match admission {
+                    HealAdmissionResult::Accepted | HealAdmissionResult::Merged => (true, None),
+                    HealAdmissionResult::Full => (false, Some("Heal request queue is full".to_string())),
+                    HealAdmissionResult::Dropped(reason) => (false, Some(format!("Heal request dropped: {}", reason.as_str()))),
+                };
 
                 let response = HealChannelResponse {
                     request_id: request.id,
-                    success: true,
-                    data: Some(format!("Task ID: {task_id}").into_bytes()),
-                    error: None,
+                    success,
+                    data: Some(
+                        format!("admission={},reason={}", admission.result_label(), admission.reason_label()).into_bytes(),
+                    ),
+                    error,
                 };
 
                 self.publish_response(response);
             }
             Err(e) => {
-                error!("Failed to submit heal request: {} - {}", request.id, e);
+                let error_text = e.to_string();
+                error!("Failed to submit heal request: {} - {}", request.id, error_text);
+                let _ = response_tx.send(Err(error_text.clone()));
 
                 // Send error response
                 let response = HealChannelResponse {
                     request_id: request.id,
                     success: false,
                     data: None,
-                    error: Some(e.to_string()),
+                    error: Some(error_text),
                 };
 
                 self.publish_response(response);
@@ -247,8 +280,9 @@ impl HealChannelProcessor {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::heal::manager::HealConfig;
     use crate::heal::storage::HealStorageAPI;
-    use rustfs_common::heal_channel::{HealChannelPriority, HealChannelRequest, HealScanMode};
+    use rustfs_common::heal_channel::{HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode};
     use std::sync::Arc;
 
     // Mock storage for testing
@@ -569,4 +603,92 @@ mod tests {
         let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
         assert!(matches!(heal_request.heal_type, HealType::Bucket { .. }));
     }
+
+    #[tokio::test]
+    async fn test_process_start_request_returns_admission_result() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = Arc::new(HealManager::new(
+            storage,
+            Some(HealConfig {
+                queue_size: 1,
+                ..HealConfig::default()
+            }),
+        ));
+        let processor = HealChannelProcessor::new(manager);
+
+        let request = HealChannelRequest {
+            id: "admission-id".to_string(),
+            bucket: "bucket".to_string(),
+            object_prefix: Some("object".to_string()),
+            object_version_id: None,
+            disk: None,
+            priority: HealChannelPriority::Low,
+            scan_mode: Some(HealScanMode::Normal),
+            remove_corrupted: None,
+            recreate_missing: None,
+            update_parity: None,
+            recursive: None,
+            dry_run: None,
+            timeout_seconds: None,
+            pool_index: None,
+            set_index: None,
+            force_start: false,
+        };
+
+        let (tx, rx) = oneshot::channel();
+        processor
+            .process_start_request(request.clone(), tx)
+            .await
+            .expect("first admission should succeed");
+        assert_eq!(
+            rx.await
+                .expect("oneshot should resolve")
+                .expect("admission should be returned"),
+            HealAdmissionResult::Accepted
+        );
+
+        let (tx, rx) = oneshot::channel();
+        processor
+            .process_start_request(request, tx)
+            .await
+            .expect("duplicate admission should succeed");
+        assert_eq!(
+            rx.await
+                .expect("oneshot should resolve")
+                .expect("admission should be returned"),
+            HealAdmissionResult::Merged
+        );
+    }
+
+    #[tokio::test]
+    async fn test_process_start_request_returns_error_on_invalid_request() {
+        let heal_manager = create_test_heal_manager();
+        let processor = HealChannelProcessor::new(heal_manager);
+
+        let request = HealChannelRequest {
+            id: "invalid-id".to_string(),
+            bucket: "bucket".to_string(),
+            object_prefix: None,
+            object_version_id: None,
+            disk: Some("invalid".to_string()),
+            priority: HealChannelPriority::Normal,
+            scan_mode: None,
+            remove_corrupted: None,
+            recreate_missing: None,
+            update_parity: None,
+            recursive: None,
+            dry_run: None,
+            timeout_seconds: None,
+            pool_index: None,
+            set_index: None,
+            force_start: false,
+        };
+
+        let (tx, rx) = oneshot::channel();
+        processor
+            .process_start_request(request, tx)
+            .await
+            .expect("processor should surface invalid request through response channel");
+        assert!(rx.await.expect("oneshot should resolve").is_err());
+    }
 }

crates/heal/src/heal/erasure_healer.rs

@@ -18,11 +18,15 @@ use crate::heal::{
     storage::HealStorageAPI,
 };
 use crate::{Error, Result};
-use futures::future::join_all;
+use futures::{StreamExt, future::join_all, stream::FuturesUnordered};
+use metrics::gauge;
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use rustfs_ecstore::disk::DiskStore;
-use std::sync::Arc;
-use tokio::sync::RwLock;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+use tokio::sync::{RwLock, Semaphore};
 use tracing::{error, info, warn};
 
 /// Erasure Set Healer
@@ -34,6 +38,29 @@ pub struct ErasureSetHealer {
 }
 
 impl ErasureSetHealer {
+    fn page_parallel_enabled() -> bool {
+        rustfs_utils::get_env_bool(
+            rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE,
+            rustfs_config::DEFAULT_HEAL_PAGE_PARALLEL_ENABLE,
+        )
+    }
+
+    fn heal_page_object_concurrency() -> usize {
+        rustfs_utils::get_env_usize(
+            rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY,
+            rustfs_config::DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY,
+        )
+        .max(1)
+    }
+
+    fn effective_heal_page_object_concurrency() -> usize {
+        if Self::page_parallel_enabled() {
+            Self::heal_page_object_concurrency()
+        } else {
+            1
+        }
+    }
+
     pub fn new(
         storage: Arc<dyn HealStorageAPI>,
         progress: Arc<RwLock<HealProgress>>,
@@ -61,7 +88,7 @@ impl ErasureSetHealer {
 
         // 3. execute heal with resume
         let result = self
-            .execute_heal_with_resume(buckets, &resume_manager, &checkpoint_manager)
+            .execute_heal_with_resume(buckets, set_disk_id, &resume_manager, &checkpoint_manager)
             .await;
 
         // 4. cleanup resume state
@@ -144,6 +171,7 @@ impl ErasureSetHealer {
     async fn execute_heal_with_resume(
         &self,
         buckets: &[String],
+        set_disk_id: &str,
         resume_manager: &ResumeManager,
         checkpoint_manager: &CheckpointManager,
     ) -> Result<()> {
@@ -182,6 +210,7 @@ impl ErasureSetHealer {
             let bucket_result = self
                 .heal_bucket_with_resume(
                     bucket,
+                    set_disk_id,
                     bucket_idx,
                     &mut current_object_index,
                     &mut processed_objects,
@@ -232,16 +261,17 @@ impl ErasureSetHealer {
 
     /// heal single bucket with resume
     #[allow(clippy::too_many_arguments)]
-    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, _skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
+    #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
     async fn heal_bucket_with_resume(
         &self,
         bucket: &str,
+        set_disk_id: &str,
         bucket_index: usize,
         current_object_index: &mut usize,
         processed_objects: &mut u64,
         successful_objects: &mut u64,
         failed_objects: &mut u64,
-        _skipped_objects: &mut u64,
+        skipped_objects: &mut u64,
         resume_manager: &ResumeManager,
         checkpoint_manager: &CheckpointManager,
     ) -> Result<()> {
@@ -259,6 +289,8 @@ impl ErasureSetHealer {
         // 2. process objects with pagination to avoid loading all objects into memory
         let mut continuation_token: Option<String> = None;
         let mut global_obj_idx = 0usize;
+        let page_concurrency_limit = Self::effective_heal_page_object_concurrency();
+        let in_flight = Arc::new(AtomicUsize::new(0));
 
         loop {
             // Get one page of objects
@@ -266,69 +298,139 @@ impl ErasureSetHealer {
                 .storage
                 .list_objects_for_heal_page(bucket, "", continuation_token.as_deref())
                 .await?;
+            let checkpoint = checkpoint_manager.get_checkpoint().await;
+            let page_resume_index = *current_object_index;
+            let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
+            let mut page_tasks = FuturesUnordered::new();
 
-            // Process objects in this page
             for object in objects {
-                // Skip objects before the checkpoint
-                if global_obj_idx < *current_object_index {
-                    global_obj_idx += 1;
+                let object_idx = global_obj_idx;
+                global_obj_idx += 1;
+
+                if object_idx < *current_object_index {
                     continue;
                 }
 
-                // check if already processed
-                if checkpoint_manager.get_checkpoint().await.processed_objects.contains(&object) {
-                    global_obj_idx += 1;
+                if checkpoint.processed_objects.contains(&object) || checkpoint.skipped_objects.contains(&object) {
                     continue;
                 }
 
-                // update current object
                 resume_manager
                     .set_current_item(Some(bucket.to_string()), Some(object.clone()))
                     .await?;
 
-                // Check if object still exists before attempting heal
-                let object_exists = match self.storage.object_exists(bucket, &object).await {
-                    Ok(exists) => exists,
-                    Err(e) => {
-                        warn!("Failed to check existence of {}/{}: {}, marking as failed", bucket, object, e);
-                        *failed_objects += 1;
-                        checkpoint_manager.add_failed_object(object.clone()).await?;
-                        global_obj_idx += 1;
-                        *current_object_index = global_obj_idx;
-                        continue;
-                    }
-                };
+                let storage = self.storage.clone();
+                let bucket_name = bucket.to_string();
+                let object_name = object.clone();
+                let cancel_token = self.cancel_token.clone();
+                let in_flight = in_flight.clone();
+                let set_label = set_disk_id.to_string();
+                let permit = semaphore
+                    .clone()
+                    .acquire_owned()
+                    .await
+                    .map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}")))?;
 
-                if !object_exists {
-                    info!(
-                        target: "rustfs:heal:heal_bucket_with_resume" ,"Object {}/{} no longer exists, skipping heal (likely deleted intentionally)",
-                        bucket, object
-                    );
-                    checkpoint_manager.add_processed_object(object.clone()).await?;
-                    *successful_objects += 1; // Treat as successful - object is gone as intended
-                    global_obj_idx += 1;
-                    *current_object_index = global_obj_idx;
-                    continue;
-                }
+                let current_in_flight = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
+                gauge!(
+                    "rustfs_heal_page_concurrency_current",
+                    "set" => set_label.clone()
+                )
+                .set(current_in_flight as f64);
 
-                // heal object
-                let heal_opts = HealOpts {
-                    scan_mode: HealScanMode::Normal,
-                    remove: true,
-                    recreate: true, // Keep recreate enabled for legitimate heal scenarios
-                    ..Default::default()
-                };
+                page_tasks.push(async move {
+                    let _permit = permit;
+                    let result = if cancel_token.is_cancelled() {
+                        Err(Error::TaskCancelled)
+                    } else {
+                        let object_exists = match storage.object_exists(&bucket_name, &object_name).await {
+                            Ok(exists) => exists,
+                            Err(err @ Error::TransientSkip { .. }) => {
+                                let current = in_flight.fetch_sub(1, Ordering::SeqCst) - 1;
+                                gauge!(
+                                    "rustfs_heal_page_concurrency_current",
+                                    "set" => set_label.clone()
+                                )
+                                .set(current as f64);
+                                return (object_name, Err(err));
+                            }
+                            Err(err) => {
+                                let object_name_for_error = object_name.clone();
+                                let current = in_flight.fetch_sub(1, Ordering::SeqCst) - 1;
+                                gauge!(
+                                    "rustfs_heal_page_concurrency_current",
+                                    "set" => set_label.clone()
+                                )
+                                .set(current as f64);
+                                return (
+                                    object_name,
+                                    Err(Error::other(format!(
+                                        "Failed to check existence of {}/{}: {}",
+                                        bucket_name, object_name_for_error, err
+                                    ))),
+                                );
+                            }
+                        };
 
-                match self.storage.heal_object(bucket, &object, None, &heal_opts).await {
-                    Ok((_result, None)) => {
+                        if !object_exists {
+                            Ok(false)
+                        } else {
+                            let heal_opts = HealOpts {
+                                scan_mode: HealScanMode::Normal,
+                                remove: true,
+                                recreate: true,
+                                ..Default::default()
+                            };
+                            match storage.heal_object(&bucket_name, &object_name, None, &heal_opts).await {
+                                Ok((_result, None)) => Ok(true),
+                                Ok((_, Some(err))) => Err(Error::other(err)),
+                                Err(err) => Err(err),
+                            }
+                        }
+                    };
+
+                    let current = in_flight.fetch_sub(1, Ordering::SeqCst) - 1;
+                    gauge!(
+                        "rustfs_heal_page_concurrency_current",
+                        "set" => set_label.clone()
+                    )
+                    .set(current as f64);
+
+                    (object_name, result)
+                });
+            }
+
+            let mut completed_in_page = 0usize;
+            while let Some((object, result)) = page_tasks.next().await {
+                match result {
+                    Ok(true) => {
                         *successful_objects += 1;
                         checkpoint_manager.add_processed_object(object.clone()).await?;
                         info!("Successfully healed object {}/{}", bucket, object);
                     }
-                    Ok((_, Some(err))) => {
-                        *failed_objects += 1;
-                        checkpoint_manager.add_failed_object(object.clone()).await?;
-                        warn!("Failed to heal object {}/{}: {}", bucket, object, err);
+                    Ok(false) => {
+                        checkpoint_manager.add_processed_object(object.clone()).await?;
+                        *successful_objects += 1;
+                        info!(
+                            target: "rustfs:heal:heal_bucket_with_resume" ,"Object {}/{} no longer exists, skipping heal (likely deleted intentionally)",
+                            bucket, object
+                        );
+                    }
+                    Err(Error::TaskCancelled) => {
+                        gauge!(
+                            "rustfs_heal_page_concurrency_current",
+                            "set" => set_disk_id.to_string()
+                        )
+                        .set(0.0);
+                        return Err(Error::TaskCancelled);
+                    }
+                    Err(Error::TransientSkip { message }) => {
+                        *skipped_objects += 1;
+                        checkpoint_manager.add_skipped_object(object.clone()).await?;
+                        warn!(
+                            "Skipping heal for object {}/{} due to transient existence check error: {}",
+                            bucket, object, message
+                        );
                     }
                     Err(err) => {
                         *failed_objects += 1;
@@ -338,23 +440,23 @@ impl ErasureSetHealer {
                 }
 
                 *processed_objects += 1;
-                global_obj_idx += 1;
-                *current_object_index = global_obj_idx;
+                completed_in_page += 1;
 
-                // check cancel status
-                if self.cancel_token.is_cancelled() {
-                    info!("Heal task cancelled during object processing");
-                    return Err(Error::TaskCancelled);
-                }
-
-                // save checkpoint periodically
-                if global_obj_idx.is_multiple_of(100) {
-                    checkpoint_manager
-                        .update_position(bucket_index, *current_object_index)
-                        .await?;
+                if completed_in_page.is_multiple_of(100) {
+                    checkpoint_manager.update_position(bucket_index, page_resume_index).await?;
                 }
             }
 
+            *current_object_index = global_obj_idx;
+            checkpoint_manager
+                .update_position(bucket_index, *current_object_index)
+                .await?;
+            gauge!(
+                "rustfs_heal_page_concurrency_current",
+                "set" => set_disk_id.to_string()
+            )
+            .set(0.0);
+
             // Check if there are more pages
             if !is_truncated {
                 break;
@@ -572,3 +674,34 @@ impl ErasureSetHealer {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::ErasureSetHealer;
+
+    #[test]
+    fn heal_page_object_concurrency_uses_default_when_env_is_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, || {
+            assert_eq!(
+                ErasureSetHealer::heal_page_object_concurrency(),
+                rustfs_config::DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY
+            );
+        });
+    }
+
+    #[test]
+    fn heal_page_object_concurrency_respects_env_override() {
+        temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("11"), || {
+            assert_eq!(ErasureSetHealer::heal_page_object_concurrency(), 11);
+        });
+    }
+
+    #[test]
+    fn effective_heal_page_object_concurrency_disables_parallelism_when_flag_is_off() {
+        temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("false"), || {
+            temp_env::with_var(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("11"), || {
+                assert_eq!(ErasureSetHealer::effective_heal_page_object_concurrency(), 1);
+            });
+        });
+    }
+}

crates/heal/src/heal/manager.rs

@@ -18,6 +18,8 @@ use crate::heal::{
     task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
 };
 use crate::{Error, Result};
+use metrics::{counter, gauge};
+use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
 use rustfs_ecstore::disk::DiskAPI;
 use rustfs_ecstore::disk::error::DiskError;
 use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
@@ -27,7 +29,7 @@ use std::{
     time::{Duration, SystemTime},
 };
 use tokio::{
-    sync::{Mutex, RwLock},
+    sync::{Mutex, Notify, RwLock},
     time::interval,
 };
 use tokio_util::sync::CancellationToken;
@@ -80,6 +82,12 @@ impl PartialOrd for PriorityQueueItem {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum QueuePushOutcome {
+    Accepted,
+    Merged,
+}
+
 impl PriorityHealQueue {
     fn new() -> Self {
         Self {
@@ -93,16 +101,24 @@ impl PriorityHealQueue {
         self.heap.len()
     }
 
+    fn pop_next(&mut self) -> Option<HealRequest> {
+        self.heap.pop().map(|item| {
+            let key = Self::make_dedup_key(&item.request);
+            self.dedup_keys.remove(&key);
+            item.request
+        })
+    }
+
     fn is_empty(&self) -> bool {
         self.heap.is_empty()
     }
 
-    fn push(&mut self, request: HealRequest) -> bool {
+    fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
         let key = Self::make_dedup_key(&request);
 
         // Check for duplicates
         if self.dedup_keys.contains(&key) {
-            return false; // Duplicate request, don't add
+            return QueuePushOutcome::Merged;
         }
 
         self.dedup_keys.insert(key);
@@ -112,7 +128,7 @@ impl PriorityHealQueue {
             sequence: self.sequence,
             request,
         });
-        true
+        QueuePushOutcome::Accepted
     }
 
     /// Get statistics about queue contents by priority
@@ -124,6 +140,7 @@ impl PriorityHealQueue {
         stats
     }
 
+    #[cfg(test)]
     fn pop(&mut self) -> Option<HealRequest> {
         self.heap.pop().map(|item| {
             let key = Self::make_dedup_key(&item.request);
@@ -132,6 +149,48 @@ impl PriorityHealQueue {
         })
     }
 
+    #[cfg(test)]
+    fn pop_runnable<F>(&mut self, can_run: F) -> Option<HealRequest>
+    where
+        F: Fn(&HealRequest) -> bool,
+    {
+        self.pop_runnable_with_skips(can_run, |_| None).0
+    }
+
+    fn pop_runnable_with_skips<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
+    where
+        F: Fn(&HealRequest) -> bool,
+        G: Fn(&HealRequest) -> Option<String>,
+    {
+        let mut deferred = Vec::new();
+        let mut selected = None;
+        let mut skipped = Vec::new();
+
+        while let Some(item) = self.heap.pop() {
+            if can_run(&item.request) {
+                selected = Some(item);
+                break;
+            }
+            if let Some(label) = skip_label(&item.request) {
+                skipped.push(label);
+            }
+            deferred.push(item);
+        }
+
+        for item in deferred {
+            self.heap.push(item);
+        }
+
+        (
+            selected.map(|item| {
+                let key = Self::make_dedup_key(&item.request);
+                self.dedup_keys.remove(&key);
+                item.request
+            }),
+            skipped,
+        )
+    }
+
     /// Create a deduplication key from a heal request
     fn make_dedup_key(request: &HealRequest) -> String {
         match &request.heal_type {
@@ -187,10 +246,22 @@ pub struct HealConfig {
     pub heal_interval: Duration,
     /// Maximum concurrent heal tasks
     pub max_concurrent_heals: usize,
+    /// Maximum concurrent heal tasks allowed for a single erasure set
+    pub max_concurrent_per_set: usize,
     /// Task timeout
     pub task_timeout: Duration,
     /// Queue size
     pub queue_size: usize,
+    /// Whether duplicate low-priority requests should merge into an existing queued request.
+    pub low_priority_merge_enable: bool,
+    /// Whether low-priority requests may be dropped when the queue is full.
+    pub low_priority_drop_when_full: bool,
+    /// Whether notify-driven scheduler wakeups are enabled.
+    pub event_driven_scheduler_enable: bool,
+    /// Whether per-set bulkhead scheduling is enabled.
+    pub set_bulkhead_enable: bool,
+    /// Whether erasure-set page parallelism is enabled.
+    pub page_parallel_enable: bool,
 }
 
 impl Default for HealConfig {
@@ -211,12 +282,42 @@ impl Default for HealConfig {
             rustfs_config::ENV_HEAL_MAX_CONCURRENT_HEALS,
             rustfs_config::DEFAULT_HEAL_MAX_CONCURRENT_HEALS,
         );
+        let max_concurrent_per_set = rustfs_utils::get_env_usize(
+            rustfs_config::ENV_HEAL_MAX_CONCURRENT_PER_SET,
+            rustfs_config::DEFAULT_HEAL_MAX_CONCURRENT_PER_SET,
+        );
+        let low_priority_merge_enable = rustfs_utils::get_env_bool(
+            rustfs_config::ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE,
+            rustfs_config::DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE,
+        );
+        let low_priority_drop_when_full = rustfs_utils::get_env_bool(
+            rustfs_config::ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL,
+            rustfs_config::DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL,
+        );
+        let event_driven_scheduler_enable = rustfs_utils::get_env_bool(
+            rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
+            rustfs_config::DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
+        );
+        let set_bulkhead_enable = rustfs_utils::get_env_bool(
+            rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE,
+            rustfs_config::DEFAULT_HEAL_SET_BULKHEAD_ENABLE,
+        );
+        let page_parallel_enable = rustfs_utils::get_env_bool(
+            rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE,
+            rustfs_config::DEFAULT_HEAL_PAGE_PARALLEL_ENABLE,
+        );
         Self {
             enable_auto_heal,
             heal_interval,        // 10 seconds
             max_concurrent_heals, // max 4,
-            task_timeout,         // 5 minutes
+            max_concurrent_per_set: std::cmp::min(max_concurrent_heals.max(1), max_concurrent_per_set.max(1)),
+            task_timeout, // 5 minutes
             queue_size,
+            low_priority_merge_enable,
+            low_priority_drop_when_full,
+            event_driven_scheduler_enable,
+            set_bulkhead_enable,
+            page_parallel_enable,
         }
     }
 }
@@ -254,9 +355,19 @@ pub struct HealManager {
     cancel_token: CancellationToken,
     /// Statistics
     statistics: Arc<RwLock<HealStatistics>>,
+    /// Scheduler wake-up notifier for event-driven dispatch
+    notify: Arc<Notify>,
 }
 
 impl HealManager {
+    fn classify_full_admission(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
+        if request.priority == HealPriority::Low && config.low_priority_drop_when_full {
+            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
+        } else {
+            HealAdmissionResult::Full
+        }
+    }
+
     /// Create new HealManager
     pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
         let config = config.unwrap_or_default();
@@ -268,6 +379,7 @@ impl HealManager {
             storage,
             cancel_token: CancellationToken::new(),
             statistics: Arc::new(RwLock::new(HealStatistics::new())),
+            notify: Arc::new(Notify::new()),
         }
     }
 
@@ -318,17 +430,63 @@ impl HealManager {
     }
 
     /// Submit heal request
-    pub async fn submit_heal_request(&self, request: HealRequest) -> Result<String> {
+    pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
         let config = self.config.read().await;
         let mut queue = self.heal_queue.lock().await;
 
         let queue_len = queue.len();
         let queue_capacity = config.queue_size;
 
+        if queue.contains_key(&request) {
+            let admission = if request.priority == HealPriority::Low && !config.low_priority_merge_enable {
+                HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
+            } else {
+                HealAdmissionResult::Merged
+            };
+
+            match admission {
+                HealAdmissionResult::Merged => {
+                    info!("Heal request already queued (duplicate merged): {}", request.id);
+                }
+                HealAdmissionResult::Dropped(reason) => {
+                    warn!(
+                        request_id = %request.id,
+                        priority = ?request.priority,
+                        reason = reason.as_str(),
+                        "Dropping duplicate heal request due to admission policy"
+                    );
+                }
+                HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
+            }
+
+            return Ok(admission);
+        }
+
         if queue_len >= queue_capacity {
-            return Err(Error::ConfigurationError {
-                message: format!("Heal queue is full ({queue_len}/{queue_capacity})"),
-            });
+            let admission = Self::classify_full_admission(&request, &config);
+            match admission {
+                HealAdmissionResult::Dropped(reason) => {
+                    warn!(
+                        request_id = %request.id,
+                        priority = ?request.priority,
+                        queue_len,
+                        queue_capacity,
+                        reason = reason.as_str(),
+                        "Dropping heal request because the queue is full"
+                    );
+                }
+                HealAdmissionResult::Full => {
+                    warn!(
+                        request_id = %request.id,
+                        priority = ?request.priority,
+                        queue_len,
+                        queue_capacity,
+                        "Rejecting heal request because the queue is full"
+                    );
+                }
+                HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
+            }
+            return Ok(admission);
         }
 
         // Warn when queue is getting full (>80% capacity)
@@ -345,8 +503,8 @@ impl HealManager {
         let request_id = request.id.clone();
         let priority = request.priority;
 
-        // Try to push the request; if it's a duplicate, still return the request_id
-        let is_new = queue.push(request);
+        let push_outcome = queue.push(request);
+        debug_assert_eq!(push_outcome, QueuePushOutcome::Accepted);
 
         // Log queue statistics periodically (when adding high/urgent priority items)
         if matches!(priority, HealPriority::High | HealPriority::Urgent) {
@@ -364,13 +522,12 @@ impl HealManager {
 
         drop(queue);
 
-        if is_new {
-            info!("Submitted heal request: {} with priority: {:?}", request_id, priority);
-        } else {
-            info!("Heal request already queued (duplicate): {}", request_id);
+        info!("Submitted heal request: {} with priority: {:?}", request_id, priority);
+        if config.event_driven_scheduler_enable {
+            self.notify.notify_one();
         }
 
-        Ok(request_id)
+        Ok(HealAdmissionResult::Accepted)
     }
 
     /// Get task status
@@ -441,18 +598,23 @@ impl HealManager {
         let cancel_token = self.cancel_token.clone();
         let statistics = self.statistics.clone();
         let storage = self.storage.clone();
+        let notify = self.notify.clone();
 
         tokio::spawn(async move {
             let mut interval = interval(config.read().await.heal_interval);
 
             loop {
+                let event_driven_scheduler_enable = config.read().await.event_driven_scheduler_enable;
                 tokio::select! {
                     _ = cancel_token.cancelled() => {
                         info!("Heal scheduler received shutdown signal");
                         break;
                     }
+                    _ = notify.notified(), if event_driven_scheduler_enable => {
+                        Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage, &notify).await;
+                    }
                     _ = interval.tick() => {
-                        Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage).await;
+                        Self::process_heal_queue(&heal_queue, &active_heals, &config, &statistics, &storage, &notify).await;
                     }
                 }
             }
@@ -468,6 +630,7 @@ impl HealManager {
         let active_heals = self.active_heals.clone();
         let cancel_token = self.cancel_token.clone();
         let storage = self.storage.clone();
+        let notify = self.notify.clone();
         let mut duration = {
             let config = config.read().await;
             config.heal_interval
@@ -567,8 +730,13 @@ impl HealManager {
                                 HealPriority::Normal,
                             );
                             let mut queue = heal_queue.lock().await;
-                            queue.push(req);
-                            info!("start_auto_disk_scanner: Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id);
+                            if matches!(queue.push(req), QueuePushOutcome::Accepted) {
+                                let config = config.read().await;
+                                if config.event_driven_scheduler_enable {
+                                    notify.notify_one();
+                                }
+                                info!("start_auto_disk_scanner: Enqueued auto erasure set heal for endpoint: {} (set_disk_id: {})", ep, set_disk_id);
+                            }
                         }
                     }
                 }
@@ -585,6 +753,7 @@ impl HealManager {
         config: &Arc<RwLock<HealConfig>>,
         statistics: &Arc<RwLock<HealStatistics>>,
         storage: &Arc<dyn HealStorageAPI>,
+        notify: &Arc<Notify>,
     ) {
         let config = config.read().await;
         let mut active_heals_guard = active_heals.lock().await;
@@ -605,28 +774,49 @@ impl HealManager {
             return;
         }
 
-        // Process multiple tasks if:
-        // 1. We have available slots
-        // 2. Queue is not empty
-        // Prioritize urgent/high priority tasks by processing up to 2 tasks per cycle if available
-        let tasks_to_process = if queue_len > 0 {
-            std::cmp::min(available_slots, std::cmp::min(2, queue_len))
-        } else {
-            0
-        };
+        let mut running_per_set = running_erasure_set_counts(&active_heals_guard);
+        let mut tasks_started = 0usize;
 
-        for _ in 0..tasks_to_process {
-            if let Some(request) = queue.pop() {
+        for _ in 0..available_slots {
+            let selected_request = if config.set_bulkhead_enable {
+                let max_concurrent_per_set = config.max_concurrent_per_set;
+                let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
+                    |request| can_schedule_request(request, &running_per_set, max_concurrent_per_set),
+                    |request| heal_request_set_key(request).map(|_| heal_request_set_metric_label(request)),
+                );
+                for skipped_set in skipped_sets {
+                    record_scheduler_skip(&skipped_set);
+                }
+                selected_request
+            } else {
+                queue.pop_next()
+            };
+
+            if let Some(request) = selected_request {
                 let task_priority = request.priority;
+                let task_type_label = heal_request_type_label(&request).to_string();
+                let task_set_label = heal_request_set_metric_label(&request);
+                if config.set_bulkhead_enable
+                    && let Some(set_key) = heal_request_set_key(&request)
+                {
+                    *running_per_set.entry(set_key).or_insert(0) += 1;
+                }
                 let task = Arc::new(HealTask::from_request(request, storage.clone()));
                 let task_id = task.id.clone();
                 active_heals_guard.insert(task_id.clone(), task.clone());
+                update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
                 let active_heals_clone = active_heals.clone();
                 let statistics_clone = statistics.clone();
+                let notify_clone = notify.clone();
+                let task_type_label_for_spawn = task_type_label.clone();
+                let task_set_label_for_spawn = task_set_label.clone();
 
                 // start heal task
                 tokio::spawn(async move {
-                    info!("Starting heal task: {} with priority: {:?}", task_id, task_priority);
+                    info!(
+                        "Starting heal task: {} with priority: {:?}, type: {}, set: {}",
+                        task_id, task_priority, task_type_label_for_spawn, task_set_label_for_spawn
+                    );
                     let result = task.execute().await;
                     match result {
                         Ok(_) => {
@@ -638,6 +828,7 @@ impl HealManager {
                     }
                     let mut active_heals_guard = active_heals_clone.lock().await;
                     if let Some(completed_task) = active_heals_guard.remove(&task_id) {
+                        update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
                         // update statistics
                         let mut stats = statistics_clone.write().await;
                         match completed_task.get_status().await {
@@ -650,7 +841,9 @@ impl HealManager {
                         }
                         stats.update_running_tasks(active_heals_guard.len() as u64);
                     }
+                    notify_clone.notify_one();
                 });
+                tasks_started += 1;
             } else {
                 break;
             }
@@ -658,7 +851,8 @@ impl HealManager {
 
         // Update statistics for all started tasks
         let mut stats = statistics.write().await;
-        stats.total_tasks += tasks_to_process as u64;
+        stats.total_tasks += tasks_started as u64;
+        stats.update_running_tasks(active_heals_guard.len() as u64);
 
         // Log queue status if items remain
         if !queue.is_empty() {
@@ -681,10 +875,180 @@ impl std::fmt::Debug for HealManager {
     }
 }
 
+fn heal_request_set_key(request: &HealRequest) -> Option<String> {
+    match &request.heal_type {
+        HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
+        _ => None,
+    }
+}
+
+fn heal_request_type_label(request: &HealRequest) -> &'static str {
+    match &request.heal_type {
+        HealType::Object { .. } => "object",
+        HealType::Bucket { .. } => "bucket",
+        HealType::ErasureSet { .. } => "erasure_set",
+        HealType::Metadata { .. } => "metadata",
+        HealType::MRF { .. } => "mrf",
+        HealType::ECDecode { .. } => "ec_decode",
+    }
+}
+
+fn heal_request_set_metric_label(request: &HealRequest) -> String {
+    heal_request_set_key(request).unwrap_or_else(|| match (request.options.pool_index, request.options.set_index) {
+        (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
+        _ => "global".to_string(),
+    })
+}
+
+fn record_scheduler_skip(set_label: &str) {
+    counter!(
+        "rustfs_heal_scheduler_skip_total",
+        "reason" => "set_limit".to_string(),
+        "set" => set_label.to_string()
+    )
+    .increment(1);
+}
+
+fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTask>>, task: &HealTask) {
+    let type_label = task.metric_type_label();
+    let set_label = task.metric_set_label();
+    let count = active_heals
+        .values()
+        .filter(|active_task| active_task.metric_type_label() == type_label && active_task.metric_set_label() == set_label)
+        .count();
+
+    gauge!(
+        "rustfs_heal_task_running",
+        "type" => type_label.to_string(),
+        "set" => set_label.clone()
+    )
+    .set(count as f64);
+}
+
+fn running_erasure_set_counts(active_heals: &HashMap<String, Arc<HealTask>>) -> HashMap<String, usize> {
+    let mut running = HashMap::new();
+    for task in active_heals.values() {
+        if let HealType::ErasureSet { set_disk_id, .. } = &task.heal_type {
+            *running.entry(set_disk_id.clone()).or_insert(0) += 1;
+        }
+    }
+    running
+}
+
+fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap<String, usize>, max_concurrent_per_set: usize) -> bool {
+    match heal_request_set_key(request) {
+        Some(set_key) => running_per_set.get(&set_key).copied().unwrap_or(0) < max_concurrent_per_set,
+        None => true,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::heal::storage::HealStorageAPI;
     use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
+    use rustfs_common::heal_channel::HealOpts;
+    use rustfs_ecstore::{
+        disk::{DiskStore, endpoint::Endpoint},
+        store_api::BucketInfo,
+    };
+    use rustfs_madmin::heal_commands::HealResultItem;
+
+    struct MockStorage;
+
+    #[async_trait::async_trait]
+    impl HealStorageAPI for MockStorage {
+        async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
+            Ok(None)
+        }
+
+        async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result<Option<Vec<u8>>> {
+            Ok(None)
+        }
+
+        async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> {
+            Ok(())
+        }
+
+        async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<()> {
+            Ok(())
+        }
+
+        async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result<bool> {
+            Ok(true)
+        }
+
+        async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result<Vec<u8>> {
+            Ok(Vec::new())
+        }
+
+        async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result<crate::heal::storage::DiskStatus> {
+            Ok(crate::heal::storage::DiskStatus::Ok)
+        }
+
+        async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> {
+            Ok(())
+        }
+
+        async fn get_bucket_info(&self, _bucket: &str) -> Result<Option<BucketInfo>> {
+            Ok(None)
+        }
+
+        async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> {
+            Ok(())
+        }
+
+        async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
+            Ok(Vec::new())
+        }
+
+        async fn object_exists(&self, _bucket: &str, _object: &str) -> Result<bool> {
+            Ok(false)
+        }
+
+        async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result<Option<u64>> {
+            Ok(None)
+        }
+
+        async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result<Option<String>> {
+            Ok(None)
+        }
+
+        async fn heal_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _version_id: Option<&str>,
+            _opts: &HealOpts,
+        ) -> Result<(HealResultItem, Option<Error>)> {
+            Ok((HealResultItem::default(), None))
+        }
+
+        async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result<HealResultItem> {
+            Ok(HealResultItem::default())
+        }
+
+        async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+            Ok((HealResultItem::default(), None))
+        }
+
+        async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result<Vec<String>> {
+            Ok(Vec::new())
+        }
+
+        async fn list_objects_for_heal_page(
+            &self,
+            _bucket: &str,
+            _prefix: &str,
+            _continuation_token: Option<&str>,
+        ) -> Result<(Vec<String>, Option<String>, bool)> {
+            Ok((Vec::new(), None, false))
+        }
+
+        async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result<DiskStore> {
+            Err(Error::other("not implemented in tests"))
+        }
+    }
 
     #[test]
     fn test_priority_queue_ordering() {
@@ -724,10 +1088,10 @@ mod tests {
         );
 
         // Add in random order: low, high, normal, urgent
-        assert!(queue.push(low_req));
-        assert!(queue.push(high_req));
-        assert!(queue.push(normal_req));
-        assert!(queue.push(urgent_req));
+        assert_eq!(queue.push(low_req), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(high_req), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(normal_req), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(urgent_req), QueuePushOutcome::Accepted);
 
         assert_eq!(queue.len(), 4);
 
@@ -780,9 +1144,9 @@ mod tests {
         let id2 = req2.id.clone();
         let id3 = req3.id.clone();
 
-        assert!(queue.push(req1));
-        assert!(queue.push(req2));
-        assert!(queue.push(req3));
+        assert_eq!(queue.push(req1), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(req2), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(req3), QueuePushOutcome::Accepted);
 
         // Should maintain FIFO order for same priority
         let popped1 = queue.pop().unwrap();
@@ -820,11 +1184,11 @@ mod tests {
         );
 
         // First request should be added
-        assert!(queue.push(req1));
+        assert_eq!(queue.push(req1), QueuePushOutcome::Accepted);
         assert_eq!(queue.len(), 1);
 
         // Second request with same object should be rejected (duplicate)
-        assert!(!queue.push(req2));
+        assert_eq!(queue.push(req2), QueuePushOutcome::Merged);
         assert_eq!(queue.len(), 1);
     }
 
@@ -841,7 +1205,7 @@ mod tests {
             HealPriority::Normal,
         );
 
-        assert!(queue.push(req));
+        assert_eq!(queue.push(req), QueuePushOutcome::Accepted);
         assert!(queue.contains_erasure_set("pool_0_set_1"));
         assert!(!queue.contains_erasure_set("pool_0_set_2"));
     }
@@ -929,7 +1293,8 @@ mod tests {
 
         for (heal_type, priority) in requests {
             let req = HealRequest::new(heal_type, HealOptions::default(), priority);
-            queue.push(req);
+            let outcome = queue.push(req);
+            assert_eq!(outcome, QueuePushOutcome::Accepted);
         }
 
         assert_eq!(queue.len(), 4);
@@ -954,32 +1319,41 @@ mod tests {
 
         // Add requests with different priorities
         for _ in 0..3 {
-            queue.push(HealRequest::new(
-                HealType::Bucket {
-                    bucket: format!("bucket-low-{}", queue.len()),
-                },
-                HealOptions::default(),
-                HealPriority::Low,
-            ));
+            assert_eq!(
+                queue.push(HealRequest::new(
+                    HealType::Bucket {
+                        bucket: format!("bucket-low-{}", queue.len()),
+                    },
+                    HealOptions::default(),
+                    HealPriority::Low,
+                )),
+                QueuePushOutcome::Accepted
+            );
         }
 
         for _ in 0..2 {
-            queue.push(HealRequest::new(
-                HealType::Bucket {
-                    bucket: format!("bucket-normal-{}", queue.len()),
-                },
-                HealOptions::default(),
-                HealPriority::Normal,
-            ));
+            assert_eq!(
+                queue.push(HealRequest::new(
+                    HealType::Bucket {
+                        bucket: format!("bucket-normal-{}", queue.len()),
+                    },
+                    HealOptions::default(),
+                    HealPriority::Normal,
+                )),
+                QueuePushOutcome::Accepted
+            );
         }
 
-        queue.push(HealRequest::new(
-            HealType::Bucket {
-                bucket: "bucket-high".to_string(),
-            },
-            HealOptions::default(),
-            HealPriority::High,
-        ));
+        assert_eq!(
+            queue.push(HealRequest::new(
+                HealType::Bucket {
+                    bucket: "bucket-high".to_string(),
+                },
+                HealOptions::default(),
+                HealPriority::High,
+            )),
+            QueuePushOutcome::Accepted
+        );
 
         let stats = queue.get_priority_stats();
 
@@ -995,13 +1369,16 @@ mod tests {
 
         assert!(queue.is_empty());
 
-        queue.push(HealRequest::new(
-            HealType::Bucket {
-                bucket: "test".to_string(),
-            },
-            HealOptions::default(),
-            HealPriority::Normal,
-        ));
+        assert_eq!(
+            queue.push(HealRequest::new(
+                HealType::Bucket {
+                    bucket: "test".to_string(),
+                },
+                HealOptions::default(),
+                HealPriority::Normal,
+            )),
+            QueuePushOutcome::Accepted
+        );
 
         assert!(!queue.is_empty());
 
@@ -1009,4 +1386,223 @@ mod tests {
 
         assert!(queue.is_empty());
     }
+
+    #[test]
+    fn test_priority_queue_pop_runnable_skips_blocked_erasure_set() {
+        let mut queue = PriorityHealQueue::new();
+
+        let blocked = HealRequest::new(
+            HealType::ErasureSet {
+                buckets: vec!["bucket-a".to_string()],
+                set_disk_id: "pool_0_set_1".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Urgent,
+        );
+        let runnable = HealRequest::new(
+            HealType::ErasureSet {
+                buckets: vec!["bucket-b".to_string()],
+                set_disk_id: "pool_0_set_2".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Normal,
+        );
+
+        assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(runnable.clone()), QueuePushOutcome::Accepted);
+
+        let mut running = HashMap::new();
+        running.insert("pool_0_set_1".to_string(), 1);
+
+        let popped = queue
+            .pop_runnable(|request| can_schedule_request(request, &running, 1))
+            .expect("should find runnable request");
+
+        match popped.heal_type {
+            HealType::ErasureSet { set_disk_id, .. } => assert_eq!(set_disk_id, "pool_0_set_2"),
+            other => panic!("expected erasure set request, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn test_can_schedule_request_respects_per_set_limit() {
+        let request = HealRequest::new(
+            HealType::ErasureSet {
+                buckets: vec!["bucket".to_string()],
+                set_disk_id: "pool_0_set_1".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Normal,
+        );
+
+        let mut running = HashMap::new();
+        running.insert("pool_0_set_1".to_string(), 1);
+
+        assert!(!can_schedule_request(&request, &running, 1));
+        assert!(can_schedule_request(&request, &running, 2));
+    }
+
+    #[tokio::test]
+    async fn test_submit_heal_request_returns_merged_for_duplicate() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(storage, None);
+
+        let request = HealRequest::new(
+            HealType::Object {
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+                version_id: None,
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+
+        assert_eq!(
+            manager
+                .submit_heal_request(request.clone())
+                .await
+                .expect("first request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .submit_heal_request(request)
+                .await
+                .expect("duplicate request should produce admission result"),
+            HealAdmissionResult::Merged
+        );
+    }
+
+    #[tokio::test]
+    async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(
+            storage,
+            Some(HealConfig {
+                queue_size: 1,
+                ..HealConfig::default()
+            }),
+        );
+
+        let request = HealRequest::new(
+            HealType::Object {
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+                version_id: None,
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+
+        assert_eq!(
+            manager
+                .submit_heal_request(request.clone())
+                .await
+                .expect("first request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .submit_heal_request(request)
+                .await
+                .expect("duplicate request should merge even when queue is full"),
+            HealAdmissionResult::Merged
+        );
+    }
+
+    #[tokio::test]
+    async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let manager = HealManager::new(
+            storage,
+            Some(HealConfig {
+                queue_size: 1,
+                low_priority_drop_when_full: true,
+                ..HealConfig::default()
+            }),
+        );
+
+        let accepted = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-a".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Normal,
+        );
+        let dropped = HealRequest::new(
+            HealType::Bucket {
+                bucket: "bucket-b".to_string(),
+            },
+            HealOptions::default(),
+            HealPriority::Low,
+        );
+
+        assert_eq!(
+            manager
+                .submit_heal_request(accepted)
+                .await
+                .expect("first request should be accepted"),
+            HealAdmissionResult::Accepted
+        );
+        assert_eq!(
+            manager
+                .submit_heal_request(dropped)
+                .await
+                .expect("low priority request should be dropped with explicit admission result"),
+            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
+        );
+    }
+
+    #[test]
+    fn test_running_erasure_set_counts_groups_only_erasure_tasks() {
+        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+        let erasure_task = Arc::new(HealTask::from_request(
+            HealRequest::new(
+                HealType::ErasureSet {
+                    buckets: vec!["bucket".to_string()],
+                    set_disk_id: "pool_0_set_1".to_string(),
+                },
+                HealOptions::default(),
+                HealPriority::Normal,
+            ),
+            storage.clone(),
+        ));
+        let object_task = Arc::new(HealTask::from_request(
+            HealRequest::new(
+                HealType::Object {
+                    bucket: "bucket".to_string(),
+                    object: "object".to_string(),
+                    version_id: None,
+                },
+                HealOptions::default(),
+                HealPriority::Normal,
+            ),
+            storage,
+        ));
+
+        let mut active = HashMap::new();
+        active.insert(erasure_task.id.clone(), erasure_task);
+        active.insert(object_task.id.clone(), object_task);
+
+        let counts = running_erasure_set_counts(&active);
+        assert_eq!(counts.get("pool_0_set_1"), Some(&1));
+        assert_eq!(counts.len(), 1);
+    }
+
+    #[test]
+    fn test_heal_config_respects_feature_flags() {
+        temp_env::with_vars(
+            [
+                (rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE, Some("false")),
+                (rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE, Some("false")),
+                (rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("false")),
+            ],
+            || {
+                let config = HealConfig::default();
+                assert!(!config.event_driven_scheduler_enable);
+                assert!(!config.set_bulkhead_enable);
+                assert!(!config.page_parallel_enable);
+            },
+        );
+    }
 }

crates/heal/src/heal/storage.rs

@@ -17,8 +17,11 @@ use async_trait::async_trait;
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use rustfs_ecstore::{
     disk::{DiskStore, endpoint::Endpoint},
+    error::StorageError,
     store::ECStore,
-    store_api::{BucketInfo, BucketOperations, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAPI},
+    store_api::{
+        BucketInfo, BucketOperations, HealOperations, ListOperations, ObjectIO, ObjectOperations, ObjectOptions, StorageAPI,
+    },
 };
 use rustfs_madmin::heal_commands::HealResultItem;
 use std::sync::Arc;
@@ -137,6 +140,37 @@ impl ECStoreHealStorage {
     }
 }
 
+fn is_transient_object_exists_message(message: &str) -> bool {
+    let message = message.to_ascii_lowercase();
+
+    [
+        "failed to acquire read lock",
+        "lock acquisition failed",
+        "lock acquisition timeout",
+        "quorum not reached",
+        "deadline has elapsed",
+        "timed out",
+        "network error",
+        "transport error",
+        "connection refused",
+    ]
+    .iter()
+    .any(|pattern| message.contains(pattern))
+}
+
+fn is_transient_object_exists_error(err: &StorageError) -> bool {
+    if err.is_quorum_error() {
+        return true;
+    }
+
+    match err {
+        StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()),
+        StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()),
+        StorageError::SlowDown | StorageError::OperationCanceled => true,
+        _ => false,
+    }
+}
+
 #[async_trait]
 impl HealStorageAPI for ECStoreHealStorage {
     async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
@@ -408,14 +442,24 @@ impl HealStorageAPI for ECStoreHealStorage {
     async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool> {
         debug!("Checking object exists: {}/{}", bucket, object);
 
-        // Use get_object_info for efficient existence check without heavy heal operations
-        match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
+        // Existence checks are best-effort for background heal scheduling, so avoid
+        // acquiring an extra namespace read lock here.
+        let opts = ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        };
+
+        match self.ecstore.get_object_info(bucket, object, &opts).await {
             Ok(_) => Ok(true), // Object exists
             Err(e) => {
-                // Map ObjectNotFound to false, other errors must be propagated!
                 if matches!(e, rustfs_ecstore::error::StorageError::ObjectNotFound(_, _)) {
                     debug!("Object not found: {}/{}", bucket, object);
                     Ok(false)
+                } else if is_transient_object_exists_error(&e) {
+                    warn!("Skipping object existence check for {}/{} due to transient error: {}", bucket, object, e);
+                    Err(Error::transient_skip(format!(
+                        "Skipped object existence check for {bucket}/{object}: {e}"
+                    )))
                 } else {
                     error!("Error checking object existence {}/{}: {}", bucket, object, e);
                     Err(Error::other(e))
@@ -594,3 +638,34 @@ impl HealStorageAPI for ECStoreHealStorage {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{is_transient_object_exists_error, is_transient_object_exists_message};
+    use rustfs_ecstore::error::StorageError;
+
+    #[test]
+    fn transient_object_exists_message_matches_lock_quorum_failures() {
+        assert!(is_transient_object_exists_message(
+            "Failed to acquire read lock: ns_loc: read lock acquisition failed on bucket/object: Quorum not reached: required 2, achieved 0"
+        ));
+        assert!(is_transient_object_exists_message("deadline has elapsed"));
+    }
+
+    #[test]
+    fn transient_object_exists_error_matches_quorum_variants() {
+        assert!(is_transient_object_exists_error(&StorageError::ErasureReadQuorum));
+        assert!(is_transient_object_exists_error(&StorageError::InsufficientReadQuorum(
+            "bucket".to_string(),
+            "object".to_string(),
+        )));
+    }
+
+    #[test]
+    fn transient_object_exists_error_does_not_treat_not_found_as_transient() {
+        assert!(!is_transient_object_exists_error(&StorageError::ObjectNotFound(
+            "bucket".to_string(),
+            "object".to_string(),
+        )));
+    }
+}

crates/heal/src/heal/task.rs

@@ -14,6 +14,7 @@
 
 use crate::heal::{ErasureSetHealer, progress::HealProgress, storage::HealStorageAPI};
 use crate::{Error, Result};
+use metrics::{counter, histogram};
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use serde::{Deserialize, Serialize};
 use std::{
@@ -133,16 +134,20 @@ pub struct HealRequest {
     pub priority: HealPriority,
     /// Created time
     pub created_at: SystemTime,
+    /// Queue admission time used for scheduler delay metrics
+    pub enqueued_at: SystemTime,
 }
 
 impl HealRequest {
     pub fn new(heal_type: HealType, options: HealOptions, priority: HealPriority) -> Self {
+        let now = SystemTime::now();
         Self {
             id: Uuid::new_v4().to_string(),
             heal_type,
             options,
             priority,
-            created_at: SystemTime::now(),
+            created_at: now,
+            enqueued_at: now,
         }
     }
 
@@ -193,6 +198,8 @@ pub struct HealTask {
     pub progress: Arc<RwLock<HealProgress>>,
     /// Created time
     pub created_at: SystemTime,
+    /// Queue admission time
+    pub enqueued_at: SystemTime,
     /// Started time
     pub started_at: Arc<RwLock<Option<SystemTime>>>,
     /// Completed time
@@ -214,6 +221,7 @@ impl HealTask {
             status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
             progress: Arc::new(RwLock::new(HealProgress::new())),
             created_at: request.created_at,
+            enqueued_at: request.enqueued_at,
             started_at: Arc::new(RwLock::new(None)),
             completed_at: Arc::new(RwLock::new(None)),
             task_start_instant: Arc::new(RwLock::new(None)),
@@ -222,6 +230,27 @@ impl HealTask {
         }
     }
 
+    pub fn metric_type_label(&self) -> &'static str {
+        match &self.heal_type {
+            HealType::Object { .. } => "object",
+            HealType::Bucket { .. } => "bucket",
+            HealType::ErasureSet { .. } => "erasure_set",
+            HealType::Metadata { .. } => "metadata",
+            HealType::MRF { .. } => "mrf",
+            HealType::ECDecode { .. } => "ec_decode",
+        }
+    }
+
+    pub fn metric_set_label(&self) -> String {
+        match &self.heal_type {
+            HealType::ErasureSet { set_disk_id, .. } => set_disk_id.clone(),
+            _ => match (self.options.pool_index, self.options.set_index) {
+                (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
+                _ => "global".to_string(),
+            },
+        }
+    }
+
     async fn remaining_timeout(&self) -> Result<Option<Duration>> {
         if let Some(total) = self.options.timeout {
             let start_instant = { *self.task_start_instant.read().await };
@@ -272,11 +301,26 @@ impl HealTask {
         }
     }
 
+    async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> {
+        warn!(
+            "Skipping heal for {}/{} due to transient object existence check error: {}",
+            bucket, object, err
+        );
+
+        let mut progress = self.progress.write().await;
+        progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
+        progress.update_progress(0, 1, 0, 0);
+        Ok(())
+    }
+
     #[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
     pub async fn execute(&self) -> Result<()> {
         // update status and timestamps atomically to avoid race conditions
         let now = SystemTime::now();
         let start_instant = Instant::now();
+        let queue_delay = now.duration_since(self.enqueued_at).unwrap_or_default();
+        let type_label = self.metric_type_label().to_string();
+        let set_label = self.metric_set_label();
         {
             let mut status = self.status.write().await;
             let mut started_at = self.started_at.write().await;
@@ -286,6 +330,19 @@ impl HealTask {
             *task_start_instant = Some(start_instant);
         }
 
+        histogram!(
+            "rustfs_heal_queue_delay_seconds",
+            "type" => type_label.clone(),
+            "set" => set_label.clone()
+        )
+        .record(queue_delay.as_secs_f64());
+        counter!(
+            "rustfs_heal_task_start_total",
+            "type" => type_label,
+            "set" => set_label
+        )
+        .increment(1);
+
         info!("Task started");
 
         let result = match &self.heal_type {
@@ -369,7 +426,13 @@ impl HealTask {
         // Step 1: Check if object exists and get metadata
         warn!("Step 1: Checking object existence and metadata");
         self.check_control_flags().await?;
-        let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
+        let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
+            Ok(exists) => exists,
+            Err(err @ Error::TransientSkip { .. }) => {
+                return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
+            }
+            Err(err) => return Err(err),
+        };
         if !object_exists {
             warn!("Object does not exist: {}/{}", bucket, object);
             if self.options.recreate_missing {
@@ -631,7 +694,13 @@ impl HealTask {
         // Step 1: Check if object exists
         info!("Step 1: Checking object existence");
         self.check_control_flags().await?;
-        let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
+        let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
+            Ok(exists) => exists,
+            Err(err @ Error::TransientSkip { .. }) => {
+                return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
+            }
+            Err(err) => return Err(err),
+        };
         if !object_exists {
             warn!("Object does not exist: {}/{}", bucket, object);
             return Err(Error::TaskExecutionFailed {
@@ -791,7 +860,13 @@ impl HealTask {
         // Step 1: Check if object exists
         info!("Step 1: Checking object existence");
         self.check_control_flags().await?;
-        let object_exists = self.await_with_control(self.storage.object_exists(bucket, object)).await?;
+        let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
+            Ok(exists) => exists,
+            Err(err @ Error::TransientSkip { .. }) => {
+                return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
+            }
+            Err(err) => return Err(err),
+        };
         if !object_exists {
             warn!("Object does not exist: {}/{}", bucket, object);
             return Err(Error::TaskExecutionFailed {

crates/heal/tests/heal_bug_fixes_test.rs

@@ -281,3 +281,152 @@ fn test_heal_task_status_atomic_update() {
     // Note: We can't directly access private fields, but creation without panic
     // confirms the fix works
 }
+
+#[tokio::test]
+async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
+    use rustfs_heal::heal::storage::{DiskStatus, HealStorageAPI};
+    use rustfs_heal::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+
+    struct MockStorage {
+        object_exists_calls: Arc<AtomicUsize>,
+        heal_object_calls: Arc<AtomicUsize>,
+    }
+
+    #[async_trait::async_trait]
+    impl HealStorageAPI for MockStorage {
+        async fn get_object_meta(
+            &self,
+            _bucket: &str,
+            _object: &str,
+        ) -> rustfs_heal::Result<Option<rustfs_ecstore::store_api::ObjectInfo>> {
+            Ok(None)
+        }
+
+        async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<Vec<u8>>> {
+            Ok(None)
+        }
+
+        async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> {
+            Ok(())
+        }
+
+        async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> {
+            Ok(())
+        }
+
+        async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
+            Ok(true)
+        }
+
+        async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Vec<u8>> {
+            Ok(Vec::new())
+        }
+
+        async fn get_disk_status(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<DiskStatus> {
+            Ok(DiskStatus::Ok)
+        }
+
+        async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<()> {
+            Ok(())
+        }
+
+        async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_ecstore::store_api::BucketInfo>> {
+            Ok(None)
+        }
+
+        async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> {
+            Ok(())
+        }
+
+        async fn list_buckets(&self) -> rustfs_heal::Result<Vec<rustfs_ecstore::store_api::BucketInfo>> {
+            Ok(Vec::new())
+        }
+
+        async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
+            self.object_exists_calls.fetch_add(1, Ordering::SeqCst);
+            Err(rustfs_heal::Error::transient_skip(
+                "Skipped object existence check for bucket/object: simulated quorum failure",
+            ))
+        }
+
+        async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<u64>> {
+            Ok(None)
+        }
+
+        async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<String>> {
+            Ok(None)
+        }
+
+        async fn heal_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _version_id: Option<&str>,
+            _opts: &rustfs_common::heal_channel::HealOpts,
+        ) -> rustfs_heal::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<rustfs_heal::Error>)> {
+            self.heal_object_calls.fetch_add(1, Ordering::SeqCst);
+            Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
+        }
+
+        async fn heal_bucket(
+            &self,
+            _bucket: &str,
+            _opts: &rustfs_common::heal_channel::HealOpts,
+        ) -> rustfs_heal::Result<rustfs_madmin::heal_commands::HealResultItem> {
+            Ok(rustfs_madmin::heal_commands::HealResultItem::default())
+        }
+
+        async fn heal_format(
+            &self,
+            _dry_run: bool,
+        ) -> rustfs_heal::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<rustfs_heal::Error>)> {
+            Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
+        }
+
+        async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result<Vec<String>> {
+            Ok(Vec::new())
+        }
+
+        async fn list_objects_for_heal_page(
+            &self,
+            _bucket: &str,
+            _prefix: &str,
+            _continuation_token: Option<&str>,
+        ) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
+            Ok((Vec::new(), None, false))
+        }
+
+        async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<rustfs_ecstore::disk::DiskStore> {
+            Err(rustfs_heal::Error::other("not implemented"))
+        }
+    }
+
+    let object_exists_calls = Arc::new(AtomicUsize::new(0));
+    let heal_object_calls = Arc::new(AtomicUsize::new(0));
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage {
+        object_exists_calls: object_exists_calls.clone(),
+        heal_object_calls: heal_object_calls.clone(),
+    });
+
+    let request = HealRequest::new(
+        HealType::Object {
+            bucket: "bucket".to_string(),
+            object: "object".to_string(),
+            version_id: None,
+        },
+        HealOptions::default(),
+        HealPriority::Normal,
+    );
+
+    let task = HealTask::from_request(request, storage);
+    task.execute().await.expect("transient existence check should be skipped");
+
+    assert_eq!(object_exists_calls.load(Ordering::SeqCst), 1);
+    assert_eq!(heal_object_calls.load(Ordering::SeqCst), 0);
+    assert_eq!(task.get_status().await, HealTaskStatus::Completed);
+    assert!(task.get_progress().await.is_completed());
+}

crates/heal/tests/heal_integration_test.rs

@@ -277,10 +277,12 @@ mod serial_tests {
             HealPriority::Normal,
         );
 
-        let task_id = heal_manager
+        let task_id = heal_request.id.clone();
+        let admission = heal_manager
             .submit_heal_request(heal_request)
             .await
             .expect("Failed to submit bucket heal request");
+        assert!(admission.is_admitted(), "bucket heal request should be admitted");
 
         info!("Submitted bucket heal request with task ID: {}", task_id);
 

crates/iam/Cargo.toml

@@ -58,6 +58,8 @@ url = { workspace = true }
 
 [dev-dependencies]
 pollster.workspace = true
+serial_test = { workspace = true }
+temp-env = { workspace = true }
 
 [lib]
 doctest = false

crates/iam/src/keyring.rs

@@ -0,0 +1,134 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_utils::get_env_opt_str;
+
+pub const ENV_IAM_MASTER_KEY: &str = "RUSTFS_IAM_MASTER_KEY";
+pub const ENV_IAM_MASTER_KEY_OLD_KEYS: &str = "RUSTFS_IAM_MASTER_KEY_OLD_KEYS";
+
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
+struct Keyring {
+    current_key: Option<Vec<u8>>,
+    decrypt_keys: Vec<Vec<u8>>,
+}
+
+fn normalize_key(value: Option<String>) -> Option<String> {
+    value.map(|v| v.trim().to_owned()).filter(|v| !v.is_empty())
+}
+
+fn parse_old_keys(value: Option<String>) -> Vec<String> {
+    let Some(value) = normalize_key(value) else {
+        return Vec::new();
+    };
+
+    value
+        .split(',')
+        .map(str::trim)
+        .filter(|v| !v.is_empty())
+        .map(str::to_owned)
+        .collect()
+}
+
+fn push_unique_key(keys: &mut Vec<Vec<u8>>, key: Vec<u8>) {
+    if !keys.iter().any(|k| k == &key) {
+        keys.push(key);
+    }
+}
+
+fn build_keyring(current_key: Option<String>, old_keys: Option<String>) -> Keyring {
+    let mut decrypt_keys = Vec::new();
+    let mut current = None;
+
+    if let Some(key) = normalize_key(current_key) {
+        let bytes = key.into_bytes();
+        current = Some(bytes.clone());
+        push_unique_key(&mut decrypt_keys, bytes);
+    }
+
+    for key in parse_old_keys(old_keys) {
+        push_unique_key(&mut decrypt_keys, key.into_bytes());
+    }
+
+    Keyring {
+        current_key: current,
+        decrypt_keys,
+    }
+}
+
+fn load_keyring() -> Keyring {
+    build_keyring(get_env_opt_str(ENV_IAM_MASTER_KEY), get_env_opt_str(ENV_IAM_MASTER_KEY_OLD_KEYS))
+}
+
+pub fn encrypt_key() -> Option<Vec<u8>> {
+    load_keyring().current_key
+}
+
+pub fn decrypt_keys() -> Vec<Vec<u8>> {
+    load_keyring().decrypt_keys
+}
+
+pub fn current_key_and_old_keys() -> (Option<Vec<u8>>, Vec<Vec<u8>>) {
+    let keyring = load_keyring();
+    let current = keyring.current_key.clone();
+    let old_keys = if current.is_some() {
+        keyring.decrypt_keys.into_iter().skip(1).collect()
+    } else {
+        keyring.decrypt_keys
+    };
+    (current, old_keys)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_keyring, parse_old_keys};
+
+    #[test]
+    fn test_parse_old_keys_ignores_empty_items() {
+        let keys = parse_old_keys(Some(" old-a, ,old-b,, old-c ".to_string()));
+        assert_eq!(keys, vec!["old-a".to_string(), "old-b".to_string(), "old-c".to_string()]);
+    }
+
+    #[test]
+    fn test_build_keyring_includes_current_then_old() {
+        let keyring = build_keyring(Some("current-key".to_string()), Some("old-key-1,old-key-2".to_string()));
+
+        assert_eq!(keyring.current_key, Some("current-key".as_bytes().to_vec()));
+        assert_eq!(
+            keyring.decrypt_keys,
+            vec![
+                "current-key".as_bytes().to_vec(),
+                "old-key-1".as_bytes().to_vec(),
+                "old-key-2".as_bytes().to_vec(),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_build_keyring_deduplicates_keys() {
+        let keyring = build_keyring(Some("k1".to_string()), Some("k1, k2, k2, k3".to_string()));
+
+        assert_eq!(
+            keyring.decrypt_keys,
+            vec!["k1".as_bytes().to_vec(), "k2".as_bytes().to_vec(), "k3".as_bytes().to_vec(),]
+        );
+    }
+
+    #[test]
+    fn test_build_keyring_uses_old_keys_without_current() {
+        let keyring = build_keyring(None, Some("legacy-a,legacy-b".to_string()));
+
+        assert_eq!(keyring.current_key, None);
+        assert_eq!(keyring.decrypt_keys, vec!["legacy-a".as_bytes().to_vec(), "legacy-b".as_bytes().to_vec()]);
+    }
+}

crates/iam/src/lib.rs

@@ -23,6 +23,7 @@ use tracing::{error, info, instrument, warn};
 
 pub mod cache;
 pub mod error;
+pub mod keyring;
 pub mod manager;
 pub mod oidc;
 pub mod oidc_state;
@@ -100,7 +101,7 @@ pub async fn init_oidc_sys() -> Result<()> {
         }
         Err(e) => {
             warn!("OIDC initialization failed (non-fatal): {}", e);
-            OidcSys::empty()
+            OidcSys::empty().map_err(Error::StringError)?
         }
     };
 

crates/iam/src/manager.rs

@@ -429,7 +429,7 @@ where
                 p.update(policy.clone());
                 p
             })
-            .unwrap_or(PolicyDoc::new(policy));
+            .unwrap_or_else(|| PolicyDoc::new(policy));
 
         self.api.save_policy_doc(name, policy_doc.clone()).await?;
 
@@ -797,7 +797,7 @@ where
                         Cache::add_or_update(&self.cache.groups, name, p, OffsetDateTime::now_utc());
                     }
 
-                    m.get(name).cloned().ok_or(Error::NoSuchGroup(name.to_string()))?
+                    m.get(name).cloned().ok_or_else(|| Error::NoSuchGroup(name.to_string()))?
                 }
             };
 
@@ -1452,7 +1452,7 @@ where
             .load()
             .get(name)
             .cloned()
-            .ok_or(Error::NoSuchGroup(name.to_string()))?;
+            .ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
 
         let mapped_policy = if let Some(policy) = self.cache.group_policies.load().get(name).cloned() {
             Some(policy)
@@ -1511,7 +1511,7 @@ where
             .load()
             .get(name)
             .cloned()
-            .ok_or(Error::NoSuchGroup(name.to_string()))?;
+            .ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
 
         let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
         let d: HashSet<&String> = HashSet::from_iter(members.iter());
@@ -1556,14 +1556,14 @@ where
             // Reload from backend so we see latest members (e.g. after user was deleted elsewhere)
             let mut m = HashMap::new();
             self.api.load_group(group, &mut m).await?;
-            m.get(group).cloned().ok_or(Error::NoSuchGroup(group.to_string()))?
+            m.get(group).cloned().ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
         } else {
             self.cache
                 .groups
                 .load()
                 .get(group)
                 .cloned()
-                .ok_or(Error::NoSuchGroup(group.to_string()))?
+                .ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
         };
 
         if members.is_empty() && !gi.members.is_empty() {

crates/iam/src/oidc.rs

@@ -24,6 +24,7 @@ use openidconnect::{
     AsyncHttpClient, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce, PkceCodeChallenge,
     PkceCodeVerifier, RedirectUrl, Scope,
 };
+use reqwest::Client;
 use rustfs_config::oidc::*;
 use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
 use rustfs_ecstore::config::{Config as ServerConfig, KVS, get_global_server_config};
@@ -32,13 +33,17 @@ use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::future::Future;
+use std::net::IpAddr;
 use std::pin::Pin;
 use std::sync::RwLock;
 use std::time::{Duration as StdDuration, Instant};
+use tokio::time::sleep;
 use tracing::{error, info, warn};
 use url::Url;
 
 const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
+const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
+const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
 
 // ---- HTTP Client Adapter ----
 
@@ -68,7 +73,46 @@ impl std::error::Error for OidcHttpError {
 }
 
 /// HTTP client adapter bridging reqwest 0.13 to the `openidconnect` `AsyncHttpClient` trait.
-pub(crate) struct ReqwestHttpClient(reqwest::Client);
+pub(crate) struct ReqwestHttpClient {
+    default: Client,
+    no_proxy: Client,
+}
+
+fn build_oidc_http_client(disable_proxy: bool) -> Result<Client, String> {
+    let mut builder = reqwest::Client::builder();
+    if disable_proxy {
+        builder = builder.no_proxy();
+    }
+    builder
+        .build()
+        .map_err(|err| format!("failed to build OIDC reqwest client: {err}"))
+}
+
+fn should_bypass_proxy_for_oidc_uri(uri: &str) -> bool {
+    let Some(host) = Url::parse(uri).ok().and_then(|url| url.host_str().map(str::to_owned)) else {
+        return false;
+    };
+    let host = host.trim_matches(['[', ']']);
+
+    host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|addr| addr.is_loopback())
+}
+
+impl ReqwestHttpClient {
+    fn new() -> Result<Self, String> {
+        Ok(Self {
+            default: build_oidc_http_client(false)?,
+            no_proxy: build_oidc_http_client(true)?,
+        })
+    }
+
+    fn client_for_uri(&self, uri: &str) -> &Client {
+        if should_bypass_proxy_for_oidc_uri(uri) {
+            &self.no_proxy
+        } else {
+            &self.default
+        }
+    }
+}
 
 impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
     type Error = OidcHttpError;
@@ -77,9 +121,10 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
     fn call(&'c self, request: http::Request<Vec<u8>>) -> Self::Future {
         Box::pin(async move {
             let (parts, body) = request.into_parts();
-            let response = self
-                .0
-                .request(parts.method, parts.uri.to_string())
+            let uri = parts.uri.to_string();
+            let client = self.client_for_uri(&uri);
+            let response = client
+                .request(parts.method, uri)
                 .headers(parts.headers)
                 .body(body)
                 .send()
@@ -193,7 +238,7 @@ pub struct OidcSys {
 impl OidcSys {
     /// Parse environment variables and discover all configured OIDC providers.
     pub async fn new() -> Result<Self, String> {
-        let http_client = ReqwestHttpClient(reqwest::Client::new());
+        let http_client = ReqwestHttpClient::new()?;
         let parsed_configs = load_effective_oidc_provider_configs(get_global_server_config().as_ref());
         let mut configs = HashMap::new();
         let mut provider_states = HashMap::new();
@@ -226,13 +271,13 @@ impl OidcSys {
     }
 
     /// Create an OidcSys with no providers (useful for when OIDC is not configured).
-    pub fn empty() -> Self {
-        Self {
+    pub fn empty() -> Result<Self, String> {
+        Ok(Self {
             configs: HashMap::new(),
             provider_states: RwLock::new(HashMap::new()),
             state_store: OidcStateStore::new(),
-            http_client: ReqwestHttpClient(reqwest::Client::new()),
-        }
+            http_client: ReqwestHttpClient::new()?,
+        })
     }
 
     /// Return true if any OIDC providers are configured and enabled.
@@ -848,22 +893,40 @@ impl OidcSys {
         for candidate_issuer in candidates.iter() {
             let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?;
 
-            match CoreProviderMetadata::discover_async(issuer_url, http_client)
-                .await
-                .map_err(|e| format!("discovery failed: {e}"))
-            {
-                Ok(metadata) => {
-                    return Ok(ProviderState {
-                        metadata,
-                        discovered_at: Instant::now(),
-                    });
-                }
-                Err(error) => {
-                    last_errors.push(format!("issuer '{candidate_issuer}': {error}"));
-                    warn!(
-                        "OIDC provider '{}' discovery attempt failed for issuer '{}': {}",
-                        config.id, candidate_issuer, error
-                    );
+            for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES {
+                match CoreProviderMetadata::discover_async(issuer_url.clone(), http_client)
+                    .await
+                    .map_err(|e| format!("discovery failed: {e}"))
+                {
+                    Ok(metadata) => {
+                        return Ok(ProviderState {
+                            metadata,
+                            discovered_at: Instant::now(),
+                        });
+                    }
+                    Err(error) => {
+                        let is_transient_transport = error.contains("Request failed");
+                        let should_retry = is_transient_transport && attempt + 1 < OIDC_DISCOVERY_TRANSPORT_RETRIES;
+                        if should_retry {
+                            warn!(
+                                "OIDC provider '{}' discovery transport attempt {}/{} failed for issuer '{}': {}",
+                                config.id,
+                                attempt + 1,
+                                OIDC_DISCOVERY_TRANSPORT_RETRIES,
+                                candidate_issuer,
+                                error
+                            );
+                            sleep(OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY).await;
+                            continue;
+                        }
+
+                        last_errors.push(format!("issuer '{candidate_issuer}': {error}"));
+                        warn!(
+                            "OIDC provider '{}' discovery attempt failed for issuer '{}': {}",
+                            config.id, candidate_issuer, error
+                        );
+                        break;
+                    }
                 }
             }
         }
@@ -924,7 +987,7 @@ pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>
 }
 
 pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
-    let http_client = ReqwestHttpClient(reqwest::Client::new());
+    let http_client = ReqwestHttpClient::new()?;
     let state = OidcSys::discover_provider(config, &http_client).await?;
 
     Ok(OidcProviderValidationResult {
@@ -1312,11 +1375,12 @@ mod tests {
         use std::io::Read;
         use std::io::Write;
         use std::net::{Shutdown, TcpListener};
+        use std::sync::mpsc;
         use std::time::{Duration, Instant};
 
         // After the last completed response, exit if no new connection arrives within this window.
-        const IDLE_SHUTDOWN: Duration = Duration::from_millis(100);
-        const ABSOLUTE_CAP: Duration = Duration::from_millis(500);
+        const IDLE_SHUTDOWN: Duration = Duration::from_millis(500);
+        const ABSOLUTE_CAP: Duration = Duration::from_secs(5);
 
         let listener = TcpListener::bind("127.0.0.1:0").unwrap();
         let base = format!("http://{}", listener.local_addr().unwrap());
@@ -1333,11 +1397,13 @@ mod tests {
         })
         .to_string();
         let jwks_body = r#"{"keys":[]}"#;
+        let (ready_tx, ready_rx) = mpsc::channel();
 
         let handle = std::thread::spawn(move || {
             listener
                 .set_nonblocking(true)
                 .expect("failed to set discovery mock listener non-blocking");
+            let _ = ready_tx.send(());
 
             let mut seen = 0usize;
             let start = Instant::now();
@@ -1404,6 +1470,9 @@ mod tests {
                 }
             }
         });
+        ready_rx
+            .recv_timeout(Duration::from_millis(100))
+            .expect("mock OIDC discovery server should become ready");
 
         (base, handle)
     }
@@ -1451,7 +1520,7 @@ mod tests {
 
     #[test]
     fn test_map_claims_to_policies_no_provider() {
-        let sys = OidcSys::empty();
+        let sys = OidcSys::empty().expect("failed to initialize empty OIDC system");
 
         let claims = OidcClaims {
             sub: "user123".to_string(),
@@ -1610,11 +1679,22 @@ mod tests {
 
     #[test]
     fn test_oidc_sys_empty() {
-        let sys = OidcSys::empty();
+        let sys = OidcSys::empty().expect("failed to initialize empty OIDC system");
         assert!(!sys.has_providers());
         assert!(sys.list_providers().is_empty());
     }
 
+    #[test]
+    fn test_should_bypass_proxy_for_oidc_uri_loopback_only() {
+        assert!(should_bypass_proxy_for_oidc_uri("http://127.0.0.1:9000/.well-known/openid-configuration"));
+        assert!(should_bypass_proxy_for_oidc_uri("http://localhost:9000/.well-known/openid-configuration"));
+        assert!(should_bypass_proxy_for_oidc_uri("http://[::1]:9000/.well-known/openid-configuration"));
+        assert!(!should_bypass_proxy_for_oidc_uri(
+            "https://idp.example.com/.well-known/openid-configuration"
+        ));
+        assert!(!should_bypass_proxy_for_oidc_uri("not-a-url"));
+    }
+
     /// Helper to create an OidcSys with configs only (no provider states needed).
     fn make_test_sys(configs: Vec<OidcProviderConfig>) -> OidcSys {
         let mut config_map = HashMap::new();
@@ -1625,7 +1705,7 @@ mod tests {
             configs: config_map,
             provider_states: RwLock::new(HashMap::new()),
             state_store: OidcStateStore::new(),
-            http_client: ReqwestHttpClient(reqwest::Client::new()),
+            http_client: ReqwestHttpClient::new().expect("failed to initialize OIDC HTTP clients"),
         }
     }
 

crates/iam/src/store/object.rs

@@ -17,23 +17,26 @@ use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group}
 use crate::{
     cache::{Cache, CacheEntity},
     error::{is_err_no_such_policy, is_err_no_such_user},
+    keyring,
     manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policyes},
 };
 use futures::future::join_all;
 use rustfs_credentials::get_global_action_cred;
+use rustfs_ecstore::error::StorageError;
 use rustfs_ecstore::store_api::{ListOperations as _, ObjectInfoOrErr, WalkOptions};
 use rustfs_ecstore::{
     config::{
         RUSTFS_CONFIG_PREFIX,
-        com::{delete_config, read_config, read_config_no_lock, read_config_with_metadata, save_config},
+        com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts},
     },
     store::ECStore,
-    store_api::{ObjectInfo, ObjectOptions},
+    store_api::{HTTPPreconditions, ObjectInfo, ObjectOptions},
 };
 use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
 use serde::{Serialize, de::DeserializeOwned};
-use std::sync::LazyLock;
+use std::sync::{LazyLock, Mutex};
+use std::time::{Duration, Instant};
 use std::{collections::HashMap, sync::Arc};
 use tokio::sync::mpsc::{self, Sender};
 use tokio_util::sync::CancellationToken;
@@ -104,6 +107,31 @@ const POLICY_DB_PREFIX: &str = "policydb/";
 const POLICY_DB_USERS_LIST_KEY: &str = "policydb/users/";
 const POLICY_DB_STS_USERS_LIST_KEY: &str = "policydb/sts-users/";
 const POLICY_DB_GROUPS_LIST_KEY: &str = "policydb/groups/";
+const IAM_LAZY_REWRITE_COOLDOWN: Duration = Duration::from_secs(60);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum DecryptSource {
+    Plaintext,
+    CurrentMasterKey,
+    OldMasterKey,
+    LegacySecretKey,
+    LegacyAccessSecretKey,
+}
+
+#[derive(Debug)]
+struct DecryptOutcome {
+    plain: Vec<u8>,
+    source: DecryptSource,
+}
+
+#[derive(Default, Clone, Copy, Debug)]
+struct LazyRewriteEntry {
+    in_flight: bool,
+    cooldown_until: Option<Instant>,
+}
+
+static IAM_LAZY_REWRITE_TRACKER: LazyLock<Mutex<HashMap<String, LazyRewriteEntry>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
 
 // split_path splits a path into a top-level directory and a child item. The
 // parent directory retains the trailing slash.
@@ -128,37 +156,83 @@ impl ObjectStore {
         Self { object_api }
     }
 
-    fn decrypt_data(data: &[u8]) -> Result<Vec<u8>> {
+    fn decrypt_data_with_source(data: &[u8]) -> Result<DecryptOutcome> {
         if Self::is_plaintext_json(data) {
-            return Ok(data.to_vec());
-        }
-
-        let cred = get_global_action_cred().unwrap_or_default();
-        let secret_key = cred.secret_key;
-        let mut keys: Vec<(Vec<u8>, bool)> = vec![(secret_key.clone().into_bytes(), false)];
-        if !cred.access_key.is_empty() && !secret_key.is_empty() {
-            keys.push((format!("{}:{secret_key}", cred.access_key).into_bytes(), true));
+            return Ok(DecryptOutcome {
+                plain: data.to_vec(),
+                source: DecryptSource::Plaintext,
+            });
         }
 
         const STREAM_IO_HEADER_LEN: usize = 41;
+        let cred = get_global_action_cred().unwrap_or_default();
         let mut last_err = None;
-        for (key, is_access_secret) in keys {
-            if is_access_secret
-                && data.len() >= STREAM_IO_HEADER_LEN
-                && let Ok(plain) = rustfs_crypto::decrypt_stream_io(&key, data)
+
+        let mut try_decrypt_with_key = |key: &[u8], source: DecryptSource| -> Option<DecryptOutcome> {
+            if data.len() >= STREAM_IO_HEADER_LEN
+                && let Ok(plain) = rustfs_crypto::decrypt_stream_io(key, data)
             {
-                return Ok(plain);
+                return Some(DecryptOutcome { plain, source });
             }
-            match rustfs_crypto::decrypt_data(&key, data) {
-                Ok(plain) => return Ok(plain),
-                Err(err) => last_err = Some(err),
+
+            match rustfs_crypto::decrypt_data(key, data) {
+                Ok(plain) => Some(DecryptOutcome { plain, source }),
+                Err(err) => {
+                    last_err = Some(err);
+                    None
+                }
+            }
+        };
+
+        let (current_key, old_keys) = keyring::current_key_and_old_keys();
+        if let Some(key) = current_key
+            && let Some(outcome) = try_decrypt_with_key(&key, DecryptSource::CurrentMasterKey)
+        {
+            return Ok(outcome);
+        }
+
+        for key in old_keys {
+            if let Some(outcome) = try_decrypt_with_key(&key, DecryptSource::OldMasterKey) {
+                return Ok(outcome);
+            }
+        }
+
+        let secret_key = cred.secret_key;
+        let mut legacy_keys = Vec::new();
+        if !secret_key.is_empty() {
+            legacy_keys.push((secret_key.clone().into_bytes(), DecryptSource::LegacySecretKey));
+        }
+        if !cred.access_key.is_empty() && !secret_key.is_empty() {
+            legacy_keys.push((
+                format!("{}:{secret_key}", cred.access_key).into_bytes(),
+                DecryptSource::LegacyAccessSecretKey,
+            ));
+        }
+
+        for (key, source) in legacy_keys {
+            if let Some(outcome) = try_decrypt_with_key(&key, source) {
+                return Ok(outcome);
             }
         }
 
         Err(last_err.unwrap_or(rustfs_crypto::Error::ErrUnexpectedHeader).into())
     }
 
+    fn encrypt_data_with_master_key(data: &[u8]) -> Result<Vec<u8>> {
+        let Some(master_key) = keyring::encrypt_key() else {
+            return Err(Error::other("iam master key is not configured"));
+        };
+
+        let encrypted = rustfs_crypto::encrypt_stream_io(&master_key, data)?;
+        Ok(encrypted)
+    }
+
     fn encrypt_data(data: &[u8]) -> Result<Vec<u8>> {
+        if keyring::encrypt_key().is_some() {
+            let encrypted = Self::encrypt_data_with_master_key(data)?;
+            return Ok(encrypted);
+        }
+
         let cred = get_global_action_cred().unwrap_or_default();
         let password = if !cred.access_key.is_empty() && !cred.secret_key.is_empty() {
             format!("{}:{}", cred.access_key, cred.secret_key).into_bytes()
@@ -169,6 +243,101 @@ impl ObjectStore {
         Ok(en)
     }
 
+    fn should_lazy_rewrite(source: DecryptSource) -> bool {
+        matches!(
+            source,
+            DecryptSource::Plaintext
+                | DecryptSource::OldMasterKey
+                | DecryptSource::LegacySecretKey
+                | DecryptSource::LegacyAccessSecretKey
+        )
+    }
+
+    fn begin_lazy_rewrite(path: &str) -> bool {
+        let Ok(mut tracker) = IAM_LAZY_REWRITE_TRACKER.lock() else {
+            return false;
+        };
+
+        let entry = tracker.entry(path.to_owned()).or_default();
+        if entry.in_flight {
+            return false;
+        }
+        if entry.cooldown_until.is_some_and(|deadline| deadline > Instant::now()) {
+            return false;
+        }
+
+        entry.in_flight = true;
+        entry.cooldown_until = None;
+        true
+    }
+
+    fn complete_lazy_rewrite(path: &str, success: bool) {
+        let Ok(mut tracker) = IAM_LAZY_REWRITE_TRACKER.lock() else {
+            return;
+        };
+
+        if success {
+            tracker.remove(path);
+            return;
+        }
+
+        let entry = tracker.entry(path.to_owned()).or_default();
+        entry.in_flight = false;
+        entry.cooldown_until = Some(Instant::now() + IAM_LAZY_REWRITE_COOLDOWN);
+    }
+
+    fn maybe_schedule_lazy_rewrite(&self, path: &str, outcome: &DecryptOutcome, object_info: &ObjectInfo) {
+        if !Self::should_lazy_rewrite(outcome.source) {
+            return;
+        }
+        if keyring::encrypt_key().is_none() {
+            return;
+        }
+
+        let Some(etag) = object_info.etag.clone() else {
+            return;
+        };
+
+        if !Self::begin_lazy_rewrite(path) {
+            return;
+        }
+
+        let path = path.to_owned();
+        let plain = outcome.plain.clone();
+        let store = self.clone();
+        tokio::spawn(async move {
+            let result = store.lazy_rewrite_iam_config(path.as_str(), &plain, etag.as_str()).await;
+            match result {
+                Ok(_) => {
+                    Self::complete_lazy_rewrite(path.as_str(), true);
+                }
+                Err(StorageError::PreconditionFailed) => {
+                    Self::complete_lazy_rewrite(path.as_str(), false);
+                    debug!("iam lazy rewrite skipped due to stale etag, path: {}", path);
+                }
+                Err(err) => {
+                    Self::complete_lazy_rewrite(path.as_str(), false);
+                    warn!("iam lazy rewrite failed, path: {}, err: {}", path, err);
+                }
+            }
+        });
+    }
+
+    async fn lazy_rewrite_iam_config(&self, path: &str, plain: &[u8], etag: &str) -> std::result::Result<(), StorageError> {
+        let encrypted = Self::encrypt_data_with_master_key(plain).map_err(StorageError::other)?;
+
+        let mut opts = ObjectOptions {
+            max_parity: true,
+            ..Default::default()
+        };
+        opts.http_preconditions = Some(HTTPPreconditions {
+            if_match: Some(etag.to_owned()),
+            ..Default::default()
+        });
+
+        save_config_with_opts(self.object_api.clone(), path, encrypted, &opts).await
+    }
+
     fn is_plaintext_json(data: &[u8]) -> bool {
         std::str::from_utf8(data).is_ok() && serde_json::from_slice::<serde_json::Value>(data).is_ok()
     }
@@ -179,9 +348,12 @@ impl ObjectStore {
     }
 
     async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef<str> + Send) -> Result<(Vec<u8>, ObjectInfo)> {
-        let (data, obj) = read_config_with_metadata(self.object_api.clone(), path.as_ref(), &ObjectOptions::default()).await?;
+        let path_ref = path.as_ref();
+        let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &ObjectOptions::default()).await?;
+        let outcome = Self::decrypt_data_with_source(&data)?;
+        self.maybe_schedule_lazy_rewrite(path_ref, &outcome, &obj);
 
-        Ok((Self::decrypt_data(&data)?, obj))
+        Ok((outcome.plain, obj))
     }
 
     async fn list_iam_config_items(&self, prefix: &str, ctx: CancellationToken, sender: Sender<StringOrErr>) {
@@ -444,18 +616,21 @@ impl Store for ObjectStore {
         false
     }
     async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
-        let mut data = read_config(self.object_api.clone(), path.as_ref()).await?;
+        let path_ref = path.as_ref();
+        let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &ObjectOptions::default()).await?;
 
-        data = match Self::decrypt_data(&data) {
+        let outcome = match Self::decrypt_data_with_source(&data) {
             Ok(v) => v,
             Err(err) => {
-                warn!("config decrypt failed, keeping file: {}, path: {}", err, path.as_ref());
+                warn!("config decrypt failed, keeping file: {}, path: {}", err, path_ref);
                 // keep the config file when decrypt failed - do not delete
                 return Err(Error::ConfigNotFound);
             }
         };
 
-        Ok(serde_json::from_slice(&data)?)
+        self.maybe_schedule_lazy_rewrite(path_ref, &outcome, &obj);
+
+        Ok(serde_json::from_slice(&outcome.plain)?)
     }
     /// Saves IAM configuration with a retry mechanism on failure.
     ///
@@ -1213,8 +1388,11 @@ impl Store for ObjectStore {
 
 #[cfg(test)]
 mod tests {
-    use super::ObjectStore;
+    use super::{DecryptSource, ObjectStore};
+    use crate::keyring;
     use rustfs_credentials::{Credentials, get_global_action_cred, init_global_action_credentials};
+    use serial_test::serial;
+    use temp_env::with_vars;
 
     fn test_cred() -> Credentials {
         if let Some(cred) = get_global_action_cred() {
@@ -1227,8 +1405,9 @@ mod tests {
     #[test]
     fn test_decrypt_data_accepts_plaintext_json() {
         let raw = br#"{"Version":1,"policy":"readonly"}"#;
-        let out = ObjectStore::decrypt_data(raw).expect("plaintext json should pass through");
-        assert_eq!(out, raw);
+        let out = ObjectStore::decrypt_data_with_source(raw).expect("plaintext json should pass through");
+        assert_eq!(out.plain, raw);
+        assert_eq!(out.source, DecryptSource::Plaintext);
     }
 
     #[test]
@@ -1236,8 +1415,8 @@ mod tests {
         let cred = test_cred();
         let plain = br#"{"accessKey":"ak","secretKey":"sk"}"#;
         let encrypted = rustfs_crypto::encrypt_data(cred.secret_key.as_bytes(), plain).expect("encrypt with rustfs secret");
-        let out = ObjectStore::decrypt_data(&encrypted).expect("decrypt rustfs legacy encryption");
-        assert_eq!(out, plain);
+        let out = ObjectStore::decrypt_data_with_source(&encrypted).expect("decrypt rustfs legacy encryption");
+        assert_eq!(out.plain, plain);
     }
 
     #[test]
@@ -1246,8 +1425,8 @@ mod tests {
         let plain = br#"{"Version":1,"updatedAt":"2025-03-07T12:00:00Z"}"#;
         let root_cred = format!("{}:{}", cred.access_key, cred.secret_key);
         let encrypted = rustfs_crypto::encrypt_stream_io(root_cred.as_bytes(), plain).expect("encrypt with stream_io");
-        let out = ObjectStore::decrypt_data(&encrypted).expect("decrypt stream_io");
-        assert_eq!(out, plain);
+        let out = ObjectStore::decrypt_data_with_source(&encrypted).expect("decrypt stream_io");
+        assert_eq!(out.plain, plain);
     }
 
     #[test]
@@ -1259,14 +1438,14 @@ mod tests {
         if encrypted.len() > 50 {
             encrypted[50] ^= 0xFF; // corrupt one byte
         }
-        let result = ObjectStore::decrypt_data(&encrypted);
+        let result = ObjectStore::decrypt_data_with_source(&encrypted);
         assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
     }
 
     #[test]
     fn test_decrypt_data_short_data_fails() {
         let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON
-        let result = ObjectStore::decrypt_data(short);
+        let result = ObjectStore::decrypt_data_with_source(short);
         assert!(result.is_err(), "short non-JSON data should fail decrypt");
     }
 
@@ -1286,7 +1465,63 @@ mod tests {
             "alg_id should be 0x00, 0x01, or 0x02"
         );
         // Round-trip: encrypt then decrypt
-        let decrypted = ObjectStore::decrypt_data(&encrypted).expect("decrypt should succeed");
-        assert_eq!(plain, decrypted.as_slice());
+        let decrypted = ObjectStore::decrypt_data_with_source(&encrypted).expect("decrypt should succeed");
+        assert_eq!(plain, decrypted.plain.as_slice());
+    }
+
+    #[test]
+    #[serial]
+    fn test_encrypt_data_prefers_iam_master_key_roundtrip() {
+        let _ = test_cred();
+        let plain = br#"{"Version":1,"policy":"master-key"}"#;
+        let master_key = "iam-master-key-roundtrip";
+
+        with_vars(
+            [
+                (keyring::ENV_IAM_MASTER_KEY, Some(master_key)),
+                (keyring::ENV_IAM_MASTER_KEY_OLD_KEYS, None),
+            ],
+            || {
+                let encrypted = ObjectStore::encrypt_data_for_test(plain).expect("encrypt with iam master key");
+
+                let by_master =
+                    rustfs_crypto::decrypt_stream_io(master_key.as_bytes(), &encrypted).expect("decrypt via master key");
+                assert_eq!(by_master, plain);
+
+                let by_object_store = ObjectStore::decrypt_data_with_source(&encrypted).expect("decrypt via object store");
+                assert_eq!(by_object_store.plain, plain);
+                assert_eq!(by_object_store.source, DecryptSource::CurrentMasterKey);
+            },
+        );
+    }
+
+    #[test]
+    #[serial]
+    fn test_decrypt_data_uses_iam_old_keys_fallback_during_rotation() {
+        let _ = test_cred();
+        let plain = br#"{"Version":1,"policy":"rotation-fallback"}"#;
+        let current_key = "iam-master-key-new";
+        let old_key_used_for_data = "iam-master-key-old-2";
+        let old_keys = format!("iam-master-key-old-1,{old_key_used_for_data}");
+
+        let encrypted = rustfs_crypto::encrypt_stream_io(old_key_used_for_data.as_bytes(), plain)
+            .expect("encrypt with old iam key for rotation simulation");
+
+        assert!(
+            rustfs_crypto::decrypt_stream_io(current_key.as_bytes(), &encrypted).is_err(),
+            "current master key should not decrypt old-key encrypted data in this test"
+        );
+
+        with_vars(
+            [
+                (keyring::ENV_IAM_MASTER_KEY, Some(current_key)),
+                (keyring::ENV_IAM_MASTER_KEY_OLD_KEYS, Some(old_keys.as_str())),
+            ],
+            || {
+                let decrypted = ObjectStore::decrypt_data_with_source(&encrypted).expect("decrypt via old-key fallback");
+                assert_eq!(decrypted.plain, plain);
+                assert_eq!(decrypted.source, DecryptSource::OldMasterKey);
+            },
+        );
     }
 }

crates/iam/src/sys.rs

@@ -134,6 +134,19 @@ impl PreparedIamAuth {
             }
         }
     }
+
+    /// Returns the resolved identity policy prepared for the current auth mode.
+    ///
+    /// This is intended for read-only views (for example `/accountinfo`) so
+    /// callers can reuse the same policy resolution path as authorization.
+    pub fn combined_policy_for_view(&self) -> Option<&Policy> {
+        match &self.mode {
+            PreparedIamMode::Regular { combined_policy } => Some(combined_policy),
+            PreparedIamMode::Sts { combined_policy, .. } => Some(combined_policy),
+            PreparedIamMode::ServiceAccount { combined_policy, .. } => Some(combined_policy),
+            PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => None,
+        }
+    }
 }
 
 impl<T: Store> IamSys<T> {
@@ -1286,6 +1299,31 @@ mod tests {
     use std::collections::HashMap;
     use time::OffsetDateTime;
 
+    #[test]
+    fn test_combined_policy_for_view_returns_regular_policy() {
+        let policy = Policy {
+            version: "2012-10-17".to_string(),
+            ..Default::default()
+        };
+        let prepared = PreparedIamAuth {
+            needs_existing_object_tag: false,
+            mode: PreparedIamMode::Regular { combined_policy: policy },
+        };
+
+        let resolved = prepared.combined_policy_for_view();
+        assert_eq!(resolved.map(|p| p.version.as_str()), Some("2012-10-17"));
+    }
+
+    #[test]
+    fn test_combined_policy_for_view_returns_none_for_deny() {
+        let prepared = PreparedIamAuth {
+            needs_existing_object_tag: false,
+            mode: PreparedIamMode::Deny,
+        };
+
+        assert!(prepared.combined_policy_for_view().is_none());
+    }
+
     /// Mock Store for STS tests: either group-attached policies via parent user, or no IAM policies.
     #[derive(Clone)]
     struct StsTestMockStore {

crates/io-core/Cargo.toml

@@ -30,7 +30,7 @@ workspace = true
 [dependencies]
 bytes = { workspace = true }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] }
+tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync","io-uring"] }
 memmap2 = { workspace = true }
 rustfs-io-metrics = { workspace = true }
 

crates/io-core/src/lock_optimizer.rs

@@ -124,11 +124,7 @@ impl LockStats {
     pub fn avg_hold_time(&self) -> Duration {
         let total = self.total_hold_time_ns.load(Ordering::Relaxed);
         let count = self.locks_acquired.load(Ordering::Relaxed);
-        if count == 0 {
-            Duration::ZERO
-        } else {
-            Duration::from_nanos(total / count)
-        }
+        total.checked_div(count).map(Duration::from_nanos).unwrap_or(Duration::ZERO)
     }
 
     /// Get maximum hold time.

crates/io-core/src/timeout_wrapper.rs

@@ -342,11 +342,7 @@ impl TimeoutStats {
     pub fn avg_wait_time(&self) -> Duration {
         let total = self.total_wait_time_ns.load(Ordering::Relaxed);
         let count = self.total_operations.load(Ordering::Relaxed);
-        if count == 0 {
-            Duration::ZERO
-        } else {
-            Duration::from_nanos(total / count)
-        }
+        total.checked_div(count).map(Duration::from_nanos).unwrap_or(Duration::ZERO)
     }
 
     /// Reset statistics.

crates/io-metrics/Cargo.toml

@@ -24,12 +24,21 @@ description = "Metrics collection and reporting for RustFS (using metrics crate
 keywords = ["metrics", "zero-copy", "rustfs", "otel", "performance"]
 categories = ["development-tools", "filesystem"]
 
+[[bench]]
+name = "metrics_pipeline"
+harness = false
+
 [dependencies]
 metrics = { workspace = true }
 num_cpus = { workspace = true }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["sync", "full"] }
+tokio = { workspace = true, features = ["sync","rt"] }
 tracing = { workspace = true }
+sysinfo = { workspace = true }
+
+[dev-dependencies]
+criterion = { workspace = true }
+tokio = { workspace = true, features = ["test-util","rt","macros"] }
 
 [lints]
 workspace = true

crates/io-metrics/README.md

@@ -31,6 +31,7 @@
 - **Bandwidth Monitoring**: Real-time bandwidth observation and analysis
 - **Performance Metrics**: I/O performance metrics collection
 - **Unified Configuration**: Centralized configuration management
+- **Exporter Boundary**: Emit via `metrics`, export via `rustfs-obs`, no Prometheus HTTP endpoint
 
 ## Features
 
@@ -199,7 +200,7 @@ cargo test --package rustfs-io-metrics
 cargo test --package rustfs-io-metrics --lib adaptive_ttl
 
 # Run benchmarks
-cargo bench --package rustfs-io-metrics
+cargo bench --package rustfs-io-metrics --bench metrics_pipeline
 ```
 
 ## Documentation

crates/io-metrics/README_zh.md

@@ -31,6 +31,7 @@
 - **带宽监控**：实时带宽观测和分析
 - **性能指标**：I/O 性能指标收集
 - **统一配置**：集中式配置管理
+- **导出边界**：通过 `metrics` 主动上报，由 `rustfs-obs` 负责 OTEL 导出，不提供 Prometheus HTTP 端点
 
 ## ✨ 核心功能
 
@@ -237,15 +238,6 @@ println!("最大并发读: {}", config.scheduler.max_concurrent_reads);
 
 ## 🔧 配置
 
-### 环境变量
-
-| 变量名 | 描述 | 默认值 |
-|--------|------|--------|
-| `RUSTFS_CACHE_MAX_CAPACITY` | 缓存最大容量 | 10000 |
-| `RUSTFS_CACHE_TTL_SECS` | 缓存 TTL 秒数 | 300 |
-| `RUSTFS_CACHE_MAX_MEMORY` | 缓存最大内存 | 104857600 |
-| `RUSTFS_ADAPTIVE_TTL_ENABLED` | 启用自适应 TTL | true |
-
 ### 代码配置
 
 ```rust
@@ -289,7 +281,7 @@ cargo test --package rustfs-io-metrics
 cargo test --package rustfs-io-metrics --lib adaptive_ttl
 
 # 运行基准测试
-cargo bench --package rustfs-io-metrics
+cargo bench --package rustfs-io-metrics --bench metrics_pipeline
 ```
 
 ## 📚 文档

crates/io-metrics/benches/metrics_pipeline.rs

@@ -0,0 +1,43 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics, record_get_object_request_started};
+use std::hint::black_box;
+use std::sync::Arc;
+use std::time::Duration;
+
+fn bench_record_get_object_request_started(c: &mut Criterion) {
+    c.bench_function("record_get_object_request_started", |b| b.iter(record_get_object_request_started));
+}
+
+fn bench_update_concurrent_requests(c: &mut Criterion) {
+    let metrics = PerformanceMetrics::new();
+
+    c.bench_function("performance_metrics_update_concurrent_requests", |b| {
+        b.iter(|| metrics.update_concurrent_requests(black_box(64)))
+    });
+}
+
+fn bench_metrics_collector_record_io_operation(c: &mut Criterion) {
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .expect("build tokio runtime for io-metrics benchmark");
+    let collector = MetricsCollector::new(Arc::new(PerformanceMetrics::new()), 256);
+
+    c.bench_function("metrics_collector_record_io_operation", |b| {
+        b.iter(|| {
+            runtime.block_on(collector.record_io_operation(
+                black_box(64 * 1024),
+                Duration::from_micros(black_box(250)),
+                black_box(true),
+            ))
+        })
+    });
+}
+
+criterion_group!(
+    benches,
+    bench_record_get_object_request_started,
+    bench_update_concurrent_requests,
+    bench_metrics_collector_record_io_operation
+);
+criterion_main!(benches);

crates/io-metrics/examples/metrics_example.rs

@@ -143,7 +143,7 @@ fn metrics_recording_example() {
 
     println!("  Recorded 10 cache operations (hits: 7, misses: 3)");
     println!("  Metrics reported via metrics crate");
-    println!("  View via Prometheus/Grafana");
+    println!("  Export via rustfs-obs OTEL pipeline");
 
     println!();
 }

crates/io-metrics/src/adaptive_ttl.rs

@@ -255,7 +255,7 @@ impl AccessTracker {
     /// Get keys sorted by access count (descending).
     pub fn top_keys(&self, n: usize) -> Vec<(&String, &AccessRecord)> {
         let mut entries: Vec<_> = self.records.iter().collect();
-        entries.sort_by(|a, b| b.1.count.cmp(&a.1.count));
+        entries.sort_by_key(|entry| std::cmp::Reverse(entry.1.count));
         entries.into_iter().take(n).collect()
     }
 

crates/io-metrics/src/lib.rs

@@ -23,6 +23,8 @@
 //! - **PerformanceMetrics**: Shared atomic counter struct for advanced use cases
 //! - **MetricsCollector**: I/O operation tracking with percentile calculation
 //! - **AutoTuner**: Automatic performance optimization based on metrics
+//! - **No HTTP metrics endpoint**: consumers emit metrics through the `metrics` crate;
+//!   `rustfs-obs` owns OTEL initialization and export
 //!
 //! # Usage
 //!
@@ -59,6 +61,8 @@ pub mod deadlock_metrics;
 pub mod io_metrics;
 pub mod lock_metrics;
 pub mod performance;
+pub mod process_lock_metrics;
+pub mod sampler;
 pub mod timeout_metrics;
 
 pub use autotuner::{AutoTuner, TunerConfig, TuningResult};
@@ -106,6 +110,16 @@ pub use lock_metrics::{
     record_spin_attempt, record_spin_count_change,
 };
 
+pub use process_lock_metrics::{
+    ProcessLockSnapshot, ProcessPlatformSnapshot, record_read_lock_held_acquire, record_read_lock_held_release,
+    record_write_lock_held_acquire, record_write_lock_held_release, snapshot_process_lock_counts,
+    snapshot_process_platform_stats,
+};
+pub use sampler::{
+    ProcessResourceSnapshot, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_platform, snapshot_process_resource,
+    snapshot_process_resource_and_system, snapshot_process_system,
+};
+
 // Timeout metrics exports
 pub use timeout_metrics::{
     TimeoutMetricsSummary, record_dynamic_timeout, record_operation_completion, record_operation_duration,

crates/io-metrics/src/process_lock_metrics.rs

@@ -0,0 +1,373 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+#[cfg(target_os = "linux")]
+use std::fs;
+#[cfg(any(
+    target_os = "windows",
+    target_os = "macos",
+    target_os = "freebsd",
+    target_os = "openbsd",
+    target_os = "netbsd",
+    target_os = "dragonfly"
+))]
+use std::process::Command;
+
+static READ_LOCKS_HELD: AtomicU64 = AtomicU64::new(0);
+static WRITE_LOCKS_HELD: AtomicU64 = AtomicU64::new(0);
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct ProcessLockSnapshot {
+    pub read_locks_held: u64,
+    pub write_locks_held: u64,
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct ProcessPlatformSnapshot {
+    pub io_rchar_bytes: Option<u64>,
+    pub io_read_bytes: Option<u64>,
+    pub io_wchar_bytes: Option<u64>,
+    pub io_write_bytes: Option<u64>,
+    pub syscall_read_total: Option<u64>,
+    pub syscall_write_total: Option<u64>,
+    pub virtual_memory_max_bytes: Option<u64>,
+}
+
+#[inline(always)]
+pub fn record_read_lock_held_acquire() {
+    READ_LOCKS_HELD.fetch_add(1, Ordering::Relaxed);
+}
+
+#[inline(always)]
+pub fn record_write_lock_held_acquire() {
+    WRITE_LOCKS_HELD.fetch_add(1, Ordering::Relaxed);
+}
+
+#[inline(always)]
+pub fn record_read_lock_held_release() {
+    let _ = READ_LOCKS_HELD.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1)));
+}
+
+#[inline(always)]
+pub fn record_write_lock_held_release() {
+    let _ = WRITE_LOCKS_HELD.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1)));
+}
+
+#[inline(always)]
+pub fn snapshot_process_lock_counts() -> ProcessLockSnapshot {
+    ProcessLockSnapshot {
+        read_locks_held: READ_LOCKS_HELD.load(Ordering::Relaxed),
+        write_locks_held: WRITE_LOCKS_HELD.load(Ordering::Relaxed),
+    }
+}
+
+#[inline]
+pub fn snapshot_process_platform_stats() -> ProcessPlatformSnapshot {
+    platform::snapshot()
+}
+
+#[cfg(any(
+    target_os = "windows",
+    target_os = "macos",
+    target_os = "freebsd",
+    target_os = "openbsd",
+    target_os = "netbsd",
+    target_os = "dragonfly"
+))]
+fn run_command(command: &str, args: &[&str]) -> Option<String> {
+    let output = Command::new(command).args(args).output().ok()?;
+    if !output.status.success() {
+        return None;
+    }
+
+    let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
+    if stdout.is_empty() { None } else { Some(stdout) }
+}
+
+#[cfg(any(target_os = "windows", test))]
+fn parse_kv_u64(content: &str, key: &str) -> Option<u64> {
+    for line in content.lines() {
+        let Some((k, v)) = line.split_once(':') else {
+            continue;
+        };
+        if k.trim().eq_ignore_ascii_case(key) {
+            return v.trim().parse::<u64>().ok();
+        }
+    }
+    None
+}
+
+#[cfg(target_os = "linux")]
+mod platform {
+    use super::*;
+
+    pub(super) fn snapshot() -> ProcessPlatformSnapshot {
+        let io = fs::read_to_string("/proc/self/io").ok();
+        let status = fs::read_to_string("/proc/self/status").ok();
+        let io_stats = io.as_deref().map(parse_proc_self_io).unwrap_or_default();
+
+        ProcessPlatformSnapshot {
+            io_rchar_bytes: io_stats.rchar_bytes,
+            io_read_bytes: io_stats.read_bytes,
+            io_wchar_bytes: io_stats.wchar_bytes,
+            io_write_bytes: io_stats.write_bytes,
+            syscall_read_total: io_stats.syscall_read_total,
+            syscall_write_total: io_stats.syscall_write_total,
+            virtual_memory_max_bytes: status.as_deref().and_then(parse_vm_peak_bytes),
+        }
+    }
+
+    #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+    struct ProcSelfIoStats {
+        rchar_bytes: Option<u64>,
+        read_bytes: Option<u64>,
+        syscall_read_total: Option<u64>,
+        syscall_write_total: Option<u64>,
+        wchar_bytes: Option<u64>,
+        write_bytes: Option<u64>,
+    }
+
+    fn parse_proc_self_io(content: &str) -> ProcSelfIoStats {
+        let mut stats = ProcSelfIoStats::default();
+
+        for line in content.lines() {
+            let Some((key, value)) = line.split_once(':') else {
+                continue;
+            };
+            let Ok(value) = value.trim().parse::<u64>() else {
+                continue;
+            };
+
+            match key.trim() {
+                "rchar" => stats.rchar_bytes = Some(value),
+                "read_bytes" => stats.read_bytes = Some(value),
+                "wchar" => stats.wchar_bytes = Some(value),
+                "write_bytes" => stats.write_bytes = Some(value),
+                "syscr" => stats.syscall_read_total = Some(value),
+                "syscw" => stats.syscall_write_total = Some(value),
+                _ => {}
+            }
+        }
+
+        stats
+    }
+
+    fn parse_vm_peak_bytes(content: &str) -> Option<u64> {
+        for line in content.lines() {
+            let Some(rest) = line.strip_prefix("VmPeak:") else {
+                continue;
+            };
+            let mut parts = rest.split_whitespace();
+            let value = parts.next()?.parse::<u64>().ok()?;
+            let unit = parts.next().unwrap_or_default();
+            return Some(match unit {
+                "kB" | "KB" | "kb" => value.saturating_mul(1024),
+                _ => value,
+            });
+        }
+        None
+    }
+
+    #[cfg(test)]
+    mod tests {
+        use super::*;
+
+        #[test]
+        fn parse_vm_peak() {
+            let status = "Name:\trustfs\nVmPeak:\t  2048 kB\nVmRSS:\t 1024 kB\n";
+            assert_eq!(parse_vm_peak_bytes(status), Some(2048 * 1024));
+        }
+
+        #[test]
+        fn parse_proc_self_io_extracts_expected_fields() {
+            let stats = parse_proc_self_io(
+                "rchar: 11\nwchar: 22\nsyscr: 33\nsyscw: 44\nread_bytes: 55\nwrite_bytes: 66\ncancelled_write_bytes: 77\n",
+            );
+
+            assert_eq!(stats.rchar_bytes, Some(11));
+            assert_eq!(stats.wchar_bytes, Some(22));
+            assert_eq!(stats.syscall_read_total, Some(33));
+            assert_eq!(stats.syscall_write_total, Some(44));
+            assert_eq!(stats.read_bytes, Some(55));
+            assert_eq!(stats.write_bytes, Some(66));
+        }
+    }
+}
+
+#[cfg(target_os = "windows")]
+mod platform {
+    use super::*;
+
+    pub(super) fn snapshot() -> ProcessPlatformSnapshot {
+        let pid = std::process::id();
+        let script = format!(
+            "Get-CimInstance Win32_Process -Filter \"ProcessId = {pid}\" | \
+             Format-List -Property ReadOperationCount,WriteOperationCount,PeakVirtualSize"
+        );
+
+        let output = run_command("powershell", &["-NoProfile", "-Command", &script]);
+        ProcessPlatformSnapshot {
+            syscall_read_total: output.as_deref().and_then(|v| parse_kv_u64(v, "ReadOperationCount")),
+            syscall_write_total: output.as_deref().and_then(|v| parse_kv_u64(v, "WriteOperationCount")),
+            virtual_memory_max_bytes: output.as_deref().and_then(|v| parse_kv_u64(v, "PeakVirtualSize")),
+            ..Default::default()
+        }
+    }
+}
+
+#[cfg(target_os = "macos")]
+mod platform {
+    use super::*;
+
+    pub(super) fn snapshot() -> ProcessPlatformSnapshot {
+        let pid = std::process::id().to_string();
+        let output = run_command("ps", &["-o", "inblock=", "-o", "oublock=", "-o", "vsz=", "-p", &pid]);
+        parse_ps_stats(output.as_deref())
+    }
+
+    fn parse_ps_stats(output: Option<&str>) -> ProcessPlatformSnapshot {
+        let Some(output) = output else {
+            return ProcessPlatformSnapshot::default();
+        };
+
+        let Some(line) = output.lines().find(|line| !line.trim().is_empty()) else {
+            return ProcessPlatformSnapshot::default();
+        };
+
+        let mut parts = line.split_whitespace();
+        let inblock = parts.next().and_then(|v| v.parse::<u64>().ok());
+        let oublock = parts.next().and_then(|v| v.parse::<u64>().ok());
+        let vsz_kb = parts.next().and_then(|v| v.parse::<u64>().ok());
+
+        ProcessPlatformSnapshot {
+            syscall_read_total: inblock,
+            syscall_write_total: oublock,
+            virtual_memory_max_bytes: vsz_kb.map(|v| v.saturating_mul(1024)),
+            ..Default::default()
+        }
+    }
+
+    #[cfg(test)]
+    mod tests {
+        use super::*;
+
+        #[test]
+        fn parse_ps_line() {
+            let snapshot = parse_ps_stats(Some("12 34 5678"));
+            assert_eq!(snapshot.syscall_read_total, Some(12));
+            assert_eq!(snapshot.syscall_write_total, Some(34));
+            assert_eq!(snapshot.virtual_memory_max_bytes, Some(5678 * 1024));
+        }
+    }
+}
+
+#[cfg(any(target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", target_os = "dragonfly"))]
+mod platform {
+    use super::*;
+
+    pub(super) fn snapshot() -> ProcessPlatformSnapshot {
+        let pid = std::process::id().to_string();
+        let output = run_command("ps", &["-o", "inblk=", "-o", "oublk=", "-o", "vsz=", "-p", &pid]);
+        parse_ps_stats(output.as_deref())
+    }
+
+    fn parse_ps_stats(output: Option<&str>) -> ProcessPlatformSnapshot {
+        let Some(output) = output else {
+            return ProcessPlatformSnapshot::default();
+        };
+
+        let Some(line) = output.lines().find(|line| !line.trim().is_empty()) else {
+            return ProcessPlatformSnapshot::default();
+        };
+
+        let mut parts = line.split_whitespace();
+        let inblock = parts.next().and_then(|v| v.parse::<u64>().ok());
+        let oublock = parts.next().and_then(|v| v.parse::<u64>().ok());
+        let vsz_kb = parts.next().and_then(|v| v.parse::<u64>().ok());
+
+        ProcessPlatformSnapshot {
+            syscall_read_total: inblock,
+            syscall_write_total: oublock,
+            virtual_memory_max_bytes: vsz_kb.map(|v| v.saturating_mul(1024)),
+            ..Default::default()
+        }
+    }
+}
+
+#[cfg(not(any(
+    target_os = "linux",
+    target_os = "windows",
+    target_os = "macos",
+    target_os = "freebsd",
+    target_os = "openbsd",
+    target_os = "netbsd",
+    target_os = "dragonfly"
+)))]
+mod platform {
+    use super::*;
+
+    pub(super) fn snapshot() -> ProcessPlatformSnapshot {
+        ProcessPlatformSnapshot::default()
+    }
+}
+
+#[cfg(test)]
+pub fn reset_process_lock_counts() {
+    READ_LOCKS_HELD.store(0, Ordering::Relaxed);
+    WRITE_LOCKS_HELD.store(0, Ordering::Relaxed);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn held_lock_counters_round_trip() {
+        reset_process_lock_counts();
+
+        record_read_lock_held_acquire();
+        record_read_lock_held_acquire();
+        record_write_lock_held_acquire();
+
+        assert_eq!(
+            snapshot_process_lock_counts(),
+            ProcessLockSnapshot {
+                read_locks_held: 2,
+                write_locks_held: 1,
+            }
+        );
+
+        record_read_lock_held_release();
+        record_write_lock_held_release();
+        record_write_lock_held_release();
+
+        assert_eq!(
+            snapshot_process_lock_counts(),
+            ProcessLockSnapshot {
+                read_locks_held: 1,
+                write_locks_held: 0,
+            }
+        );
+    }
+
+    #[test]
+    fn parse_kv_u64_round_trip() {
+        let content = "syscr: 123\nsyscw: 456\n";
+        assert_eq!(parse_kv_u64(content, "syscr"), Some(123));
+        assert_eq!(parse_kv_u64(content, "syscw"), Some(456));
+        assert_eq!(parse_kv_u64(content, "missing"), None);
+    }
+}

crates/io-metrics/src/sampler/mod.rs

@@ -0,0 +1,22 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod process;
+pub mod system;
+
+pub use process::{
+    ProcessResourceSnapshot, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_resource,
+    snapshot_process_resource_and_system, snapshot_process_system,
+};
+pub use system::snapshot_process_platform;

crates/io-metrics/src/sampler/process.rs

@@ -0,0 +1,166 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Mutex, OnceLock};
+use sysinfo::{Pid, ProcessRefreshKind, ProcessStatus, ProcessesToUpdate, System};
+
+static PROCESS_SYSTEM: OnceLock<Mutex<System>> = OnceLock::new();
+
+#[inline]
+fn current_pid() -> Pid {
+    Pid::from_u32(std::process::id())
+}
+
+#[inline]
+fn process_system() -> &'static Mutex<System> {
+    PROCESS_SYSTEM.get_or_init(|| {
+        let pid = current_pid();
+        let mut system = System::new();
+        system.refresh_processes_specifics(ProcessesToUpdate::Some(&[pid]), true, ProcessRefreshKind::everything());
+        Mutex::new(system)
+    })
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq)]
+pub struct ProcessResourceSnapshot {
+    pub cpu_percent: f64,
+    pub memory_bytes: u64,
+    pub uptime_seconds: u64,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum ProcessStatusSnapshot {
+    Running = 0,
+    Sleeping = 1,
+    Zombie = 2,
+    #[default]
+    Other = 3,
+}
+
+impl From<ProcessStatus> for ProcessStatusSnapshot {
+    fn from(status: ProcessStatus) -> Self {
+        match status {
+            ProcessStatus::Run => ProcessStatusSnapshot::Running,
+            ProcessStatus::Sleep => ProcessStatusSnapshot::Sleeping,
+            ProcessStatus::Zombie => ProcessStatusSnapshot::Zombie,
+            _ => ProcessStatusSnapshot::Other,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq)]
+pub struct ProcessSystemSnapshot {
+    pub locks_read_total: u64,
+    pub locks_write_total: u64,
+    pub cpu_total_seconds: f64,
+    pub go_routine_total: u64,
+    pub disk_read_bytes: u64,
+    pub disk_write_bytes: u64,
+    pub io_rchar_bytes: u64,
+    pub io_read_bytes: u64,
+    pub io_wchar_bytes: u64,
+    pub io_write_bytes: u64,
+    pub start_time_seconds: u64,
+    pub uptime_seconds: u64,
+    pub file_descriptor_limit_total: u64,
+    pub file_descriptor_open_total: u64,
+    pub syscall_read_total: u64,
+    pub syscall_write_total: u64,
+    pub resident_memory_bytes: u64,
+    pub virtual_memory_bytes: u64,
+    pub virtual_memory_max_bytes: u64,
+    pub status: ProcessStatusSnapshot,
+    pub status_value: i64,
+}
+
+/// Collect resource-only process snapshot for the current process.
+#[inline]
+pub fn snapshot_process_resource() -> ProcessResourceSnapshot {
+    snapshot_process_resource_and_system().0
+}
+
+/// Collect system-level process snapshot for the current process.
+#[inline]
+pub fn snapshot_process_system() -> ProcessSystemSnapshot {
+    snapshot_process_resource_and_system().1
+}
+
+/// Collect both resource and system snapshots in one sysinfo refresh.
+#[inline]
+pub fn snapshot_process_resource_and_system() -> (ProcessResourceSnapshot, ProcessSystemSnapshot) {
+    let platform_stats = crate::snapshot_process_platform_stats();
+    let lock_snapshot = crate::snapshot_process_lock_counts();
+    let pid = current_pid();
+    let mut sys = process_system().lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+    sys.refresh_processes_specifics(ProcessesToUpdate::Some(&[pid]), true, ProcessRefreshKind::everything());
+
+    if let Some(process) = sys.process(pid) {
+        let disk_usage = process.disk_usage();
+        let status = ProcessStatusSnapshot::from(process.status());
+        let uptime_seconds = process.run_time();
+
+        let resource_stats = ProcessResourceSnapshot {
+            cpu_percent: process.cpu_usage() as f64,
+            memory_bytes: process.memory(),
+            uptime_seconds,
+        };
+
+        let process_stats = ProcessSystemSnapshot {
+            locks_read_total: lock_snapshot.read_locks_held,
+            locks_write_total: lock_snapshot.write_locks_held,
+            cpu_total_seconds: process.accumulated_cpu_time() as f64 / 1000.0,
+            disk_read_bytes: disk_usage.read_bytes,
+            disk_write_bytes: disk_usage.written_bytes,
+            file_descriptor_limit_total: process.open_files_limit().map_or(0, |value| value as u64),
+            file_descriptor_open_total: process.open_files().map_or(0, |value| value as u64),
+            go_routine_total: process.tasks().map_or(0, |tasks| tasks.len() as u64),
+            io_rchar_bytes: platform_stats.io_rchar_bytes.unwrap_or(disk_usage.total_read_bytes),
+            io_read_bytes: platform_stats.io_read_bytes.unwrap_or(disk_usage.total_read_bytes),
+            io_wchar_bytes: platform_stats.io_wchar_bytes.unwrap_or(disk_usage.total_written_bytes),
+            io_write_bytes: platform_stats.io_write_bytes.unwrap_or(disk_usage.total_written_bytes),
+            resident_memory_bytes: process.memory(),
+            start_time_seconds: process.start_time(),
+            status,
+            status_value: status as i64,
+            syscall_read_total: platform_stats.syscall_read_total.unwrap_or(0),
+            syscall_write_total: platform_stats.syscall_write_total.unwrap_or(0),
+            uptime_seconds,
+            virtual_memory_bytes: process.virtual_memory(),
+            virtual_memory_max_bytes: platform_stats.virtual_memory_max_bytes.unwrap_or(0),
+        };
+
+        (resource_stats, process_stats)
+    } else {
+        (ProcessResourceSnapshot::default(), ProcessSystemSnapshot::default())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn process_status_snapshot_maps_sysinfo_values() {
+        assert_eq!(ProcessStatusSnapshot::from(ProcessStatus::Run), ProcessStatusSnapshot::Running);
+        assert_eq!(ProcessStatusSnapshot::from(ProcessStatus::Sleep), ProcessStatusSnapshot::Sleeping);
+        assert_eq!(ProcessStatusSnapshot::from(ProcessStatus::Zombie), ProcessStatusSnapshot::Zombie);
+    }
+
+    #[test]
+    fn process_snapshots_are_collectable() {
+        let _ = snapshot_process_resource();
+        let _ = snapshot_process_system();
+        let _ = snapshot_process_resource_and_system();
+    }
+}